MDL-70897 various: uasort callback can not return bool
[moodle.git] / search / classes / manager.php
CommitLineData
db48207e
DM
1<?php
2// This file is part of Moodle - http://moodle.org/
3//
4// Moodle is free software: you can redistribute it and/or modify
5// it under the terms of the GNU General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8//
9// Moodle is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12// GNU General Public License for more details.
13//
14// You should have received a copy of the GNU General Public License
15// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
16
17/**
18 * Search subsystem manager.
19 *
20 * @package core_search
21 * @copyright Prateek Sachan {@link http://prateeksachan.com}
22 * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
23 */
24
25namespace core_search;
26
27defined('MOODLE_INTERNAL') || die;
28
29require_once($CFG->dirroot . '/lib/accesslib.php');
30
31/**
32 * Search subsystem manager.
33 *
34 * @package core_search
35 * @copyright Prateek Sachan {@link http://prateeksachan.com}
36 * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
37 */
38class manager {
39
40 /**
41 * @var int Text contents.
42 */
43 const TYPE_TEXT = 1;
44
091973db
EM
45 /**
46 * @var int File contents.
47 */
48 const TYPE_FILE = 2;
49
db48207e
DM
50 /**
51 * @var int User can not access the document.
52 */
53 const ACCESS_DENIED = 0;
54
55 /**
56 * @var int User can access the document.
57 */
58 const ACCESS_GRANTED = 1;
59
60 /**
61 * @var int The document was deleted.
62 */
63 const ACCESS_DELETED = 2;
64
65 /**
66 * @var int Maximum number of results that will be retrieved from the search engine.
67 */
68 const MAX_RESULTS = 100;
69
70 /**
71 * @var int Number of results per page.
72 */
73 const DISPLAY_RESULTS_PER_PAGE = 10;
74
f6b425e2
EM
75 /**
76 * @var int The id to be placed in owneruserid when there is no owner.
77 */
78 const NO_OWNER_ID = 0;
79
1b8cf12a 80 /**
81 * @var float If initial query takes longer than N seconds, this will be shown in cron log.
82 */
83 const DISPLAY_LONG_QUERY_TIME = 5.0;
84
85 /**
86 * @var float Adds indexing progress within one search area to cron log every N seconds.
87 */
88 const DISPLAY_INDEXING_PROGRESS_EVERY = 30.0;
89
8736fbc1 90 /**
91 * @var int Context indexing: normal priority.
92 */
93 const INDEX_PRIORITY_NORMAL = 100;
94
95 /**
96 * @var int Context indexing: low priority for reindexing.
97 */
98 const INDEX_PRIORITY_REINDEXING = 50;
99
e9074ee3
DM
100 /**
101 * @var string Core search area category for all results.
102 */
103 const SEARCH_AREA_CATEGORY_ALL = 'core-all';
104
105 /**
106 * @var string Core search area category for course content.
107 */
108 const SEARCH_AREA_CATEGORY_COURSE_CONTENT = 'core-course-content';
109
110 /**
111 * @var string Core search area category for courses.
112 */
113 const SEARCH_AREA_CATEGORY_COURSES = 'core-courses';
114
115 /**
116 * @var string Core search area category for users.
117 */
118 const SEARCH_AREA_CATEGORY_USERS = 'core-users';
119
120 /**
121 * @var string Core search area category for results that do not fit into any of existing categories.
122 */
123 const SEARCH_AREA_CATEGORY_OTHER = 'core-other';
124
db48207e 125 /**
0bd8383a 126 * @var \core_search\base[] Enabled search areas.
db48207e
DM
127 */
128 protected static $enabledsearchareas = null;
129
130 /**
0bd8383a 131 * @var \core_search\base[] All system search areas.
db48207e
DM
132 */
133 protected static $allsearchareas = null;
134
e9074ee3
DM
135 /**
136 * @var \core_search\area_category[] A list of search area categories.
137 */
138 protected static $searchareacategories = null;
139
db48207e
DM
140 /**
141 * @var \core_search\manager
142 */
143 protected static $instance = null;
144
7ba2a201 145 /**
146 * @var array IDs (as keys) of course deletions in progress in this requuest, if any.
147 */
148 protected static $coursedeleting = [];
149
db48207e
DM
150 /**
151 * @var \core_search\engine
152 */
153 protected $engine = null;
154
82735dec 155 /**
156 * Note: This should be removed once possible (see MDL-60644).
157 *
158 * @var float Fake current time for use in PHPunit tests
159 */
160 protected static $phpunitfaketime = 0;
161
ed73ff2b 162 /**
163 * @var int Result count when used with mock results for Behat tests.
164 */
165 protected $behatresultcount = 0;
166
db48207e
DM
167 /**
168 * Constructor, use \core_search\manager::instance instead to get a class instance.
169 *
0bd8383a 170 * @param \core_search\base The search engine to use
db48207e
DM
171 */
172 public function __construct($engine) {
173 $this->engine = $engine;
174 }
175
73fd5666 176 /**
177 * @var int Record time of each successful schema check, but not more than once per 10 minutes.
178 */
179 const SCHEMA_CHECK_TRACKING_DELAY = 10 * 60;
180
181 /**
182 * @var int Require a new schema check at least every 4 hours.
183 */
184 const SCHEMA_CHECK_REQUIRED_EVERY = 4 * 3600;
185
db48207e
DM
186 /**
187 * Returns an initialised \core_search instance.
188 *
73fd5666 189 * While constructing the instance, checks on the search schema may be carried out. The $fast
190 * parameter provides a way to skip those checks on pages which are used frequently. It has
191 * no effect if an instance has already been constructed in this request.
192 *
679e8d8b 193 * The $query parameter indicates that the page is used for queries rather than indexing. If
194 * configured, this will cause the query-only search engine to be used instead of the 'normal'
195 * one.
196 *
23fc1be8
DM
197 * @see \core_search\engine::is_installed
198 * @see \core_search\engine::is_server_ready
73fd5666 199 * @param bool $fast Set to true when calling on a page that requires high performance
679e8d8b 200 * @param bool $query Set true on a page that is used for querying
db48207e
DM
201 * @throws \core_search\engine_exception
202 * @return \core_search\manager
203 */
679e8d8b 204 public static function instance(bool $fast = false, bool $query = false) {
db48207e
DM
205 global $CFG;
206
207 // One per request, this should be purged during testing.
208 if (static::$instance !== null) {
209 return static::$instance;
210 }
211
379ca986
DM
212 if (empty($CFG->searchengine)) {
213 throw new \core_search\engine_exception('enginenotselected', 'search');
214 }
215
679e8d8b 216 if (!$engine = static::search_engine_instance($query)) {
db48207e
DM
217 throw new \core_search\engine_exception('enginenotfound', 'search', '', $CFG->searchengine);
218 }
219
73fd5666 220 // Get time now and at last schema check.
221 $now = (int)self::get_current_time();
222 $lastschemacheck = get_config($engine->get_plugin_name(), 'lastschemacheck');
223
224 // On pages where performance matters, tell the engine to skip schema checks.
225 $skipcheck = false;
226 if ($fast && $now < $lastschemacheck + self::SCHEMA_CHECK_REQUIRED_EVERY) {
227 $skipcheck = true;
228 $engine->skip_schema_check();
229 }
230
db48207e
DM
231 if (!$engine->is_installed()) {
232 throw new \core_search\engine_exception('enginenotinstalled', 'search', '', $CFG->searchengine);
233 }
234
235 $serverstatus = $engine->is_server_ready();
236 if ($serverstatus !== true) {
e36eefae 237 // Skip this error in Behat when faking seach results.
238 if (!defined('BEHAT_SITE_RUNNING') || !get_config('core_search', 'behat_fakeresult')) {
73fd5666 239 // Clear the record of successful schema checks since it might have failed.
240 unset_config('lastschemacheck', $engine->get_plugin_name());
e36eefae 241 // Error message with no details as this is an exception that any user may find if the server crashes.
242 throw new \core_search\engine_exception('engineserverstatus', 'search');
243 }
db48207e
DM
244 }
245
73fd5666 246 // If we did a successful schema check, record this, but not more than once per 10 minutes
247 // (to avoid updating the config db table/cache too often in case it gets called frequently).
248 if (!$skipcheck && $now >= $lastschemacheck + self::SCHEMA_CHECK_TRACKING_DELAY) {
249 set_config('lastschemacheck', $now, $engine->get_plugin_name());
250 }
251
db48207e
DM
252 static::$instance = new \core_search\manager($engine);
253 return static::$instance;
254 }
255
256 /**
257 * Returns whether global search is enabled or not.
258 *
259 * @return bool
260 */
261 public static function is_global_search_enabled() {
262 global $CFG;
263 return !empty($CFG->enableglobalsearch);
264 }
265
36667548
SG
266 /**
267 * Returns the search URL for course search
268 *
269 * @return moodle_url
270 */
271 public static function get_course_search_url() {
272 if (self::is_global_search_enabled()) {
273 $searchurl = '/search/index.php';
274 } else {
275 $searchurl = '/course/search.php';
276 }
277
278 return new \moodle_url($searchurl);
279 }
280
d761b3fb 281 /**
282 * Returns whether indexing is enabled or not (you can enable indexing even when search is not
283 * enabled at the moment, so as to have it ready for students).
284 *
285 * @return bool True if indexing is enabled.
286 */
287 public static function is_indexing_enabled() {
288 global $CFG;
289 return !empty($CFG->enableglobalsearch) || !empty($CFG->searchindexwhendisabled);
290 }
291
db48207e
DM
292 /**
293 * Returns an instance of the search engine.
294 *
679e8d8b 295 * @param bool $query If true, gets the query-only search engine (where configured)
db48207e
DM
296 * @return \core_search\engine
297 */
679e8d8b 298 public static function search_engine_instance(bool $query = false) {
db48207e
DM
299 global $CFG;
300
679e8d8b 301 if ($query && $CFG->searchenginequeryonly) {
302 return self::search_engine_instance_from_setting($CFG->searchenginequeryonly);
303 } else {
304 return self::search_engine_instance_from_setting($CFG->searchengine);
305 }
306 }
307
308 /**
309 * Loads a search engine based on the name given in settings, which can optionally
310 * include '-alternate' to indicate that an alternate version should be used.
311 *
312 * @param string $setting
313 * @return engine|null
314 */
315 protected static function search_engine_instance_from_setting(string $setting): ?engine {
316 if (preg_match('~^(.*)-alternate$~', $setting, $matches)) {
317 $enginename = $matches[1];
318 $alternate = true;
319 } else {
320 $enginename = $setting;
321 $alternate = false;
322 }
323
324 $classname = '\\search_' . $enginename . '\\engine';
db48207e 325 if (!class_exists($classname)) {
679e8d8b 326 return null;
db48207e
DM
327 }
328
679e8d8b 329 if ($alternate) {
330 return new $classname(true);
331 } else {
332 // Use the constructor with no parameters for compatibility.
333 return new $classname();
334 }
db48207e
DM
335 }
336
337 /**
338 * Returns the search engine.
339 *
340 * @return \core_search\engine
341 */
342 public function get_engine() {
343 return $this->engine;
344 }
345
346 /**
347 * Returns a search area class name.
348 *
349 * @param string $areaid
350 * @return string
351 */
352 protected static function get_area_classname($areaid) {
353 list($componentname, $areaname) = static::extract_areaid_parts($areaid);
354 return '\\' . $componentname . '\\search\\' . $areaname;
355 }
356
357 /**
358 * Returns a new area search indexer instance.
359 *
360 * @param string $areaid
0bd8383a 361 * @return \core_search\base|bool False if the area is not available.
db48207e
DM
362 */
363 public static function get_search_area($areaid) {
364
b805d3f8 365 // We have them all here.
db48207e
DM
366 if (!empty(static::$allsearchareas[$areaid])) {
367 return static::$allsearchareas[$areaid];
368 }
db48207e
DM
369
370 $classname = static::get_area_classname($areaid);
f3d38863
DM
371
372 if (class_exists($classname) && static::is_search_area($classname)) {
db48207e
DM
373 return new $classname();
374 }
375
376 return false;
377 }
378
379 /**
380 * Return the list of available search areas.
381 *
382 * @param bool $enabled Return only the enabled ones.
0bd8383a 383 * @return \core_search\base[]
db48207e
DM
384 */
385 public static function get_search_areas_list($enabled = false) {
386
387 // Two different arrays, we don't expect these arrays to be big.
b805d3f8
DM
388 if (static::$allsearchareas !== null) {
389 if (!$enabled) {
390 return static::$allsearchareas;
391 } else {
392 return static::$enabledsearchareas;
393 }
db48207e
DM
394 }
395
b805d3f8
DM
396 static::$allsearchareas = array();
397 static::$enabledsearchareas = array();
0d6a45d2 398 $searchclasses = \core_component::get_component_classes_in_namespace(null, 'search');
db48207e 399
0d6a45d2
TD
400 foreach ($searchclasses as $classname => $classpath) {
401 $areaname = substr(strrchr($classname, '\\'), 1);
402 $componentname = strstr($classname, '\\', 1);
403 if (!static::is_search_area($classname)) {
404 continue;
db48207e 405 }
396d6f0a 406
0d6a45d2
TD
407 $areaid = static::generate_areaid($componentname, $areaname);
408 $searchclass = new $classname();
409 static::$allsearchareas[$areaid] = $searchclass;
410 if ($searchclass->is_enabled()) {
411 static::$enabledsearchareas[$areaid] = $searchclass;
db48207e
DM
412 }
413 }
414
db48207e 415 if ($enabled) {
b805d3f8 416 return static::$enabledsearchareas;
db48207e 417 }
b805d3f8 418 return static::$allsearchareas;
db48207e
DM
419 }
420
e9074ee3
DM
421 /**
422 * Return search area category instance by category name.
423 *
424 * @param string $name Category name. If name is not valid will return default category.
425 *
426 * @return \core_search\area_category
427 */
428 public static function get_search_area_category_by_name($name) {
429 if (key_exists($name, self::get_search_area_categories())) {
430 return self::get_search_area_categories()[$name];
431 } else {
432 return self::get_search_area_categories()[self::get_default_area_category_name()];
433 }
434 }
435
436 /**
437 * Return a list of existing search area categories.
438 *
439 * @return \core_search\area_category[]
440 */
441 public static function get_search_area_categories() {
442 if (!isset(static::$searchareacategories)) {
443 $categories = self::get_core_search_area_categories();
444
445 // Go through all existing search areas and get categories they are assigned to.
446 $areacategories = [];
447 foreach (self::get_search_areas_list() as $searcharea) {
448 foreach ($searcharea->get_category_names() as $categoryname) {
449 if (!key_exists($categoryname, $areacategories)) {
450 $areacategories[$categoryname] = [];
451 }
452
453 $areacategories[$categoryname][] = $searcharea;
454 }
455 }
456
457 // Populate core categories by areas.
458 foreach ($areacategories as $name => $searchareas) {
459 if (key_exists($name, $categories)) {
460 $categories[$name]->set_areas($searchareas);
461 } else {
462 throw new \coding_exception('Unknown core search area category ' . $name);
463 }
464 }
465
466 // Get additional categories.
467 $additionalcategories = self::get_additional_search_area_categories();
468 foreach ($additionalcategories as $additionalcategory) {
469 if (!key_exists($additionalcategory->get_name(), $categories)) {
470 $categories[$additionalcategory->get_name()] = $additionalcategory;
471 }
472 }
473
474 // Remove categories without areas.
475 foreach ($categories as $key => $category) {
476 if (empty($category->get_areas())) {
477 unset($categories[$key]);
478 }
479 }
480
481 // Sort categories by order.
482 uasort($categories, function($category1, $category2) {
881b4258 483 return $category1->get_order() <=> $category2->get_order();
e9074ee3
DM
484 });
485
486 static::$searchareacategories = $categories;
487 }
488
489 return static::$searchareacategories;
490 }
491
492 /**
493 * Get list of core search area categories.
494 *
495 * @return \core_search\area_category[]
496 */
497 protected static function get_core_search_area_categories() {
498 $categories = [];
499
500 $categories[self::SEARCH_AREA_CATEGORY_ALL] = new area_category(
501 self::SEARCH_AREA_CATEGORY_ALL,
502 get_string('core-all', 'search'),
503 0,
504 self::get_search_areas_list(true)
505 );
506
507 $categories[self::SEARCH_AREA_CATEGORY_COURSE_CONTENT] = new area_category(
508 self::SEARCH_AREA_CATEGORY_COURSE_CONTENT,
509 get_string('core-course-content', 'search'),
510 1
511 );
512
513 $categories[self::SEARCH_AREA_CATEGORY_COURSES] = new area_category(
514 self::SEARCH_AREA_CATEGORY_COURSES,
515 get_string('core-courses', 'search'),
516 2
517 );
518
519 $categories[self::SEARCH_AREA_CATEGORY_USERS] = new area_category(
520 self::SEARCH_AREA_CATEGORY_USERS,
521 get_string('core-users', 'search'),
522 3
523 );
524
525 $categories[self::SEARCH_AREA_CATEGORY_OTHER] = new area_category(
526 self::SEARCH_AREA_CATEGORY_OTHER,
527 get_string('core-other', 'search'),
528 4
529 );
530
531 return $categories;
532 }
533
534 /**
535 * Gets a list of additional search area categories.
536 *
537 * @return \core_search\area_category[]
538 */
539 protected static function get_additional_search_area_categories() {
540 $additionalcategories = [];
541
542 // Allow plugins to add custom search area categories.
543 if ($pluginsfunction = get_plugins_with_function('search_area_categories')) {
544 foreach ($pluginsfunction as $plugintype => $plugins) {
545 foreach ($plugins as $pluginfunction) {
546 $plugincategories = $pluginfunction();
547 // We're expecting a list of valid area categories.
548 if (is_array($plugincategories)) {
549 foreach ($plugincategories as $plugincategory) {
550 if (self::is_valid_area_category($plugincategory)) {
551 $additionalcategories[] = $plugincategory;
552 } else {
553 throw new \coding_exception('Invalid search area category!');
554 }
555 }
556 } else {
557 throw new \coding_exception($pluginfunction . ' should return a list of search area categories!');
558 }
559 }
560 }
561 }
562
563 return $additionalcategories;
564 }
565
566 /**
567 * Check if provided instance of area category is valid.
568 *
569 * @param mixed $areacategory Area category instance. Potentially could be anything.
570 *
571 * @return bool
572 */
573 protected static function is_valid_area_category($areacategory) {
574 return $areacategory instanceof area_category;
575 }
576
db48207e
DM
577 /**
578 * Clears all static caches.
579 *
580 * @return void
581 */
582 public static function clear_static() {
583
584 static::$enabledsearchareas = null;
585 static::$allsearchareas = null;
586 static::$instance = null;
e9074ee3 587 static::$searchareacategories = null;
a96faa49 588
589 base_block::clear_static();
26b86f31 590 engine::clear_users_cache();
db48207e
DM
591 }
592
593 /**
594 * Generates an area id from the componentname and the area name.
595 *
596 * There should not be any naming conflict as the area name is the
597 * class name in component/classes/search/.
598 *
599 * @param string $componentname
600 * @param string $areaname
601 * @return void
602 */
603 public static function generate_areaid($componentname, $areaname) {
604 return $componentname . '-' . $areaname;
605 }
606
607 /**
608 * Returns all areaid string components (component name and area name).
609 *
610 * @param string $areaid
611 * @return array Component name (Frankenstyle) and area name (search area class name)
612 */
613 public static function extract_areaid_parts($areaid) {
614 return explode('-', $areaid);
615 }
616
2085e860
DM
617 /**
618 * Parse a search area id and get plugin name and config name prefix from it.
619 *
620 * @param string $areaid Search area id.
621 * @return array Where the first element is a plugin name and the second is config names prefix.
622 */
623 public static function parse_areaid($areaid) {
624 $parts = self::extract_areaid_parts($areaid);
625
626 if (empty($parts[1])) {
627 throw new \coding_exception('Trying to parse invalid search area id ' . $areaid);
628 }
629
630 $component = $parts[0];
631 $area = $parts[1];
632
633 if (strpos($component, 'core') === 0) {
634 $plugin = 'core_search';
635 $configprefix = str_replace('-', '_', $areaid);
636 } else {
637 $plugin = $component;
638 $configprefix = 'search_' . $area;
639 }
640
641 return [$plugin, $configprefix];
642 }
643
db48207e 644 /**
4359ef18 645 * Returns information about the areas which the user can access.
db48207e 646 *
4359ef18 647 * The returned value is a stdClass object with the following fields:
648 * - everything (bool, true for admin only)
649 * - usercontexts (indexed by area identifier then context
650 * - separategroupscontexts (contexts within which group restrictions apply)
651 * - visiblegroupscontextsareas (overrides to the above when the same contexts also have
652 * 'visible groups' for certain search area ids - hopefully rare)
653 * - usergroups (groups which the current user belongs to)
db48207e 654 *
cfa00fc5 655 * The areas can be limited by course id and context id. If specifying context ids, results
656 * are limited to the exact context ids specified and not their children (for example, giving
657 * the course context id would result in including search items with the course context id, and
658 * not anything from a context inside the course). For performance, you should also specify
659 * course id(s) when using context ids.
660 *
427e3cbc 661 * @param array|false $limitcourseids An array of course ids to limit the search to. False for no limiting.
cfa00fc5 662 * @param array|false $limitcontextids An array of context ids to limit the search to. False for no limiting.
4359ef18 663 * @return \stdClass Object as described above
db48207e 664 */
cfa00fc5 665 protected function get_areas_user_accesses($limitcourseids = false, $limitcontextids = false) {
a96faa49 666 global $DB, $USER;
db48207e 667
cfa00fc5 668 // All results for admins (unless they have chosen to limit results). Eventually we could
669 // add a new capability for managers.
670 if (is_siteadmin() && !$limitcourseids && !$limitcontextids) {
4359ef18 671 return (object)array('everything' => true);
db48207e
DM
672 }
673
674 $areasbylevel = array();
675
676 // Split areas by context level so we only iterate only once through courses and cms.
677 $searchareas = static::get_search_areas_list(true);
678 foreach ($searchareas as $areaid => $unused) {
679 $classname = static::get_area_classname($areaid);
680 $searcharea = new $classname();
681 foreach ($classname::get_levels() as $level) {
682 $areasbylevel[$level][$areaid] = $searcharea;
683 }
684 }
685
686 // This will store area - allowed contexts relations.
687 $areascontexts = array();
688
4359ef18 689 // Initialise two special-case arrays for storing other information related to the contexts.
690 $separategroupscontexts = array();
691 $visiblegroupscontextsareas = array();
692 $usergroups = array();
693
427e3cbc 694 if (empty($limitcourseids) && !empty($areasbylevel[CONTEXT_SYSTEM])) {
db48207e
DM
695 // We add system context to all search areas working at this level. Here each area is fully responsible of
696 // the access control as we can not automate much, we can not even check guest access as some areas might
697 // want to allow guests to retrieve data from them.
698
699 $systemcontextid = \context_system::instance()->id;
cfa00fc5 700 if (!$limitcontextids || in_array($systemcontextid, $limitcontextids)) {
701 foreach ($areasbylevel[CONTEXT_SYSTEM] as $areaid => $searchclass) {
702 $areascontexts[$areaid][$systemcontextid] = $systemcontextid;
703 }
25ba053f
DM
704 }
705 }
706
707 if (!empty($areasbylevel[CONTEXT_USER])) {
708 if ($usercontext = \context_user::instance($USER->id, IGNORE_MISSING)) {
cfa00fc5 709 if (!$limitcontextids || in_array($usercontext->id, $limitcontextids)) {
710 // Extra checking although only logged users should reach this point, guest users have a valid context id.
711 foreach ($areasbylevel[CONTEXT_USER] as $areaid => $searchclass) {
712 $areascontexts[$areaid][$usercontext->id] = $usercontext->id;
713 }
25ba053f 714 }
db48207e
DM
715 }
716 }
717
cfa00fc5 718 if (is_siteadmin()) {
2085e860 719 $allcourses = $this->get_all_courses($limitcourseids);
cfa00fc5 720 } else {
2085e860
DM
721 $allcourses = $mycourses = $this->get_my_courses((bool)get_config('core', 'searchallavailablecourses'));
722
723 if (self::include_all_courses()) {
724 $allcourses = $this->get_all_courses($limitcourseids);
725 }
cfa00fc5 726 }
427e3cbc
EM
727
728 if (empty($limitcourseids) || in_array(SITEID, $limitcourseids)) {
2085e860
DM
729 $allcourses[SITEID] = get_course(SITEID);
730 if (isset($mycourses)) {
731 $mycourses[SITEID] = get_course(SITEID);
732 }
427e3cbc
EM
733 }
734
a96faa49 735 // Keep a list of included course context ids (needed for the block calculation below).
736 $coursecontextids = [];
4359ef18 737 $modulecms = [];
a96faa49 738
2085e860 739 foreach ($allcourses as $course) {
427e3cbc
EM
740 if (!empty($limitcourseids) && !in_array($course->id, $limitcourseids)) {
741 // Skip non-included courses.
742 continue;
743 }
db48207e 744
a96faa49 745 $coursecontext = \context_course::instance($course->id);
4359ef18 746 $hasgrouprestrictions = false;
a96faa49 747
cfa00fc5 748 if (!empty($areasbylevel[CONTEXT_COURSE]) &&
749 (!$limitcontextids || in_array($coursecontext->id, $limitcontextids))) {
db48207e 750 // Add the course contexts the user can view.
db48207e 751 foreach ($areasbylevel[CONTEXT_COURSE] as $areaid => $searchclass) {
beff3806 752 if (!empty($mycourses[$course->id]) || \core_course_category::can_view_course_info($course)) {
db48207e
DM
753 $areascontexts[$areaid][$coursecontext->id] = $coursecontext->id;
754 }
755 }
756 }
757
2085e860
DM
758 // Skip module context if a user can't access related course.
759 if (isset($mycourses) && !key_exists($course->id, $mycourses)) {
760 continue;
761 }
762
763 $coursecontextids[] = $coursecontext->id;
764
765 // Info about the course modules.
766 $modinfo = get_fast_modinfo($course);
767
db48207e
DM
768 if (!empty($areasbylevel[CONTEXT_MODULE])) {
769 // Add the module contexts the user can view (cm_info->uservisible).
770
771 foreach ($areasbylevel[CONTEXT_MODULE] as $areaid => $searchclass) {
772
773 // Removing the plugintype 'mod_' prefix.
774 $modulename = substr($searchclass->get_component_name(), 4);
775
776 $modinstances = $modinfo->get_instances_of($modulename);
777 foreach ($modinstances as $modinstance) {
cfa00fc5 778 // Skip module context if not included in list of context ids.
779 if ($limitcontextids && !in_array($modinstance->context->id, $limitcontextids)) {
780 continue;
781 }
db48207e 782 if ($modinstance->uservisible) {
4359ef18 783 $contextid = $modinstance->context->id;
784 $areascontexts[$areaid][$contextid] = $contextid;
785 $modulecms[$modinstance->id] = $modinstance;
786
787 if (!has_capability('moodle/site:accessallgroups', $modinstance->context) &&
788 ($searchclass instanceof base_mod) &&
789 $searchclass->supports_group_restriction()) {
790 if ($searchclass->restrict_cm_access_by_group($modinstance)) {
791 $separategroupscontexts[$contextid] = $contextid;
792 $hasgrouprestrictions = true;
793 } else {
794 // Track a list of anything that has a group id (so might get
795 // filtered) and doesn't want to be, in this context.
796 if (!array_key_exists($contextid, $visiblegroupscontextsareas)) {
797 $visiblegroupscontextsareas[$contextid] = array();
798 }
799 $visiblegroupscontextsareas[$contextid][$areaid] = $areaid;
800 }
801 }
db48207e
DM
802 }
803 }
804 }
805 }
4359ef18 806
807 // Insert group information for course (unless there aren't any modules restricted by
808 // group for this user in this course, in which case don't bother).
809 if ($hasgrouprestrictions) {
810 $groups = groups_get_all_groups($course->id, $USER->id, 0, 'g.id');
811 foreach ($groups as $group) {
812 $usergroups[$group->id] = $group->id;
813 }
814 }
815 }
816
817 // Chuck away all the 'visible groups contexts' data unless there is actually something
818 // that does use separate groups in the same context (this data is only used as an
819 // 'override' in cases where the search is restricting to separate groups).
820 foreach ($visiblegroupscontextsareas as $contextid => $areas) {
821 if (!array_key_exists($contextid, $separategroupscontexts)) {
822 unset($visiblegroupscontextsareas[$contextid]);
823 }
db48207e
DM
824 }
825
e7e62a4b
PH
826 // Add all supported block contexts for course contexts that user can access, in a single query for performance.
827 if (!empty($areasbylevel[CONTEXT_BLOCK]) && !empty($coursecontextids)) {
a96faa49 828 // Get list of all block types we care about.
829 $blocklist = [];
830 foreach ($areasbylevel[CONTEXT_BLOCK] as $areaid => $searchclass) {
831 $blocklist[$searchclass->get_block_name()] = true;
832 }
833 list ($blocknamesql, $blocknameparams) = $DB->get_in_or_equal(array_keys($blocklist));
834
835 // Get list of course contexts.
836 list ($contextsql, $contextparams) = $DB->get_in_or_equal($coursecontextids);
837
cfa00fc5 838 // Get list of block context (if limited).
839 $blockcontextwhere = '';
840 $blockcontextparams = [];
841 if ($limitcontextids) {
842 list ($blockcontextsql, $blockcontextparams) = $DB->get_in_or_equal($limitcontextids);
843 $blockcontextwhere = 'AND x.id ' . $blockcontextsql;
844 }
845
a96faa49 846 // Query all blocks that are within an included course, and are set to be visible, and
847 // in a supported page type (basically just course view). This query could be
848 // extended (or a second query added) to support blocks that are within a module
849 // context as well, and we could add more page types if required.
850 $blockrecs = $DB->get_records_sql("
851 SELECT x.*, bi.blockname AS blockname, bi.id AS blockinstanceid
852 FROM {block_instances} bi
853 JOIN {context} x ON x.instanceid = bi.id AND x.contextlevel = ?
854 LEFT JOIN {block_positions} bp ON bp.blockinstanceid = bi.id
855 AND bp.contextid = bi.parentcontextid
856 AND bp.pagetype LIKE 'course-view-%'
857 AND bp.subpage = ''
858 AND bp.visible = 0
859 WHERE bi.parentcontextid $contextsql
cfa00fc5 860 $blockcontextwhere
a96faa49 861 AND bi.blockname $blocknamesql
862 AND bi.subpagepattern IS NULL
863 AND (bi.pagetypepattern = 'site-index'
864 OR bi.pagetypepattern LIKE 'course-view-%'
865 OR bi.pagetypepattern = 'course-*'
866 OR bi.pagetypepattern = '*')
867 AND bp.id IS NULL",
cfa00fc5 868 array_merge([CONTEXT_BLOCK], $contextparams, $blockcontextparams, $blocknameparams));
a96faa49 869 $blockcontextsbyname = [];
870 foreach ($blockrecs as $blockrec) {
871 if (empty($blockcontextsbyname[$blockrec->blockname])) {
872 $blockcontextsbyname[$blockrec->blockname] = [];
873 }
874 \context_helper::preload_from_record($blockrec);
875 $blockcontextsbyname[$blockrec->blockname][] = \context_block::instance(
876 $blockrec->blockinstanceid);
877 }
878
879 // Add the block contexts the user can view.
880 foreach ($areasbylevel[CONTEXT_BLOCK] as $areaid => $searchclass) {
881 if (empty($blockcontextsbyname[$searchclass->get_block_name()])) {
882 continue;
883 }
884 foreach ($blockcontextsbyname[$searchclass->get_block_name()] as $context) {
885 if (has_capability('moodle/block:view', $context)) {
886 $areascontexts[$areaid][$context->id] = $context->id;
887 }
888 }
889 }
890 }
891
4359ef18 892 // Return all the data.
893 return (object)array('everything' => false, 'usercontexts' => $areascontexts,
894 'separategroupscontexts' => $separategroupscontexts, 'usergroups' => $usergroups,
895 'visiblegroupscontextsareas' => $visiblegroupscontextsareas);
db48207e
DM
896 }
897
053118a1
EM
898 /**
899 * Returns requested page of documents plus additional information for paging.
900 *
901 * This function does not perform any kind of security checking for access, the caller code
902 * should check that the current user have moodle/search:query capability.
903 *
904 * If a page is requested that is beyond the last result, the last valid page is returned in
905 * results, and actualpage indicates which page was returned.
906 *
907 * @param stdClass $formdata
908 * @param int $pagenum The 0 based page number.
909 * @return object An object with 3 properties:
910 * results => An array of \core_search\documents for the actual page.
911 * totalcount => Number of records that are possibly available, to base paging on.
912 * actualpage => The actual page returned.
913 */
914 public function paged_search(\stdClass $formdata, $pagenum) {
915 $out = new \stdClass();
916
e9074ee3
DM
917 if (self::is_search_area_categories_enabled() && !empty($formdata->cat)) {
918 $cat = self::get_search_area_category_by_name($formdata->cat);
919 if (empty($formdata->areaids)) {
920 $formdata->areaids = array_keys($cat->get_areas());
921 } else {
922 foreach ($formdata->areaids as $key => $areaid) {
923 if (!key_exists($areaid, $cat->get_areas())) {
924 unset($formdata->areaids[$key]);
925 }
926 }
927 }
928 }
929
053118a1
EM
930 $perpage = static::DISPLAY_RESULTS_PER_PAGE;
931
932 // Make sure we only allow request up to max page.
933 $pagenum = min($pagenum, (static::MAX_RESULTS / $perpage) - 1);
934
935 // Calculate the first and last document number for the current page, 1 based.
936 $mindoc = ($pagenum * $perpage) + 1;
937 $maxdoc = ($pagenum + 1) * $perpage;
938
939 // Get engine documents, up to max.
940 $docs = $this->search($formdata, $maxdoc);
941
942 $resultcount = count($docs);
943 if ($resultcount < $maxdoc) {
944 // This means it couldn't give us results to max, so the count must be the max.
945 $out->totalcount = $resultcount;
946 } else {
947 // Get the possible count reported by engine, and limit to our max.
948 $out->totalcount = $this->engine->get_query_total_count();
ed73ff2b 949 if (defined('BEHAT_SITE_RUNNING') && $this->behatresultcount) {
950 // Override results when using Behat mock results.
951 $out->totalcount = $this->behatresultcount;
952 }
053118a1
EM
953 $out->totalcount = min($out->totalcount, static::MAX_RESULTS);
954 }
955
956 // Determine the actual page.
957 if ($resultcount < $mindoc) {
958 // We couldn't get the min docs for this page, so determine what page we can get.
959 $out->actualpage = floor(($resultcount - 1) / $perpage);
960 } else {
961 $out->actualpage = $pagenum;
962 }
963
964 // Split the results to only return the page.
965 $out->results = array_slice($docs, $out->actualpage * $perpage, $perpage, true);
966
967 return $out;
968 }
969
db48207e
DM
970 /**
971 * Returns documents from the engine based on the data provided.
972 *
69d66020
DM
973 * This function does not perform any kind of security checking, the caller code
974 * should check that the current user have moodle/search:query capability.
975 *
db48207e
DM
976 * It might return the results from the cache instead.
977 *
cfa00fc5 978 * Valid formdata options include:
979 * - q (query text)
980 * - courseids (optional list of course ids to restrict)
981 * - contextids (optional list of context ids to restrict)
fc440796 982 * - context (Moodle context object for location user searched from)
983 * - order (optional ordering, one of the types supported by the search engine e.g. 'relevance')
222a97ce 984 * - userids (optional list of user ids to restrict)
cfa00fc5 985 *
986 * @param \stdClass $formdata Query input data (usually from search form)
987 * @param int $limit The maximum number of documents to return
db48207e
DM
988 * @return \core_search\document[]
989 */
053118a1 990 public function search(\stdClass $formdata, $limit = 0) {
e36eefae 991 // For Behat testing, the search results can be faked using a special step.
992 if (defined('BEHAT_SITE_RUNNING')) {
993 $fakeresult = get_config('core_search', 'behat_fakeresult');
994 if ($fakeresult) {
995 // Clear config setting.
996 unset_config('core_search', 'behat_fakeresult');
997
998 // Check query matches expected value.
999 $details = json_decode($fakeresult);
1000 if ($formdata->q !== $details->query) {
1001 throw new \coding_exception('Unexpected search query: ' . $formdata->q);
1002 }
1003
1004 // Create search documents from the JSON data.
1005 $docs = [];
1006 foreach ($details->results as $result) {
1007 $doc = new \core_search\document($result->itemid, $result->componentname,
1008 $result->areaname);
1009 foreach ((array)$result->fields as $field => $value) {
1010 $doc->set($field, $value);
1011 }
1012 foreach ((array)$result->extrafields as $field => $value) {
1013 $doc->set_extra($field, $value);
1014 }
1015 $area = $this->get_search_area($doc->get('areaid'));
1016 $doc->set_doc_url($area->get_doc_url($doc));
1017 $doc->set_context_url($area->get_context_url($doc));
1018 $docs[] = $doc;
1019 }
1020
ed73ff2b 1021 // Store the mock count, and apply the limit to the returned results.
1022 $this->behatresultcount = count($docs);
1023 if ($this->behatresultcount > $limit) {
1024 $docs = array_slice($docs, 0, $limit);
1025 }
1026
e36eefae 1027 return $docs;
1028 }
1029 }
db48207e 1030
2085e860 1031 $limitcourseids = $this->build_limitcourseids($formdata);
427e3cbc 1032
cfa00fc5 1033 $limitcontextids = false;
1034 if (!empty($formdata->contextids)) {
1035 $limitcontextids = $formdata->contextids;
1036 }
1037
db48207e
DM
1038 // Clears previous query errors.
1039 $this->engine->clear_query_error();
1040
4359ef18 1041 $contextinfo = $this->get_areas_user_accesses($limitcourseids, $limitcontextids);
1042 if (!$contextinfo->everything && !$contextinfo->usercontexts) {
db48207e
DM
1043 // User can not access any context.
1044 $docs = array();
1045 } else {
4359ef18 1046 // If engine does not support groups, remove group information from the context info -
1047 // use the old format instead (true = admin, array = user contexts).
1048 if (!$this->engine->supports_group_filtering()) {
1049 $contextinfo = $contextinfo->everything ? true : $contextinfo->usercontexts;
1050 }
1051
1052 // Execute the actual query.
1053 $docs = $this->engine->execute_query($formdata, $contextinfo, $limit);
db48207e
DM
1054 }
1055
db48207e
DM
1056 return $docs;
1057 }
1058
2085e860
DM
1059 /**
1060 * Build a list of course ids to limit the search based on submitted form data.
1061 *
1062 * @param \stdClass $formdata Submitted search form data.
1063 *
1064 * @return array|bool
1065 */
1066 protected function build_limitcourseids(\stdClass $formdata) {
1067 $limitcourseids = false;
1068
1069 if (!empty($formdata->mycoursesonly)) {
1070 $limitcourseids = array_keys($this->get_my_courses(false));
1071 }
1072
1073 if (!empty($formdata->courseids)) {
1074 if (empty($limitcourseids)) {
1075 $limitcourseids = $formdata->courseids;
1076 } else {
1077 $limitcourseids = array_intersect($limitcourseids, $formdata->courseids);
1078 }
1079 }
1080
1081 return $limitcourseids;
1082 }
1083
db48207e
DM
1084 /**
1085 * Merge separate index segments into one.
1086 */
1087 public function optimize_index() {
1088 $this->engine->optimize();
1089 }
1090
1091 /**
1092 * Index all documents.
1093 *
1094 * @param bool $fullindex Whether we should reindex everything or not.
67d64795 1095 * @param float $timelimit Time limit in seconds (0 = no time limit)
4ba11aa9 1096 * @param \progress_trace|null $progress Optional class for tracking progress
db48207e
DM
1097 * @throws \moodle_exception
1098 * @return bool Whether there was any updated document or not.
1099 */
67d64795 1100 public function index($fullindex = false, $timelimit = 0, \progress_trace $progress = null) {
eacb2bd1 1101 global $DB;
1102
67d64795 1103 // Cannot combine time limit with reindex.
1104 if ($timelimit && $fullindex) {
1105 throw new \coding_exception('Cannot apply time limit when reindexing');
1106 }
1107 if (!$progress) {
1108 $progress = new \null_progress_trace();
1109 }
db48207e
DM
1110
1111 // Unlimited time.
1112 \core_php_time_limit::raise();
1113
075fa912
EM
1114 // Notify the engine that an index starting.
1115 $this->engine->index_starting($fullindex);
1116
bf2235bb 1117 $sumdocs = 0;
db48207e
DM
1118
1119 $searchareas = $this->get_search_areas_list(true);
67d64795 1120
1121 if ($timelimit) {
1122 // If time is limited (and therefore we're not just indexing everything anyway), select
1123 // an order for search areas. The intention here is to avoid a situation where a new
1124 // large search area is enabled, and this means all our other search areas go out of
1125 // date while that one is being indexed. To do this, we order by the time we spent
1126 // indexing them last time we ran, meaning anything that took a very long time will be
1127 // done last.
1128 uasort($searchareas, function(\core_search\base $area1, \core_search\base $area2) {
1129 return (int)$area1->get_last_indexing_duration() - (int)$area2->get_last_indexing_duration();
1130 });
1131
1132 // Decide time to stop.
82735dec 1133 $stopat = self::get_current_time() + $timelimit;
67d64795 1134 }
1135
db48207e
DM
1136 foreach ($searchareas as $areaid => $searcharea) {
1137
67d64795 1138 $progress->output('Processing area: ' . $searcharea->get_visible_name());
db48207e 1139
075fa912
EM
1140 // Notify the engine that an area is starting.
1141 $this->engine->area_index_starting($searcharea, $fullindex);
1142
7f83e99e 1143 $indexingstart = (int)self::get_current_time();
82735dec 1144 $elapsed = self::get_current_time();
db48207e
DM
1145
1146 // This is used to store this component config.
1147 list($componentconfigname, $varname) = $searcharea->get_config_var_name();
1148
091973db
EM
1149 $prevtimestart = intval(get_config($componentconfigname, $varname . '_indexingstart'));
1150
db48207e 1151 if ($fullindex === true) {
091973db 1152 $referencestarttime = 0;
eacb2bd1 1153
1154 // For full index, we delete any queued context index requests, as those will
1155 // obviously be met by the full index.
1156 $DB->delete_records('search_index_requests');
db48207e 1157 } else {
67d64795 1158 $partial = get_config($componentconfigname, $varname . '_partial');
1159 if ($partial) {
1160 // When the previous index did not complete all data, we start from the time of the
1161 // last document that was successfully indexed. (Note this will result in
1162 // re-indexing that one document, but we can't avoid that because there may be
1163 // other documents in the same second.)
1164 $referencestarttime = intval(get_config($componentconfigname, $varname . '_lastindexrun'));
1165 } else {
1166 $referencestarttime = $prevtimestart;
1167 }
db48207e
DM
1168 }
1169
1170 // Getting the recordset from the area.
091973db 1171 $recordset = $searcharea->get_recordset_by_timestamp($referencestarttime);
1b8cf12a 1172 $initialquerytime = self::get_current_time() - $elapsed;
1173 if ($initialquerytime > self::DISPLAY_LONG_QUERY_TIME) {
1174 $progress->output('Initial query took ' . round($initialquerytime, 1) .
1175 ' seconds.', 1);
1176 }
db48207e
DM
1177
1178 // Pass get_document as callback.
091973db
EM
1179 $fileindexing = $this->engine->file_indexing_enabled() && $searcharea->uses_file_indexing();
1180 $options = array('indexfiles' => $fileindexing, 'lastindexedtime' => $prevtimestart);
67d64795 1181 if ($timelimit) {
1182 $options['stopat'] = $stopat;
1183 }
1b8cf12a 1184 $options['progress'] = $progress;
2d2fcc1c 1185 $iterator = new skip_future_documents_iterator(new \core\dml\recordset_walk(
1186 $recordset, array($searcharea, 'get_document'), $options));
67d64795 1187 $result = $this->engine->add_documents($iterator, $searcharea, $options);
2d2fcc1c 1188 $recordset->close();
0deb1946 1189 $batchinfo = '';
1190 if (count($result) === 6) {
1191 [$numrecords, $numdocs, $numdocsignored, $lastindexeddoc, $partial, $batches] = $result;
1192 // Only show the batch count if we actually batched any requests.
1193 if ($batches !== $numdocs + $numdocsignored) {
1194 $batchinfo = ' (' . $batches . ' batch' . ($batches === 1 ? '' : 'es') . ')';
1195 }
1196 } else if (count($result) === 5) {
1197 // Backward compatibility for engines that don't return a batch count.
1198 [$numrecords, $numdocs, $numdocsignored, $lastindexeddoc, $partial] = $result;
6a601111 1199 // Deprecated since Moodle 3.10 MDL-68690.
19070b2e 1200 // TODO: MDL-68776 This will be deleted in Moodle 4.2.
0deb1946 1201 debugging('engine::add_documents() should return $batches (5-value return is deprecated)',
1202 DEBUG_DEVELOPER);
67d64795 1203 } else {
c5b2f2b3 1204 throw new coding_exception('engine::add_documents() should return $partial (4-value return is deprecated)');
67d64795 1205 }
1206
1207 if ($numdocs > 0) {
1b8cf12a 1208 $elapsed = round((self::get_current_time() - $elapsed), 1);
1209
1210 $partialtext = '';
1211 if ($partial) {
1212 $partialtext = ' (not complete; done to ' . userdate($lastindexeddoc,
1213 get_string('strftimedatetimeshort', 'langconfig')) . ')';
1214 }
1215
67d64795 1216 $progress->output('Processed ' . $numrecords . ' records containing ' . $numdocs .
0deb1946 1217 ' documents' . $batchinfo . ', in ' . $elapsed . ' seconds' . $partialtext . '.', 1);
67d64795 1218 } else {
1219 $progress->output('No new documents to index.', 1);
db48207e
DM
1220 }
1221
075fa912
EM
1222 // Notify the engine this area is complete, and only mark times if true.
1223 if ($this->engine->area_index_complete($searcharea, $numdocs, $fullindex)) {
1224 $sumdocs += $numdocs;
1225
67d64795 1226 // Store last index run once documents have been committed to the search engine.
075fa912 1227 set_config($varname . '_indexingstart', $indexingstart, $componentconfigname);
7f83e99e 1228 set_config($varname . '_indexingend', (int)self::get_current_time(), $componentconfigname);
075fa912
EM
1229 set_config($varname . '_docsignored', $numdocsignored, $componentconfigname);
1230 set_config($varname . '_docsprocessed', $numdocs, $componentconfigname);
1231 set_config($varname . '_recordsprocessed', $numrecords, $componentconfigname);
1232 if ($lastindexeddoc > 0) {
1233 set_config($varname . '_lastindexrun', $lastindexeddoc, $componentconfigname);
1234 }
67d64795 1235 if ($partial) {
1236 set_config($varname . '_partial', 1, $componentconfigname);
1237 } else {
1238 unset_config($varname . '_partial', $componentconfigname);
1239 }
1240 } else {
1241 $progress->output('Engine reported error.');
1242 }
1243
82735dec 1244 if ($timelimit && (self::get_current_time() >= $stopat)) {
67d64795 1245 $progress->output('Stopping indexing due to time limit.');
1246 break;
db48207e
DM
1247 }
1248 }
1249
bf2235bb 1250 if ($sumdocs > 0) {
db48207e
DM
1251 $event = \core\event\search_indexed::create(
1252 array('context' => \context_system::instance()));
1253 $event->trigger();
1254 }
1255
bf2235bb
EM
1256 $this->engine->index_complete($sumdocs, $fullindex);
1257
1258 return (bool)$sumdocs;
db48207e
DM
1259 }
1260
4ba11aa9 1261 /**
1262 * Indexes or reindexes a specific context of the system, e.g. one course.
1263 *
1264 * The function returns an object with field 'complete' (true or false).
1265 *
1266 * This function supports partial indexing via the time limit parameter. If the time limit
1267 * expires, it will return values for $startfromarea and $startfromtime which can be passed
1268 * next time to continue indexing.
1269 *
1270 * @param \context $context Context to restrict index.
1271 * @param string $singleareaid If specified, indexes only the given area.
1272 * @param float $timelimit Time limit in seconds (0 = no time limit)
1273 * @param \progress_trace|null $progress Optional class for tracking progress
1274 * @param string $startfromarea Area to start from
1275 * @param int $startfromtime Timestamp to start from
1276 * @return \stdClass Object indicating success
1277 */
1278 public function index_context($context, $singleareaid = '', $timelimit = 0,
1279 \progress_trace $progress = null, $startfromarea = '', $startfromtime = 0) {
1280 if (!$progress) {
1281 $progress = new \null_progress_trace();
1282 }
1283
1284 // Work out time to stop, if limited.
1285 if ($timelimit) {
1286 // Decide time to stop.
82735dec 1287 $stopat = self::get_current_time() + $timelimit;
4ba11aa9 1288 }
1289
1290 // No PHP time limit.
1291 \core_php_time_limit::raise();
1292
1293 // Notify the engine that an index starting.
1294 $this->engine->index_starting(false);
1295
1296 $sumdocs = 0;
1297
1298 // Get all search areas, in consistent order.
1299 $searchareas = $this->get_search_areas_list(true);
1300 ksort($searchareas);
1301
1302 // Are we skipping past some that were handled previously?
1303 $skipping = $startfromarea ? true : false;
1304
1305 foreach ($searchareas as $areaid => $searcharea) {
1306 // If we're only processing one area id, skip all the others.
1307 if ($singleareaid && $singleareaid !== $areaid) {
1308 continue;
1309 }
1310
1311 // If we're skipping to a later area, continue through the loop.
1312 $referencestarttime = 0;
1313 if ($skipping) {
1314 if ($areaid !== $startfromarea) {
1315 continue;
1316 }
1317 // Stop skipping and note the reference start time.
1318 $skipping = false;
1319 $referencestarttime = $startfromtime;
1320 }
1321
1322 $progress->output('Processing area: ' . $searcharea->get_visible_name());
1323
82735dec 1324 $elapsed = self::get_current_time();
4ba11aa9 1325
1326 // Get the recordset of all documents from the area for this context.
1327 $recordset = $searcharea->get_document_recordset($referencestarttime, $context);
1328 if (!$recordset) {
1329 if ($recordset === null) {
1330 $progress->output('Skipping (not relevant to context).', 1);
1331 } else {
1332 $progress->output('Skipping (does not support context indexing).', 1);
1333 }
1334 continue;
1335 }
1336
1337 // Notify the engine that an area is starting.
1338 $this->engine->area_index_starting($searcharea, false);
1339
1340 // Work out search options.
1341 $options = [];
1342 $options['indexfiles'] = $this->engine->file_indexing_enabled() &&
1343 $searcharea->uses_file_indexing();
1344 if ($timelimit) {
1345 $options['stopat'] = $stopat;
1346 }
1347
1348 // Construct iterator which will use get_document on the recordset results.
1349 $iterator = new \core\dml\recordset_walk($recordset,
1350 array($searcharea, 'get_document'), $options);
1351
1352 // Use this iterator to add documents.
1353 $result = $this->engine->add_documents($iterator, $searcharea, $options);
0deb1946 1354 $batchinfo = '';
1355 if (count($result) === 6) {
1356 [$numrecords, $numdocs, $numdocsignored, $lastindexeddoc, $partial, $batches] = $result;
1357 // Only show the batch count if we actually batched any requests.
1358 if ($batches !== $numdocs + $numdocsignored) {
1359 $batchinfo = ' (' . $batches . ' batch' . ($batches === 1 ? '' : 'es') . ')';
1360 }
1361 } else if (count($result) === 5) {
1362 // Backward compatibility for engines that don't return a batch count.
1363 [$numrecords, $numdocs, $numdocsignored, $lastindexeddoc, $partial] = $result;
6a601111 1364 // Deprecated since Moodle 3.10 MDL-68690.
19070b2e 1365 // TODO: MDL-68776 This will be deleted in Moodle 4.2 (as should the below bit).
0deb1946 1366 debugging('engine::add_documents() should return $batches (5-value return is deprecated)',
1367 DEBUG_DEVELOPER);
4ba11aa9 1368 } else {
1369 // Backward compatibility for engines that don't support partial adding.
1370 list($numrecords, $numdocs, $numdocsignored, $lastindexeddoc) = $result;
1371 debugging('engine::add_documents() should return $partial (4-value return is deprecated)',
1372 DEBUG_DEVELOPER);
1373 $partial = false;
1374 }
1375
1376 if ($numdocs > 0) {
82735dec 1377 $elapsed = round((self::get_current_time() - $elapsed), 3);
4ba11aa9 1378 $progress->output('Processed ' . $numrecords . ' records containing ' . $numdocs .
0deb1946 1379 ' documents' . $batchinfo . ', in ' . $elapsed . ' seconds' .
4ba11aa9 1380 ($partial ? ' (not complete)' : '') . '.', 1);
1381 } else {
1382 $progress->output('No documents to index.', 1);
1383 }
1384
1385 // Notify the engine this area is complete, but don't store any times as this is not
1386 // part of the 'normal' search index.
1387 if (!$this->engine->area_index_complete($searcharea, $numdocs, false)) {
1388 $progress->output('Engine reported error.', 1);
1389 }
1390
82735dec 1391 if ($partial && $timelimit && (self::get_current_time() >= $stopat)) {
4ba11aa9 1392 $progress->output('Stopping indexing due to time limit.');
1393 break;
1394 }
1395 }
1396
1397 if ($sumdocs > 0) {
1398 $event = \core\event\search_indexed::create(
1399 array('context' => $context));
1400 $event->trigger();
1401 }
1402
1403 $this->engine->index_complete($sumdocs, false);
1404
1405 // Indicate in result whether we completed indexing, or only part of it.
1406 $result = new \stdClass();
1407 if ($partial) {
1408 $result->complete = false;
1409 $result->startfromarea = $areaid;
1410 $result->startfromtime = $lastindexeddoc;
1411 } else {
1412 $result->complete = true;
1413 }
1414 return $result;
1415 }
1416
db48207e
DM
1417 /**
1418 * Resets areas config.
1419 *
1420 * @throws \moodle_exception
1421 * @param string $areaid
1422 * @return void
1423 */
1424 public function reset_config($areaid = false) {
1425
1426 if (!empty($areaid)) {
1427 $searchareas = array();
1428 if (!$searchareas[$areaid] = static::get_search_area($areaid)) {
1429 throw new \moodle_exception('errorareanotavailable', 'search', '', $areaid);
1430 }
1431 } else {
1432 // Only the enabled ones.
1433 $searchareas = static::get_search_areas_list(true);
1434 }
1435
1436 foreach ($searchareas as $searcharea) {
69d66020
DM
1437 list($componentname, $varname) = $searcharea->get_config_var_name();
1438 $config = $searcharea->get_config();
db48207e 1439
69d66020
DM
1440 foreach ($config as $key => $value) {
1441 // We reset them all but the enable/disabled one.
1442 if ($key !== $varname . '_enabled') {
1443 set_config($key, 0, $componentname);
1444 }
1445 }
db48207e
DM
1446 }
1447 }
1448
1449 /**
1450 * Deletes an area's documents or all areas documents.
1451 *
1452 * @param string $areaid The area id or false for all
1453 * @return void
1454 */
1455 public function delete_index($areaid = false) {
1456 if (!empty($areaid)) {
1457 $this->engine->delete($areaid);
1458 $this->reset_config($areaid);
1459 } else {
1460 $this->engine->delete();
1461 $this->reset_config();
1462 }
db48207e
DM
1463 }
1464
1465 /**
1466 * Deletes index by id.
1467 *
1468 * @param int Solr Document string $id
1469 */
1470 public function delete_index_by_id($id) {
1471 $this->engine->delete_by_id($id);
db48207e
DM
1472 }
1473
1474 /**
1475 * Returns search areas configuration.
1476 *
0bd8383a 1477 * @param \core_search\base[] $searchareas
db48207e
DM
1478 * @return \stdClass[] $configsettings
1479 */
1480 public function get_areas_config($searchareas) {
1481
67d64795 1482 $vars = array('indexingstart', 'indexingend', 'lastindexrun', 'docsignored',
1483 'docsprocessed', 'recordsprocessed', 'partial');
db48207e 1484
0a9a10f0 1485 $configsettings = [];
db48207e
DM
1486 foreach ($searchareas as $searcharea) {
1487
1488 $areaid = $searcharea->get_area_id();
1489
1490 $configsettings[$areaid] = new \stdClass();
1491 list($componentname, $varname) = $searcharea->get_config_var_name();
1492
1493 if (!$searcharea->is_enabled()) {
1494 // We delete all indexed data on disable so no info.
1495 foreach ($vars as $var) {
1496 $configsettings[$areaid]->{$var} = 0;
1497 }
1498 } else {
1499 foreach ($vars as $var) {
1500 $configsettings[$areaid]->{$var} = get_config($componentname, $varname .'_' . $var);
1501 }
1502 }
1503
1504 // Formatting the time.
1505 if (!empty($configsettings[$areaid]->lastindexrun)) {
1506 $configsettings[$areaid]->lastindexrun = userdate($configsettings[$areaid]->lastindexrun);
1507 } else {
1508 $configsettings[$areaid]->lastindexrun = get_string('never');
1509 }
1510 }
1511 return $configsettings;
1512 }
396d6f0a 1513
e71061a2
DM
1514 /**
1515 * Triggers search_results_viewed event
1516 *
1517 * Other data required:
1518 * - q: The query string
1519 * - page: The page number
1520 * - title: Title filter
1521 * - areaids: Search areas filter
1522 * - courseids: Courses filter
1523 * - timestart: Time start filter
1524 * - timeend: Time end filter
1525 *
1526 * @since Moodle 3.2
1527 * @param array $other Other info for the event.
1528 * @return \core\event\search_results_viewed
1529 */
1530 public static function trigger_search_results_viewed($other) {
1531 $event = \core\event\search_results_viewed::create([
1532 'context' => \context_system::instance(),
1533 'other' => $other
1534 ]);
1535 $event->trigger();
1536
1537 return $event;
1538 }
1539
396d6f0a
DG
1540 /**
1541 * Checks whether a classname is of an actual search area.
1542 *
f3d38863 1543 * @param string $classname
396d6f0a
DG
1544 * @return bool
1545 */
f3d38863
DM
1546 protected static function is_search_area($classname) {
1547 if (is_subclass_of($classname, 'core_search\base')) {
1548 return (new \ReflectionClass($classname))->isInstantiable();
396d6f0a
DG
1549 }
1550
1551 return false;
1552 }
eacb2bd1 1553
1554 /**
1555 * Requests that a specific context is indexed by the scheduled task. The context will be
1556 * added to a queue which is processed by the task.
1557 *
1558 * This is used after a restore to ensure that restored items are indexed, even though their
65da6840 1559 * modified time will be older than the latest indexed. It is also used by the 'Gradual reindex'
1560 * admin feature from the search areas screen.
eacb2bd1 1561 *
1562 * @param \context $context Context to index within
1563 * @param string $areaid Area to index, '' = all areas
8736fbc1 1564 * @param int $priority Priority (INDEX_PRIORITY_xx constant)
eacb2bd1 1565 */
8736fbc1 1566 public static function request_index(\context $context, $areaid = '',
1567 $priority = self::INDEX_PRIORITY_NORMAL) {
eacb2bd1 1568 global $DB;
1569
1570 // Check through existing requests for this context or any parent context.
1571 list ($contextsql, $contextparams) = $DB->get_in_or_equal(
1572 $context->get_parent_context_ids(true));
1573 $existing = $DB->get_records_select('search_index_requests',
8736fbc1 1574 'contextid ' . $contextsql, $contextparams, '',
1575 'id, searcharea, partialarea, indexpriority');
eacb2bd1 1576 foreach ($existing as $rec) {
1577 // If we haven't started processing the existing request yet, and it covers the same
1578 // area (or all areas) then that will be sufficient so don't add anything else.
1579 if ($rec->partialarea === '' && ($rec->searcharea === $areaid || $rec->searcharea === '')) {
8736fbc1 1580 // If the existing request has the same (or higher) priority, no need to add anything.
1581 if ($rec->indexpriority >= $priority) {
1582 return;
1583 }
1584 // The existing request has lower priority. If it is exactly the same, then just
1585 // adjust the priority of the existing request.
1586 if ($rec->searcharea === $areaid) {
1587 $DB->set_field('search_index_requests', 'indexpriority', $priority,
1588 ['id' => $rec->id]);
1589 return;
1590 }
1591 // The existing request would cover this area but is a lower priority. We need to
1592 // add the new request even though that means we will index part of it twice.
eacb2bd1 1593 }
1594 }
1595
1596 // No suitable existing request, so add a new one.
1597 $newrecord = [ 'contextid' => $context->id, 'searcharea' => $areaid,
8736fbc1 1598 'timerequested' => (int)self::get_current_time(),
1599 'partialarea' => '', 'partialtime' => 0,
1600 'indexpriority' => $priority ];
eacb2bd1 1601 $DB->insert_record('search_index_requests', $newrecord);
1602 }
74b7a42d 1603
1604 /**
8736fbc1 1605 * Processes outstanding index requests. This will take the first item from the queue (taking
1606 * account the indexing priority) and process it, continuing until an optional time limit is
1607 * reached.
74b7a42d 1608 *
1609 * If there are no index requests, the function will do nothing.
1610 *
1611 * @param float $timelimit Time limit (0 = none)
1612 * @param \progress_trace|null $progress Optional progress indicator
1613 */
1614 public function process_index_requests($timelimit = 0.0, \progress_trace $progress = null) {
1615 global $DB;
1616
1617 if (!$progress) {
1618 $progress = new \null_progress_trace();
1619 }
1620
82735dec 1621 $before = self::get_current_time();
74b7a42d 1622 if ($timelimit) {
1623 $stopat = $before + $timelimit;
1624 }
1625 while (true) {
1626 // Retrieve first request, using fully defined ordering.
1627 $requests = $DB->get_records('search_index_requests', null,
8736fbc1 1628 'indexpriority DESC, timerequested, contextid, searcharea',
74b7a42d 1629 'id, contextid, searcharea, partialarea, partialtime', 0, 1);
1630 if (!$requests) {
1631 // If there are no more requests, stop.
74b7a42d 1632 break;
1633 }
1634 $request = reset($requests);
1635
1636 // Calculate remaining time.
1637 $remainingtime = 0;
82735dec 1638 $beforeindex = self::get_current_time();
74b7a42d 1639 if ($timelimit) {
1640 $remainingtime = $stopat - $beforeindex;
8736fbc1 1641
1642 // If the time limit expired already, stop now. (Otherwise we might accidentally
1643 // index with no time limit or a negative time limit.)
1644 if ($remainingtime <= 0) {
1645 break;
1646 }
74b7a42d 1647 }
1648
1649 // Show a message before each request, indicating what will be indexed.
a539e560 1650 $context = \context::instance_by_id($request->contextid, IGNORE_MISSING);
1651 if (!$context) {
1652 $DB->delete_records('search_index_requests', ['id' => $request->id]);
1653 $progress->output('Skipped deleted context: ' . $request->contextid);
1654 continue;
1655 }
74b7a42d 1656 $contextname = $context->get_context_name();
1657 if ($request->searcharea) {
1658 $contextname .= ' (search area: ' . $request->searcharea . ')';
1659 }
1660 $progress->output('Indexing requested context: ' . $contextname);
1661
1662 // Actually index the context.
1663 $result = $this->index_context($context, $request->searcharea, $remainingtime,
1664 $progress, $request->partialarea, $request->partialtime);
1665
1666 // Work out shared part of message.
82735dec 1667 $endmessage = $contextname . ' (' . round(self::get_current_time() - $beforeindex, 1) . 's)';
74b7a42d 1668
1669 // Update database table and continue/stop as appropriate.
1670 if ($result->complete) {
1671 // If we completed the request, remove it from the table.
1672 $DB->delete_records('search_index_requests', ['id' => $request->id]);
1673 $progress->output('Completed requested context: ' . $endmessage);
1674 } else {
1675 // If we didn't complete the request, store the partial details (how far it got).
1676 $DB->update_record('search_index_requests', ['id' => $request->id,
1677 'partialarea' => $result->startfromarea,
1678 'partialtime' => $result->startfromtime]);
1679 $progress->output('Ending requested context: ' . $endmessage);
1680
1681 // The time limit must have expired, so stop looping.
1682 break;
1683 }
1684 }
1685 }
1686
65da6840 1687 /**
1688 * Gets information about the request queue, in the form of a plain object suitable for passing
1689 * to a template for rendering.
1690 *
1691 * @return \stdClass Information about queued index requests
1692 */
1693 public function get_index_requests_info() {
1694 global $DB;
1695
1696 $result = new \stdClass();
1697
1698 $result->total = $DB->count_records('search_index_requests');
1699 $result->topten = $DB->get_records('search_index_requests', null,
1700 'indexpriority DESC, timerequested, contextid, searcharea',
1701 'id, contextid, timerequested, searcharea, partialarea, partialtime, indexpriority',
1702 0, 10);
1703 foreach ($result->topten as $item) {
1704 $context = \context::instance_by_id($item->contextid);
1705 $item->contextlink = \html_writer::link($context->get_url(),
1706 s($context->get_context_name()));
1707 if ($item->searcharea) {
1708 $item->areaname = $this->get_search_area($item->searcharea)->get_visible_name();
1709 }
1710 if ($item->partialarea) {
1711 $item->partialareaname = $this->get_search_area($item->partialarea)->get_visible_name();
1712 }
1713 switch ($item->indexpriority) {
1714 case self::INDEX_PRIORITY_REINDEXING :
1715 $item->priorityname = get_string('priority_reindexing', 'search');
1716 break;
1717 case self::INDEX_PRIORITY_NORMAL :
1718 $item->priorityname = get_string('priority_normal', 'search');
1719 break;
1720 }
1721 }
1722
1723 // Normalise array indices.
1724 $result->topten = array_values($result->topten);
1725
1726 if ($result->total > 10) {
1727 $result->ellipsis = true;
1728 }
1729
1730 return $result;
1731 }
1732
82735dec 1733 /**
1734 * Gets current time for use in search system.
1735 *
1736 * Note: This should be replaced with generic core functionality once possible (see MDL-60644).
1737 *
1738 * @return float Current time in seconds (with decimals)
1739 */
1740 public static function get_current_time() {
1741 if (PHPUNIT_TEST && self::$phpunitfaketime) {
1742 return self::$phpunitfaketime;
1743 }
1744 return microtime(true);
1745 }
e9074ee3
DM
1746
1747 /**
1748 * Check if search area categories functionality is enabled.
1749 *
1750 * @return bool
1751 */
1752 public static function is_search_area_categories_enabled() {
1753 return !empty(get_config('core', 'searchenablecategories'));
1754 }
1755
1756 /**
1757 * Check if all results category should be hidden.
1758 *
1759 * @return bool
1760 */
1761 public static function should_hide_all_results_category() {
1762 return get_config('core', 'searchhideallcategory');
1763 }
1764
1765 /**
1766 * Returns default search area category name.
1767 *
1768 * @return string
1769 */
1770 public static function get_default_area_category_name() {
1771 $default = get_config('core', 'searchdefaultcategory');
1772
1773 if (empty($default)) {
1774 $default = self::SEARCH_AREA_CATEGORY_ALL;
1775 }
1776
1777 if ($default == self::SEARCH_AREA_CATEGORY_ALL && self::should_hide_all_results_category()) {
1778 $default = self::SEARCH_AREA_CATEGORY_COURSE_CONTENT;
1779 }
1780
1781 return $default;
1782 }
2085e860
DM
1783
1784 /**
1785 * Get a list of all courses limited by ids if required.
1786 *
1787 * @param array|false $limitcourseids An array of course ids to limit the search to. False for no limiting.
1788 * @return array
1789 */
1790 protected function get_all_courses($limitcourseids) {
1791 global $DB;
1792
1793 if ($limitcourseids) {
1794 list ($coursesql, $courseparams) = $DB->get_in_or_equal($limitcourseids);
1795 $coursesql = 'id ' . $coursesql;
1796 } else {
1797 $coursesql = '';
1798 $courseparams = [];
1799 }
1800
1801 // Get courses using the same list of fields from enrol_get_my_courses.
1802 return $DB->get_records_select('course', $coursesql, $courseparams, '',
1803 'id, category, sortorder, shortname, fullname, idnumber, startdate, visible, ' .
1804 'groupmode, groupmodeforce, cacherev');
1805 }
1806
1807 /**
1808 * Get a list of courses as user can access.
1809 *
1810 * @param bool $allaccessible Include courses user is not enrolled in, but can access.
1811 * @return array
1812 */
1813 protected function get_my_courses($allaccessible) {
1814 return enrol_get_my_courses(array('id', 'cacherev'), 'id', 0, [], $allaccessible);
1815 }
1816
1817 /**
1818 * Check if search all courses setting is enabled.
1819 *
1820 * @return bool
1821 */
1822 public static function include_all_courses() {
1823 return !empty(get_config('core', 'searchincludeallcourses'));
1824 }
1825
1826 /**
1827 * Cleans up non existing search area.
1828 *
1829 * 1. Remove all configs from {config_plugins} table.
1830 * 2. Delete all related indexed documents.
1831 *
1832 * @param string $areaid Search area id.
1833 */
1834 public static function clean_up_non_existing_area($areaid) {
1835 global $DB;
1836
1837 if (!empty(self::get_search_area($areaid))) {
1838 throw new \coding_exception("Area $areaid exists. Please use appropriate search area class to manipulate the data.");
1839 }
1840
1841 $parts = self::parse_areaid($areaid);
1842
1843 $plugin = $parts[0];
1844 $configprefix = $parts[1];
1845
1846 foreach (base::get_settingnames() as $settingname) {
1847 $name = $configprefix. $settingname;
1848 $DB->delete_records('config_plugins', ['name' => $name, 'plugin' => $plugin]);
1849 }
1850
1851 $engine = self::instance()->get_engine();
1852 $engine->delete($areaid);
1853 }
1854
7ba2a201 1855 /**
1856 * Informs the search system that a context has been deleted.
1857 *
1858 * This will clear the data from the search index, where the search engine supports that.
1859 *
1860 * This function does not usually throw an exception (so as not to get in the way of the
1861 * context deletion finishing).
1862 *
1863 * This is called for all types of context deletion.
1864 *
1865 * @param \context $context Context object that has just been deleted
1866 */
1867 public static function context_deleted(\context $context) {
1868 if (self::is_indexing_enabled()) {
1869 try {
1870 // Hold on, are we deleting a course? If so, and this context is part of the course,
1871 // then don't bother to send a delete because we delete the whole course at once
1872 // later.
1873 if (!empty(self::$coursedeleting)) {
1874 $coursecontext = $context->get_course_context(false);
1875 if ($coursecontext && array_key_exists($coursecontext->instanceid, self::$coursedeleting)) {
1876 // Skip further processing.
1877 return;
1878 }
1879 }
1880
1881 $engine = self::instance()->get_engine();
1882 $engine->delete_index_for_context($context->id);
1883 } catch (\moodle_exception $e) {
1884 debugging('Error deleting search index data for context ' . $context->id . ': ' . $e->getMessage());
1885 }
1886 }
1887 }
1888
1889 /**
1890 * Informs the search system that a course is about to be deleted.
1891 *
1892 * This prevents it from sending hundreds of 'delete context' updates for all the individual
1893 * contexts that are deleted.
1894 *
1895 * If you call this, you must call course_deleting_finish().
1896 *
1897 * @param int $courseid Course id that is being deleted
1898 */
1899 public static function course_deleting_start(int $courseid) {
1900 self::$coursedeleting[$courseid] = true;
1901 }
1902
1903 /**
1904 * Informs the search engine that a course has now been deleted.
1905 *
1906 * This causes the search engine to actually delete the index for the whole course.
1907 *
1908 * @param int $courseid Course id that no longer exists
1909 */
1910 public static function course_deleting_finish(int $courseid) {
1911 if (!array_key_exists($courseid, self::$coursedeleting)) {
1912 // Show a debug warning. It doesn't actually matter very much, as we will now delete
1913 // the course data anyhow.
1914 debugging('course_deleting_start not called before deletion of ' . $courseid, DEBUG_DEVELOPER);
1915 }
1916 unset(self::$coursedeleting[$courseid]);
1917
1918 if (self::is_indexing_enabled()) {
1919 try {
1920 $engine = self::instance()->get_engine();
1921 $engine->delete_index_for_course($courseid);
1922 } catch (\moodle_exception $e) {
1923 debugging('Error deleting search index data for course ' . $courseid . ': ' . $e->getMessage());
1924 }
1925 }
1926 }
db48207e 1927}