MDL-61102 core_search: Improve performance by reducing schema checks
[moodle.git] / search / engine / solr / classes / engine.php
CommitLineData
95c6aeaf
DM
1<?php
2// This file is part of Moodle - http://moodle.org/
3//
4// Moodle is free software: you can redistribute it and/or modify
5// it under the terms of the GNU General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8//
9// Moodle is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12// GNU General Public License for more details.
13//
14// You should have received a copy of the GNU General Public License
15// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
16
17/**
18 * Solr engine.
19 *
20 * @package search_solr
21 * @copyright 2015 Daniel Neis Araujo
22 * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
23 */
24
25namespace search_solr;
26
27defined('MOODLE_INTERNAL') || die();
28
29/**
30 * Solr engine.
31 *
32 * @package search_solr
33 * @copyright 2015 Daniel Neis Araujo
34 * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
35 */
36class engine extends \core_search\engine {
37
38 /**
39 * @var string The date format used by solr.
40 */
41 const DATE_FORMAT = 'Y-m-d\TH:i:s\Z';
42
43 /**
44 * @var int Commit documents interval (number of miliseconds).
45 */
46 const AUTOCOMMIT_WITHIN = 15000;
47
053118a1
EM
48 /**
49 * The maximum number of results to fetch at a time.
50 */
51 const QUERY_SIZE = 120;
52
95c6aeaf 53 /**
4894840d 54 * Highlighting fragsize. Slightly larger than output size (500) to allow for ... appending.
95c6aeaf 55 */
4894840d
EM
56 const FRAG_SIZE = 510;
57
58 /**
59 * Marker for the start of a highlight.
60 */
61 const HIGHLIGHT_START = '@@HI_S@@';
62
63 /**
64 * Marker for the end of a highlight.
65 */
66 const HIGHLIGHT_END = '@@HI_E@@';
95c6aeaf 67
fc440796 68 /** @var float Boost value for matching course in location-ordered searches */
69 const COURSE_BOOST = 1;
70
71 /** @var float Boost value for matching context (in addition to course boost) */
72 const CONTEXT_BOOST = 0.5;
73
95c6aeaf
DM
74 /**
75 * @var \SolrClient
76 */
77 protected $client = null;
78
7a4a0bc8
EM
79 /**
80 * @var bool True if we should reuse SolrClients, false if not.
81 */
82 protected $cacheclient = true;
83
5dc4624c
EM
84 /**
85 * @var \curl Direct curl object.
86 */
87 protected $curl = null;
88
95c6aeaf
DM
89 /**
90 * @var array Fields that can be highlighted.
91 */
4894840d 92 protected $highlightfields = array('title', 'content', 'description1', 'description2');
95c6aeaf 93
053118a1
EM
94 /**
95 * @var int Number of total docs reported by Sorl for the last query.
96 */
97 protected $totalenginedocs = 0;
98
99 /**
100 * @var int Number of docs we have processed for the last query.
101 */
102 protected $processeddocs = 0;
103
104 /**
105 * @var int Number of docs that have been skipped while processing the last query.
106 */
107 protected $skippeddocs = 0;
108
895e0059
DM
109 /**
110 * Solr server major version.
111 *
112 * @var int
113 */
114 protected $solrmajorversion = null;
115
7a4a0bc8
EM
116 /**
117 * Initialises the search engine configuration.
118 *
119 * @return void
120 */
121 public function __construct() {
122 parent::__construct();
123
124 $curlversion = curl_version();
125 if (isset($curlversion['version']) && stripos($curlversion['version'], '7.35.') === 0) {
126 // There is a flaw with curl 7.35.0 that causes problems with client reuse.
127 $this->cacheclient = false;
128 }
129 }
130
95c6aeaf
DM
131 /**
132 * Prepares a Solr query, applies filters and executes it returning its results.
133 *
134 * @throws \core_search\engine_exception
4359ef18 135 * @param \stdClass $filters Containing query and filters.
136 * @param \stdClass $accessinfo Information about areas user can access.
053118a1 137 * @param int $limit The maximum number of results to return.
95c6aeaf
DM
138 * @return \core_search\document[] Results or false if no results
139 */
4359ef18 140 public function execute_query($filters, $accessinfo, $limit = 0) {
f6b425e2 141 global $USER;
95c6aeaf 142
053118a1
EM
143 if (empty($limit)) {
144 $limit = \core_search\manager::MAX_RESULTS;
145 }
95c6aeaf
DM
146
147 // If there is any problem we trigger the exception as soon as possible.
7a4a0bc8 148 $client = $this->get_search_client();
95c6aeaf 149
053118a1 150 // Create the query object.
4359ef18 151 $query = $this->create_user_query($filters, $accessinfo);
053118a1 152
cfa00fc5 153 // If the query cannot have results, return none.
154 if (!$query) {
155 return [];
156 }
157
053118a1
EM
158 // We expect good match rates, so for our first get, we will get a small number of records.
159 // This significantly speeds solr response time for first few pages.
160 $query->setRows(min($limit * 3, static::QUERY_SIZE));
161 $response = $this->get_query_response($query);
162
163 // Get count data out of the response, and reset our counters.
164 list($included, $found) = $this->get_response_counts($response);
165 $this->totalenginedocs = $found;
166 $this->processeddocs = 0;
167 $this->skippeddocs = 0;
168 if ($included == 0 || $this->totalenginedocs == 0) {
169 // No results.
170 return array();
171 }
172
173 // Get valid documents out of the response.
174 $results = $this->process_response($response, $limit);
175
176 // We have processed all the docs in the response at this point.
177 $this->processeddocs += $included;
178
179 // If we haven't reached the limit, and there are more docs left in Solr, lets keep trying.
180 while (count($results) < $limit && ($this->totalenginedocs - $this->processeddocs) > 0) {
181 // Offset the start of the query, and since we are making another call, get more per call.
182 $query->setStart($this->processeddocs);
183 $query->setRows(static::QUERY_SIZE);
184
185 $response = $this->get_query_response($query);
186 list($included, $found) = $this->get_response_counts($response);
187 if ($included == 0 || $found == 0) {
188 // No new results were found. Found being empty would be weird, so we will just return.
189 return $results;
190 }
191 $this->totalenginedocs = $found;
192
193 // Get the new response docs, limiting to remaining we need, then add it to the end of the results array.
194 $newdocs = $this->process_response($response, $limit - count($results));
195 $results = array_merge($results, $newdocs);
196
197 // Add to our processed docs count.
198 $this->processeddocs += $included;
199 }
200
201 return $results;
202 }
203
204 /**
205 * Takes a query and returns the response in SolrObject format.
206 *
207 * @param SolrQuery $query Solr query object.
208 * @return SolrObject|false Response document or false on error.
209 */
210 protected function get_query_response($query) {
211 try {
212 return $this->get_search_client()->query($query)->getResponse();
213 } catch (\SolrClientException $ex) {
214 debugging('Error executing the provided query: ' . $ex->getMessage(), DEBUG_DEVELOPER);
215 $this->queryerror = $ex->getMessage();
216 return false;
217 } catch (\SolrServerException $ex) {
218 debugging('Error executing the provided query: ' . $ex->getMessage(), DEBUG_DEVELOPER);
219 $this->queryerror = $ex->getMessage();
220 return false;
cd894f84 221 }
053118a1
EM
222 }
223
224 /**
225 * Returns the total number of documents available for the most recently call to execute_query.
226 *
227 * @return int
228 */
229 public function get_query_total_count() {
230 // Return the total engine count minus the docs we have determined are bad.
231 return $this->totalenginedocs - $this->skippeddocs;
232 }
233
234 /**
235 * Returns count information for a provided response. Will return 0, 0 for invalid or empty responses.
236 *
237 * @param SolrDocument $response The response document from Solr.
238 * @return array A two part array. First how many response docs are in the response.
239 * Second, how many results are vailable in the engine.
240 */
241 protected function get_response_counts($response) {
242 $found = 0;
243 $included = 0;
244
245 if (isset($response->grouped->solr_filegroupingid->ngroups)) {
246 // Get the number of results for file grouped queries.
247 $found = $response->grouped->solr_filegroupingid->ngroups;
248 $included = count($response->grouped->solr_filegroupingid->groups);
249 } else if (isset($response->response->numFound)) {
250 // Get the number of results for standard queries.
251 $found = $response->response->numFound;
93b22672
DM
252 if ($found > 0 && is_array($response->response->docs)) {
253 $included = count($response->response->docs);
254 }
cd894f84 255 }
053118a1
EM
256
257 return array($included, $found);
258 }
259
260 /**
261 * Prepares a new query object with needed limits, filters, etc.
262 *
4359ef18 263 * @param \stdClass $filters Containing query and filters.
264 * @param \stdClass $accessinfo Information about contexts the user can access
cfa00fc5 265 * @return \SolrDisMaxQuery|null Query object or null if they can't get any results
053118a1 266 */
4359ef18 267 protected function create_user_query($filters, $accessinfo) {
053118a1
EM
268 global $USER;
269
270 // Let's keep these changes internal.
271 $data = clone $filters;
272
273 $query = new \SolrDisMaxQuery();
274
275 $this->set_query($query, $data->q);
95c6aeaf
DM
276 $this->add_fields($query);
277
278 // Search filters applied, we don't cache these filters as we don't want to pollute the cache with tmp filters
279 // we are really interested in caching contexts filters instead.
280 if (!empty($data->title)) {
281 $query->addFilterQuery('{!field cache=false f=title}' . $data->title);
282 }
501801a2
EM
283 if (!empty($data->areaids)) {
284 // If areaids are specified, we want to get any that match.
285 $query->addFilterQuery('{!cache=false}areaid:(' . implode(' OR ', $data->areaids) . ')');
95c6aeaf 286 }
427e3cbc
EM
287 if (!empty($data->courseids)) {
288 $query->addFilterQuery('{!cache=false}courseid:(' . implode(' OR ', $data->courseids) . ')');
289 }
4359ef18 290 if (!empty($data->groupids)) {
291 $query->addFilterQuery('{!cache=false}groupid:(' . implode(' OR ', $data->groupids) . ')');
292 }
95c6aeaf
DM
293
294 if (!empty($data->timestart) or !empty($data->timeend)) {
295 if (empty($data->timestart)) {
296 $data->timestart = '*';
297 } else {
298 $data->timestart = \search_solr\document::format_time_for_engine($data->timestart);
299 }
300 if (empty($data->timeend)) {
301 $data->timeend = '*';
302 } else {
303 $data->timeend = \search_solr\document::format_time_for_engine($data->timeend);
304 }
305
306 // No cache.
307 $query->addFilterQuery('{!cache=false}modified:[' . $data->timestart . ' TO ' . $data->timeend . ']');
308 }
309
f6b425e2
EM
310 // Restrict to users who are supposed to be able to see a particular result.
311 $query->addFilterQuery('owneruserid:(' . \core_search\manager::NO_OWNER_ID . ' OR ' . $USER->id . ')');
312
95c6aeaf
DM
313 // And finally restrict it to the context where the user can access, we want this one cached.
314 // If the user can access all contexts $usercontexts value is just true, we don't need to filter
315 // in that case.
4359ef18 316 if (!$accessinfo->everything && is_array($accessinfo->usercontexts)) {
427e3cbc
EM
317 // Join all area contexts into a single array and implode.
318 $allcontexts = array();
4359ef18 319 foreach ($accessinfo->usercontexts as $areaid => $areacontexts) {
501801a2 320 if (!empty($data->areaids) && !in_array($areaid, $data->areaids)) {
427e3cbc
EM
321 // Skip unused areas.
322 continue;
95c6aeaf 323 }
427e3cbc
EM
324 foreach ($areacontexts as $contextid) {
325 // Ensure they are unique.
326 $allcontexts[$contextid] = $contextid;
327 }
328 }
329 if (empty($allcontexts)) {
330 // This means there are no valid contexts for them, so they get no results.
cfa00fc5 331 return null;
95c6aeaf 332 }
427e3cbc 333 $query->addFilterQuery('contextid:(' . implode(' OR ', $allcontexts) . ')');
95c6aeaf
DM
334 }
335
4359ef18 336 if (!$accessinfo->everything && $accessinfo->separategroupscontexts) {
337 // Add another restriction to handle group ids. If there are any contexts using separate
338 // groups, then results in that context will not show unless you belong to the group.
339 // (Note: Access all groups is taken care of earlier, when computing these arrays.)
340
341 // This special exceptions list allows for particularly pig-headed developers to create
342 // multiple search areas within the same module, where one of them uses separate
343 // groups and the other uses visible groups. It is a little inefficient, but this should
344 // be rare.
345 $exceptions = '';
346 if ($accessinfo->visiblegroupscontextsareas) {
347 foreach ($accessinfo->visiblegroupscontextsareas as $contextid => $areaids) {
348 $exceptions .= ' OR (contextid:' . $contextid . ' AND areaid:(' .
349 implode(' OR ', $areaids) . '))';
350 }
351 }
352
353 if ($accessinfo->usergroups) {
354 // Either the document has no groupid, or the groupid is one that the user
355 // belongs to, or the context is not one of the separate groups contexts.
356 $query->addFilterQuery('(*:* -groupid:[* TO *]) OR ' .
357 'groupid:(' . implode(' OR ', $accessinfo->usergroups) . ') OR ' .
358 '(*:* -contextid:(' . implode(' OR ', $accessinfo->separategroupscontexts) . '))' .
359 $exceptions);
360 } else {
361 // Either the document has no groupid, or the context is not a restricted one.
362 $query->addFilterQuery('(*:* -groupid:[* TO *]) OR ' .
363 '(*:* -contextid:(' . implode(' OR ', $accessinfo->separategroupscontexts) . '))' .
364 $exceptions);
365 }
366 }
367
053118a1
EM
368 if ($this->file_indexing_enabled()) {
369 // Now group records by solr_filegroupingid. Limit to 3 results per group.
370 $query->setGroup(true);
371 $query->setGroupLimit(3);
372 $query->setGroupNGroups(true);
373 $query->addGroupField('solr_filegroupingid');
e0867b22
EM
374 } else {
375 // Make sure we only get text files, in case the index has pre-existing files.
376 $query->addFilterQuery('type:'.\core_search\manager::TYPE_TEXT);
95c6aeaf
DM
377 }
378
fc440796 379 // If ordering by location, add in boost for the relevant course or context ids.
380 if (!empty($filters->order) && $filters->order === 'location') {
381 $coursecontext = $filters->context->get_course_context();
382 $query->addBoostQuery('courseid', $coursecontext->instanceid, self::COURSE_BOOST);
383 if ($filters->context->contextlevel !== CONTEXT_COURSE) {
384 // If it's a block or activity, also add a boost for the specific context id.
385 $query->addBoostQuery('contextid', $filters->context->id, self::CONTEXT_BOOST);
386 }
387 }
388
053118a1 389 return $query;
95c6aeaf
DM
390 }
391
392 /**
393 * Prepares a new query by setting the query, start offset and rows to return.
3744ceb6 394 *
95c6aeaf 395 * @param SolrQuery $query
cd894f84 396 * @param object $q Containing query and filters.
95c6aeaf 397 */
053118a1 398 protected function set_query($query, $q) {
95c6aeaf
DM
399 // Set hightlighting.
400 $query->setHighlight(true);
401 foreach ($this->highlightfields as $field) {
402 $query->addHighlightField($field);
403 }
404 $query->setHighlightFragsize(static::FRAG_SIZE);
4894840d
EM
405 $query->setHighlightSimplePre(self::HIGHLIGHT_START);
406 $query->setHighlightSimplePost(self::HIGHLIGHT_END);
407 $query->setHighlightMergeContiguous(true);
95c6aeaf
DM
408
409 $query->setQuery($q);
410
411 // A reasonable max.
053118a1 412 $query->setRows(static::QUERY_SIZE);
95c6aeaf
DM
413 }
414
415 /**
416 * Sets fields to be returned in the result.
417 *
3744ceb6 418 * @param SolrDisMaxQuery|SolrQuery $query object.
95c6aeaf
DM
419 */
420 public function add_fields($query) {
421 $documentclass = $this->get_document_classname();
3744ceb6
EM
422 $fields = $documentclass::get_default_fields_definition();
423
424 $dismax = false;
546c0af5 425 if ($query instanceof \SolrDisMaxQuery) {
3744ceb6
EM
426 $dismax = true;
427 }
428
429 foreach ($fields as $key => $field) {
430 $query->addField($key);
431 if ($dismax && !empty($field['mainquery'])) {
432 // Add fields the main query should be run against.
433 $query->addQueryField($key);
434 }
95c6aeaf
DM
435 }
436 }
437
438 /**
439 * Finds the key common to both highlighing and docs array returned from response.
440 * @param object $response containing results.
441 */
442 public function add_highlight_content($response) {
cd894f84
EM
443 if (!isset($response->highlighting)) {
444 // There is no highlighting to add.
445 return;
446 }
447
95c6aeaf
DM
448 $highlightedobject = $response->highlighting;
449 foreach ($response->response->docs as $doc) {
450 $x = $doc->id;
451 $highlighteddoc = $highlightedobject->$x;
452 $this->merge_highlight_field_values($doc, $highlighteddoc);
453 }
454 }
455
456 /**
457 * Adds the highlighting array values to docs array values.
458 *
459 * @throws \core_search\engine_exception
460 * @param object $doc containing the results.
461 * @param object $highlighteddoc containing the highlighted results values.
462 */
463 public function merge_highlight_field_values($doc, $highlighteddoc) {
464
465 foreach ($this->highlightfields as $field) {
466 if (!empty($doc->$field)) {
467
468 // Check that the returned value is not an array. No way we can make this work with multivalued solr fields.
469 if (is_array($doc->{$field})) {
470 throw new \core_search\engine_exception('multivaluedfield', 'search_solr', '', $field);
471 }
472
473 if (!empty($highlighteddoc->$field)) {
474 // Replace by the highlighted result.
475 $doc->$field = reset($highlighteddoc->$field);
476 }
477 }
478 }
479 }
480
481 /**
482 * Filters the response on Moodle side.
483 *
053118a1
EM
484 * @param SolrObject $response Solr object containing the response return from solr server.
485 * @param int $limit The maximum number of results to return. 0 for all.
486 * @param bool $skipaccesscheck Don't use check_access() on results. Only to be used when results have known access.
95c6aeaf
DM
487 * @return array $results containing final results to be displayed.
488 */
053118a1 489 protected function process_response($response, $limit = 0, $skipaccesscheck = false) {
f6b425e2
EM
490 global $USER;
491
053118a1
EM
492 if (empty($response)) {
493 return array();
494 }
495
496 if (isset($response->grouped)) {
497 return $this->grouped_files_process_response($response, $limit);
498 }
499
f6b425e2
EM
500 $userid = $USER->id;
501 $noownerid = \core_search\manager::NO_OWNER_ID;
95c6aeaf 502
95c6aeaf
DM
503 $numgranted = 0;
504
505 if (!$docs = $response->response->docs) {
506 return array();
507 }
508
053118a1 509 $out = array();
95c6aeaf
DM
510 if (!empty($response->response->numFound)) {
511 $this->add_highlight_content($response);
512
513 // Iterate through the results checking its availability and whether they are available for the user or not.
514 foreach ($docs as $key => $docdata) {
f6b425e2
EM
515 if ($docdata['owneruserid'] != $noownerid && $docdata['owneruserid'] != $userid) {
516 // If owneruserid is set, no other user should be able to access this record.
f6b425e2
EM
517 continue;
518 }
519
95c6aeaf 520 if (!$searcharea = $this->get_search_area($docdata->areaid)) {
95c6aeaf
DM
521 continue;
522 }
523
524 $docdata = $this->standarize_solr_obj($docdata);
525
053118a1
EM
526 if ($skipaccesscheck) {
527 $access = \core_search\manager::ACCESS_GRANTED;
528 } else {
529 $access = $searcharea->check_access($docdata['itemid']);
530 }
95c6aeaf
DM
531 switch ($access) {
532 case \core_search\manager::ACCESS_DELETED:
533 $this->delete_by_id($docdata['id']);
053118a1
EM
534 // Remove one from our processed and total counters, since we promptly deleted.
535 $this->processeddocs--;
536 $this->totalenginedocs--;
95c6aeaf
DM
537 break;
538 case \core_search\manager::ACCESS_DENIED:
053118a1 539 $this->skippeddocs++;
95c6aeaf
DM
540 break;
541 case \core_search\manager::ACCESS_GRANTED:
542 $numgranted++;
543
544 // Add the doc.
053118a1 545 $out[] = $this->to_document($searcharea, $docdata);
95c6aeaf
DM
546 break;
547 }
548
053118a1
EM
549 // Stop when we hit our limit.
550 if (!empty($limit) && count($out) >= $limit) {
95c6aeaf
DM
551 break;
552 }
553 }
554 }
555
053118a1 556 return $out;
95c6aeaf
DM
557 }
558
cd894f84
EM
559 /**
560 * Processes grouped file results into documents, with attached matching files.
561 *
053118a1
EM
562 * @param SolrObject $response The response returned from solr server
563 * @param int $limit The maximum number of results to return. 0 for all.
cd894f84
EM
564 * @return array Final results to be displayed.
565 */
053118a1 566 protected function grouped_files_process_response($response, $limit = 0) {
cd894f84
EM
567 // If we can't find the grouping, or there are no matches in the grouping, return empty.
568 if (!isset($response->grouped->solr_filegroupingid) || empty($response->grouped->solr_filegroupingid->matches)) {
569 return array();
570 }
571
572 $numgranted = 0;
573 $orderedids = array();
574 $completedocs = array();
575 $incompletedocs = array();
576
577 $highlightingobj = $response->highlighting;
578
579 // Each group represents a "master document".
580 $groups = $response->grouped->solr_filegroupingid->groups;
581 foreach ($groups as $group) {
582 $groupid = $group->groupValue;
583 $groupdocs = $group->doclist->docs;
584 $firstdoc = reset($groupdocs);
585
586 if (!$searcharea = $this->get_search_area($firstdoc->areaid)) {
587 // Well, this is a problem.
588 continue;
589 }
590
591 // Check for access.
592 $access = $searcharea->check_access($firstdoc->itemid);
593 switch ($access) {
594 case \core_search\manager::ACCESS_DELETED:
595 // If deleted from Moodle, delete from index and then continue.
596 $this->delete_by_id($firstdoc->id);
053118a1
EM
597 // Remove one from our processed and total counters, since we promptly deleted.
598 $this->processeddocs--;
599 $this->totalenginedocs--;
cd894f84
EM
600 continue 2;
601 break;
602 case \core_search\manager::ACCESS_DENIED:
603 // This means we should just skip for the current user.
053118a1 604 $this->skippeddocs++;
cd894f84
EM
605 continue 2;
606 break;
607 }
608 $numgranted++;
609
610 $maindoc = false;
611 $fileids = array();
612 // Seperate the main document and any files returned.
613 foreach ($groupdocs as $groupdoc) {
614 if ($groupdoc->id == $groupid) {
615 $maindoc = $groupdoc;
616 } else if (isset($groupdoc->solr_fileid)) {
617 $fileids[] = $groupdoc->solr_fileid;
618 }
619 }
620
621 // Store the id of this group, in order, for later merging.
622 $orderedids[] = $groupid;
623
624 if (!$maindoc) {
625 // We don't have the main doc, store what we know for later building.
626 $incompletedocs[$groupid] = $fileids;
627 } else {
628 if (isset($highlightingobj->$groupid)) {
629 // Merge the highlighting for this doc.
630 $this->merge_highlight_field_values($maindoc, $highlightingobj->$groupid);
631 }
632 $docdata = $this->standarize_solr_obj($maindoc);
633 $doc = $this->to_document($searcharea, $docdata);
634 // Now we need to attach the result files to the doc.
635 foreach ($fileids as $fileid) {
636 $doc->add_stored_file($fileid);
637 }
638 $completedocs[$groupid] = $doc;
639 }
640
053118a1 641 if (!empty($limit) && $numgranted >= $limit) {
cd894f84
EM
642 // We have hit the max results, we will just ignore the rest.
643 break;
644 }
645 }
646
647 $incompletedocs = $this->get_missing_docs($incompletedocs);
648
649 $out = array();
650 // Now merge the complete and incomplete documents, in results order.
651 foreach ($orderedids as $docid) {
652 if (isset($completedocs[$docid])) {
653 $out[] = $completedocs[$docid];
654 } else if (isset($incompletedocs[$docid])) {
655 $out[] = $incompletedocs[$docid];
656 }
657 }
658
659 return $out;
660 }
661
662 /**
663 * Retreive any missing main documents and attach provided files.
664 *
665 * The missingdocs array should be an array, indexed by document id, of main documents we need to retrieve. The value
666 * associated to the key should be an array of stored_files or stored file ids to attach to the result document.
667 *
668 * Return array also indexed by document id.
669 *
670 * @param array() $missingdocs An array, indexed by document id, with arrays of files/ids to attach.
671 * @return document[]
672 */
673 protected function get_missing_docs($missingdocs) {
674 if (empty($missingdocs)) {
675 return array();
676 }
677
678 $docids = array_keys($missingdocs);
679
680 // Build a custom query that will get all the missing documents.
681 $query = new \SolrQuery();
053118a1 682 $this->set_query($query, '*');
cd894f84 683 $this->add_fields($query);
053118a1 684 $query->setRows(count($docids));
cd894f84
EM
685 $query->addFilterQuery('{!cache=false}id:(' . implode(' OR ', $docids) . ')');
686
053118a1
EM
687 $response = $this->get_query_response($query);
688 // We know the missing docs have already been checked for access, so don't recheck.
689 $results = $this->process_response($response, 0, true);
cd894f84
EM
690
691 $out = array();
692 foreach ($results as $result) {
693 $resultid = $result->get('id');
694 if (!isset($missingdocs[$resultid])) {
695 // We got a result we didn't expect. Skip it.
696 continue;
697 }
698 // Attach the files.
699 foreach ($missingdocs[$resultid] as $filedoc) {
700 $result->add_stored_file($filedoc);
701 }
702 $out[$resultid] = $result;
703 }
704
705 return $out;
706 }
707
95c6aeaf
DM
708 /**
709 * Returns a standard php array from a \SolrObject instance.
710 *
711 * @param \SolrObject $obj
712 * @return array The returned document as an array.
713 */
714 public function standarize_solr_obj(\SolrObject $obj) {
715 $properties = $obj->getPropertyNames();
716
717 $docdata = array();
718 foreach($properties as $name) {
719 // http://php.net/manual/en/solrobject.getpropertynames.php#98018.
720 $name = trim($name);
721 $docdata[$name] = $obj->offsetGet($name);
722 }
723 return $docdata;
724 }
725
726 /**
727 * Adds a document to the search engine.
728 *
729 * This does not commit to the search engine.
730 *
091973db
EM
731 * @param document $document
732 * @param bool $fileindexing True if file indexing is to be used
733 * @return bool
95c6aeaf 734 */
091973db
EM
735 public function add_document($document, $fileindexing = false) {
736 $docdata = $document->export_for_engine();
737
cd894f84 738 if (!$this->add_solr_document($docdata)) {
091973db
EM
739 return false;
740 }
741
cd894f84
EM
742 if ($fileindexing) {
743 // This will take care of updating all attached files in the index.
744 $this->process_document_files($document);
745 }
746
091973db
EM
747 return true;
748 }
95c6aeaf 749
091973db
EM
750 /**
751 * Adds a text document to the search engine.
752 *
cd894f84 753 * @param array $doc
091973db
EM
754 * @return bool
755 */
cd894f84 756 protected function add_solr_document($doc) {
95c6aeaf
DM
757 $solrdoc = new \SolrInputDocument();
758 foreach ($doc as $field => $value) {
759 $solrdoc->addField($field, $value);
760 }
761
762 try {
763 $result = $this->get_search_client()->addDocument($solrdoc, true, static::AUTOCOMMIT_WITHIN);
091973db 764 return true;
95c6aeaf
DM
765 } catch (\SolrClientException $e) {
766 debugging('Solr client error adding document with id ' . $doc['id'] . ': ' . $e->getMessage(), DEBUG_DEVELOPER);
63d5007b
EM
767 } catch (\SolrServerException $e) {
768 // We only use the first line of the message, as it's a fully java stacktrace behind it.
769 $msg = strtok($e->getMessage(), "\n");
770 debugging('Solr server error adding document with id ' . $doc['id'] . ': ' . $msg, DEBUG_DEVELOPER);
95c6aeaf 771 }
091973db
EM
772
773 return false;
95c6aeaf
DM
774 }
775
cd894f84
EM
776 /**
777 * Index files attached to the docuemnt, ensuring the index matches the current document files.
778 *
779 * For documents that aren't known to be new, we check the index for existing files.
780 * - New files we will add.
781 * - Existing and unchanged files we will skip.
782 * - File that are in the index but not on the document will be deleted from the index.
783 * - Files that have changed will be re-indexed.
784 *
785 * @param document $document
786 */
787 protected function process_document_files($document) {
788 if (!$this->file_indexing_enabled()) {
789 return;
790 }
791
792 // Maximum rows to process at a time.
793 $rows = 500;
794
795 // Get the attached files.
796 $files = $document->get_files();
797
798 // If this isn't a new document, we need to check the exiting indexed files.
799 if (!$document->get_is_new()) {
800 // We do this progressively, so we can handle lots of files cleanly.
801 list($numfound, $indexedfiles) = $this->get_indexed_files($document, 0, $rows);
802 $count = 0;
803 $idstodelete = array();
804
805 do {
806 // Go through each indexed file. We want to not index any stored and unchanged ones, delete any missing ones.
807 foreach ($indexedfiles as $indexedfile) {
808 $fileid = $indexedfile->solr_fileid;
809
810 if (isset($files[$fileid])) {
811 // Check for changes that would mean we need to re-index the file. If so, just leave in $files.
812 // Filelib does not guarantee time modified is updated, so we will check important values.
1aaead91 813 if ($indexedfile->modified != $files[$fileid]->get_timemodified()) {
cd894f84
EM
814 continue;
815 }
816 if (strcmp($indexedfile->title, $files[$fileid]->get_filename()) !== 0) {
817 continue;
818 }
819 if ($indexedfile->solr_filecontenthash != $files[$fileid]->get_contenthash()) {
820 continue;
821 }
f6b4ec7b 822 if ($indexedfile->solr_fileindexstatus == document::INDEXED_FILE_FALSE &&
cd894f84
EM
823 $this->file_is_indexable($files[$fileid])) {
824 // This means that the last time we indexed this file, filtering blocked it.
825 // Current settings say it is indexable, so we will allow it to be indexed.
826 continue;
827 }
828
829 // If the file is already indexed, we can just remove it from the files array and skip it.
830 unset($files[$fileid]);
831 } else {
832 // This means we have found a file that is no longer attached, so we need to delete from the index.
833 // We do it later, since this is progressive, and it could reorder results.
834 $idstodelete[] = $indexedfile->id;
835 }
836 }
837 $count += $rows;
838
839 if ($count < $numfound) {
840 // If we haven't hit the total count yet, fetch the next batch.
841 list($numfound, $indexedfiles) = $this->get_indexed_files($document, $count, $rows);
842 }
843
844 } while ($count < $numfound);
845
846 // Delete files that are no longer attached.
847 foreach ($idstodelete as $id) {
848 // We directly delete the item using the client, as the engine delete_by_id won't work on file docs.
849 $this->get_search_client()->deleteById($id);
850 }
851 }
852
853 // Now we can actually index all the remaining files.
854 foreach ($files as $file) {
855 $this->add_stored_file($document, $file);
856 }
857 }
858
859 /**
860 * Get the currently indexed files for a particular document, returns the total count, and a subset of files.
861 *
862 * @param document $document
863 * @param int $start The row to start the results on. Zero indexed.
864 * @param int $rows The number of rows to fetch
865 * @return array A two element array, the first is the total number of availble results, the second is an array
866 * of documents for the current request.
867 */
868 protected function get_indexed_files($document, $start = 0, $rows = 500) {
869 // Build a custom query that will get any document files that are in our solr_filegroupingid.
870 $query = new \SolrQuery();
871
872 // We want to get all file records tied to a document.
873 // For efficiency, we are building our own, stripped down, query.
874 $query->setQuery('*');
875 $query->setRows($rows);
876 $query->setStart($start);
877 // We want a consistent sorting.
878 $query->addSortField('id');
879
880 // We only want the bare minimum of fields.
881 $query->addField('id');
882 $query->addField('modified');
883 $query->addField('title');
884 $query->addField('solr_fileid');
885 $query->addField('solr_filecontenthash');
f6b4ec7b 886 $query->addField('solr_fileindexstatus');
cd894f84
EM
887
888 $query->addFilterQuery('{!cache=false}solr_filegroupingid:(' . $document->get('id') . ')');
889 $query->addFilterQuery('type:' . \core_search\manager::TYPE_FILE);
890
053118a1
EM
891 $response = $this->get_query_response($query);
892 if (empty($response->response->numFound)) {
cd894f84
EM
893 return array(0, array());
894 }
053118a1
EM
895
896 return array($response->response->numFound, $this->convert_file_results($response));
cd894f84
EM
897 }
898
899 /**
900 * A very lightweight handler for getting information about already indexed files from a Solr response.
901 *
902 * @param SolrObject $responsedoc A Solr response document
903 * @return stdClass[] An array of objects that contain the basic information for file processing.
904 */
905 protected function convert_file_results($responsedoc) {
906 if (!$docs = $responsedoc->response->docs) {
907 return array();
908 }
909
910 $out = array();
911
912 foreach ($docs as $doc) {
913 // Copy the bare minimim needed info.
914 $result = new \stdClass();
915 $result->id = $doc->id;
916 $result->modified = document::import_time_from_engine($doc->modified);
917 $result->title = $doc->title;
918 $result->solr_fileid = $doc->solr_fileid;
919 $result->solr_filecontenthash = $doc->solr_filecontenthash;
f6b4ec7b 920 $result->solr_fileindexstatus = $doc->solr_fileindexstatus;
cd894f84
EM
921 $out[] = $result;
922 }
923
924 return $out;
925 }
926
927 /**
928 * Adds a file to the search engine.
929 *
930 * Notes about Solr and Tika indexing. We do not send the mime type, only the filename.
931 * Tika has much better content type detection than Moodle, and we will have many more doc failures
932 * if we try to send mime types.
933 *
934 * @param document $document
935 * @param \stored_file $storedfile
936 * @return void
937 */
938 protected function add_stored_file($document, $storedfile) {
939 $filedoc = $document->export_file_for_engine($storedfile);
940
941 if (!$this->file_is_indexable($storedfile)) {
942 // For files that we don't consider indexable, we will still place a reference in the search engine.
f6b4ec7b 943 $filedoc['solr_fileindexstatus'] = document::INDEXED_FILE_FALSE;
cd894f84
EM
944 $this->add_solr_document($filedoc);
945 return;
946 }
947
948 $curl = $this->get_curl_object();
949
950 $url = $this->get_connection_url('/update/extract');
951
895e0059
DM
952 // Return results as XML.
953 $url->param('wt', 'xml');
954
cd894f84
EM
955 // This will prevent solr from automatically making fields for every tika output.
956 $url->param('uprefix', 'ignored_');
957
546c0af5
EM
958 // Control how content is captured. This will keep our file content clean of non-important metadata.
959 $url->param('captureAttr', 'true');
960 // Move the content to a field for indexing.
961 $url->param('fmap.content', 'solr_filecontent');
962
cd894f84
EM
963 // These are common fields that matches the standard *_point dynamic field and causes an error.
964 $url->param('fmap.media_white_point', 'ignored_mwp');
965 $url->param('fmap.media_black_point', 'ignored_mbp');
966
967 // Copy each key to the url with literal.
968 // We place in a temp name then copy back to the true field, which prevents errors or Tika overwriting common field names.
969 foreach ($filedoc as $key => $value) {
970 // This will take any fields from tika that match our schema and discard them, so they don't overwrite ours.
971 $url->param('fmap.'.$key, 'ignored_'.$key);
972 // Place data in a tmp field.
973 $url->param('literal.mdltmp_'.$key, $value);
974 // Then move to the final field.
975 $url->param('fmap.mdltmp_'.$key, $key);
976 }
977
978 // This sets the true filename for Tika.
979 $url->param('resource.name', $storedfile->get_filename());
980
981 // A giant block of code that is really just error checking around the curl request.
982 try {
983 // Now actually do the request.
984 $result = $curl->post($url->out(false), array('myfile' => $storedfile));
985
986 $code = $curl->get_errno();
987 $info = $curl->get_info();
988
989 // Now error handling. It is just informational, since we aren't tracking per file/doc results.
990 if ($code != 0) {
991 // This means an internal cURL error occurred error is in result.
992 $message = 'Curl error '.$code.' while indexing file with document id '.$filedoc['id'].': '.$result.'.';
993 debugging($message, DEBUG_DEVELOPER);
994 } else if (isset($info['http_code']) && ($info['http_code'] !== 200)) {
995 // Unexpected HTTP response code.
996 $message = 'Error while indexing file with document id '.$filedoc['id'];
997 // Try to get error message out of msg or title if it exists.
998 if (preg_match('|<str [^>]*name="msg"[^>]*>(.*?)</str>|i', $result, $matches)) {
999 $message .= ': '.$matches[1];
1000 } else if (preg_match('|<title[^>]*>([^>]*)</title>|i', $result, $matches)) {
1001 $message .= ': '.$matches[1];
1002 }
1003 // This is a common error, happening whenever a file fails to index for any reason, so we will make it quieter.
1004 if (CLI_SCRIPT && !PHPUNIT_TEST) {
1005 mtrace($message);
1006 }
1007 } else {
1008 // Check for the expected status field.
1009 if (preg_match('|<int [^>]*name="status"[^>]*>(\d*)</int>|i', $result, $matches)) {
1010 // Now check for the expected status of 0, if not, error.
1011 if ((int)$matches[1] !== 0) {
1012 $message = 'Unexpected Solr status code '.(int)$matches[1];
1013 $message .= ' while indexing file with document id '.$filedoc['id'].'.';
1014 debugging($message, DEBUG_DEVELOPER);
1015 } else {
1016 // The document was successfully indexed.
1017 return;
1018 }
1019 } else {
1020 // We received an unprocessable response.
1021 $message = 'Unexpected Solr response while indexing file with document id '.$filedoc['id'].': ';
1022 $message .= strtok($result, "\n");
1023 debugging($message, DEBUG_DEVELOPER);
1024 }
1025 }
1026 } catch (\Exception $e) {
1027 // There was an error, but we are not tracking per-file success, so we just continue on.
1028 debugging('Unknown exception while indexing file "'.$storedfile->get_filename().'".', DEBUG_DEVELOPER);
1029 }
1030
1031 // If we get here, the document was not indexed due to an error. So we will index just the base info without the file.
f6b4ec7b 1032 $filedoc['solr_fileindexstatus'] = document::INDEXED_FILE_ERROR;
cd894f84
EM
1033 $this->add_solr_document($filedoc);
1034 }
1035
1036 /**
1037 * Checks to see if a passed file is indexable.
1038 *
1039 * @param \stored_file $file The file to check
1040 * @return bool True if the file can be indexed
1041 */
1042 protected function file_is_indexable($file) {
1043 if (!empty($this->config->maxindexfilekb) && ($file->get_filesize() > ($this->config->maxindexfilekb * 1024))) {
1044 // The file is too big to index.
1045 return false;
1046 }
1047
1048 $mime = $file->get_mimetype();
1049
1050 if ($mime == 'application/vnd.moodle.backup') {
1051 // We don't index Moodle backup files. There is nothing usefully indexable in them.
1052 return false;
1053 }
1054
1055 return true;
1056 }
1057
95c6aeaf
DM
1058 /**
1059 * Commits all pending changes.
1060 *
1061 * @return void
1062 */
075fa912 1063 protected function commit() {
95c6aeaf
DM
1064 $this->get_search_client()->commit();
1065 }
1066
075fa912
EM
1067 /**
1068 * Do any area cleanup needed, and do anything to confirm contents.
1069 *
1070 * Return false to prevent the search area completed time and stats from being updated.
1071 *
0bd8383a 1072 * @param \core_search\base $searcharea The search area that was complete
075fa912
EM
1073 * @param int $numdocs The number of documents that were added to the index
1074 * @param bool $fullindex True if a full index is being performed
1075 * @return bool True means that data is considered indexed
1076 */
1077 public function area_index_complete($searcharea, $numdocs = 0, $fullindex = false) {
1078 $this->commit();
1079
1080 return true;
1081 }
1082
cd894f84
EM
1083 /**
1084 * Return true if file indexing is supported and enabled. False otherwise.
1085 *
1086 * @return bool
1087 */
1088 public function file_indexing_enabled() {
1089 return (bool)$this->config->fileindexing;
1090 }
1091
95c6aeaf
DM
1092 /**
1093 * Defragments the index.
1094 *
1095 * @return void
1096 */
1097 public function optimize() {
bfd6c78f 1098 $this->get_search_client()->optimize(1, true, false);
95c6aeaf
DM
1099 }
1100
1101 /**
1102 * Deletes the specified document.
1103 *
1104 * @param string $id The document id to delete
1105 * @return void
1106 */
1107 public function delete_by_id($id) {
cd894f84
EM
1108 // We need to make sure we delete the item and all related files, which can be done with solr_filegroupingid.
1109 $this->get_search_client()->deleteByQuery('solr_filegroupingid:' . $id);
075fa912 1110 $this->commit();
95c6aeaf
DM
1111 }
1112
1113 /**
1114 * Delete all area's documents.
1115 *
1116 * @param string $areaid
1117 * @return void
1118 */
1119 public function delete($areaid = null) {
1120 if ($areaid) {
1121 $this->get_search_client()->deleteByQuery('areaid:' . $areaid);
1122 } else {
1123 $this->get_search_client()->deleteByQuery('*:*');
1124 }
075fa912 1125 $this->commit();
95c6aeaf
DM
1126 }
1127
1128 /**
1129 * Pings the Solr server using search_solr config
1130 *
1131 * @return true|string Returns true if all good or an error string.
1132 */
1133 public function is_server_ready() {
1134
23fc1be8
DM
1135 $configured = $this->is_server_configured();
1136 if ($configured !== true) {
1137 return $configured;
1138 }
1139
73fd5666 1140 // As part of the above we have already checked that we can contact the server. For pages
1141 // where performance is important, we skip doing a full schema check as well.
1142 if ($this->should_skip_schema_check()) {
1143 return true;
1144 }
1145
4359ef18 1146 // Update schema if required/possible.
1147 $schemalatest = $this->check_latest_schema();
1148 if ($schemalatest !== true) {
1149 return $schemalatest;
1150 }
1151
23fc1be8
DM
1152 // Check that the schema is already set up.
1153 try {
1154 $schema = new \search_solr\schema();
1155 $schema->validate_setup();
1156 } catch (\moodle_exception $e) {
1157 return $e->getMessage();
1158 }
1159
1160 return true;
1161 }
1162
1163 /**
1164 * Is the solr server properly configured?.
1165 *
1166 * @return true|string Returns true if all good or an error string.
1167 */
1168 public function is_server_configured() {
1169
95c6aeaf
DM
1170 if (empty($this->config->server_hostname) || empty($this->config->indexname)) {
1171 return 'No solr configuration found';
1172 }
1173
7a4a0bc8 1174 if (!$client = $this->get_search_client(false)) {
95c6aeaf
DM
1175 return get_string('engineserverstatus', 'search');
1176 }
1177
1178 try {
23fc1be8
DM
1179 if ($this->get_solr_major_version() < 4) {
1180 // Minimum solr 4.0.
1181 return get_string('minimumsolr4', 'search_solr');
1182 }
95c6aeaf 1183 } catch (\SolrClientException $ex) {
d0b4772c
DM
1184 debugging('Solr client error: ' . html_to_text($ex->getMessage()), DEBUG_DEVELOPER);
1185 return get_string('engineserverstatus', 'search');
95c6aeaf 1186 } catch (\SolrServerException $ex) {
d0b4772c
DM
1187 debugging('Solr server error: ' . html_to_text($ex->getMessage()), DEBUG_DEVELOPER);
1188 return get_string('engineserverstatus', 'search');
95c6aeaf
DM
1189 }
1190
95c6aeaf
DM
1191 return true;
1192 }
1193
23fc1be8
DM
1194 /**
1195 * Returns the solr server major version.
1196 *
1197 * @return int
1198 */
1199 public function get_solr_major_version() {
895e0059
DM
1200 if ($this->solrmajorversion !== null) {
1201 return $this->solrmajorversion;
1202 }
1203
d0b4772c
DM
1204 // We should really ping first the server to see if the specified indexname is valid but
1205 // we want to minimise solr server requests as they are expensive. system() emits a warning
1206 // if it can not connect to the configured index in the configured server.
1207 $systemdata = @$this->get_search_client()->system();
23fc1be8 1208 $solrversion = $systemdata->getResponse()->offsetGet('lucene')->offsetGet('solr-spec-version');
895e0059
DM
1209 $this->solrmajorversion = intval(substr($solrversion, 0, strpos($solrversion, '.')));
1210
1211 return $this->solrmajorversion;
23fc1be8
DM
1212 }
1213
95c6aeaf
DM
1214 /**
1215 * Checks if the PHP Solr extension is available.
1216 *
1217 * @return bool
1218 */
1219 public function is_installed() {
1220 return function_exists('solr_get_version');
1221 }
1222
1223 /**
1224 * Returns the solr client instance.
1225 *
7a4a0bc8
EM
1226 * We don't reuse SolrClient if we are on libcurl 7.35.0, due to a bug in that version of curl.
1227 *
95c6aeaf
DM
1228 * @throws \core_search\engine_exception
1229 * @param bool $triggerexception
1230 * @return \SolrClient
1231 */
1232 protected function get_search_client($triggerexception = true) {
558b6306 1233 global $CFG;
95c6aeaf
DM
1234
1235 // Type comparison as it is set to false if not available.
1236 if ($this->client !== null) {
1237 return $this->client;
1238 }
1239
1240 $options = array(
1241 'hostname' => $this->config->server_hostname,
1242 'path' => '/solr/' . $this->config->indexname,
1243 'login' => !empty($this->config->server_username) ? $this->config->server_username : '',
1244 'password' => !empty($this->config->server_password) ? $this->config->server_password : '',
1245 'port' => !empty($this->config->server_port) ? $this->config->server_port : '',
5dc4624c 1246 'secure' => !empty($this->config->secure) ? true : false,
95c6aeaf 1247 'ssl_cert' => !empty($this->config->ssl_cert) ? $this->config->ssl_cert : '',
95c6aeaf 1248 'ssl_key' => !empty($this->config->ssl_key) ? $this->config->ssl_key : '',
5dc4624c 1249 'ssl_keypassword' => !empty($this->config->ssl_keypassword) ? $this->config->ssl_keypassword : '',
95c6aeaf
DM
1250 'ssl_cainfo' => !empty($this->config->ssl_cainfo) ? $this->config->ssl_cainfo : '',
1251 'ssl_capath' => !empty($this->config->ssl_capath) ? $this->config->ssl_capath : '',
bfd6c78f 1252 'timeout' => !empty($this->config->server_timeout) ? $this->config->server_timeout : '30'
95c6aeaf
DM
1253 );
1254
558b6306 1255 if ($CFG->proxyhost && !is_proxybypass('http://' . $this->config->server_hostname . '/')) {
1256 $options['proxy_host'] = $CFG->proxyhost;
1257 $options['proxy_port'] = $CFG->proxyport;
1258 }
1259
e240a613
DM
1260 if (!class_exists('\SolrClient')) {
1261 throw new \core_search\engine_exception('enginenotinstalled', 'search', '', 'solr');
1262 }
1263
7a4a0bc8 1264 $client = new \SolrClient($options);
95c6aeaf 1265
7a4a0bc8 1266 if ($client === false && $triggerexception) {
95c6aeaf
DM
1267 throw new \core_search\engine_exception('engineserverstatus', 'search');
1268 }
1269
7a4a0bc8
EM
1270 if ($this->cacheclient) {
1271 $this->client = $client;
1272 }
1273
1274 return $client;
95c6aeaf 1275 }
5dc4624c
EM
1276
1277 /**
1278 * Returns a curl object for conntecting to solr.
1279 *
1280 * @return \curl
1281 */
1282 public function get_curl_object() {
1283 if (!is_null($this->curl)) {
1284 return $this->curl;
1285 }
1286
402a8e7a
MG
1287 // Connection to Solr is allowed to use 'localhost' and other potentially blocked hosts/ports.
1288 $this->curl = new \curl(['ignoresecurity' => true]);
5dc4624c
EM
1289
1290 $options = array();
1291 // Build the SSL options. Based on pecl-solr and general testing.
1292 if (!empty($this->config->secure)) {
1293 if (!empty($this->config->ssl_cert)) {
1294 $options['CURLOPT_SSLCERT'] = $this->config->ssl_cert;
1295 $options['CURLOPT_SSLCERTTYPE'] = 'PEM';
1296 }
1297
1298 if (!empty($this->config->ssl_key)) {
1299 $options['CURLOPT_SSLKEY'] = $this->config->ssl_key;
1300 $options['CURLOPT_SSLKEYTYPE'] = 'PEM';
1301 }
1302
1303 if (!empty($this->config->ssl_keypassword)) {
1304 $options['CURLOPT_KEYPASSWD'] = $this->config->ssl_keypassword;
1305 }
1306
1307 if (!empty($this->config->ssl_cainfo)) {
1308 $options['CURLOPT_CAINFO'] = $this->config->ssl_cainfo;
1309 }
1310
1311 if (!empty($this->config->ssl_capath)) {
1312 $options['CURLOPT_CAPATH'] = $this->config->ssl_capath;
1313 }
1314 }
1315
411b5f3a 1316 // Set timeout as for Solr client.
1317 $options['CURLOPT_TIMEOUT'] = !empty($this->config->server_timeout) ? $this->config->server_timeout : '30';
1318
5dc4624c
EM
1319 $this->curl->setopt($options);
1320
1321 if (!empty($this->config->server_username) && !empty($this->config->server_password)) {
1322 $authorization = $this->config->server_username . ':' . $this->config->server_password;
c7203847 1323 $this->curl->setHeader('Authorization: Basic ' . base64_encode($authorization));
5dc4624c
EM
1324 }
1325
1326 return $this->curl;
1327 }
1328
1329 /**
1330 * Return a Moodle url object for the server connection.
1331 *
1332 * @param string $path The solr path to append.
1333 * @return \moodle_url
1334 */
1335 public function get_connection_url($path) {
1336 // Must use the proper protocol, or SSL will fail.
1337 $protocol = !empty($this->config->secure) ? 'https' : 'http';
1338 $url = $protocol . '://' . rtrim($this->config->server_hostname, '/');
1339 if (!empty($this->config->server_port)) {
1340 $url .= ':' . $this->config->server_port;
1341 }
1342 $url .= '/solr/' . $this->config->indexname . '/' . ltrim($path, '/');
1343
1344 return new \moodle_url($url);
1345 }
4359ef18 1346
1347 /**
1348 * Solr includes group support in the execute_query function.
1349 *
1350 * @return bool True
1351 */
1352 public function supports_group_filtering() {
1353 return true;
1354 }
1355
1356 protected function update_schema($oldversion, $newversion) {
1357 // Construct schema.
1358 $schema = new schema();
1359 $cansetup = $schema->can_setup_server();
1360 if ($cansetup !== true) {
1361 return $cansetup;
1362 }
1363
1364 switch ($newversion) {
1365 // This version just requires a setup call to add new fields.
1366 case 2017091700:
1367 $setup = true;
1368 break;
1369
1370 // If we don't know about the schema version we might not have implemented the
1371 // change correctly, so return.
1372 default:
1373 return get_string('schemaversionunknown', 'search');
1374 }
1375
1376 if ($setup) {
1377 $schema->setup();
1378 }
1379
1380 return true;
1381 }
fc440796 1382
1383 /**
1384 * Solr supports sort by location within course contexts or below.
1385 *
1386 * @param \context $context Context that the user requested search from
1387 * @return array Array from order name => display text
1388 */
1389 public function get_supported_orders(\context $context) {
1390 $orders = parent::get_supported_orders($context);
1391
1392 // If not within a course, no other kind of sorting supported.
1393 $coursecontext = $context->get_course_context(false);
1394 if ($coursecontext) {
1395 // Within a course or activity/block, support sort by location.
1396 $orders['location'] = get_string('order_location', 'search',
1397 $context->get_context_name());
1398 }
1399
1400 return $orders;
1401 }
95c6aeaf 1402}