From 95c6aeaf1c5dabc21ca6e1a7a9567b0b73721de9 Mon Sep 17 00:00:00 2001 From: David Monllao Date: Fri, 19 Feb 2016 14:44:41 +0800 Subject: [PATCH] MDL-31989 search_solr: Solr search engine --- search/engine/solr/classes/document.php | 77 ++++ search/engine/solr/classes/engine.php | 437 +++++++++++++++++++++ search/engine/solr/classes/schema.php | 301 ++++++++++++++ search/engine/solr/cli/setup_schema.php | 48 +++ search/engine/solr/lang/en/search_solr.php | 74 ++++ search/engine/solr/settings.php | 52 +++ search/engine/solr/setup_schema.php | 48 +++ search/engine/solr/tests/engine_test.php | 179 +++++++++ search/engine/solr/version.php | 29 ++ 9 files changed, 1245 insertions(+) create mode 100644 search/engine/solr/classes/document.php create mode 100644 search/engine/solr/classes/engine.php create mode 100644 search/engine/solr/classes/schema.php create mode 100644 search/engine/solr/cli/setup_schema.php create mode 100644 search/engine/solr/lang/en/search_solr.php create mode 100644 search/engine/solr/settings.php create mode 100644 search/engine/solr/setup_schema.php create mode 100644 search/engine/solr/tests/engine_test.php create mode 100644 search/engine/solr/version.php diff --git a/search/engine/solr/classes/document.php b/search/engine/solr/classes/document.php new file mode 100644 index 00000000000..c4ef6a89eb3 --- /dev/null +++ b/search/engine/solr/classes/document.php @@ -0,0 +1,77 @@ +. + +/** + * Document representation. + * + * @package search_solr + * @copyright 2015 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace search_solr; + +defined('MOODLE_INTERNAL') || die(); + +/** + * Respresents a document to index. + * + * @copyright 2015 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class document extends \core_search\document { + + /** + * Formats the timestamp according to the search engine needs. + * + * @param int $timestamp + * @return string + */ + public static function format_time_for_engine($timestamp) { + return gmdate(\search_solr\engine::DATE_FORMAT, $timestamp); + } + + /** + * Formats the timestamp according to the search engine needs. + * + * @param int $timestamp + * @return string + */ + public static function format_string_for_engine($string) { + // 2^15 default. We could convert this to a setting as is possible to + // change the max in solr. + return substr($string, 0, 32766); + } + + /** + * Returns a timestamp from the value stored in the search engine. + * + * @param string $time + * @return int + */ + public static function import_time_from_engine($time) { + return strtotime($time); + } + + /** + * Overwritten to use markdown format as we use markdown for solr highlighting. + * + * @return int + */ + protected function get_text_format() { + return FORMAT_MARKDOWN; + } +} diff --git a/search/engine/solr/classes/engine.php b/search/engine/solr/classes/engine.php new file mode 100644 index 00000000000..1e45829519d --- /dev/null +++ b/search/engine/solr/classes/engine.php @@ -0,0 +1,437 @@ +. + +/** + * Solr engine. + * + * @package search_solr + * @copyright 2015 Daniel Neis Araujo + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace search_solr; + +defined('MOODLE_INTERNAL') || die(); + +/** + * Solr engine. + * + * @package search_solr + * @copyright 2015 Daniel Neis Araujo + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class engine extends \core_search\engine { + + /** + * @var string The date format used by solr. + */ + const DATE_FORMAT = 'Y-m-d\TH:i:s\Z'; + + /** + * @var int Commit documents interval (number of miliseconds). + */ + const AUTOCOMMIT_WITHIN = 15000; + + /** + * @var int Highlighting fragsize. + */ + const FRAG_SIZE = 500; + + /** + * @var \SolrClient + */ + protected $client = null; + + /** + * @var array Fields that can be highlighted. + */ + protected $highlightfields = array('content', 'description1', 'description2'); + + /** + * Prepares a Solr query, applies filters and executes it returning its results. + * + * @throws \core_search\engine_exception + * @param stdClass $filters Containing query and filters. + * @param array $usercontexts Contexts where the user has access. True if the user can access all contexts. + * @return \core_search\document[] Results or false if no results + */ + public function execute_query($filters, $usercontexts) { + + // Let's keep these changes internal. + $data = clone $filters; + + // If there is any problem we trigger the exception as soon as possible. + $this->client = $this->get_search_client(); + + $serverstatus = $this->is_server_ready(); + if ($serverstatus !== true) { + throw new \core_search\engine_exception('engineserverstatus', 'search'); + } + + $query = new \SolrQuery(); + $this->set_query($query, $data->q); + $this->add_fields($query); + + // Search filters applied, we don't cache these filters as we don't want to pollute the cache with tmp filters + // we are really interested in caching contexts filters instead. + if (!empty($data->title)) { + $query->addFilterQuery('{!field cache=false f=title}' . $data->title); + } + if (!empty($data->areaid)) { + // Even if it is only supposed to contain PARAM_ALPHANUMEXT, better to prevent. + $query->addFilterQuery('{!field cache=false f=areaid}' . $data->areaid); + } + + if (!empty($data->timestart) or !empty($data->timeend)) { + if (empty($data->timestart)) { + $data->timestart = '*'; + } else { + $data->timestart = \search_solr\document::format_time_for_engine($data->timestart); + } + if (empty($data->timeend)) { + $data->timeend = '*'; + } else { + $data->timeend = \search_solr\document::format_time_for_engine($data->timeend); + } + + // No cache. + $query->addFilterQuery('{!cache=false}modified:[' . $data->timestart . ' TO ' . $data->timeend . ']'); + } + + // And finally restrict it to the context where the user can access, we want this one cached. + // If the user can access all contexts $usercontexts value is just true, we don't need to filter + // in that case. + if ($usercontexts && is_array($usercontexts)) { + if (!empty($data->areaid)) { + $query->addFilterQuery('contextid:(' . implode(' OR ', $usercontexts[$data->areaid]) . ')'); + } else { + // Join all area contexts into a single array and implode. + $allcontexts = array(); + foreach ($usercontexts as $areacontexts) { + foreach ($areacontexts as $contextid) { + // Ensure they are unique. + $allcontexts[$contextid] = $contextid; + } + } + $query->addFilterQuery('contextid:(' . implode(' OR ', $allcontexts) . ')'); + } + } + + try { + return $this->query_response($this->client->query($query)); + } catch (\SolrClientException $ex) { + debugging('Error executing the provided query: ' . $ex->getMessage(), DEBUG_DEVELOPER); + $this->queryerror = $ex->getMessage(); + return array(); + } catch (\SolrServerException $ex) { + debugging('Error executing the provided query: ' . $ex->getMessage(), DEBUG_DEVELOPER); + $this->queryerror = $ex->getMessage(); + return array(); + } + + } + + /** + * Prepares a new query by setting the query, start offset and rows to return. + * @param SolrQuery $query + * @param object $q Containing query and filters. + */ + protected function set_query($query, $q) { + + // Set hightlighting. + $query->setHighlight(true); + foreach ($this->highlightfields as $field) { + $query->addHighlightField($field); + } + $query->setHighlightFragsize(static::FRAG_SIZE); + $query->setHighlightSimplePre('__'); + $query->setHighlightSimplePost('__'); + + $query->setQuery($q); + + // A reasonable max. + $query->setRows(\core_search\manager::MAX_RESULTS); + } + + /** + * Sets fields to be returned in the result. + * + * @param SolrQuery $query object. + */ + public function add_fields($query) { + $documentclass = $this->get_document_classname(); + $fields = array_keys($documentclass::get_default_fields_definition()); + foreach ($fields as $field) { + $query->addField($field); + } + } + + /** + * Finds the key common to both highlighing and docs array returned from response. + * @param object $response containing results. + */ + public function add_highlight_content($response) { + $highlightedobject = $response->highlighting; + foreach ($response->response->docs as $doc) { + $x = $doc->id; + $highlighteddoc = $highlightedobject->$x; + $this->merge_highlight_field_values($doc, $highlighteddoc); + } + } + + /** + * Adds the highlighting array values to docs array values. + * + * @throws \core_search\engine_exception + * @param object $doc containing the results. + * @param object $highlighteddoc containing the highlighted results values. + */ + public function merge_highlight_field_values($doc, $highlighteddoc) { + + foreach ($this->highlightfields as $field) { + if (!empty($doc->$field)) { + + // Check that the returned value is not an array. No way we can make this work with multivalued solr fields. + if (is_array($doc->{$field})) { + throw new \core_search\engine_exception('multivaluedfield', 'search_solr', '', $field); + } + + if (!empty($highlighteddoc->$field)) { + // Replace by the highlighted result. + $doc->$field = reset($highlighteddoc->$field); + } + } + } + } + + /** + * Filters the response on Moodle side. + * + * @param object $queryresponse containing the response return from solr server. + * @return array $results containing final results to be displayed. + */ + public function query_response($queryresponse) { + + $response = $queryresponse->getResponse(); + $numgranted = 0; + + if (!$docs = $response->response->docs) { + return array(); + } + + if (!empty($response->response->numFound)) { + $this->add_highlight_content($response); + + // Iterate through the results checking its availability and whether they are available for the user or not. + foreach ($docs as $key => $docdata) { + if (!$searcharea = $this->get_search_area($docdata->areaid)) { + unset($docs[$key]); + continue; + } + + $docdata = $this->standarize_solr_obj($docdata); + + $access = $searcharea->check_access($docdata['itemid']); + switch ($access) { + case \core_search\manager::ACCESS_DELETED: + $this->delete_by_id($docdata['id']); + unset($docs[$key]); + break; + case \core_search\manager::ACCESS_DENIED: + unset($docs[$key]); + break; + case \core_search\manager::ACCESS_GRANTED: + $numgranted++; + + // Add the doc. + $docs[$key] = $this->to_document($searcharea, $docdata); + break; + } + + // This should never happen. + if ($numgranted >= \core_search\manager::MAX_RESULTS) { + $docs = array_slice($docs, 0, \core_search\manager::MAX_RESULTS, true); + break; + } + } + } + + return $docs; + } + + /** + * Returns a standard php array from a \SolrObject instance. + * + * @param \SolrObject $obj + * @return array The returned document as an array. + */ + public function standarize_solr_obj(\SolrObject $obj) { + $properties = $obj->getPropertyNames(); + + $docdata = array(); + foreach($properties as $name) { + // http://php.net/manual/en/solrobject.getpropertynames.php#98018. + $name = trim($name); + $docdata[$name] = $obj->offsetGet($name); + } + return $docdata; + } + + /** + * Adds a document to the search engine. + * + * This does not commit to the search engine. + * + * @param array $doc + * @return void + */ + public function add_document($doc) { + + $solrdoc = new \SolrInputDocument(); + foreach ($doc as $field => $value) { + $solrdoc->addField($field, $value); + } + + try { + $result = $this->get_search_client()->addDocument($solrdoc, true, static::AUTOCOMMIT_WITHIN); + } catch (\SolrClientException $e) { + debugging('Solr client error adding document with id ' . $doc['id'] . ': ' . $e->getMessage(), DEBUG_DEVELOPER); + } + } + + /** + * Commits all pending changes. + * + * @return void + */ + public function commit() { + $this->get_search_client()->commit(); + } + + /** + * Defragments the index. + * + * @return void + */ + public function optimize() { + $this->get_search_client()->optimize(); + } + + /** + * Deletes the specified document. + * + * @param string $id The document id to delete + * @return void + */ + public function delete_by_id($id) { + $this->get_search_client()->deleteById($id); + } + + /** + * Delete all area's documents. + * + * @param string $areaid + * @return void + */ + public function delete($areaid = null) { + if ($areaid) { + $this->get_search_client()->deleteByQuery('areaid:' . $areaid); + } else { + $this->get_search_client()->deleteByQuery('*:*'); + } + } + + /** + * Pings the Solr server using search_solr config + * + * @return true|string Returns true if all good or an error string. + */ + public function is_server_ready() { + + if (empty($this->config->server_hostname) || empty($this->config->indexname)) { + return 'No solr configuration found'; + } + + if (!$this->client = $this->get_search_client(false)) { + return get_string('engineserverstatus', 'search'); + } + + try { + @$this->client->ping(); + } catch (\SolrClientException $ex) { + return 'Solr client error: ' . $ex->getMessage(); + } catch (\SolrServerException $ex) { + return 'Solr server error: ' . $ex->getMessage(); + } + + // Check that setup schema has already run. + try { + $schema = new \search_solr\schema(); + $schema->validate_setup(); + } catch (\moodle_exception $e) { + return $e->getMessage(); + } + + return true; + } + + /** + * Checks if the PHP Solr extension is available. + * + * @return bool + */ + public function is_installed() { + return function_exists('solr_get_version'); + } + + /** + * Returns the solr client instance. + * + * @throws \core_search\engine_exception + * @param bool $triggerexception + * @return \SolrClient + */ + protected function get_search_client($triggerexception = true) { + + // Type comparison as it is set to false if not available. + if ($this->client !== null) { + return $this->client; + } + + $options = array( + 'hostname' => $this->config->server_hostname, + 'path' => '/solr/' . $this->config->indexname, + 'login' => !empty($this->config->server_username) ? $this->config->server_username : '', + 'password' => !empty($this->config->server_password) ? $this->config->server_password : '', + 'port' => !empty($this->config->server_port) ? $this->config->server_port : '', + 'issecure' => !empty($this->config->secure) ? $this->config->secure : '', + 'ssl_cert' => !empty($this->config->ssl_cert) ? $this->config->ssl_cert : '', + 'ssl_cert_only' => !empty($this->config->ssl_cert_only) ? $this->config->ssl_cert_only : '', + 'ssl_key' => !empty($this->config->ssl_key) ? $this->config->ssl_key : '', + 'ssl_password' => !empty($this->config->ssl_keypassword) ? $this->config->ssl_keypassword : '', + 'ssl_cainfo' => !empty($this->config->ssl_cainfo) ? $this->config->ssl_cainfo : '', + 'ssl_capath' => !empty($this->config->ssl_capath) ? $this->config->ssl_capath : '', + ); + + $this->client = new \SolrClient($options); + + if ($this->client === false && $triggerexception) { + throw new \core_search\engine_exception('engineserverstatus', 'search'); + } + + return $this->client; + } +} diff --git a/search/engine/solr/classes/schema.php b/search/engine/solr/classes/schema.php new file mode 100644 index 00000000000..5adff6df3f4 --- /dev/null +++ b/search/engine/solr/classes/schema.php @@ -0,0 +1,301 @@ +. + +/** + * Solr schema manipulation manager. + * + * @package search_solr + * @copyright 2015 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace search_solr; + +defined('MOODLE_INTERNAL') || die(); + +require_once($CFG->dirroot . '/lib/filelib.php'); + +/** + * Schema class to interact with Solr schema. + * + * At the moment it only implements create which should be enough for a basic + * moodle configuration in Solr. + * + * @package search_solr + * @copyright 2015 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class schema { + + /** + * @var stdClass + */ + protected $config = null; + + /** + * cUrl instance. + * @var \curl + */ + protected $curl = null; + + /** + * The URL. + * @var string + */ + protected $url = null; + + /** + * The schema URL. + * @var string + */ + protected $schemaurl = null; + + /** + * Constructor. + * + * @throws \moodle_exception + * @return void + */ + public function __construct() { + if (!$this->config = get_config('search_solr')) { + throw new \moodle_exception('missingconfig', 'search_solr'); + } + + if (empty($this->config->server_hostname) || empty($this->config->indexname)) { + throw new \moodle_exception('missingconfig', 'search_solr'); + } + + $this->curl = new \curl(); + + // HTTP headers. + $this->curl->setHeader('Content-type: application/json'); + if (!empty($this->config->server_username) && !empty($this->config->server_password)) { + $authorization = $this->config->server_username . ':' . $this->config->server_password; + $this->curl->setHeader('Authorization', 'Basic ' . base64_encode($authorization)); + } + + $this->url = rtrim($this->config->server_hostname, '/'); + if (!empty($this->config->server_port)) { + $this->url .= ':' . $this->config->server_port; + } + $this->url .= '/solr/' . $this->config->indexname; + $this->schemaurl = $this->url . '/schema'; + + + } + + /** + * Setup solr stuff required by moodle. + * + * @param bool $checkexisting Whether to check if the fields already exist or not + * @return bool + */ + public function setup($checkexisting = true) { + $fields = \search_solr\document::get_default_fields_definition(); + + // Field id is already there. + unset($fields['id']); + + $this->check_index(); + + return $this->add_fields($fields, $checkexisting); + } + + /** + * Checks the schema is properly set up. + * + * @throws \moodle_exception + * @return void + */ + public function validate_setup() { + $fields = \search_solr\document::get_default_fields_definition(); + + // Field id is already there. + unset($fields['id']); + + $this->check_index(); + $this->validate_fields($fields, true); + } + + /** + * Checks if the index is ready, triggers an exception otherwise. + * + * @throws \moodle_exception + * @return void + */ + protected function check_index() { + + // Check that the server is available and the index exists. + $result = $this->curl->get($this->url . '/select?wt=json'); + if ($this->curl->error) { + throw new \moodle_exception('connectionerror', 'search_solr'); + } + if ($this->curl->info['http_code'] === 404) { + throw new \moodle_exception('connectionerror', 'search_solr'); + } + } + + /** + * Adds the provided fields to Solr schema. + * + * Intentionally separated from create(), it can be called to add extra fields. + * fields separately. + * + * @throws \coding_exception + * @throws \moodle_exception + * @param array $fields \core_search\document::$requiredfields format + * @param bool $checkexisting Whether to check if the fields already exist or not + * @return bool + */ + protected function add_fields($fields, $checkexisting = true) { + + if ($checkexisting) { + // Check that non of them exists. + $this->validate_fields($fields, false); + } + + // Add all fields. + foreach ($fields as $fieldname => $data) { + + if (!isset($data['type']) || !isset($data['stored']) || !isset($data['indexed'])) { + throw new \coding_exception($fieldname . ' does not define all required field params: type, stored and indexed.'); + } + // Changing default multiValued value to false as we want to match values easily. + $params = array( + 'add-field' => array( + 'name' => $fieldname, + 'type' => $data['type'], + 'stored' => $data['stored'], + 'multiValued' => false, + 'indexed' => $data['indexed'] + ) + ); + $results = $this->curl->post($this->schemaurl, json_encode($params)); + + // We only validate if we are interested on it. + if ($checkexisting) { + if ($this->curl->error) { + throw new \moodle_exception('errorcreatingschema', 'search_solr', '', $this->curl->error); + } + $this->validate_add_field_result($results); + } + } + + return true; + } + + /** + * Checks if the schema existing fields are properly set, triggers an exception otherwise. + * + * @throws \moodle_exception + * @param array $fields + * @param bool $requireexisting Require the fields to exist, otherwise exception. + * @return void + */ + protected function validate_fields(&$fields, $requireexisting = false) { + global $CFG; + + foreach ($fields as $fieldname => $data) { + $results = $this->curl->get($this->schemaurl . '/fields/' . $fieldname); + + if ($this->curl->error) { + throw new \moodle_exception('errorcreatingschema', 'search_solr', '', $this->curl->error); + } + + if (!$results) { + throw new \moodle_exception('errorcreatingschema', 'search_solr', '', get_string('nodatafromserver', 'search_solr')); + } + $results = json_decode($results); + + if ($requireexisting && !empty($results->error) && $results->error->code === 404) { + $a = new \stdClass(); + $a->fieldname = $fieldname; + $a->setupurl = $CFG->wwwroot . '/search/engine/solr/setup_schema.php'; + throw new \moodle_exception('errorvalidatingschema', 'search_solr', '', $a); + } + + // The field should not exist so we only accept 404 errors. + if (empty($results->error) || (!empty($results->error) && $results->error->code !== 404)) { + if (!empty($results->error)) { + throw new \moodle_exception('errorcreatingschema', 'search_solr', '', $results->error->msg); + } else { + // All these field attributes are set when fields are added through this script and should + // be returned and match the defined field's values. + + if (empty($results->field) || !isset($results->field->type) || + !isset($results->field->multiValued) || !isset($results->field->indexed) || + !isset($results->field->stored)) { + + throw new \moodle_exception('errorcreatingschema', 'search_solr', '', + get_string('schemafieldautocreated', 'search_solr', $fieldname)); + + } else if ($results->field->type !== $data['type'] || + $results->field->multiValued !== false || + $results->field->indexed !== $data['indexed'] || + $results->field->stored !== $data['stored']) { + + throw new \moodle_exception('errorcreatingschema', 'search_solr', '', + get_string('schemafieldautocreated', 'search_solr', $fieldname)); + } else { + // The field already exists and it is properly defined, no need to create it. + unset($fields[$fieldname]); + } + } + } + } + } + + /** + * Checks that the field results do not contain errors. + * + * @throws \moodle_exception + * @param string $results curl response body + * @return void + */ + protected function validate_add_field_result($result) { + + if (!$result) { + throw new \moodle_exception('errorcreatingschema', 'search_solr', '', get_string('nodatafromserver', 'search_solr')); + } + + $results = json_decode($result); + if (!$results) { + if (is_scalar($result)) { + $errormsg = $result; + } else { + $errormsg = json_encode($result); + } + throw new \moodle_exception('errorcreatingschema', 'search_solr', '', $errormsg); + } + + // It comes as error when fetching fields data. + if (!empty($results->error)) { + throw new \moodle_exception('errorcreatingschema', 'search_solr', '', $results->error); + } + + // It comes as errors when adding fields. + if (!empty($results->errors)) { + + // We treat this error separately. + $errorstr = ''; + foreach ($results->errors as $error) { + $errorstr .= implode(', ', $error->errorMessages); + } + throw new \moodle_exception('errorcreatingschema', 'search_solr', '', $errorstr); + } + + } +} diff --git a/search/engine/solr/cli/setup_schema.php b/search/engine/solr/cli/setup_schema.php new file mode 100644 index 00000000000..b6697f38905 --- /dev/null +++ b/search/engine/solr/cli/setup_schema.php @@ -0,0 +1,48 @@ +. + +/** + * Adds moodle fields to solr schema. + * + * Schema REST API write actions are only available from Solr 4.4 onwards. + * + * The schema should be managed and mutable to allow this script + * to add new fields to the schema. + * + * @link https://cwiki.apache.org/confluence/display/solr/Managed+Schema+Definition+in+SolrConfig + * @package search_solr + * @copyright 2015 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +define('CLI_SCRIPT', true); + +require_once(__DIR__ . '/../../../../config.php'); + +echo get_string('addingfields', 'search_solr') . "\n"; + +if (!\core_search\manager::is_global_search_enabled()) { + throw new moodle_exception('globalsearchdisabled', 'search'); +} + +if ($CFG->searchengine !== 'solr') { + throw new moodle_exception('solrnotselected', 'search_solr'); +} + +$schema = new \search_solr\schema(); +$schema->setup(); + +echo get_string('setupok', 'search_solr') . "\n"; diff --git a/search/engine/solr/lang/en/search_solr.php b/search/engine/solr/lang/en/search_solr.php new file mode 100644 index 00000000000..2ee4ccd9752 --- /dev/null +++ b/search/engine/solr/lang/en/search_solr.php @@ -0,0 +1,74 @@ +. + +/** + * Strings for component 'search_solr'. + * + * @package core_search + * @copyright Prateek Sachan {@link http://prateeksachan.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +$string['addingfields'] = 'Adding Moodle fields to the schema'; +$string['connectionerror'] = 'The specified Solr server is not available or the specified index does not exist'; +$string['errorcreatingschema'] = 'Error creating the Solr schema: {$a}'; +$string['errorvalidatingschema'] = 'Error validating Solr schema, field {$a->fieldname} does not exist. Please follow this link to setup the fields required by Moodle.'; +$string['extensionerror'] = 'The Apache Solr PHP extension is not installed. Please check the documentation.'; +$string['missingconfig'] = 'Your Apache Solr server is not yet configured in Moodle.'; +$string['multivaluedfield'] = 'Field "{$a}" returned an array instead of a scalar, the field is probably defined in Solr with "Multivalued" to true, this means that Solr autocreated the field for you when you indexed data because you forgot to run search/engine/solr/cli/setup_schema.php. Please delete the current index, create a new one and run setup_schema.php before indexing data in Solr.'; +$string['nodatafromserver'] = 'No data from server'; +$string['pluginname'] = 'Solr'; +$string['schemafieldautocreated'] = 'Field "{$a}" already exists in Solr schema. You probably forgot to run this script before indexing data and fields were autocreated by Solr. Please delete the current index, create a new one and run setup_schema.php again before indexing data in Solr.'; +$string['searchinfo'] = 'Search queries'; +$string['searchinfo_help'] = 'Features you can use while performing search queries: + +* Fields: You can specify which fields you want results from.
+ * title:(moodle + perth): returns all records that contains both "moodle" and "perth" in the title. Available fields: title, content, name, intro +* Boolean Operators ("AND", "OR", "NOT"):
+ * (moodle AND perth) OR (moodle AND australia) +* Wildcards ("*", "?"):
+ * "mo??l*" returns both "moodle" and "moodledata". +* Proximity Searches ("~"):
+ * mood~2 returns "moodle". (2 alphabets away from "mood"). + * moodle australia~3 returns results containing "moodle hq at perth australia" (the queried terms were within 3 words of each other) +* Boosting Terms ("^"): To boost certain words/phrases:
+ * "perth australia"^5 "australia" will make results with the phrase "perth australia" more relevant. +'; +$string['setupok'] = 'The schema is ready to be used.'; +$string['solrauthpassword'] = 'HTTP Authentication Password'; +$string['solrauthuser'] = 'HTTP Authentication username'; +$string['solrindexname'] = 'Index name'; +$string['solrhttpconnectionport'] = 'Port'; +$string['solrhttpconnectiontimeout'] = 'Timeout'; +$string['solrhttpconnectiontimeout_desc'] = 'The HTTP connection timeout is the maximum time in seconds allowed for the HTTP data transfer operation.'; +$string['solrinfo'] = 'Solr'; +$string['solrnotselected'] = 'Solr engine is not the configured search engine'; +$string['solrserverhostname'] = 'Host Name'; +$string['solrserverhostname_desc'] = 'Domain name of the Solr server.'; +$string['solrsecuremode'] = 'Secure mode'; +$string['solrsetting'] = 'Solr settings'; +$string['solrsslcainfo'] = 'SSL CA certificates name'; +$string['solrsslcainfo_desc'] = 'File name holding one or more CA certificates to verify peer with'; +$string['solrsslcapath'] = 'SSL CA certificates path'; +$string['solrsslcapath_desc'] = 'Directory path holding multiple CA certificates to verify peer with'; +$string['solrsslcert'] = 'SSL key & certificate'; +$string['solrsslcert_desc'] = 'File name to a PEM-formatted private key + private certificate (concatenated in that order)'; +$string['solrsslcertonly'] = 'SSL certificate'; +$string['solrsslcertonly_desc'] = 'File name to a PEM-formatted private certificate only'; +$string['solrsslkey'] = 'SSL key'; +$string['solrsslkey_desc'] = 'File name to a PEM-formatted private key'; +$string['solrsslkeypassword'] = 'SSL Key password'; +$string['solrsslkeypassword_desc'] = 'Password for PEM-formatted private key file'; diff --git a/search/engine/solr/settings.php b/search/engine/solr/settings.php new file mode 100644 index 00000000000..995005f0e6b --- /dev/null +++ b/search/engine/solr/settings.php @@ -0,0 +1,52 @@ +. + +/** + * Solr search engine settings. + * + * @package search_solr + * @copyright 2015 Daniel Neis + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +if ($ADMIN->fulltree) { + + if (!during_initial_install()) { + if (!function_exists('solr_get_version')) { + $settings->add(new admin_setting_heading('search_solr_settings', '', get_string('extensionerror', 'search_solr'))); + + } else { + $settings->add(new admin_setting_configtext('search_solr/server_hostname', new lang_string('solrserverhostname', 'search_solr'), new lang_string('solrserverhostname_desc', 'search_solr'), '127.0.0.1', PARAM_TEXT)); + $settings->add(new admin_setting_configtext('search_solr/indexname', new lang_string('solrindexname', 'search_solr'), '', 'moodle', PARAM_TEXT)); + $settings->add(new admin_setting_configcheckbox('search_solr/secure', new lang_string('solrsecuremode', 'search_solr'), '', 0, 1, 0)); + + $secure = get_config('search_solr', 'secure'); + $defaultport = !empty($secure) ? 8443 : 8983; + $settings->add(new admin_setting_configtext('search_solr/server_port', new lang_string('solrhttpconnectionport', 'search_solr'), '', $defaultport, PARAM_INT)); + $settings->add(new admin_setting_configtext('search_solr/server_username', new lang_string('solrauthuser', 'search_solr'), '', '', PARAM_RAW)); + $settings->add(new admin_setting_configtext('search_solr/server_password', new lang_string('solrauthpassword', 'search_solr'), '', '', PARAM_RAW)); + $settings->add(new admin_setting_configtext('search_solr/server_timeout', new lang_string('solrhttpconnectiontimeout', 'search_solr'), new lang_string('solrhttpconnectiontimeout_desc', 'search_solr'), 30, PARAM_INT)); + $settings->add(new admin_setting_configtext('search_solr/ssl_cert', new lang_string('solrsslcert', 'search_solr'), new lang_string('solrsslcert_desc', 'search_solr'), '', PARAM_RAW)); + $settings->add(new admin_setting_configtext('search_solr/ssl_cert_only', new lang_string('solrsslcertonly', 'search_solr'), new lang_string('solrsslcertonly_desc', 'search_solr'), '', PARAM_RAW)); + $settings->add(new admin_setting_configtext('search_solr/ssl_key', new lang_string('solrsslkey', 'search_solr'), new lang_string('solrsslkey_desc', 'search_solr'), '', PARAM_RAW)); + $settings->add(new admin_setting_configtext('search_solr/ssl_keypassword', new lang_string('solrsslkeypassword', 'search_solr'), new lang_string('solrsslkeypassword_desc', 'search_solr'), '', PARAM_RAW)); + $settings->add(new admin_setting_configtext('search_solr/ssl_cainfo', new lang_string('solrsslcainfo', 'search_solr'), new lang_string('solrsslcainfo_desc', 'search_solr'), '', PARAM_RAW)); + $settings->add(new admin_setting_configtext('search_solr/ssl_capath', new lang_string('solrsslcapath', 'search_solr'), new lang_string('solrsslcapath_desc', 'search_solr'), '', PARAM_RAW)); + } + } +} diff --git a/search/engine/solr/setup_schema.php b/search/engine/solr/setup_schema.php new file mode 100644 index 00000000000..3c9cf3a0358 --- /dev/null +++ b/search/engine/solr/setup_schema.php @@ -0,0 +1,48 @@ +. + +/** + * Adds moodle fields to solr schema. + * + * Schema REST API write actions are only available from Solr 4.4 onwards. + * + * The schema should be managed and mutable to allow this script + * to add new fields to the schema. + * + * @link https://cwiki.apache.org/confluence/display/solr/Managed+Schema+Definition+in+SolrConfig + * @package search_solr + * @copyright 2015 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +require_once(__DIR__ . '/../../../config.php'); +require_once($CFG->libdir.'/adminlib.php'); + +require_capability('moodle/site:config', context_system::instance()); + +if (!\core_search\manager::is_global_search_enabled()) { + throw new moodle_exception('globalsearchdisabled', 'search'); +} + +if ($CFG->searchengine !== 'solr') { + throw new moodle_exception('solrnotselected', 'search_solr'); +} + +$schema = new \search_solr\schema(); +$schema->setup(); + +$url = new moodle_url('/admin/settings.php', array('section' => 'searchsolr')); +redirect($url, get_string('setupok', 'search_solr'), 4); diff --git a/search/engine/solr/tests/engine_test.php b/search/engine/solr/tests/engine_test.php new file mode 100644 index 00000000000..da9234db71e --- /dev/null +++ b/search/engine/solr/tests/engine_test.php @@ -0,0 +1,179 @@ +. + +/** + * Solr earch engine base unit tests. + * + * Required params: + * - define('TEST_SEARCH_SOLR_HOSTNAME', '127.0.0.1'); + * - define('TEST_SEARCH_SOLR_PORT', '8983'); + * - define('TEST_SEARCH_SOLR_INDEXNAME', 'unittest'); + * + * Optional params: + * - define('TEST_SEARCH_SOLR_USERNAME', ''); + * - define('TEST_SEARCH_SOLR_PASSWORD', ''); + * + * @package core_search + * @category phpunit + * @copyright 2015 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +global $CFG; +require_once($CFG->dirroot . '/search/tests/fixtures/testable_core_search.php'); +require_once($CFG->dirroot . '/search/tests/fixtures/mock_search_area.php'); + +/** + * Solr search engine base unit tests. + * + * @package core_search + * @category phpunit + * @copyright 2015 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class search_solr_engine_testcase extends advanced_testcase { + + /** + * @var \core_search::manager + */ + protected $search = null; + + public function setUp() { + $this->resetAfterTest(); + set_config('enableglobalsearch', true); + + if (!function_exists('solr_get_version')) { + $this->markTestSkipped('Solr extension is not loaded.'); + } + + if (!defined('TEST_SEARCH_SOLR_HOSTNAME') || !defined('TEST_SEARCH_SOLR_INDEXNAME') || + !defined('TEST_SEARCH_SOLR_PORT')) { + $this->markTestSkipped('Solr extension test server not set.'); + } + + set_config('hostname', TEST_SEARCH_SOLR_HOSTNAME, 'search_solr'); + set_config('port', TEST_SEARCH_SOLR_PORT, 'search_solr'); + set_config('indexname', TEST_SEARCH_SOLR_INDEXNAME, 'search_solr'); + + if (defined('TEST_SEARCH_SOLR_USERNAME')) { + set_config('server_username', TEST_SEARCH_SOLR_USERNAME); + } + + if (defined('TEST_SEARCH_SOLR_PASSWORD')) { + set_config('server_password', TEST_SEARCH_SOLR_PASSWORD); + } + + // Inject search solr engine into the testable core search as we need to add the mock + // search component to it. + $searchengine = new \search_solr\engine(); + $this->search = testable_core_search::instance($searchengine); + $areaid = \core_search\manager::generate_areaid('core_mocksearch', 'role_capabilities'); + $this->search->add_search_area($areaid, new core_mocksearch\search\role_capabilities()); + + $this->setAdminUser(); + + // Cleanup before doing anything on it as the index it is out of this test control. + $this->search->delete_index(); + + // Add moodle fields if they don't exist. + $schema = new \search_solr\schema(); + $schema->setup(false); + } + + public function test_connection() { + $this->assertTrue($this->search->get_engine()->is_server_ready()); + } + + public function test_index() { + global $DB; + + $noneditingteacherid = $DB->get_field('role', 'id', array('shortname' => 'teacher')); + + // Data gets into the search engine. + $this->assertTrue($this->search->index()); + + // Not anymore as everything was already added. + sleep(1); + $this->assertFalse($this->search->index()); + + assign_capability('moodle/course:renameroles', CAP_ALLOW, $noneditingteacherid, context_system::instance()->id); + accesslib_clear_all_caches_for_unit_testing(); + + // Indexing again once there is new data. + $this->assertTrue($this->search->index()); + } + + /** + * Better keep this not very strict about which or how many results are returned as may depend on solr engine config. + * + * @return void + */ + public function test_search() { + global $USER, $DB; + + $noneditingteacherid = $DB->get_field('role', 'id', array('shortname' => 'teacher')); + + $this->search->index(); + + $querydata = new stdClass(); + $querydata->q = 'message'; + $results = $this->search->search($querydata); + $this->assertCount(2, $results); + + // Based on core_mocksearch\search\indexer. + $this->assertEquals($USER->id, $results[0]->get('userid')); + $this->assertEquals(\context_system::instance()->id, $results[0]->get('contextid')); + + // Testing filters we don't purge cache in between assertions because cache key depends on the whole filters set + // and they are different. + sleep(1); + $beforeadding = time(); + sleep(1); + assign_capability('moodle/course:renameroles', CAP_ALLOW, $noneditingteacherid, context_system::instance()->id); + accesslib_clear_all_caches_for_unit_testing(); + $this->search->index(); + + // Timestart. + $querydata->timestart = $beforeadding; + $this->assertCount(1, $this->search->search($querydata)); + + // Timeend. + unset($querydata->timestart); + $querydata->timeend = $beforeadding; + $this->assertCount(2, $this->search->search($querydata)); + + // Title. + unset($querydata->timeend); + $querydata->title = 'moodle/course:renameroles roleid 1'; + $this->assertCount(1, $this->search->search($querydata)); + } + + public function test_delete() { + $this->search->index(); + + $querydata = new stdClass(); + $querydata->q = 'message'; + + $this->assertCount(2, $this->search->search($querydata)); + + $areaid = \core_search\manager::generate_areaid('core_mocksearch', 'role_capabilities'); + $this->search->delete_index($areaid); + cache_helper::purge_by_definition('core', 'search_results'); + $this->assertCount(0, $this->search->search($querydata)); + } +} diff --git a/search/engine/solr/version.php b/search/engine/solr/version.php new file mode 100644 index 00000000000..443ddc68de7 --- /dev/null +++ b/search/engine/solr/version.php @@ -0,0 +1,29 @@ +. + +/** + * Version info. + * + * @package search_solr + * @copyright 2015 Daniel Neis Araujo + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +$plugin->version = 2016012000; +$plugin->requires = 2015111000; +$plugin->component = 'search_solr'; -- 2.17.1