global search MDL-25009 indexing failures in each module are now handled so that...

diff --git a/search/add.php b/search/add.php
index f15a349..5f04782 100644 (file)
--- a/search/add.php
+++ b/search/add.php
@@ -61,9 +61,7 @@
     }
     $dbcontrol = new IndexDBControl();
     $addition_count = 0;
-    $startindextime = time();
-
-    $indexdate = @$CFG->search_indexer_run_date;
+    $mainstartindextime = time();
 
     mtrace('Starting index update (additions)...');
     mtrace('Index size before: '.$CFG->search_index_size."\n");
@@ -74,6 +72,14 @@
 /// append virtual modules onto array
 
         foreach ($mods as $mod) {
+
+            $indexdate = 0;
+            $indexdatestring = 'search_indexer_run_date_'.$mod->name;
+            $startrundate = time();
+            if (isset($CFG->$indexdatestring)) {
+                $indexdate = $CFG->$indexdatestring;
+            }
+
             //build include file and function names
             $class_file = $CFG->dirroot.'/search/documents/'.$mod->name.'_document.php';
             $db_names_function = $mod->name.'_db_names';
@@ -138,21 +144,37 @@
                         // foreach document, add it to the index and database table
                         foreach ($additions as $add) {
                             ++$addition_count;
+                            // try the addDocument() so possible dml_write_exception don't block other modules running.
+                            // also we can list all the new documents that are failing.
+                            try {
+                                // object to insert into db
+                                $dbid = $dbcontrol->addDocument($add);
+
+                                // synchronise db with index
+                                $add->addField(Zend_Search_Lucene_Field::Keyword('dbid', $dbid));
 
-                            // object to insert into db
-                            $dbid = $dbcontrol->addDocument($add);
+                                $index->addDocument($add);
 
-                            // synchronise db with index
-                            $add->addField(Zend_Search_Lucene_Field::Keyword('dbid', $dbid));
+                                mtrace("  Add: $add->title (database id = $add->dbid, moodle instance id = $add->docid)");
+                            }
 
-                            mtrace("  Add: $add->title (database id = $add->dbid, moodle instance id = $add->docid)");
+                            catch (dml_write_exception $e) {
+                                mtrace(" Add: FAILED adding '$add->title' ,  moodle instance id = $add->docid , Error: $e->error ");
+                                mtrace($e);
+                            }
 
-                            $index->addDocument($add);
                         }
                     }
                     else{
                         mtrace("No types to add.\n");
                     }
+
+                    //commit changes
+                    $index->commit();
+
+                    //update index date
+                    set_config($indexdatestring, $startrundate);
+
                     mtrace("Finished $mod->name.\n");
                 }
             }
@@ -165,7 +187,7 @@
 
 /// update index date and size
 
-    set_config('search_indexer_run_date', $startindextime);
+    set_config('search_indexer_run_date', $mainstartindextime);
     set_config('search_index_size', (int)$CFG->search_index_size + (int)$addition_count);
 
 /// print some additional info

diff --git a/search/indexer.php b/search/indexer.php
index 53c8b4f..3a33ef4 100644 (file)
--- a/search/indexer.php
+++ b/search/indexer.php
@@ -125,6 +125,12 @@ require_once($CFG->dirroot.'/search/lib.php');
 
     if ($searchables){
         foreach ($searchables as $mod) {
+
+            //mark last update times for mods to now.
+            $indexdatestring = 'search_indexer_update_date_'.$mod->name;
+            set_config($indexdatestring, time());
+            $indexdatestring = 'search_indexer_run_date_'.$mod->name;
+            set_config($indexdatestring, time());
         
             mtrace("starting indexing {$mod->name}\n");
         
@@ -211,13 +217,6 @@ require_once($CFG->dirroot.'/search/lib.php');
 
     set_config('search_indexer_run_date', time());
 
-    //mark last update times for mods to now.
-    if ($mods = search_collect_searchables(false, true)){
-        foreach($mods as $mod) {
-            $indexdatestring = 'search_indexer_update_date_'.$mod->name;
-            set_config($indexdatestring, time());
-        }
-    }
 /// and the index size
 
     set_config('search_index_size', (int)$index->count());

diff --git a/search/update.php b/search/update.php
index c11f9c3..eb75f55 100644 (file)
--- a/search/update.php
+++ b/search/update.php
@@ -128,29 +128,50 @@
 
                         foreach ($updates as $update) {
                             ++$update_count;
-                            
-                            //delete old document
+                            $added_doc = false;
+
+                            //get old document for deletion later
                             // change from default text only search to include numerals for this search.
                             Zend_Search_Lucene_Analysis_Analyzer::setDefault(new Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum_CaseInsensitive());
                             $doc = $index->find("+docid:{$update->id} +doctype:{$mod->name} +itemtype:{$update->itemtype}");
                             
-                            //get the record, should only be one
-                            foreach ($doc as $thisdoc) {
-                                mtrace("  Delete: $thisdoc->title (database id = $thisdoc->dbid, index id = $thisdoc->id, moodle instance id = $thisdoc->docid)");
-                                $dbcontrol->delDocument($thisdoc);
-                                $index->delete($thisdoc->id);
-                            } 
-                            
-                            //add new modified document back into index
-                            $add = $get_document_function($update->id, $update->itemtype);
-                            
-                            //object to insert into db
-                            $dbid = $dbcontrol->addDocument($add);
+                            try {
+                                //add new modified document back into index
+                                $add = $get_document_function($update->id, $update->itemtype);
+
+                                //object to insert into db
+                                $dbid = $dbcontrol->addDocument($add);
+
+                                //synchronise db with index
+                                $add->addField(Zend_Search_Lucene_Field::Keyword('dbid', $dbid));
+                                mtrace("  Add: $add->title (database id = $add->dbid, moodle instance id = $add->docid)");
+                                $index->addDocument($add);
+                                $added_doc = true;
+                            }
+
+                            catch (dml_write_exception $e) {
+                                mtrace(" Add: FAILED adding '$add->title' ,  moodle instance id = $add->docid , Error: $e->error ");
+                                mtrace($e);
+                                $added_doc = false;
+                            }
 
-                            //synchronise db with index
-                            $add->addField(Zend_Search_Lucene_Field::Keyword('dbid', $dbid));
-                            mtrace("  Add: $add->title (database id = $add->dbid, moodle instance id = $add->docid)");
-                            $index->addDocument($add);
+                            if ($added_doc) {
+                                // ok we've successfully added the new document so far
+                                // delete single previous old document
+                                try {
+                                    //get the record, should only be one
+                                    foreach ($doc as $thisdoc) {
+                                        mtrace("  Delete: $thisdoc->title (database id = $thisdoc->dbid, index id = $thisdoc->id, moodle instance id = $thisdoc->docid)");
+                                        $dbcontrol->delDocument($thisdoc);
+                                        $index->delete($thisdoc->id);
+                                    }
+                                }
+
+                                catch (dml_write_exception $e) {
+                                    mtrace(" Delete: FAILED deleting '$thisdoc->title' ,  moodle instance id = $thisdoc->docid , Error: $e->error ");
+                                    mtrace($e);
+                                }
+                            }
                         } 
                     }
                     else{