Merge branch 'MDL-66496-master' of git://github.com/tobiasreischmann/moodle
authorAndrew Nicols <andrew@nicols.co.uk>
Thu, 17 Oct 2019 03:05:34 +0000 (11:05 +0800)
committerAndrew Nicols <andrew@nicols.co.uk>
Thu, 17 Oct 2019 03:05:34 +0000 (11:05 +0800)
mod/data/import.php
mod/data/lib.php
mod/data/tests/fixtures/test_data_import.csv [new file with mode: 0644]
mod/data/tests/fixtures/test_data_import_with_field_username.csv [new file with mode: 0644]
mod/data/tests/fixtures/test_data_import_with_userdata.csv [new file with mode: 0644]
mod/data/tests/import_test.php [new file with mode: 0644]

index c2c0c37..fb5ad44 100644 (file)
@@ -91,93 +91,8 @@ if (!$formdata = $form->get_data()) {
     echo $OUTPUT->footer();
     die;
 } else {
-    // Large files are likely to take their time and memory. Let PHP know
-    // that we'll take longer, and that the process should be recycled soon
-    // to free up memory.
-    core_php_time_limit::raise();
-    raise_memory_limit(MEMORY_EXTRA);
-
-    $iid = csv_import_reader::get_new_iid('moddata');
-    $cir = new csv_import_reader($iid, 'moddata');
-
     $filecontent = $form->get_file_content('recordsfile');
-    $readcount = $cir->load_csv_content($filecontent, $formdata->encoding, $formdata->fielddelimiter);
-    unset($filecontent);
-    if (empty($readcount)) {
-        print_error('csvfailed','data',"{$CFG->wwwroot}/mod/data/edit.php?d={$data->id}");
-    } else {
-        if (!$fieldnames = $cir->get_columns()) {
-            print_error('cannotreadtmpfile', 'error');
-        }
-        $fieldnames = array_flip($fieldnames);
-        // check the fieldnames are valid
-        $rawfields = $DB->get_records('data_fields', array('dataid' => $data->id), '', 'name, id, type');
-        $fields = array();
-        $errorfield = '';
-        $safetoskipfields = array(get_string('user'), get_string('username'), get_string('email'),
-            get_string('timeadded', 'data'), get_string('timemodified', 'data'),
-            get_string('approved', 'data'), get_string('tags', 'data'));
-        foreach ($fieldnames as $name => $id) {
-            if (!isset($rawfields[$name])) {
-                if (!in_array($name, $safetoskipfields)) {
-                    $errorfield .= "'$name' ";
-                }
-            } else {
-                $field = $rawfields[$name];
-                require_once("$CFG->dirroot/mod/data/field/$field->type/field.class.php");
-                $classname = 'data_field_' . $field->type;
-                $fields[$name] = new $classname($field, $data, $cm);
-            }
-        }
-
-        if (!empty($errorfield)) {
-            print_error('fieldnotmatched','data',"{$CFG->wwwroot}/mod/data/edit.php?d={$data->id}",$errorfield);
-        }
-
-        $cir->init();
-        $recordsadded = 0;
-        while ($record = $cir->next()) {
-            if ($recordid = data_add_record($data, 0)) {  // add instance to data_record
-                foreach ($fields as $field) {
-                    $fieldid = $fieldnames[$field->field->name];
-                    if (isset($record[$fieldid])) {
-                        $value = $record[$fieldid];
-                    } else {
-                        $value = '';
-                    }
-
-                    if (method_exists($field, 'update_content_import')) {
-                        $field->update_content_import($recordid, $value, 'field_' . $field->field->id);
-                    } else {
-                        $content = new stdClass();
-                        $content->fieldid = $field->field->id;
-                        $content->content = $value;
-                        $content->recordid = $recordid;
-                        $DB->insert_record('data_content', $content);
-                    }
-                }
-
-                if (core_tag_tag::is_enabled('mod_data', 'data_records') &&
-                        isset($fieldnames[get_string('tags', 'data')])) {
-                    $columnindex = $fieldnames[get_string('tags', 'data')];
-                    $rawtags = $record[$columnindex];
-                    $tags = explode(',', $rawtags);
-                    foreach ($tags as $tag) {
-                        $tag = trim($tag);
-                        if (empty($tag)) {
-                            continue;
-                        }
-                        core_tag_tag::add_item_tag('mod_data', 'data_records', $recordid, $context, $tag);
-                    }
-                }
-
-                $recordsadded++;
-                print get_string('added', 'moodle', $recordsadded) . ". " . get_string('entry', 'data') . " (ID $recordid)<br />\n";
-            }
-        }
-        $cir->close();
-        $cir->cleanup(true);
-    }
+    $recordsadded = data_import_csv($cm, $data, $filecontent, $formdata->encoding, $formdata->fielddelimiter);
 }
 
 if ($recordsadded > 0) {
index bd562c9..c9a7c33 100644 (file)
@@ -973,16 +973,17 @@ function data_numentries($data, $userid=null) {
  * @global object
  * @param object $data
  * @param int $groupid
+ * @param int $userid
  * @return bool
  */
-function data_add_record($data, $groupid=0){
+function data_add_record($data, $groupid=0, $userid=null) {
     global $USER, $DB;
 
     $cm = get_coursemodule_from_instance('data', $data->id);
     $context = context_module::instance($cm->id);
 
     $record = new stdClass();
-    $record->userid = $USER->id;
+    $record->userid = $userid ?? $USER->id;
     $record->dataid = $data->id;
     $record->groupid = $groupid;
     $record->timecreated = $record->timemodified = time();
@@ -3044,6 +3045,135 @@ function data_supports($feature) {
         default: return null;
     }
 }
+
+/**
+ * Import records for a data instance from csv data.
+ *
+ * @param object $cm Course module of the data instance.
+ * @param object $data The data instance.
+ * @param string $csvdata The csv data to be imported.
+ * @param string $encoding The encoding of csv data.
+ * @param string $fielddelimiter The delimiter of the csv data.
+ * @return int Number of records added.
+ */
+function data_import_csv($cm, $data, &$csvdata, $encoding, $fielddelimiter) {
+    global $CFG, $DB;
+    // Large files are likely to take their time and memory. Let PHP know
+    // that we'll take longer, and that the process should be recycled soon
+    // to free up memory.
+    core_php_time_limit::raise();
+    raise_memory_limit(MEMORY_EXTRA);
+
+    $iid = csv_import_reader::get_new_iid('moddata');
+    $cir = new csv_import_reader($iid, 'moddata');
+
+    $context = context_module::instance($cm->id);
+
+    $readcount = $cir->load_csv_content($csvdata, $encoding, $fielddelimiter);
+    $csvdata = null; // Free memory.
+    if (empty($readcount)) {
+        print_error('csvfailed', 'data', "{$CFG->wwwroot}/mod/data/edit.php?d={$data->id}");
+    } else {
+        if (!$fieldnames = $cir->get_columns()) {
+            print_error('cannotreadtmpfile', 'error');
+        }
+
+        // Check the fieldnames are valid.
+        $rawfields = $DB->get_records('data_fields', array('dataid' => $data->id), '', 'name, id, type');
+        $fields = array();
+        $errorfield = '';
+        $usernamestring = get_string('username');
+        $safetoskipfields = array(get_string('user'), get_string('email'),
+            get_string('timeadded', 'data'), get_string('timemodified', 'data'),
+            get_string('approved', 'data'), get_string('tags', 'data'));
+        $userfieldid = null;
+        foreach ($fieldnames as $id => $name) {
+            if (!isset($rawfields[$name])) {
+                if ($name == $usernamestring) {
+                    $userfieldid = $id;
+                } else if (!in_array($name, $safetoskipfields)) {
+                    $errorfield .= "'$name' ";
+                }
+            } else {
+                // If this is the second time, a field with this name comes up, it must be a field not provided by the user...
+                // like the username.
+                if (isset($fields[$name])) {
+                    if ($name == $usernamestring) {
+                        $userfieldid = $id;
+                    }
+                    unset($fieldnames[$id]); // To ensure the user provided content fields remain in the array once flipped.
+                } else {
+                    $field = $rawfields[$name];
+                    require_once("$CFG->dirroot/mod/data/field/$field->type/field.class.php");
+                    $classname = 'data_field_' . $field->type;
+                    $fields[$name] = new $classname($field, $data, $cm);
+                }
+            }
+        }
+
+        if (!empty($errorfield)) {
+            print_error('fieldnotmatched', 'data',
+                "{$CFG->wwwroot}/mod/data/edit.php?d={$data->id}", $errorfield);
+        }
+
+        $fieldnames = array_flip($fieldnames);
+
+        $cir->init();
+        $recordsadded = 0;
+        while ($record = $cir->next()) {
+            $authorid = null;
+            if ($userfieldid) {
+                if (!($author = core_user::get_user_by_username($record[$userfieldid], 'id'))) {
+                    $authorid = null;
+                } else {
+                    $authorid = $author->id;
+                }
+            }
+            if ($recordid = data_add_record($data, 0, $authorid)) {  // Add instance to data_record.
+                foreach ($fields as $field) {
+                    $fieldid = $fieldnames[$field->field->name];
+                    if (isset($record[$fieldid])) {
+                        $value = $record[$fieldid];
+                    } else {
+                        $value = '';
+                    }
+
+                    if (method_exists($field, 'update_content_import')) {
+                        $field->update_content_import($recordid, $value, 'field_' . $field->field->id);
+                    } else {
+                        $content = new stdClass();
+                        $content->fieldid = $field->field->id;
+                        $content->content = $value;
+                        $content->recordid = $recordid;
+                        $DB->insert_record('data_content', $content);
+                    }
+                }
+
+                if (core_tag_tag::is_enabled('mod_data', 'data_records') &&
+                    isset($fieldnames[get_string('tags', 'data')])) {
+                    $columnindex = $fieldnames[get_string('tags', 'data')];
+                    $rawtags = $record[$columnindex];
+                    $tags = explode(',', $rawtags);
+                    foreach ($tags as $tag) {
+                        $tag = trim($tag);
+                        if (empty($tag)) {
+                            continue;
+                        }
+                        core_tag_tag::add_item_tag('mod_data', 'data_records', $recordid, $context, $tag);
+                    }
+                }
+
+                $recordsadded++;
+                print get_string('added', 'moodle', $recordsadded) . ". " . get_string('entry', 'data') . " (ID $recordid)<br />\n";
+            }
+        }
+        $cir->close();
+        $cir->cleanup(true);
+        return $recordsadded;
+    }
+    return 0;
+}
+
 /**
  * @global object
  * @param array $export
diff --git a/mod/data/tests/fixtures/test_data_import.csv b/mod/data/tests/fixtures/test_data_import.csv
new file mode 100644 (file)
index 0000000..0ac9f1c
--- /dev/null
@@ -0,0 +1,3 @@
+ID,Param2
+1,"My first entry"
+2,"My second entry"
\ No newline at end of file
diff --git a/mod/data/tests/fixtures/test_data_import_with_field_username.csv b/mod/data/tests/fixtures/test_data_import_with_field_username.csv
new file mode 100644 (file)
index 0000000..215bf6c
--- /dev/null
@@ -0,0 +1,4 @@
+ID,Username,Param2,Username,"Email address"
+1,otherusername1,"My first entry",student,student@moodle.org
+2,otherusername2,"My second entry",student2,student@moodle.org
+3,otherusername3,"My third entry",student,student@moodle.org
\ No newline at end of file
diff --git a/mod/data/tests/fixtures/test_data_import_with_userdata.csv b/mod/data/tests/fixtures/test_data_import_with_userdata.csv
new file mode 100644 (file)
index 0000000..dd5ef63
--- /dev/null
@@ -0,0 +1,3 @@
+ID,Param2,Username,"Email address"
+1,"My first entry",student,student@moodle.org
+2,"My second entry",student2,student@moodle.org
\ No newline at end of file
diff --git a/mod/data/tests/import_test.php b/mod/data/tests/import_test.php
new file mode 100644 (file)
index 0000000..570f29d
--- /dev/null
@@ -0,0 +1,263 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Unit tests for importing csv files.
+ *
+ * @package    mod_data
+ * @category   test
+ * @copyright  2019 Tobias Reischmann
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+global $CFG;
+require_once($CFG->dirroot . '/mod/data/lib.php');
+require_once($CFG->dirroot . '/lib/datalib.php');
+require_once($CFG->dirroot . '/lib/csvlib.class.php');
+require_once($CFG->dirroot . '/search/tests/fixtures/testable_core_search.php');
+require_once($CFG->dirroot . '/mod/data/tests/generator/lib.php');
+
+/**
+ * Unit tests for import.php.
+ *
+ * @package    mod_data
+ * @copyright  2019 Tobias Reischmann
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class mod_data_import_test extends advanced_testcase {
+
+    /** @var object $cm Course module of data instance. */
+    private $cm;
+
+    /** @var object $data Data instance. */
+    private $data;
+
+    /** @var mod_data_generator $generator */
+    private $generator;
+
+    /** @var object $student Student object */
+    private $student;
+
+    /** @var object $teacher Teacher object */
+    private $teacher;
+
+    /**
+     * Set up function. In this instance we are setting up database
+     * records to be used in the unit tests.
+     */
+    protected function setUp() {
+        parent::setUp();
+
+        $this->resetAfterTest(true);
+
+        $this->generator = $this->getDataGenerator()->get_plugin_generator('mod_data');
+        $course = $this->getDataGenerator()->create_course();
+        $this->teacher = $this->getDataGenerator()->create_and_enrol($course, 'teacher');
+        $this->setUser($this->teacher);
+        $this->student = $this->getDataGenerator()->create_and_enrol($course, 'student', array('username' => 'student'));
+
+        $this->data = $this->generator->create_instance(array('course' => $course->id));
+        $this->cm = get_coursemodule_from_instance('data', $this->data->id);
+
+        // Add fields.
+        $fieldrecord = new StdClass();
+        $fieldrecord->name = 'ID'; // Identifier of the records for testing.
+        $fieldrecord->type = 'number';
+        $this->generator->create_field($fieldrecord, $this->data);
+
+        $fieldrecord->name = 'Param2';
+        $fieldrecord->type = 'text';
+        $this->generator->create_field($fieldrecord, $this->data);
+    }
+
+    /**
+     * Test uploading entries for a data instance without userdata.
+     * @throws dml_exception
+     */
+    public function test_import() {
+        $filecontent = file_get_contents(__DIR__ . '/fixtures/test_data_import.csv');
+        ob_start();
+        data_import_csv($this->cm, $this->data, $filecontent, 'UTF-8', 'comma');
+        ob_end_clean();
+
+        // No userdata is present in the file: Fallback is to assign the uploading user as author.
+        $expecteduserids = array();
+        $expecteduserids[1] = $this->teacher->id;
+        $expecteduserids[2] = $this->teacher->id;
+
+        $records = $this->get_data_records($this->data->id);
+        $this->assertCount(2, $records);
+        foreach ($records as $record) {
+            $identifier = $record->items['ID']->content;
+            $this->assertEquals($expecteduserids[$identifier], $record->userid);
+        }
+    }
+
+    /**
+     * Test uploading entries for a data instance with userdata.
+     *
+     * At least one entry has an identifiable user, which is assigned as author.
+     * @throws dml_exception
+     */
+    public function test_import_with_userdata() {
+        $filecontent = file_get_contents(__DIR__ . '/fixtures/test_data_import_with_userdata.csv');
+        ob_start();
+        data_import_csv($this->cm, $this->data, $filecontent, 'UTF-8', 'comma');
+        ob_end_clean();
+
+        $expecteduserids = array();
+        $expecteduserids[1] = $this->student->id; // User student exists and is assigned as author.
+        $expecteduserids[2] = $this->teacher->id; // User student2 does not exist. Fallback is the uploading user.
+
+        $records = $this->get_data_records($this->data->id);
+        $this->assertCount(2, $records);
+        foreach ($records as $record) {
+            $identifier = $record->items['ID']->content;
+            $this->assertEquals($expecteduserids[$identifier], $record->userid);
+        }
+    }
+
+    /**
+     * Test uploading entries for a data instance with userdata and a defined field 'Username'.
+     *
+     * This should test the corner case, in which a user has defined a data fields, which has the same name
+     * as the current lang string for username. In that case, the first Username entry is used for the field.
+     * The second one is used to identify the author.
+     * @throws coding_exception
+     * @throws dml_exception
+     */
+    public function test_import_with_field_username() {
+
+        // Add username field.
+        $fieldrecord = new StdClass();
+        $fieldrecord->name = 'Username';
+        $fieldrecord->type = 'text';
+        $this->generator->create_field($fieldrecord, $this->data);
+
+        $filecontent = file_get_contents(__DIR__ . '/fixtures/test_data_import_with_field_username.csv');
+        ob_start();
+        data_import_csv($this->cm, $this->data, $filecontent, 'UTF-8', 'comma');
+        ob_end_clean();
+
+        $expecteduserids = array();
+        $expecteduserids[1] = $this->student->id; // User student exists and is assigned as author.
+        $expecteduserids[2] = $this->teacher->id; // User student2 does not exist. Fallback is the uploading user.
+        $expecteduserids[3] = $this->student->id; // User student exists and is assigned as author.
+
+        $expectedcontent = array();
+        $expectedcontent[1] = array(
+            'Username' => 'otherusername1',
+            'Param2' => 'My first entry',
+        );
+        $expectedcontent[2] = array(
+            'Username' => 'otherusername2',
+            'Param2' => 'My second entry',
+        );
+        $expectedcontent[3] = array(
+            'Username' => 'otherusername3',
+            'Param2' => 'My third entry',
+        );
+
+        $records = $this->get_data_records($this->data->id);
+        $this->assertCount(3, $records);
+        foreach ($records as $record) {
+            $identifier = $record->items['ID']->content;
+            $this->assertEquals($expecteduserids[$identifier], $record->userid);
+
+            foreach ($expectedcontent[$identifier] as $field => $value) {
+                $this->assertEquals($value, $record->items[$field]->content,
+                    "The value of field \"$field\" for the record at position \"$identifier\" ".
+                    "which is \"{$record->items[$field]->content}\" does not match the expected value \"$value\".");
+            }
+        }
+    }
+
+    /**
+     * Test uploading entries for a data instance with a field 'Username' but only one occurrence in the csv file.
+     *
+     * This should test the corner case, in which a user has defined a data fields, which has the same name
+     * as the current lang string for username. In that case, the only Username entry is used for the field.
+     * The author should not be set.
+     * @throws coding_exception
+     * @throws dml_exception
+     */
+    public function test_import_with_field_username_without_userdata() {
+
+        // Add username field.
+        $fieldrecord = new StdClass();
+        $fieldrecord->name = 'Username';
+        $fieldrecord->type = 'text';
+        $this->generator->create_field($fieldrecord, $this->data);
+
+        $filecontent = file_get_contents(__DIR__ . '/fixtures/test_data_import_with_userdata.csv');
+        ob_start();
+        data_import_csv($this->cm, $this->data, $filecontent, 'UTF-8', 'comma');
+        ob_end_clean();
+
+        // No userdata is present in the file: Fallback is to assign the uploading user as author.
+        $expecteduserids = array();
+        $expecteduserids[1] = $this->teacher->id;
+        $expecteduserids[2] = $this->teacher->id;
+
+        $expectedcontent = array();
+        $expectedcontent[1] = array(
+            'Username' => 'student',
+            'Param2' => 'My first entry',
+        );
+        $expectedcontent[2] = array(
+            'Username' => 'student2',
+            'Param2' => 'My second entry',
+        );
+
+        $records = $this->get_data_records($this->data->id);
+        $this->assertCount(2, $records);
+        foreach ($records as $record) {
+            $identifier = $record->items['ID']->content;
+            $this->assertEquals($expecteduserids[$identifier], $record->userid);
+
+            foreach ($expectedcontent[$identifier] as $field => $value) {
+                $this->assertEquals($value, $record->items[$field]->content,
+                    "The value of field \"$field\" for the record at position \"$identifier\" ".
+                    "which is \"{$record->items[$field]->content}\" does not match the expected value \"$value\".");
+            }
+        }
+    }
+
+    /**
+     * Returns the records of the data instance.
+     *
+     * Each records has an item entry, which contains all fields associated with this item.
+     * Each fields has the parameters name, type and content.
+     * @param int $dataid Id of the data instance.
+     * @return array The records of the data instance.
+     * @throws dml_exception
+     *
+     */
+    private function get_data_records($dataid) {
+        global $DB;
+        $records = $DB->get_records('data_records', ['dataid' => $dataid]);
+        foreach ($records as $record) {
+            $sql = 'SELECT f.name, f.type, con.content FROM
+                {data_content} con JOIN {data_fields} f ON con.fieldid = f.id
+                WHERE con.recordid = :recordid';
+            $items = $DB->get_records_sql($sql, array('recordid' => $record->id));
+            $record->items = $items;
+        }
+        return $records;
+    }
+}