From 92dfce6be7a01618f5f665391d5e9c7e582a7b93 Mon Sep 17 00:00:00 2001 From: Adrian Greeve Date: Fri, 27 Jul 2012 12:10:42 +0800 Subject: [PATCH] MDL-34075 - lib - Alteration to the csv import lib to include rfc-4180 compliance --- lib/csvlib.class.php | 89 ++++++++++++++++++----------------- lib/tests/csvtest.php | 107 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+), 44 deletions(-) create mode 100644 lib/tests/csvtest.php diff --git a/lib/csvlib.class.php b/lib/csvlib.class.php index 92296fd481a..8fd13271e62 100644 --- a/lib/csvlib.class.php +++ b/lib/csvlib.class.php @@ -55,7 +55,14 @@ class csv_import_reader { * @var object file handle used during import */ var $_fp; - + /** + * @var string delimiter of the records. + */ + var $_delimiter; + /** + * @var string enclosure of each field. + */ + var $_enclosure; /** * Contructor * @@ -78,7 +85,7 @@ class csv_import_reader { * @param string $column_validation name of function for columns validation, must have one param $columns * @return bool false if error, count of data lines if ok; use get_error() to get error string */ - function load_csv_content(&$content, $encoding, $delimiter_name, $column_validation=null) { + function load_csv_content(&$content, $encoding, $delimiter_name, $column_validation=null, $enclosure='"') { global $USER, $CFG; $this->close(); @@ -89,62 +96,56 @@ class csv_import_reader { $content = textlib::trim_utf8_bom($content); // Fix mac/dos newlines $content = preg_replace('!\r\n?!', "\n", $content); - // is there anyting in file? - $columns = strtok($content, "\n"); - if ($columns === false) { - $this->_error = get_string('csvemptyfile', 'error'); - return false; - } + $csv_delimiter = csv_import_reader::get_delimiter($delimiter_name); - $csv_encode = csv_import_reader::get_encoded_delimiter($delimiter_name); +// $csv_encode = csv_import_reader::get_encoded_delimiter($delimiter_name); + $this->_delimiter = $csv_delimiter; + $this->_enclosure = $enclosure; + + // create a temporary file and store the csv file there in csv file format. + $filename = $CFG->tempdir.'/csvimport/'.$this->_type.'/'.$USER->id.'/'.$this->_iid; + $fp = fopen($filename, 'w+'); + fwrite($fp, $content); + fseek($fp, 0); + // Create an array to store the imported data for error checking. + $columns = array(); + while ($fgetdata = fgetcsv($fp, 0, $csv_delimiter, $enclosure)) { + $columns[] = $fgetdata; + } + $col_count = 0; // process header - list of columns - $columns = explode($csv_delimiter, $columns); - $col_count = count($columns); - if ($col_count === 0) { + if (!isset($columns[0])) { $this->_error = get_string('csvemptyfile', 'error'); + fclose($fp); + $this->cleanup(); return false; + } else { + $col_count = count($columns[0]); } - - foreach ($columns as $key=>$value) { - $columns[$key] = str_replace($csv_encode, $csv_delimiter, trim($value)); - } + // column validation if ($column_validation) { - $result = $column_validation($columns); + $result = $column_validation($columns[0]); if ($result !== true) { $this->_error = $result; + fclose($fp); + $this->cleanup(); return false; } } - $this->_columns = $columns; // cached columns - // open file for writing - $filename = $CFG->tempdir.'/csvimport/'.$this->_type.'/'.$USER->id.'/'.$this->_iid; - $fp = fopen($filename, "w"); - fwrite($fp, serialize($columns)."\n"); - - // again - do we have any data for processing? - $line = strtok("\n"); - $data_count = 0; - while ($line !== false) { - $line = explode($csv_delimiter, $line); - foreach ($line as $key=>$value) { - $line[$key] = str_replace($csv_encode, $csv_delimiter, trim($value)); - } - if (count($line) !== $col_count) { - // this is critical!! + $this->_columns = $columns[0]; // cached columns + // check to make sure that the data columns match up with the headers. + foreach ($columns as $rowdata) { + if (count($rowdata) !== $col_count) { $this->_error = get_string('csvweirdcolumns', 'error'); fclose($fp); $this->cleanup(); return false; } - fwrite($fp, serialize($line)."\n"); - $data_count++; - $line = strtok("\n"); } - - fclose($fp); - return $data_count; + $datacount = count($columns); + return $datacount; } /** @@ -164,12 +165,12 @@ class csv_import_reader { return false; } $fp = fopen($filename, "r"); - $line = fgets($fp); + $line = fgetcsv($fp, 0, $this->_delimiter, $this->_enclosure); fclose($fp); if ($line === false) { return false; } - $this->_columns = unserialize($line); + $this->_columns = $line; return $this->_columns; } @@ -194,7 +195,7 @@ class csv_import_reader { return false; } //skip header - return (fgets($this->_fp) !== false); + return (fgetcsv($this->_fp, 0, $this->_delimiter, $this->_enclosure) !== false); } /** @@ -206,8 +207,8 @@ class csv_import_reader { if (empty($this->_fp) or feof($this->_fp)) { return false; } - if ($ser = fgets($this->_fp)) { - return unserialize($ser); + if ($ser = fgetcsv($this->_fp, 0, $this->_delimiter, $this->_enclosure)) { + return $ser; } else { return false; } diff --git a/lib/tests/csvtest.php b/lib/tests/csvtest.php new file mode 100644 index 00000000000..db7d1bdf1bb --- /dev/null +++ b/lib/tests/csvtest.php @@ -0,0 +1,107 @@ +. + +/** + * Tests csv import and export functions + * + * @package core + * @category phpunit + * @copyright 2012 Adrian Greeve + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +global $CFG; +require_once($CFG->dirroot . '/lib/csvlib.class.php'); + +class csv_testcase extends advanced_testcase { + + var $testdata = array(); + var $teststring = ''; + var $teststring2 = ''; + var $teststring3 = ''; + + protected function setUp(){ + + $this->resetAfterTest(true); + + $csvdata = array(); + $csvdata[0][] = 'fullname'; + $csvdata[0][] = 'description of things'; + $csvdata[0][] = 'beer'; + $csvdata[1][] = 'William H T Macey'; + $csvdata[1][] = '

A field that contains "double quotes"

'; + $csvdata[1][] = 'Asahi'; + $csvdata[2][] = 'Phillip Jenkins'; + $csvdata[2][] = '

This field has

+

Multiple lines

+

and also contains "double quotes"

'; + $csvdata[2][] = 'Yebisu'; + $this->testdata = $csvdata; + + // Please note that each line needs a carriage return. + $this->teststring = 'fullname,"description of things",beer +"William H T Macey","

A field that contains ""double quotes""

",Asahi +"Phillip Jenkins","

This field has

+

Multiple lines

+

and also contains ""double quotes""

",Yebisu +'; + + $this->teststring2 = 'fullname,"description of things",beer +"Fred Flint","

Find the stone inside the box

",Asahi,"A fourth column" +"Sarah Smith","

How are the people next door?

,Yebisu,"Forget the next" +'; + } + + public function test_csv_functions() { + // Testing that the content is imported correctly. + $iid = csv_import_reader::get_new_iid('lib'); + $csvimport = new csv_import_reader($iid, 'lib'); + $contentcount = $csvimport->load_csv_content($this->teststring, 'utf-8', 'comma'); + $csvimport->init(); + $dataset = array(); + $dataset[] = $csvimport->get_columns(); + while ($record = $csvimport->next()) { + $dataset[] = $record; + } + $csvimport->cleanup(); + $csvimport->close(); + $this->assertEquals($dataset, $this->testdata); + + // Testing for the wrong count of columns. + $errortext = get_string('csvweirdcolumns', 'error'); + + $iid = csv_import_reader::get_new_iid('lib'); + $csvimport = new csv_import_reader($iid, 'lib'); + $contentcount = $csvimport->load_csv_content($this->teststring2, 'utf-8', 'comma'); + $importerror = $csvimport->get_error(); + $csvimport->cleanup(); + $csvimport->close(); + $this->assertEquals($importerror, $errortext); + + // Testing for empty content + $errortext = get_string('csvemptyfile', 'error'); + + $iid = csv_import_reader::get_new_iid('lib'); + $csvimport = new csv_import_reader($iid, 'lib'); + $contentcount = $csvimport->load_csv_content($this->teststring3, 'utf-8', 'comma'); + $importerror = $csvimport->get_error(); + $csvimport->cleanup(); + $csvimport->close(); + $this->assertEquals($importerror, $errortext); + } +} \ No newline at end of file -- 2.43.0