MDL-24751 fixed default memory limit when deciding to use apache_child_terminate...
[moodle.git] / grade / import / csv / index.php
CommitLineData
e060e33d 1<?php
2
3// This file is part of Moodle - http://moodle.org/
4//
5// Moodle is free software: you can redistribute it and/or modify
6// it under the terms of the GNU General Public License as published by
7// the Free Software Foundation, either version 3 of the License, or
8// (at your option) any later version.
9//
10// Moodle is distributed in the hope that it will be useful,
11// but WITHOUT ANY WARRANTY; without even the implied warranty of
12// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13// GNU General Public License for more details.
14//
15// You should have received a copy of the GNU General Public License
16// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
8ad36f4c 17
e77bcaa3 18require_once '../../../config.php';
235b07fb 19require_once $CFG->libdir.'/gradelib.php';
e77bcaa3 20require_once $CFG->dirroot.'/grade/lib.php';
21require_once '../grade_import_form.php';
22require_once '../lib.php';
b4355273 23
c69b02bb 24$id = required_param('id', PARAM_INT); // course id
25$separator = optional_param('separator', '', PARAM_ALPHA);
26$verbosescales = optional_param('verbosescales', 1, PARAM_BOOL);
7f999ccb 27
a6855934 28$url = new moodle_url('/grade/import/csv/index.php', array('id'=>$id));
beebcf26 29if ($separator !== '') {
30 $url->param('separator', $separator);
31}
32if ($verbosescales !== 1) {
33 $url->param('verbosescales', $verbosescales);
34}
35$PAGE->set_url($url);
36
233ae5a3 37define('GRADE_CSV_LINE_LENGTH', 4096);
38
f400841b 39if (!$course = $DB->get_record('course', array('id'=>$id))) {
e77bcaa3 40 print_error('nocourseid');
41}
42
43require_login($course);
44$context = get_context_instance(CONTEXT_COURSE, $id);
45require_capability('moodle/grade:import', $context);
46require_capability('gradeimport/csv:view', $context);
741bde47 47
1dc9f2e2 48$separatemode = (groups_get_course_groupmode($COURSE) == SEPARATEGROUPS and !has_capability('moodle/site:accessallgroups', $context));
49$currentgroup = groups_get_course_group($course);
50
741bde47 51// sort out delimiter
7f999ccb 52if (isset($CFG->CSV_DELIMITER)) {
d5b36b40 53 $csv_delimiter = $CFG->CSV_DELIMITER;
7f999ccb 54
55 if (isset($CFG->CSV_ENCODE)) {
56 $csv_encode = '/\&\#' . $CFG->CSV_ENCODE . '/';
57 }
c69b02bb 58} else if ($separator == 'tab') {
59 $csv_delimiter = "\t";
c69b02bb 60 $csv_encode = "";
7f999ccb 61} else {
d5b36b40 62 $csv_delimiter = ",";
c69b02bb 63 $csv_encode = '/\&\#44/';
7f999ccb 64}
65
dc482cfa 66print_grade_page_head($course->id, 'import', 'csv', get_string('importcsv', 'grades'));
f0b9c95b 67
68// set up import form
c69b02bb 69$mform = new grade_import_form(null, array('includeseparator'=>!isset($CFG->CSV_DELIMITER), 'verbosescales'=>true));
f0b9c95b 70
71// set up grade import mapping form
72$header = '';
73$gradeitems = array();
74if ($id) {
75 if ($grade_items = grade_item::fetch_all(array('courseid'=>$id))) {
76 foreach ($grade_items as $grade_item) {
77 // skip course type and category type
78 if ($grade_item->itemtype == 'course' || $grade_item->itemtype == 'category') {
79 continue;
80 }
81
82 // this was idnumber
83 $gradeitems[$grade_item->id] = $grade_item->get_name();
84 }
85 }
86}
87
88if ($importcode = optional_param('importcode', '', PARAM_FILE)) {
89 $filename = $CFG->dataroot.'/temp/gradeimport/cvs/'.$USER->id.'/'.$importcode;
90 $fp = fopen($filename, "r");
d5b36b40 91 $header = explode($csv_delimiter, fgets($fp,GRADE_CSV_LINE_LENGTH), PARAM_RAW);
f0b9c95b 92}
93
94$mform2 = new grade_import_mapping_form(null, array('gradeitems'=>$gradeitems, 'header'=>$header));
95
96// if import form is submitted
294ce987 97if ($formdata = $mform->get_data()) {
f0b9c95b 98
99 // Large files are likely to take their time and memory. Let PHP know
100 // that we'll take longer, and that the process should be recycled soon
101 // to free up memory.
102 @set_time_limit(0);
103 @raise_memory_limit("192M");
f0b9c95b 104
105 // use current (non-conflicting) time stamp
106 $importcode = get_new_importcode();
71904f4d 107 $filename = make_upload_directory('temp/gradeimport/cvs/'.$USER->id);
f0b9c95b 108 $filename = $filename.'/'.$importcode;
109
110 $text = $mform->get_file_content('userfile');
111 // trim utf-8 bom
112 $textlib = textlib_get_instance();
113 /// normalize line endings and do the encoding conversion
114 $text = $textlib->convert($text, $formdata->encoding);
115 $text = $textlib->trim_utf8_bom($text);
116 // Fix mac/dos newlines
117 $text = preg_replace('!\r\n?!',"\n",$text);
118 $fp = fopen($filename, "w");
119 fwrite($fp,$text);
120 fclose($fp);
121
122 $fp = fopen($filename, "r");
123
124 // --- get header (field names) ---
a12f8571 125 $header = explode($csv_delimiter, fgets($fp,GRADE_CSV_LINE_LENGTH));
f0b9c95b 126
127 // print some preview
128 $numlines = 0; // 0 preview lines displayed
129
c018f973 130 echo $OUTPUT->heading(get_string('importpreview', 'grades'));
f0b9c95b 131 echo '<table>';
132 echo '<tr>';
133 foreach ($header as $h) {
134 $h = clean_param($h, PARAM_RAW);
135 echo '<th>'.$h.'</th>';
136 }
137 echo '</tr>';
138 while (!feof ($fp) && $numlines <= $formdata->previewrows) {
d5b36b40 139 $lines = explode($csv_delimiter, fgets($fp,GRADE_CSV_LINE_LENGTH));
f0b9c95b 140 echo '<tr>';
141 foreach ($lines as $line) {
142 echo '<td>'.$line.'</td>';;
143 }
144 $numlines ++;
145 echo '</tr>';
146 }
147 echo '</table>';
148
149 // display the mapping form with header info processed
150 $mform2 = new grade_import_mapping_form(null, array('gradeitems'=>$gradeitems, 'header'=>$header));
c69b02bb 151 $mform2->set_data(array('importcode'=>$importcode, 'id'=>$id, 'verbosescales'=>$verbosescales, 'separator'=>$separator));
f0b9c95b 152 $mform2->display();
153
294ce987 154//} else if (($formdata = data_submitted()) && !empty($formdata->map)) {
d24832f9 155
f0b9c95b 156// else if grade import mapping form is submitted
294ce987 157} else if ($formdata = $mform2->get_data()) {
ba74762b 158
8108909a 159 $importcode = clean_param($formdata->importcode, PARAM_FILE);
160 $filename = $CFG->dataroot.'/temp/gradeimport/cvs/'.$USER->id.'/'.$importcode;
161
162 if (!file_exists($filename)) {
f39c16e0 163 print_error('cannotuploadfile');
8108909a 164 }
eff9c473 165
b4355273 166 if ($fp = fopen($filename, "r")) {
167 // --- get header (field names) ---
d5b36b40 168 $header = explode($csv_delimiter, clean_param(fgets($fp,GRADE_CSV_LINE_LENGTH), PARAM_RAW));
ba74762b 169
b4355273 170 foreach ($header as $i => $h) {
171 $h = trim($h); $header[$i] = $h; // remove whitespace
ba74762b 172 }
b4355273 173 } else {
771dc7b2 174 print_error('cannotopenfile');
b4355273 175 }
ba74762b 176
b263e539 177 $map = array();
b4355273 178 // loops mapping_0, mapping_1 .. mapping_n and construct $map array
b263e539 179 foreach ($header as $i => $head) {
ba74762b 180 $map[$i] = $formdata->{'mapping_'.$i};
eff9c473 181 }
85e287de 182
6ef4878b 183 // if mapping information is supplied
eff9c473 184 $map[clean_param($formdata->mapfrom, PARAM_RAW)] = clean_param($formdata->mapto, PARAM_RAW);
85e287de 185
ffe8ee55 186 // check for mapto collisions
187 $maperrors = array();
188 foreach ($map as $i=>$j) {
189 if ($j == 0) {
190 // you can have multiple ignores
ba74762b 191 continue;
ffe8ee55 192 } else {
193 if (!isset($maperrors[$j])) {
ba74762b 194 $maperrors[$j] = true;
ffe8ee55 195 } else {
ba74762b 196 // collision
ec69f4ab 197 fclose($fp);
ffe8ee55 198 unlink($filename); // needs to be uploaded again, sorry
f39c16e0 199 print_error('cannotmapfield', '', '', $j);
ffe8ee55 200 }
201 }
202 }
203
3f8bcf7c 204 // Large files are likely to take their time and memory. Let PHP know
205 // that we'll take longer, and that the process should be recycled soon
206 // to free up memory.
7f999ccb 207 @set_time_limit(0);
208 @raise_memory_limit("192M");
ba74762b 209
d9617050 210 // we only operate if file is readable
211 if ($fp = fopen($filename, "r")) {
ba74762b 212
b4355273 213 // read the first line makes sure this doesn't get read again
d5b36b40 214 $header = explode($csv_delimiter, fgets($fp,GRADE_CSV_LINE_LENGTH));
ba74762b 215
b89ef1db 216 $newgradeitems = array(); // temporary array to keep track of what new headers are processed
f115f8c8 217 $status = true;
ba74762b 218
d9617050 219 while (!feof ($fp)) {
220 // add something
d5b36b40 221 $line = explode($csv_delimiter, fgets($fp,GRADE_CSV_LINE_LENGTH));
ba74762b 222
46decdf5 223 if(count($line) <= 1){
224 // there is no data on this line, move on
225 continue;
226 }
227
741bde47 228 // array to hold all grades to be inserted
b89ef1db 229 $newgrades = array();
b4355273 230 // array to hold all feedback
ba74762b 231 $newfeedbacks = array();
741bde47 232 // each line is a student record
ba74762b 233 foreach ($line as $key => $value) {
d9617050 234 //decode encoded commas
eff9c473 235 $value = clean_param($value, PARAM_RAW);
c69b02bb 236 $value = trim($value);
d5b36b40
JH
237 if (!empty($csv_encode)) {
238 $value = preg_replace($csv_encode, $csv_delimiter, $value);
c69b02bb 239 }
eff9c473 240
b4355273 241 /*
242 * the options are
243 * 1) userid, useridnumber, usermail, username - used to identify user row
244 * 2) new - new grade item
245 * 3) id - id of the old grade item to map onto
246 * 3) feedback_id - feedback for grade item id
247 */
248
249 $t = explode("_", $map[$key]);
250 $t0 = $t[0];
251 if (isset($t[1])) {
8108909a 252 $t1 = (int)$t[1];
b4355273 253 } else {
ba74762b 254 $t1 = '';
b4355273 255 }
ba74762b 256
b4355273 257 switch ($t0) {
f115f8c8 258 case 'userid': //
5c75a0a3 259 if (!$user = $DB->get_record('user', array('id' => $value))) {
6ef4878b 260 // user not found, abort whole import
f115f8c8 261 import_cleanup($importcode);
c3b834b4 262 echo $OUTPUT->notification("user mapping error, could not find user with id \"$value\"");
f115f8c8 263 $status = false;
ba74762b 264 break 3;
f115f8c8 265 }
d9617050 266 $studentid = $value;
267 break;
268 case 'useridnumber':
5c75a0a3 269 if (!$user = $DB->get_record('user', array('idnumber' => $value))) {
6ef4878b 270 // user not found, abort whole import
f115f8c8 271 import_cleanup($importcode);
c3b834b4 272 echo $OUTPUT->notification("user mapping error, could not find user with idnumber \"$value\"");
f115f8c8 273 $status = false;
ba74762b 274 break 3;
f115f8c8 275 }
d9617050 276 $studentid = $user->id;
277 break;
278 case 'useremail':
5c75a0a3 279 if (!$user = $DB->get_record('user', array('email' => $value))) {
f115f8c8 280 import_cleanup($importcode);
c3b834b4 281 echo $OUTPUT->notification("user mapping error, could not find user with email address \"$value\"");
f115f8c8 282 $status = false;
ba74762b 283 break 3;
f115f8c8 284 }
ba74762b 285 $studentid = $user->id;
d9617050 286 break;
287 case 'username':
5c75a0a3 288 if (!$user = $DB->get_record('user', array('username' => $value))) {
f115f8c8 289 import_cleanup($importcode);
c3b834b4 290 echo $OUTPUT->notification("user mapping error, could not find user with username \"$value\"");
f115f8c8 291 $status = false;
ba74762b 292 break 3;
f115f8c8 293 }
d9617050 294 $studentid = $user->id;
295 break;
b89ef1db 296 case 'new':
297 // first check if header is already in temp database
ba74762b 298
299 if (empty($newgradeitems[$key])) {
300
ace9051c 301 $newgradeitem = new stdClass();
b89ef1db 302 $newgradeitem->itemname = $header[$key];
8108909a 303 $newgradeitem->importcode = $importcode;
304 $newgradeitem->importer = $USER->id;
ba74762b 305
f115f8c8 306 // failed to insert into new grade item buffer
a8f3a651 307 $newgradeitems[$key] = $DB->insert_record('grade_import_newitem', $newgradeitem);
b89ef1db 308 // add this to grade_import_newitem table
ba74762b 309 // add the new id to $newgradeitem[$key]
310 }
ace9051c 311 $newgrade = new stdClass();
c22e335d 312 $newgrade->newgradeitem = $newgradeitems[$key];
313 $newgrade->finalgrade = $value;
b89ef1db 314 $newgrades[] = $newgrade;
ba74762b 315
316 // if not, put it in
b89ef1db 317 // else, insert grade into the table
318 break;
ffe8ee55 319 case 'feedback':
b4355273 320 if ($t1) {
8108909a 321 // case of an id, only maps id of a grade_item
322 // this was idnumber
323 if (!$gradeitem = new grade_item(array('id'=>$t1, 'courseid'=>$course->id))) {
324 // supplied bad mapping, should not be possible since user
325 // had to pick mapping
326 $status = false;
327 import_cleanup($importcode);
c3b834b4 328 echo $OUTPUT->notification(get_string('importfailed', 'grades'));
8108909a 329 break 3;
330 }
331
b4355273 332 // t1 is the id of the grade item
ace9051c 333 $feedback = new stdClass();
c22e335d 334 $feedback->itemid = $t1;
335 $feedback->feedback = $value;
ffe8ee55 336 $newfeedbacks[] = $feedback;
b4355273 337 }
ba74762b 338 break;
d9617050 339 default:
b89ef1db 340 // existing grade items
8108909a 341 if (!empty($map[$key])) {
ba74762b 342 // case of an id, only maps id of a grade_item
b4355273 343 // this was idnumber
235b07fb 344 if (!$gradeitem = new grade_item(array('id'=>$map[$key], 'courseid'=>$course->id))) {
f115f8c8 345 // supplied bad mapping, should not be possible since user
346 // had to pick mapping
347 $status = false;
348 import_cleanup($importcode);
c3b834b4 349 echo $OUTPUT->notification(get_string('importfailed', 'grades'));
b263e539 350 break 3;
351 }
ba74762b 352
b263e539 353 // check if grade item is locked if so, abort
8108909a 354 if ($gradeitem->is_locked()) {
b263e539 355 $status = false;
356 import_cleanup($importcode);
c3b834b4 357 echo $OUTPUT->notification(get_string('gradeitemlocked', 'grades'));
ba74762b 358 break 3;
b4355273 359 }
360
ace9051c 361 $newgrade = new stdClass();
c22e335d 362 $newgrade->itemid = $gradeitem->id;
c69b02bb 363 if ($gradeitem->gradetype == GRADE_TYPE_SCALE and $verbosescales) {
364 if ($value === '' or $value == '-') {
365 $value = null; // no grade
366 } else {
367 $scale = $gradeitem->load_scale();
368 $scales = explode(',', $scale->scale);
dc482cfa 369 $scales = array_map('trim', $scales); //hack - trim whitespace around scale options
c69b02bb 370 array_unshift($scales, '-'); // scales start at key 1
371 $key = array_search($value, $scales);
372 if ($key === false) {
373 echo "<br/>t0 is $t0";
374 echo "<br/>grade is $value";
375 $status = false;
376 import_cleanup($importcode);
c3b834b4 377 echo $OUTPUT->notification(get_string('badgrade', 'grades'));
c69b02bb 378 break 3;
379 }
380 $value = $key;
381 }
382 $newgrade->finalgrade = $value;
383 } else {
384 if ($value === '' or $value == '-') {
385 $value = null; // no grade
dc482cfa 386
c69b02bb 387 } else if (!is_numeric($value)) {
388 // non numeric grade value supplied, possibly mapped wrong column
389 echo "<br/>t0 is $t0";
390 echo "<br/>grade is $value";
391 $status = false;
392 import_cleanup($importcode);
c3b834b4 393 echo $OUTPUT->notification(get_string('badgrade', 'grades'));
c69b02bb 394 break 3;
395 }
396 $newgrade->finalgrade = $value;
397 }
b89ef1db 398 $newgrades[] = $newgrade;
ba74762b 399 } // otherwise, we ignore this column altogether
741bde47 400 // because user has chosen to ignore them (e.g. institution, address etc)
b263e539 401 break;
d9617050 402 }
7f999ccb 403 }
b4355273 404
f115f8c8 405 // no user mapping supplied at all, or user mapping failed
ce40b793 406 if (empty($studentid) || !is_numeric($studentid)) {
6ef4878b 407 // user not found, abort whole import
f115f8c8 408 $status = false;
ce40b793 409 import_cleanup($importcode);
c3b834b4 410 echo $OUTPUT->notification('user mapping error, could not find user!');
b263e539 411 break;
ce40b793 412 }
b4355273 413
1dc9f2e2 414 if ($separatemode and !groups_is_member($currentgroup, $studentid)) {
415 // not allowed to import into this group, abort
416 $status = false;
417 import_cleanup($importcode);
c3b834b4 418 echo $OUTPUT->notification('user not member of current group, can not update!');
1dc9f2e2 419 break;
420 }
421
b89ef1db 422 // insert results of this students into buffer
8108909a 423 if ($status and !empty($newgrades)) {
ba74762b 424
b89ef1db 425 foreach ($newgrades as $newgrade) {
ba74762b 426
3ee5c201 427 // check if grade_grade is locked and if so, abort
8108909a 428 if (!empty($newgrade->itemid) and $grade_grade = new grade_grade(array('itemid'=>$newgrade->itemid, 'userid'=>$studentid))) {
429 if ($grade_grade->is_locked()) {
b263e539 430 // individual grade locked
431 $status = false;
432 import_cleanup($importcode);
c3b834b4 433 echo $OUTPUT->notification(get_string('gradelocked', 'grades'));
b263e539 434 break 2;
435 }
436 }
437
8108909a 438 $newgrade->importcode = $importcode;
439 $newgrade->userid = $studentid;
440 $newgrade->importer = $USER->id;
fc29e51b 441 $DB->insert_record('grade_import_values', $newgrade);
b89ef1db 442 }
85e287de 443 }
b4355273 444
445 // updating/inserting all comments here
8108909a 446 if ($status and !empty($newfeedbacks)) {
b4355273 447 foreach ($newfeedbacks as $newfeedback) {
8108909a 448 $sql = "SELECT *
d24832f9 449 FROM {grade_import_values}
5c75a0a3 450 WHERE importcode=? AND userid=? AND itemid=? AND importer=?";
451 if ($feedback = $DB->get_record_sql($sql, array($importcode, $studentid, $newfeedback->itemid, $USER->id))) {
8108909a 452 $newfeedback->id = $feedback->id;
5c75a0a3 453 $DB->update_record('grade_import_values', $newfeedback);
8108909a 454
b4355273 455 } else {
456 // the grade item for this is not updated
8108909a 457 $newfeedback->importcode = $importcode;
458 $newfeedback->userid = $studentid;
459 $newfeedback->importer = $USER->id;
5c75a0a3 460 $DB->insert_record('grade_import_values', $newfeedback);
b4355273 461 }
462 }
463 }
7f999ccb 464 }
b4355273 465
ba74762b 466 /// at this stage if things are all ok, we commit the changes from temp table
f115f8c8 467 if ($status) {
468 grade_import_commit($course->id, $importcode);
469 }
d9617050 470 // temporary file can go now
ec69f4ab 471 fclose($fp);
d9617050 472 unlink($filename);
473 } else {
771dc7b2 474 print_error('cannotreadfil');
85e287de 475 }
d9617050 476
7f999ccb 477} else {
1dc9f2e2 478 groups_print_course_menu($course, 'index.php?id='.$id);
479 echo '<div class="clearer"></div>';
480
741bde47 481 // display the standard upload file form
7f999ccb 482 $mform->display();
483}
b4355273 484
5a931394 485echo $OUTPUT->footer();
6c3ef410 486