MDL-29027 fallback to mb_string on errors if available and simplify charset parsing
[moodle.git] / lib / textlib.class.php
CommitLineData
72fb21b6 1<?php
72fb21b6 2// This file is part of Moodle - http://moodle.org/
3//
4// Moodle is free software: you can redistribute it and/or modify
5// it under the terms of the GNU General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8//
9// Moodle is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12// GNU General Public License for more details.
13//
14// You should have received a copy of the GNU General Public License
15// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
779dc9ac 16
72fb21b6 17/**
78bfb562
PS
18 * @package core
19 * @subpackage lib
20 * @copyright (C) 2001-3001 Eloy Lafuente (stronk7) {@link http://contiento.com}
21 * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
72fb21b6 22 */
6aaa17c7 23
78bfb562
PS
24defined('MOODLE_INTERNAL') || die();
25
72fb21b6 26/**
5a7e4c11
PS
27 * Original singleton helper function, please use static methods instead,
28 * ex: textlib::convert()
29 *
30 * @deprecated
31 * @return textlib instance
32 */
33function textlib_get_instance() {
34 return new textlib();
35}
36
37
38/**
39 * This class is used to manipulate strings under Moodle 1.6 an later. As
40 * utf-8 text become mandatory a pool of safe functions under this encoding
41 * become necessary. The name of the methods is exactly the
42 * same than their PHP originals.
43 *
44 * A big part of this class acts as a wrapper over the Typo3 charset library,
45 * really a cool group of utilities to handle texts and encoding conversion.
46 *
47 * Take a look to its own copyright and license details.
72fb21b6 48 *
49 * IMPORTANT Note: Typo3 libraries always expect lowercase charsets to use 100%
50 * its capabilities so, don't forget to make the conversion
51 * from every wrapper function!
52 *
5a7e4c11
PS
53 * @package core
54 * @subpackage lib
55 * @copyright 1999 onwards Martin Dougiamas {@link http://moodle.com}
56 * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
72fb21b6 57 */
5a7e4c11
PS
58class textlib {
59
60 /**
61 * Return t3lib helper class
62 * @return t3lib_cs
63 */
64 protected static function typo3() {
65 static $typo3cs = null;
3b093310 66
5a7e4c11
PS
67 if (isset($typo3cs)) {
68 return $typo3cs;
69 }
3b093310 70
5a7e4c11 71 global $CFG;
3b093310 72
5a7e4c11 73 // Required files
3b093310 74 require_once($CFG->libdir.'/typo3/class.t3lib_cs.php');
75 require_once($CFG->libdir.'/typo3/class.t3lib_div.php');
76
5a7e4c11 77 // If ICONV is available, lets Typo3 library use it for convert
3b093310 78 if (extension_loaded('iconv')) {
79 $GLOBALS['TYPO3_CONF_VARS']['SYS']['t3lib_cs_convMethod'] = 'iconv';
5a7e4c11 80 // Else if mbstring is available, lets Typo3 library use it
3b093310 81 } else if (extension_loaded('mbstring')) {
82 $GLOBALS['TYPO3_CONF_VARS']['SYS']['t3lib_cs_convMethod'] = 'mbstring';
5a7e4c11 83 // Else if recode is available, lets Typo3 library use it
3b093310 84 } else if (extension_loaded('recode')) {
85 $GLOBALS['TYPO3_CONF_VARS']['SYS']['t3lib_cs_convMethod'] = 'recode';
86 } else {
87 $GLOBALS['TYPO3_CONF_VARS']['SYS']['t3lib_cs_convMethod'] = '';
88 }
89
5a7e4c11 90 // If mbstring is available, lets Typo3 library use it for functions
3b093310 91 if (extension_loaded('mbstring')) {
92 $GLOBALS['TYPO3_CONF_VARS']['SYS']['t3lib_cs_utils'] = 'mbstring';
93 } else {
94 $GLOBALS['TYPO3_CONF_VARS']['SYS']['t3lib_cs_utils'] = '';
95 }
96
5a7e4c11 97 // Tell Typo3 we are curl enabled always (mandatory since 2.0)
850ea1ee
EL
98 $GLOBALS['TYPO3_CONF_VARS']['SYS']['curlUse'] = '1';
99
5a7e4c11
PS
100 // And this directory must exist to allow Typo to cache conversion
101 // tables when using internal functions
3b093310 102 make_upload_directory('temp/typo3temp/cs');
103
5a7e4c11 104 // Make sure typo is using our dir permissions
8d3bf3e0
PS
105 $GLOBALS['TYPO3_CONF_VARS']['BE']['folderCreateMask'] = decoct($CFG->directorypermissions);
106
5a7e4c11 107 // Default mask for Typo
3b093310 108 $GLOBALS['TYPO3_CONF_VARS']['BE']['fileCreateMask'] = $CFG->directorypermissions;
109
5a7e4c11
PS
110 // This full path constants must be defined too, transforming backslashes
111 // to forward slashed because Typo3 requires it.
3b093310 112 define ('PATH_t3lib', str_replace('\\','/',$CFG->libdir.'/typo3/'));
113 define ('PATH_typo3', str_replace('\\','/',$CFG->libdir.'/typo3/'));
114 define ('PATH_site', str_replace('\\','/',$CFG->dataroot.'/temp/'));
115 define ('TYPO3_OS', stristr(PHP_OS,'win')&&!stristr(PHP_OS,'darwin')?'WIN':'');
116
5a7e4c11 117 $typo3cs = new t3lib_cs();
779dc9ac 118
5a7e4c11
PS
119 return $typo3cs;
120 }
779dc9ac 121
e4023a4c 122 /**
5a7e4c11
PS
123 * Standardise charset name
124 *
125 * Please note it does not mean the returned charset is actually supported.
779dc9ac 126 *
5a7e4c11
PS
127 * @static
128 * @param string $charset raw charset name
129 * @return string normalised lowercase charset name
779dc9ac 130 */
5a7e4c11 131 public static function parse_charset($charset) {
5a7e4c11 132 $charset = strtolower($charset);
5a7e4c11 133
a7a20faf
PS
134 // shortcuts so that we do not have to load typo3 on every page
135
136 if ($charset === 'utf8' or $charset === 'utf-8') {
137 return 'utf-8';
138 }
139
140 if (preg_match('/^(cp|win|windows)-?(12[0-9]{2})$/', $charset, $matches)) {
141 return 'windows-'.$matches[2];
142 }
143
144 if (preg_match('/^iso-8859-[0-9]+$/', $charset, $matches)) {
145 return $charset;
5a7e4c11
PS
146 }
147
148 // fallback to typo3
149 return self::typo3()->parse_charset($charset);
779dc9ac 150 }
151
e4023a4c 152 /**
a7a20faf
PS
153 * Converts the text between different encodings. It uses iconv extension with //TRANSLIT parameter
154 * or mb_string if iconv conversion fails.
5a7e4c11
PS
155 * Returns false if fails.
156 *
157 * @param string $text
158 * @param string $fromCS source encoding
159 * @param string $toCS result encoding
a7a20faf 160 * @return string|bool converted string or false on error
e4023a4c 161 */
5a7e4c11
PS
162 public static function convert($text, $fromCS, $toCS='utf-8') {
163 $fromCS = self::parse_charset($fromCS);
164 $toCS = self::parse_charset($toCS);
165
a7a20faf
PS
166 $text = (string)$text; // we can work only with strings
167
168 if ($text === '') {
169 return '';
170 }
171
172 $result = iconv($fromCS, $toCS.'//TRANSLIT', $text);
173
174 if ($result === false or $result === '') {
175 // note: iconv is prone to return empty string when invalid char encountered,
176 // mbstring ignores invalid chars completely, mbstring may support more encodings too
177 if (function_exists('mb_convert_encoding')) {
178 $result = mb_convert_encoding($text, $toCS, $fromCS);
179 }
180 }
181
182 return $result;
779dc9ac 183 }
184
e4023a4c 185 /**
a7a20faf
PS
186 * Multibyte safe substr() function, uses iconv,
187 * falls back on error to mbstring if available.
5a7e4c11
PS
188 *
189 * @param string $text
190 * @param int $start negative value means from end
191 * @param int $len
192 * @param string $charset encoding of the text
193 * @return string
e4023a4c 194 */
5a7e4c11
PS
195 public static function substr($text, $start, $len=null, $charset='utf-8') {
196 $charset = self::parse_charset($charset);
197
198 return iconv_substr($text, $start, $len, $charset);
a7a20faf
PS
199
200 if ($result === false) {
201 if (function_exists('mb_substr')) {
202 $result = mb_substr($text, $start, $len, $charset);
203 }
204 }
205
206 return $result;
779dc9ac 207 }
208
e4023a4c 209 /**
a7a20faf
PS
210 * Multibyte safe strlen() function, uses iconv,
211 * falls back on error to mbstring if available.
5a7e4c11
PS
212 *
213 * @param string $text
214 * @param string $charset encoding of the text
215 * @return int number of characters
e4023a4c 216 */
5a7e4c11
PS
217 public static function strlen($text, $charset='utf-8') {
218 $charset = self::parse_charset($charset);
219
a7a20faf
PS
220 $result = iconv_strlen($text, $charset);
221
222 if ($result === false) {
223 if (function_exists('mb_strlen')) {
224 $result = mb_strlen($text, $charset);
225 }
226 }
227
228 return $result;
779dc9ac 229 }
230
e4023a4c 231 /**
e8515774 232 * Multibyte safe strtolower() function, uses mbstring if available.
5a7e4c11
PS
233 *
234 * @param string $text
235 * @param string $charset encoding of the text (may not work for all encodings)
236 * @return string lower case text
e4023a4c 237 */
5a7e4c11
PS
238 public static function strtolower($text, $charset='utf-8') {
239 $charset = self::parse_charset($charset);
240
241 if (function_exists('mb_strtolower')) {
242 return mb_strtolower($text, $charset);
243 }
244
245 // Avoid some notices from Typo3 code
779dc9ac 246 $oldlevel = error_reporting(E_PARSE);
5a7e4c11
PS
247 // Call Typo3 conv_case() function. It will do all the work
248 $result = self::typo3()->conv_case($charset, $text, 'toLower');
249 // Restore original debug level
779dc9ac 250 error_reporting($oldlevel);
251 return $result;
252 }
253
e4023a4c 254 /**
e8515774 255 * Multibyte safe strtoupper() function, uses mbstring if available.
5a7e4c11
PS
256 *
257 * @param string $text
258 * @param string $charset encoding of the text (may not work for all encodings)
259 * @return string upper case text
e4023a4c 260 */
5a7e4c11
PS
261 public static function strtoupper($text, $charset='utf-8') {
262 $charset = self::parse_charset($charset);
263
264 if (function_exists('mb_strtoupper')) {
265 return mb_strtoupper($text, $charset);
266 }
267
268 // Avoid some notices from Typo3 code
779dc9ac 269 $oldlevel = error_reporting(E_PARSE);
5a7e4c11
PS
270 // Call Typo3 conv_case() function. It will do all the work
271 $result = self::typo3()->conv_case($charset, $text, 'toUpper');
272 // Restore original debug level
779dc9ac 273 error_reporting($oldlevel);
274 return $result;
275 }
276
e4023a4c 277 /**
5a7e4c11
PS
278 * UTF-8 ONLY safe strpos().
279 *
280 * @param string $haystack
281 * @param string $needle
282 * @param int $offset
283 * @return string
e4023a4c 284 */
5a7e4c11
PS
285 public static function strpos($haystack, $needle, $offset=0) {
286 return iconv_strpos($haystack, $needle, $offset, 'utf-8');
779dc9ac 287 }
288
e4023a4c 289 /**
5a7e4c11
PS
290 * UTF-8 ONLY safe strrpos().
291 *
292 * @param string $haystack
293 * @param string $needle
294 * @return string
e4023a4c 295 */
5a7e4c11
PS
296 public static function strrpos($haystack, $needle) {
297 return iconv_strrpos($haystack, $needle, 'utf-8');
779dc9ac 298 }
299
e4023a4c 300 /**
301 * Try to convert upper unicode characters to plain ascii,
5a7e4c11
PS
302 * the returned string may contain unconverted unicode characters.
303 *
304 * @param string $text
305 * @param string $charset encoding of the text
306 * @return string
e4023a4c 307 */
5a7e4c11
PS
308 public static function specialtoascii($text, $charset='utf-8') {
309 $charset = self::parse_charset($charset);
310 // Avoid some notices from Typo3 code
e05fc498 311 $oldlevel = error_reporting(E_PARSE);
5a7e4c11
PS
312 $result = self::typo3()->specCharsToASCII($charset, $text);
313 // Restore original debug level
e05fc498 314 error_reporting($oldlevel);
315 return $result;
46b81130 316 }
317
e4023a4c 318 /**
319 * Generate a correct base64 encoded header to be used in MIME mail messages.
3ec5ec15 320 * This function seems to be 100% compliant with RFC1342. Credits go to:
321 * paravoid (http://www.php.net/manual/en/function.mb-encode-mimeheader.php#60283).
5a7e4c11
PS
322 *
323 * @param string $text
324 * @param string $charset encoding of the text
325 * @return string
3ec5ec15 326 */
5a7e4c11 327 public static function encode_mimeheader($text, $charset='utf-8') {
53146827 328 if (empty($text)) {
329 return (string)$text;
330 }
5a7e4c11
PS
331 // Normalize charset
332 $charset = self::parse_charset($charset);
333 // If the text is pure ASCII, we don't need to encode it
334 if (self::convert($text, $charset, 'ascii') == $text) {
c539d622 335 return $text;
336 }
5a7e4c11
PS
337 // Although RFC says that line feed should be \r\n, it seems that
338 // some mailers double convert \r, so we are going to use \n alone
3ec5ec15 339 $linefeed="\n";
5a7e4c11 340 // Define start and end of every chunk
3ec5ec15 341 $start = "=?$charset?B?";
342 $end = "?=";
5a7e4c11 343 // Accumulate results
3ec5ec15 344 $encoded = '';
5a7e4c11 345 // Max line length is 75 (including start and end)
3ec5ec15 346 $length = 75 - strlen($start) - strlen($end);
5a7e4c11
PS
347 // Multi-byte ratio
348 $multilength = self::strlen($text, $charset);
349 // Detect if strlen and friends supported
53146827 350 if ($multilength === false) {
351 if ($charset == 'GB18030' or $charset == 'gb18030') {
352 while (strlen($text)) {
353 // try to encode first 22 chars - we expect most chars are two bytes long
354 if (preg_match('/^(([\x00-\x7f])|([\x81-\xfe][\x40-\x7e])|([\x81-\xfe][\x80-\xfe])|([\x81-\xfe][\x30-\x39]..)){1,22}/m', $text, $matches)) {
355 $chunk = $matches[0];
356 $encchunk = base64_encode($chunk);
357 if (strlen($encchunk) > $length) {
358 // find first 11 chars - each char in 4 bytes - worst case scenario
e8515774 359 preg_match('/^(([\x00-\x7f])|([\x81-\xfe][\x40-\x7e])|([\x81-\xfe][\x80-\xfe])|([\x81-\xfe][\x30-\x39]..)){1,11}/m', $text, $matches);
53146827 360 $chunk = $matches[0];
361 $encchunk = base64_encode($chunk);
362 }
363 $text = substr($text, strlen($chunk));
364 $encoded .= ' '.$start.$encchunk.$end.$linefeed;
365 } else {
366 break;
367 }
368 }
369 $encoded = trim($encoded);
e8515774 370 return $encoded;
53146827 371 } else {
372 return false;
373 }
374 }
375 $ratio = $multilength / strlen($text);
5a7e4c11 376 // Base64 ratio
3ec5ec15 377 $magic = $avglength = floor(3 * $length * $ratio / 4);
5a7e4c11 378 // basic infinite loop protection
684663f5 379 $maxiterations = strlen($text)*2;
e8515774 380 $iteration = 0;
5a7e4c11 381 // Iterate over the string in magic chunks
53146827 382 for ($i=0; $i <= $multilength; $i+=$magic) {
684663f5 383 if ($iteration++ > $maxiterations) {
e8515774 384 return false; // probably infinite loop
385 }
3ec5ec15 386 $magic = $avglength;
387 $offset = 0;
5a7e4c11 388 // Ensure the chunk fits in length, reducing magic if necessary
3ec5ec15 389 do {
390 $magic -= $offset;
5a7e4c11 391 $chunk = self::substr($text, $i, $magic, $charset);
3ec5ec15 392 $chunk = base64_encode($chunk);
393 $offset++;
394 } while (strlen($chunk) > $length);
5a7e4c11 395 // This chunk doesn't break any multi-byte char. Use it.
3ec5ec15 396 if ($chunk)
397 $encoded .= ' '.$start.$chunk.$end.$linefeed;
398 }
5a7e4c11 399 // Strip the first space and the last linefeed
3ec5ec15 400 $encoded = substr($encoded, 1, -strlen($linefeed));
401
402 return $encoded;
403 }
9c1cd39d 404
4e2cb0e3 405 /**
406 * Converts all the numeric entities &#nnnn; or &#xnnn; to UTF-8
407 * Original from laurynas dot butkus at gmail at:
408 * http://php.net/manual/en/function.html-entity-decode.php#75153
409 * with some custom mods to provide more functionality
e8515774 410 *
4e2cb0e3 411 * @param string $str input string
412 * @param boolean $htmlent convert also html entities (defaults to true)
5a7e4c11 413 * @return string
4e2cb0e3 414 *
415 * NOTE: we could have used typo3 entities_to_utf8() here
416 * but the direct alternative used runs 400% quicker
417 * and uses 0.5Mb less memory, so, let's use it
5a7e4c11 418 * (tested against 10^6 conversions)
4e2cb0e3 419 */
5a7e4c11
PS
420 public static function entities_to_utf8($str, $htmlent=true) {
421 static $trans_tbl; // Going to use static transliteration table
4e2cb0e3 422
5a7e4c11 423 // Replace numeric entities
4e2cb0e3 424 $result = preg_replace('~&#x([0-9a-f]+);~ei', 'textlib::code2utf8(hexdec("\\1"))', $str);
425 $result = preg_replace('~&#([0-9]+);~e', 'textlib::code2utf8(\\1)', $result);
426
5a7e4c11 427 // Replace literal entities (if desired)
4e2cb0e3 428 if ($htmlent) {
5a7e4c11 429 // Generate/create $trans_tbl
4e2cb0e3 430 if (!isset($trans_tbl)) {
431 $trans_tbl = array();
432 foreach (get_html_translation_table(HTML_ENTITIES) as $val=>$key) {
433 $trans_tbl[$key] = utf8_encode($val);
434 }
435 }
436 $result = strtr($result, $trans_tbl);
437 }
5a7e4c11 438 // Return utf8-ised string
4e2cb0e3 439 return $result;
440 }
441
9c1cd39d 442 /**
443 * Converts all Unicode chars > 127 to numeric entities &#nnnn; or &#xnnn;.
444 *
5a7e4c11
PS
445 * @param string $str input string
446 * @param boolean $dec output decadic only number entities
447 * @param boolean $nonnum remove all non-numeric entities
448 * @return string converted string
9c1cd39d 449 */
5a7e4c11
PS
450 public static function utf8_to_entities($str, $dec=false, $nonnum=false) {
451 // Avoid some notices from Typo3 code
9c1cd39d 452 $oldlevel = error_reporting(E_PARSE);
453 if ($nonnum) {
5a7e4c11 454 $str = self::typo3()->entities_to_utf8($str, true);
9c1cd39d 455 }
5a7e4c11 456 $result = self::typo3()->utf8_to_entities($str);
9c1cd39d 457 if ($dec) {
458 $result = preg_replace('/&#x([0-9a-f]+);/ie', "'&#'.hexdec('$1').';'", $result);
459 }
5a7e4c11 460 // Restore original debug level
9c1cd39d 461 error_reporting($oldlevel);
462 return $result;
463 }
464
77b4d4be 465 /**
466 * Removes the BOM from unicode string - see http://unicode.org/faq/utf_bom.html
5a7e4c11
PS
467 *
468 * @param string $str
469 * @return string
77b4d4be 470 */
5a7e4c11 471 public static function trim_utf8_bom($str) {
77b4d4be 472 $bom = "\xef\xbb\xbf";
473 if (strpos($str, $bom) === 0) {
474 return substr($str, strlen($bom));
475 }
476 return $str;
477 }
e8515774 478
4d40aa14 479 /**
480 * Returns encoding options for select boxes, utf-8 and platform encoding first
e8515774 481 * @return array encodings
4d40aa14 482 */
5a7e4c11 483 public static function get_encodings() {
4d40aa14 484 $encodings = array();
485 $encodings['UTF-8'] = 'UTF-8';
bf69b06d 486 $winenc = strtoupper(get_string('localewincharset', 'langconfig'));
4d40aa14 487 if ($winenc != '') {
488 $encodings[$winenc] = $winenc;
489 }
bf69b06d 490 $nixenc = strtoupper(get_string('oldcharset', 'langconfig'));
4d40aa14 491 $encodings[$nixenc] = $nixenc;
e8515774 492
5a7e4c11 493 foreach (self::typo3()->synonyms as $enc) {
4d40aa14 494 $enc = strtoupper($enc);
495 $encodings[$enc] = $enc;
496 }
497 return $encodings;
498 }
4e2cb0e3 499
500 /**
e8515774 501 * Returns the utf8 string corresponding to the unicode value
4e2cb0e3 502 * (from php.net, courtesy - romans@void.lv)
e8515774 503 *
4e2cb0e3 504 * @param int $num one unicode value
505 * @return string the UTF-8 char corresponding to the unicode value
506 */
5a7e4c11 507 public static function code2utf8($num) {
4e2cb0e3 508 if ($num < 128) {
509 return chr($num);
510 }
511 if ($num < 2048) {
512 return chr(($num >> 6) + 192) . chr(($num & 63) + 128);
513 }
514 if ($num < 65536) {
515 return chr(($num >> 12) + 224) . chr((($num >> 6) & 63) + 128) . chr(($num & 63) + 128);
516 }
517 if ($num < 2097152) {
518 return chr(($num >> 18) + 240) . chr((($num >> 12) & 63) + 128) . chr((($num >> 6) & 63) + 128) . chr(($num & 63) + 128);
519 }
520 return '';
521 }
e8515774 522
523 /**
524 * Makes first letter of each word capital - words must be separated by spaces.
525 * Use with care, this function does not work properly in many locales!!!
5a7e4c11 526 *
e8515774 527 * @param string $text
528 * @return string
529 */
5a7e4c11 530 public static function strtotitle($text) {
e8515774 531 if (empty($text)) {
532 return $text;
533 }
534
535 if (function_exists('mb_convert_case')) {
5a7e4c11 536 return mb_convert_case($text, MB_CASE_TITLE, 'UTF-8');
e8515774 537 }
538
5a7e4c11 539 $text = self::strtolower($text);
e8515774 540 $words = explode(' ', $text);
541 foreach ($words as $i=>$word) {
5a7e4c11 542 $length = self::strlen($word);
e8515774 543 if (!$length) {
544 continue;
545
546 } else if ($length == 1) {
5a7e4c11 547 $words[$i] = self::strtoupper($word);
e8515774 548
549 } else {
5a7e4c11
PS
550 $letter = self::substr($word, 0, 1);
551 $letter = self::strtoupper($letter);
552 $rest = self::substr($word, 1);
e8515774 553 $words[$i] = $letter.$rest;
554 }
555 }
556 return implode(' ', $words);
557 }
878bcc7f
PS
558
559 /**
560 * Locale aware sorting, the key associations are kept, values are sorted alphabetically.
5a7e4c11
PS
561 *
562 * Note: this function is using current moodle locale.
563 *
878bcc7f 564 * @param array $arr array to be sorted
3650b83d 565 * @return void, modifies parameter
878bcc7f 566 */
5a7e4c11 567 public static function asort(array &$arr) {
878bcc7f 568 if (function_exists('collator_asort')) {
0b661c7d
PS
569 if ($coll = collator_create(get_string('locale', 'langconfig'))) {
570 collator_asort($coll, $arr);
571 return;
572 }
878bcc7f 573 }
0b661c7d 574 asort($arr, SORT_LOCALE_STRING);
878bcc7f 575 }
779dc9ac 576}