From 39e5102f8b8a4975fd1a8046264d736a89eedd34 Mon Sep 17 00:00:00 2001 From: sam marshall Date: Tue, 24 Sep 2013 18:14:09 +0100 Subject: [PATCH] MDL-41838 Backup/restore: Support .tar.gz format for .mbz (2 of 2) The new experimental setting enabletgzbackups allows backups to be created so that the internal format for .mbz files is .tar.gz. Restore transparently supports .mbz files with either internal formats (.zip or .tar.gz). The .tar.gz format has the following benefits for backup: - Supports larger files (no limit on total size, 8GB on single file vs. 4GB limit on total size) - Compresses text better, resulting in smaller .mbz files. - Reports progress regularly during compression of single files, reducing the chance of timeouts during backups that include a very large file. Time performance may also be improved although I haven't done a direct comparison. --- admin/settings/development.php | 9 + backup/moodle2/backup_stepslib.php | 2 +- .../helper/backup_general_helper.class.php | 3 +- backup/util/ui/restore_ui_stage.class.php | 2 +- lang/en/admin.php | 2 + lib/filestorage/mbz_packer.php | 170 ++++++++++++++++++ lib/filestorage/tests/mbz_packer_test.php | 91 ++++++++++ lib/moodlelib.php | 6 +- 8 files changed, 281 insertions(+), 4 deletions(-) create mode 100644 lib/filestorage/mbz_packer.php create mode 100644 lib/filestorage/tests/mbz_packer_test.php diff --git a/admin/settings/development.php b/admin/settings/development.php index 44d25462017..6be22df803b 100644 --- a/admin/settings/development.php +++ b/admin/settings/development.php @@ -19,6 +19,15 @@ if ($hassiteconfig) { // speedup for non-admins, add all caps used on this page $enablecssoptimiser->set_updatedcallback('theme_reset_all_caches'); $temp->add($enablecssoptimiser); + // Backup archive .mbz format: switching to .tar.gz enables larger files, better + // progress reporting and possibly better performance. This is an experimental + // setting but if successful, should be removed and enabled by default in a future + // version. Note: this setting controls newly-created backups only; restore always + // supports both formats. + $temp->add(new admin_setting_configcheckbox('enabletgzbackups', + new lang_string('enabletgzbackups', 'admin'), + new lang_string('enabletgzbackups_desc', 'admin'), 0)); + $ADMIN->add('experimental', $temp); // "debugging" settingpage diff --git a/backup/moodle2/backup_stepslib.php b/backup/moodle2/backup_stepslib.php index 922318c349b..0a7faef802b 100644 --- a/backup/moodle2/backup_stepslib.php +++ b/backup/moodle2/backup_stepslib.php @@ -1704,7 +1704,7 @@ class backup_zip_contents extends backup_execution_step implements file_progress $zipfile = $basepath . '/backup.mbz'; // Get the zip packer - $zippacker = get_file_packer('application/zip'); + $zippacker = get_file_packer('application/vnd.moodle.backup'); // Zip files $zippacker->archive_to_pathname($files, $zipfile, true, $this); diff --git a/backup/util/helper/backup_general_helper.class.php b/backup/util/helper/backup_general_helper.class.php index 03225a38aae..9e9d2c289f1 100644 --- a/backup/util/helper/backup_general_helper.class.php +++ b/backup/util/helper/backup_general_helper.class.php @@ -243,7 +243,8 @@ abstract class backup_general_helper extends backup_helper { // Extract moodle_backup.xml. $tmpname = 'info_from_mbz_' . time() . '_' . random_string(4); $tmpdir = $CFG->tempdir . '/backup/' . $tmpname; - $fp = get_file_packer('application/vnd.moodle.backup'); + $packer = get_file_packer('application/vnd.moodle.backup'); + $extracted = $fp->extract_to_pathname($filepath, $tmpdir, array('moodle_backup.xml')); $moodlefile = $tmpdir . '/' . 'moodle_backup.xml'; if (!$extracted || !is_readable($moodlefile)) { diff --git a/backup/util/ui/restore_ui_stage.class.php b/backup/util/ui/restore_ui_stage.class.php index 327722e72e3..dd58ca876ce 100644 --- a/backup/util/ui/restore_ui_stage.class.php +++ b/backup/util/ui/restore_ui_stage.class.php @@ -212,7 +212,7 @@ class restore_ui_stage_confirm extends restore_ui_independent_stage implements f $this->filepath = restore_controller::get_tempdir_name($this->contextid, $USER->id); - $fb = get_file_packer(); + $fb = get_file_packer('application/vnd.moodle.backup'); $result = $fb->extract_to_pathname("$CFG->tempdir/backup/".$this->filename, "$CFG->tempdir/backup/$this->filepath/", null, $this); diff --git a/lang/en/admin.php b/lang/en/admin.php index 71a73d89c36..2a2205c3039 100644 --- a/lang/en/admin.php +++ b/lang/en/admin.php @@ -487,6 +487,8 @@ $string['enablerecordcache'] = 'Enable record cache'; $string['enablerssfeeds'] = 'Enable RSS feeds'; $string['enablesafebrowserintegration'] = 'Enable Safe Exam Browser integration'; $string['enablestats'] = 'Enable statistics'; +$string['enabletgzbackups'] = 'Enable new backup format'; +$string['enabletgzbackups_desc'] = 'If enabled, future backups will be created in a new compression format for .mbz files (internally stored as a .tar.gz file). This removes the 4GB backup size restriction and may improve performance. Restore supports both formats and the difference should be transparent to users.'; $string['enabletrusttext'] = 'Enable trusted content'; $string['enablewebservices'] = 'Enable web services'; $string['enablewsdocumentation'] = 'Web services documentation'; diff --git a/lib/filestorage/mbz_packer.php b/lib/filestorage/mbz_packer.php new file mode 100644 index 00000000000..e149e35b284 --- /dev/null +++ b/lib/filestorage/mbz_packer.php @@ -0,0 +1,170 @@ +. + +/** + * Implementation of .mbz packer. + * + * This packer supports .mbz files which can be either .zip or .tar.gz format + * internally. A suitable format is chosen depending on system option when + * creating new files. + * + * Internally this packer works by wrapping the existing .zip/.tar.gz packers. + * + * Backup filenames do not contain non-ASCII characters so packers that do not + * support UTF-8 (like the current .tar.gz packer, and possibly external zip + * software in some cases if used) can be used by this packer. + * + * @package core_files + * @copyright 2013 The Open University + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +require_once("$CFG->libdir/filestorage/file_packer.php"); + +/** + * Utility class - handles all packing/unpacking of .mbz files. + * + * @package core_files + * @category files + * @copyright 2013 The Open University + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class mbz_packer extends file_packer { + /** + * Archive files and store the result in file storage. + * + * Any existing file at that location will be overwritten. + * + * @param array $files array from archive path => pathname or stored_file + * @param int $contextid context ID + * @param string $component component + * @param string $filearea file area + * @param int $itemid item ID + * @param string $filepath file path + * @param string $filename file name + * @param int $userid user ID + * @param bool $ignoreinvalidfiles true means ignore missing or invalid files, false means abort on any error + * @param file_progress $progress Progress indicator callback or null if not required + * @return stored_file|bool false if error stored_file instance if ok + * @throws file_exception If file operations fail + * @throws coding_exception If any archive paths do not meet the restrictions + */ + public function archive_to_storage(array $files, $contextid, + $component, $filearea, $itemid, $filepath, $filename, + $userid = null, $ignoreinvalidfiles = true, file_progress $progress = null) { + return $this->get_packer_for_archive_operation()->archive_to_storage($files, + $contextid, $component, $filearea, $itemid, $filepath, $filename, + $userid, $ignoreinvalidfiles, $progress); + } + + /** + * Archive files and store the result in an OS file. + * + * @param array $files array from archive path => pathname or stored_file + * @param string $archivefile path to target zip file + * @param bool $ignoreinvalidfiles true means ignore missing or invalid files, false means abort on any error + * @param file_progress $progress Progress indicator callback or null if not required + * @return bool true if file created, false if not + * @throws coding_exception If any archive paths do not meet the restrictions + */ + public function archive_to_pathname(array $files, $archivefile, + $ignoreinvalidfiles=true, file_progress $progress = null) { + return $this->get_packer_for_archive_operation()->archive_to_pathname($files, + $archivefile, $ignoreinvalidfiles, $progress); + } + + /** + * Extract file to given file path (real OS filesystem), existing files are overwritten. + * + * @param stored_file|string $archivefile full pathname of zip file or stored_file instance + * @param string $pathname target directory + * @param array $onlyfiles only extract files present in the array + * @param file_progress $progress Progress indicator callback or null if not required + * @return array list of processed files (name=>true) + * @throws moodle_exception If error + */ + public function extract_to_pathname($archivefile, $pathname, + array $onlyfiles = null, file_progress $progress = null) { + return $this->get_packer_for_read_operation($archivefile)->extract_to_pathname( + $archivefile, $pathname, $onlyfiles, $progress); + } + + /** + * Extract file to given file path (real OS filesystem), existing files are overwritten. + * + * @param string|stored_file $archivefile full pathname of zip file or stored_file instance + * @param int $contextid context ID + * @param string $component component + * @param string $filearea file area + * @param int $itemid item ID + * @param string $pathbase file path + * @param int $userid user ID + * @param file_progress $progress Progress indicator callback or null if not required + * @return array list of processed files (name=>true) + * @throws moodle_exception If error + */ + public function extract_to_storage($archivefile, $contextid, + $component, $filearea, $itemid, $pathbase, $userid = null, + file_progress $progress = null) { + return $this->get_packer_for_read_operation($archivefile)->extract_to_storage( + $archivefile, $contextid, $component, $filearea, $itemid, $pathbase, + $userid, $progress); + } + + /** + * Returns array of info about all files in archive. + * + * @param string|stored_file $archivefile + * @return array of file infos + */ + public function list_files($archivefile) { + return $this->get_packer_for_read_operation($archivefile)->list_files($archivefile); + } + + /** + * Selects appropriate packer for new archive depending on system option. + * + * @return file_packer Suitable packer + */ + protected function get_packer_for_archive_operation() { + global $CFG; + + if ($CFG->enabletgzbackups) { + return get_file_packer('application/x-gzip'); + } else { + return get_file_packer('application/zip'); + } + } + + /** + * Selects appropriate packer for existing archive depending on file contents. + * + * @param string|stored_file $archivefile full pathname of zip file or stored_file instance + * @return file_packer Suitable packer + */ + protected function get_packer_for_read_operation($archivefile) { + global $CFG; + require_once($CFG->dirroot . '/lib/filestorage/tgz_packer.php'); + + if (tgz_packer::is_tgz_file($archivefile)) { + return get_file_packer('application/x-gzip'); + } else { + return get_file_packer('application/zip'); + } + } +} diff --git a/lib/filestorage/tests/mbz_packer_test.php b/lib/filestorage/tests/mbz_packer_test.php new file mode 100644 index 00000000000..cdb7bef92fb --- /dev/null +++ b/lib/filestorage/tests/mbz_packer_test.php @@ -0,0 +1,91 @@ +. + +/** + * Unit tests for /lib/filestorage/mbz_packer.php. + * + * @package core_files + * @copyright 2013 The Open University + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +global $CFG; +require_once($CFG->libdir . '/filestorage/file_progress.php'); + +class core_files_mbz_packer_testcase extends advanced_testcase { + + public function test_archive_with_both_options() { + global $CFG; + $this->resetAfterTest(); + + // Get backup packer. + $packer = get_file_packer('application/vnd.moodle.backup'); + + // Set up basic archive contents. + $files = array('1.txt' => array('frog')); + + // Create 2 archives (each with one file in) in default mode. + $CFG->enabletgzbackups = false; + $filefalse = $CFG->tempdir . '/false.mbz'; + $this->assertNotEmpty($packer->archive_to_pathname($files, $filefalse)); + $context = context_system::instance(); + $this->assertNotEmpty($storagefalse = $packer->archive_to_storage( + $files, $context->id, 'phpunit', 'data', 0, '/', 'false.mbz')); + + // Create 2 archives in tgz mode. + $CFG->enabletgzbackups = true; + $filetrue = $CFG->tempdir . '/true.mbz'; + $this->assertNotEmpty($packer->archive_to_pathname($files, $filetrue)); + $context = context_system::instance(); + $this->assertNotEmpty($storagetrue = $packer->archive_to_storage( + $files, $context->id, 'phpunit', 'data', 0, '/', 'false.mbz')); + + // Check the sizes are different (indicating different formats). + $this->assertNotEquals(filesize($filefalse), filesize($filetrue)); + $this->assertNotEquals($storagefalse->get_filesize(), $storagetrue->get_filesize()); + + // Extract files into storage and into filesystem from both formats. + // (Note: the setting does not matter, but set to false just to check.) + $CFG->enabletgzbackups = false; + + // Extract to path (zip). + $packer->extract_to_pathname($filefalse, $CFG->tempdir); + $onefile = $CFG->tempdir . '/1.txt'; + $this->assertEquals('frog', file_get_contents($onefile)); + unlink($onefile); + + // Extract to path (tgz). + $packer->extract_to_pathname($filetrue, $CFG->tempdir); + $onefile = $CFG->tempdir . '/1.txt'; + $this->assertEquals('frog', file_get_contents($onefile)); + unlink($onefile); + + // Extract to storage (zip). + $packer->extract_to_storage($storagefalse, $context->id, 'phpunit', 'data', 1, '/'); + $fs = get_file_storage(); + $out = $fs->get_file($context->id, 'phpunit', 'data', 1, '/', '1.txt'); + $this->assertNotEmpty($out); + $this->assertEquals('frog', $out->get_content()); + + // Extract to storage (tgz). + $packer->extract_to_storage($storagetrue, $context->id, 'phpunit', 'data', 2, '/'); + $out = $fs->get_file($context->id, 'phpunit', 'data', 2, '/', '1.txt'); + $this->assertNotEmpty($out); + $this->assertEquals('frog', $out->get_content()); + } +} diff --git a/lib/moodlelib.php b/lib/moodlelib.php index d4c47b81e23..6df44149b17 100644 --- a/lib/moodlelib.php +++ b/lib/moodlelib.php @@ -6149,14 +6149,18 @@ function get_file_packer($mimetype='application/zip') { switch ($mimetype) { case 'application/zip': - case 'application/vnd.moodle.backup': case 'application/vnd.moodle.profiling': $classname = 'zip_packer'; break; + case 'application/x-gzip' : $classname = 'tgz_packer'; break; + case 'application/vnd.moodle.backup': + $classname = 'mbz_packer'; + break; + default: return false; } -- 2.17.1