MDL-46375 core_files: Split parts of file_storage into new file system
authorAndrew Nicols <andrew@nicols.co.uk>
Tue, 5 Jan 2016 10:14:16 +0000 (18:14 +0800)
committerAndrew Nicols <andrew@nicols.co.uk>
Tue, 14 Feb 2017 00:42:28 +0000 (08:42 +0800)
This change moves all operations which deal with the fetching/updating, or
setting of files from the file_storage class into a new file_system class.

A new file_system can be specified in the config.php and used to replace
all relevant methods in order to move the file system component to an
alternative solution.

14 files changed:
composer.json
composer.lock
lib/filestorage/file_storage.php
lib/filestorage/file_system.php [new file with mode: 0644]
lib/filestorage/file_system_filedir.php [new file with mode: 0644]
lib/filestorage/stored_file.php
lib/filestorage/tests/file_storage_test.php
lib/filestorage/tests/file_system_filedir_test.php [new file with mode: 0644]
lib/filestorage/tests/file_system_test.php [new file with mode: 0644]
lib/filestorage/tests/fixtures/test.tgz [new file with mode: 0644]
lib/moodlelib.php
lib/upgrade.txt
question/format/blackboard_six/formatbase.php
repository/lib.php

index 9c795b6..92dfb41 100644 (file)
@@ -7,6 +7,7 @@
     "require-dev": {
         "phpunit/phpunit": "5.5.*",
         "phpunit/dbUnit": "1.4.*",
-        "moodlehq/behat-extension": "3.33.1"
+        "moodlehq/behat-extension": "3.33.1",
+        "mikey179/vfsStream": "^1.6"
     }
 }
index c757473..1fca2b9 100644 (file)
@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
         "This file is @generated automatically"
     ],
-    "content-hash": "f4cfcd74744fbbced495458ea82fd314",
+    "content-hash": "751fc6623b264b33856167019b391053",
     "packages": [],
     "packages-dev": [
         {
             ],
             "time": "2015-06-15T20:19:33+00:00"
         },
+        {
+            "name": "mikey179/vfsStream",
+            "version": "v1.6.4",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/mikey179/vfsStream.git",
+                "reference": "0247f57b2245e8ad2e689d7cee754b45fbabd592"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/mikey179/vfsStream/zipball/0247f57b2245e8ad2e689d7cee754b45fbabd592",
+                "reference": "0247f57b2245e8ad2e689d7cee754b45fbabd592",
+                "shasum": ""
+            },
+            "require": {
+                "php": ">=5.3.0"
+            },
+            "require-dev": {
+                "phpunit/phpunit": "~4.5"
+            },
+            "type": "library",
+            "extra": {
+                "branch-alias": {
+                    "dev-master": "1.6.x-dev"
+                }
+            },
+            "autoload": {
+                "psr-0": {
+                    "org\\bovigo\\vfs\\": "src/main/php"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "BSD-3-Clause"
+            ],
+            "authors": [
+                {
+                    "name": "Frank Kleine",
+                    "homepage": "http://frankkleine.de/",
+                    "role": "Developer"
+                }
+            ],
+            "description": "Virtual file system to mock the real file system in unit tests.",
+            "homepage": "http://vfs.bovigo.org/",
+            "time": "2016-07-18T14:02:57+00:00"
+        },
         {
             "name": "moodlehq/behat-extension",
             "version": "v3.33.1",
         },
         {
             "name": "myclabs/deep-copy",
-            "version": "1.5.5",
+            "version": "1.6.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/myclabs/DeepCopy.git",
-                "reference": "399c1f9781e222f6eb6cc238796f5200d1b7f108"
+                "reference": "5a5a9fc8025a08d8919be87d6884d5a92520cefe"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/399c1f9781e222f6eb6cc238796f5200d1b7f108",
-                "reference": "399c1f9781e222f6eb6cc238796f5200d1b7f108",
+                "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/5a5a9fc8025a08d8919be87d6884d5a92520cefe",
+                "reference": "5a5a9fc8025a08d8919be87d6884d5a92520cefe",
                 "shasum": ""
             },
             "require": {
                 "object",
                 "object graph"
             ],
-            "time": "2016-10-31T17:19:45+00:00"
+            "time": "2017-01-26T22:05:40+00:00"
         },
         {
             "name": "phpdocumentor/reflection-common",
         },
         {
             "name": "phpunit/php-code-coverage",
-            "version": "4.0.4",
+            "version": "4.0.5",
             "source": {
                 "type": "git",
                 "url": "https://github.com/sebastianbergmann/php-code-coverage.git",
-                "reference": "c14196e64a78570034afd0b7a9f3757ba71c2a0a"
+                "reference": "c19cfc7cbb0e9338d8c469c7eedecc2a428b0971"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/c14196e64a78570034afd0b7a9f3757ba71c2a0a",
-                "reference": "c14196e64a78570034afd0b7a9f3757ba71c2a0a",
+                "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/c19cfc7cbb0e9338d8c469c7eedecc2a428b0971",
+                "reference": "c19cfc7cbb0e9338d8c469c7eedecc2a428b0971",
                 "shasum": ""
             },
             "require": {
                 "testing",
                 "xunit"
             ],
-            "time": "2016-12-20T15:22:42+00:00"
+            "time": "2017-01-20T15:06:43+00:00"
         },
         {
             "name": "phpunit/php-file-iterator",
         },
         {
             "name": "sebastian/comparator",
-            "version": "1.2.2",
+            "version": "1.2.4",
             "source": {
                 "type": "git",
                 "url": "https://github.com/sebastianbergmann/comparator.git",
-                "reference": "6a1ed12e8b2409076ab22e3897126211ff8b1f7f"
+                "reference": "2b7424b55f5047b47ac6e5ccb20b2aea4011d9be"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/6a1ed12e8b2409076ab22e3897126211ff8b1f7f",
-                "reference": "6a1ed12e8b2409076ab22e3897126211ff8b1f7f",
+                "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/2b7424b55f5047b47ac6e5ccb20b2aea4011d9be",
+                "reference": "2b7424b55f5047b47ac6e5ccb20b2aea4011d9be",
                 "shasum": ""
             },
             "require": {
                 "compare",
                 "equality"
             ],
-            "time": "2016-11-19T09:18:40+00:00"
+            "time": "2017-01-29T09:50:25+00:00"
         },
         {
             "name": "sebastian/diff",
index c481ebe..3282796 100644 (file)
@@ -43,16 +43,13 @@ require_once("$CFG->libdir/filestorage/stored_file.php");
  * @since     Moodle 2.0
  */
 class file_storage {
-    /** @var string Directory with file contents */
-    private $filedir;
-    /** @var string Contents of deleted files not needed any more */
-    private $trashdir;
+
     /** @var string tempdir */
     private $tempdir;
-    /** @var int Permissions for new directories */
-    private $dirpermissions;
-    /** @var int Permissions for new files */
-    private $filepermissions;
+
+    /** @var file_system filesystem */
+    private $filesystem;
+
     /** @var array List of formats supported by unoconv */
     private $unoconvformats;
 
@@ -74,43 +71,46 @@ class file_storage {
     /** Any other error */
     const UNOCONVPATH_ERROR = 'error';
 
-
     /**
      * Constructor - do not use directly use {@link get_file_storage()} call instead.
-     *
-     * @param string $filedir full path to pool directory
-     * @param string $trashdir temporary storage of deleted area
-     * @param string $tempdir temporary storage of various files
-     * @param int $dirpermissions new directory permissions
-     * @param int $filepermissions new file permissions
      */
-    public function __construct($filedir, $trashdir, $tempdir, $dirpermissions, $filepermissions) {
-        global $CFG;
+    public function __construct() {
+        // The tempdir must always remain on disk, but shared between all ndoes in a cluster. Its content is not subject
+        // to the file_system abstraction.
+        $this->tempdir = make_temp_directory('filestorage');
 
-        $this->filedir         = $filedir;
-        $this->trashdir        = $trashdir;
-        $this->tempdir         = $tempdir;
-        $this->dirpermissions  = $dirpermissions;
-        $this->filepermissions = $filepermissions;
+        $this->setup_file_system();
+    }
 
-        // make sure the file pool directory exists
-        if (!is_dir($this->filedir)) {
-            if (!mkdir($this->filedir, $this->dirpermissions, true)) {
-                throw new file_exception('storedfilecannotcreatefiledirs'); // permission trouble
-            }
-            // place warning file in file pool root
-            if (!file_exists($this->filedir.'/warning.txt')) {
-                file_put_contents($this->filedir.'/warning.txt',
-                                  'This directory contains the content of uploaded files and is controlled by Moodle code. Do not manually move, change or rename any of the files and subdirectories here.');
-                chmod($this->filedir.'/warning.txt', $CFG->filepermissions);
-            }
-        }
-        // make sure the file pool directory exists
-        if (!is_dir($this->trashdir)) {
-            if (!mkdir($this->trashdir, $this->dirpermissions, true)) {
-                throw new file_exception('storedfilecannotcreatefiledirs'); // permission trouble
+    /**
+     * Complete setup procedure for the file_system component.
+     *
+     * @return file_system
+     */
+    public function setup_file_system() {
+        global $CFG;
+        if ($this->filesystem === null) {
+            require_once($CFG->libdir . '/filestorage/file_system.php');
+            if (!empty($CFG->alternative_file_system_class)) {
+                $class = $CFG->alternative_file_system_class;
+            } else {
+                // The default file_system is the filedir.
+                require_once($CFG->libdir . '/filestorage/file_system_filedir.php');
+                $class = file_system_filedir::class;
             }
+            $this->filesystem = new $class();
         }
+
+        return $this->filesystem;
+    }
+
+    /**
+     * Return the file system instance.
+     *
+     * @return file_system
+     */
+    public function get_file_system() {
+        return $this->filesystem;
     }
 
     /**
@@ -173,7 +173,7 @@ class file_storage {
      * @return stored_file instance of file abstraction class
      */
     public function get_file_instance(stdClass $filerecord) {
-        $storedfile = new stored_file($this, $filerecord, $this->filedir);
+        $storedfile = new stored_file($this, $filerecord);
         return $storedfile;
     }
 
@@ -1502,7 +1502,7 @@ class file_storage {
             $newrecord->id = $DB->insert_record('files', $newrecord);
         } catch (dml_exception $e) {
             if ($newfile) {
-                $this->deleted_file_cleanup($newrecord->contenthash);
+                $this->move_to_trash($newrecord->contenthash);
             }
             throw new stored_file_creation_exception($newrecord->contextid, $newrecord->component, $newrecord->filearea, $newrecord->itemid,
                                                     $newrecord->filepath, $newrecord->filename, $e->debuginfo);
@@ -1609,9 +1609,11 @@ class file_storage {
         $newrecord->sortorder    = $filerecord->sortorder;
 
         list($newrecord->contenthash, $newrecord->filesize, $newfile) = $this->add_string_to_pool($content);
-        $filepathname = $this->path_from_hash($newrecord->contenthash) . '/' . $newrecord->contenthash;
-        // get mimetype by magic bytes
-        $newrecord->mimetype = empty($filerecord->mimetype) ? $this->mimetype($filepathname, $filerecord->filename) : $filerecord->mimetype;
+        if (empty($filerecord->mimetype)) {
+            $newrecord->mimetype = $this->filesystem->mimetype_from_hash($newrecord->contenthash, $newrecord->filename);
+        } else {
+            $newrecord->mimetype = $filerecord->mimetype;
+        }
 
         $newrecord->pathnamehash = $this->get_pathname_hash($newrecord->contextid, $newrecord->component, $newrecord->filearea, $newrecord->itemid, $newrecord->filepath, $newrecord->filename);
 
@@ -1619,7 +1621,7 @@ class file_storage {
             $newrecord->id = $DB->insert_record('files', $newrecord);
         } catch (dml_exception $e) {
             if ($newfile) {
-                $this->deleted_file_cleanup($newrecord->contenthash);
+                $this->move_to_trash($newrecord->contenthash);
             }
             throw new stored_file_creation_exception($newrecord->contextid, $newrecord->component, $newrecord->filearea, $newrecord->itemid,
                                                     $newrecord->filepath, $newrecord->filename, $e->debuginfo);
@@ -1723,16 +1725,19 @@ class file_storage {
             throw new file_reference_exception($repositoryid, $reference, null, null, $e->getMessage());
         }
 
-        if (isset($filerecord->contenthash) && $this->content_exists($filerecord->contenthash)) {
-            // there was specified the contenthash for a file already stored in moodle filepool
+        $existingfile = null;
+        if (isset($filerecord->contenthash)) {
+            $existingfile = $DB->get_record('files', array('contenthash' => $filerecord->contenthash));
+        }
+        if (!empty($existingfile)) {
+            // There is an existing file already available.
             if (empty($filerecord->filesize)) {
-                $filepathname = $this->path_from_hash($filerecord->contenthash) . '/' . $filerecord->contenthash;
-                $filerecord->filesize = filesize($filepathname);
+                $filerecord->filesize = $existingfile->filesize;
             } else {
                 $filerecord->filesize = clean_param($filerecord->filesize, PARAM_INT);
             }
         } else {
-            // atempt to get the result of last synchronisation for this reference
+            // Attempt to get the result of last synchronisation for this reference.
             $lastcontent = $DB->get_record('files', array('referencefileid' => $filerecord->referencefileid),
                     'id, contenthash, filesize', IGNORE_MULTIPLE);
             if ($lastcontent) {
@@ -1751,7 +1756,7 @@ class file_storage {
             $filerecord->id = $DB->insert_record('files', $filerecord);
         } catch (dml_exception $e) {
             if (!empty($newfile)) {
-                $this->deleted_file_cleanup($filerecord->contenthash);
+                $this->move_to_trash($filerecord->contenthash);
             }
             throw new stored_file_creation_exception($filerecord->contextid, $filerecord->component, $filerecord->filearea, $filerecord->itemid,
                                                     $filerecord->filepath, $filerecord->filename, $e->debuginfo);
@@ -1934,98 +1939,7 @@ class file_storage {
      * @return array (contenthash, filesize, newfile)
      */
     public function add_file_to_pool($pathname, $contenthash = NULL) {
-        global $CFG;
-
-        if (!is_readable($pathname)) {
-            throw new file_exception('storedfilecannotread', '', $pathname);
-        }
-
-        $filesize = filesize($pathname);
-        if ($filesize === false) {
-            throw new file_exception('storedfilecannotread', '', $pathname);
-        }
-
-        if (is_null($contenthash)) {
-            $contenthash = sha1_file($pathname);
-        } else if ($CFG->debugdeveloper) {
-            $filehash = sha1_file($pathname);
-            if ($filehash === false) {
-                throw new file_exception('storedfilecannotread', '', $pathname);
-            }
-            if ($filehash !== $contenthash) {
-                // Hopefully this never happens, if yes we need to fix calling code.
-                debugging("Invalid contenthash submitted for file $pathname", DEBUG_DEVELOPER);
-                $contenthash = $filehash;
-            }
-        }
-        if ($contenthash === false) {
-            throw new file_exception('storedfilecannotread', '', $pathname);
-        }
-
-        if ($filesize > 0 and $contenthash === sha1('')) {
-            // Did the file change or is sha1_file() borked for this file?
-            clearstatcache();
-            $contenthash = sha1_file($pathname);
-            $filesize = filesize($pathname);
-
-            if ($contenthash === false or $filesize === false) {
-                throw new file_exception('storedfilecannotread', '', $pathname);
-            }
-            if ($filesize > 0 and $contenthash === sha1('')) {
-                // This is very weird...
-                throw new file_exception('storedfilecannotread', '', $pathname);
-            }
-        }
-
-        $hashpath = $this->path_from_hash($contenthash);
-        $hashfile = "$hashpath/$contenthash";
-
-        $newfile = true;
-
-        if (file_exists($hashfile)) {
-            if (filesize($hashfile) === $filesize) {
-                return array($contenthash, $filesize, false);
-            }
-            if (sha1_file($hashfile) === $contenthash) {
-                // Jackpot! We have a sha1 collision.
-                mkdir("$this->filedir/jackpot/", $this->dirpermissions, true);
-                copy($pathname, "$this->filedir/jackpot/{$contenthash}_1");
-                copy($hashfile, "$this->filedir/jackpot/{$contenthash}_2");
-                throw new file_pool_content_exception($contenthash);
-            }
-            debugging("Replacing invalid content file $contenthash");
-            unlink($hashfile);
-            $newfile = false;
-        }
-
-        if (!is_dir($hashpath)) {
-            if (!mkdir($hashpath, $this->dirpermissions, true)) {
-                // Permission trouble.
-                throw new file_exception('storedfilecannotcreatefiledirs');
-            }
-        }
-
-        // Let's try to prevent some race conditions.
-
-        $prev = ignore_user_abort(true);
-        @unlink($hashfile.'.tmp');
-        if (!copy($pathname, $hashfile.'.tmp')) {
-            // Borked permissions or out of disk space.
-            ignore_user_abort($prev);
-            throw new file_exception('storedfilecannotcreatefile');
-        }
-        if (filesize($hashfile.'.tmp') !== $filesize) {
-            // This should not happen.
-            unlink($hashfile.'.tmp');
-            ignore_user_abort($prev);
-            throw new file_exception('storedfilecannotcreatefile');
-        }
-        rename($hashfile.'.tmp', $hashfile);
-        chmod($hashfile, $this->filepermissions); // Fix permissions if needed.
-        @unlink($hashfile.'.tmp'); // Just in case anything fails in a weird way.
-        ignore_user_abort($prev);
-
-        return array($contenthash, $filesize, $newfile);
+        return $this->filesystem->add_file_from_path($pathname, $contenthash);
     }
 
     /**
@@ -2035,66 +1949,7 @@ class file_storage {
      * @return array (contenthash, filesize, newfile)
      */
     public function add_string_to_pool($content) {
-        global $CFG;
-
-        $contenthash = sha1($content);
-        $filesize = strlen($content); // binary length
-
-        $hashpath = $this->path_from_hash($contenthash);
-        $hashfile = "$hashpath/$contenthash";
-
-        $newfile = true;
-
-        if (file_exists($hashfile)) {
-            if (filesize($hashfile) === $filesize) {
-                return array($contenthash, $filesize, false);
-            }
-            if (sha1_file($hashfile) === $contenthash) {
-                // Jackpot! We have a sha1 collision.
-                mkdir("$this->filedir/jackpot/", $this->dirpermissions, true);
-                copy($hashfile, "$this->filedir/jackpot/{$contenthash}_1");
-                file_put_contents("$this->filedir/jackpot/{$contenthash}_2", $content);
-                throw new file_pool_content_exception($contenthash);
-            }
-            debugging("Replacing invalid content file $contenthash");
-            unlink($hashfile);
-            $newfile = false;
-        }
-
-        if (!is_dir($hashpath)) {
-            if (!mkdir($hashpath, $this->dirpermissions, true)) {
-                // Permission trouble.
-                throw new file_exception('storedfilecannotcreatefiledirs');
-            }
-        }
-
-        // Hopefully this works around most potential race conditions.
-
-        $prev = ignore_user_abort(true);
-
-        if (!empty($CFG->preventfilelocking)) {
-            $newsize = file_put_contents($hashfile.'.tmp', $content);
-        } else {
-            $newsize = file_put_contents($hashfile.'.tmp', $content, LOCK_EX);
-        }
-
-        if ($newsize === false) {
-            // Borked permissions most likely.
-            ignore_user_abort($prev);
-            throw new file_exception('storedfilecannotcreatefile');
-        }
-        if (filesize($hashfile.'.tmp') !== $filesize) {
-            // Out of disk space?
-            unlink($hashfile.'.tmp');
-            ignore_user_abort($prev);
-            throw new file_exception('storedfilecannotcreatefile');
-        }
-        rename($hashfile.'.tmp', $hashfile);
-        chmod($hashfile, $this->filepermissions); // Fix permissions if needed.
-        @unlink($hashfile.'.tmp'); // Just in case anything fails in a weird way.
-        ignore_user_abort($prev);
-
-        return array($contenthash, $filesize, $newfile);
+        return $this->filesystem->add_file_from_string($content);
     }
 
     /**
@@ -2106,11 +1961,7 @@ class file_storage {
      * @return bool success
      */
     public function xsendfile($contenthash) {
-        global $CFG;
-        require_once("$CFG->libdir/xsendfilelib.php");
-
-        $hashpath = $this->path_from_hash($contenthash);
-        return xsendfile("$hashpath/$contenthash");
+        return $this->filesystem->xsendfile($contenthash);
     }
 
     /**
@@ -2118,39 +1969,12 @@ class file_storage {
      *
      * @param string $contenthash
      * @return bool
+     * @deprecated since 3.3
      */
     public function content_exists($contenthash) {
-        $dir = $this->path_from_hash($contenthash);
-        $filepath = $dir . '/' . $contenthash;
-        return file_exists($filepath);
-    }
+        debugging('The content_exists function has been deprecated and should no longer be used.', DEBUG_DEVELOPER);
 
-    /**
-     * Return path to file with given hash.
-     *
-     * NOTE: must not be public, files in pool must not be modified
-     *
-     * @param string $contenthash content hash
-     * @return string expected file location
-     */
-    protected function path_from_hash($contenthash) {
-        $l1 = $contenthash[0].$contenthash[1];
-        $l2 = $contenthash[2].$contenthash[3];
-        return "$this->filedir/$l1/$l2";
-    }
-
-    /**
-     * Return path to file with given hash.
-     *
-     * NOTE: must not be public, files in pool must not be modified
-     *
-     * @param string $contenthash content hash
-     * @return string expected file location
-     */
-    protected function trash_path_from_hash($contenthash) {
-        $l1 = $contenthash[0].$contenthash[1];
-        $l2 = $contenthash[2].$contenthash[3];
-        return "$this->trashdir/$l1/$l2";
+        return false;
     }
 
     /**
@@ -2158,79 +1982,12 @@ class file_storage {
      *
      * @param stored_file $file stored_file instance
      * @return bool success
+     * @deprecated since 3.3
      */
     public function try_content_recovery($file) {
-        $contenthash = $file->get_contenthash();
-        $trashfile = $this->trash_path_from_hash($contenthash).'/'.$contenthash;
-        if (!is_readable($trashfile)) {
-            if (!is_readable($this->trashdir.'/'.$contenthash)) {
-                return false;
-            }
-            // nice, at least alternative trash file in trash root exists
-            $trashfile = $this->trashdir.'/'.$contenthash;
-        }
-        if (filesize($trashfile) != $file->get_filesize() or sha1_file($trashfile) != $contenthash) {
-            //weird, better fail early
-            return false;
-        }
-        $contentdir  = $this->path_from_hash($contenthash);
-        $contentfile = $contentdir.'/'.$contenthash;
-        if (file_exists($contentfile)) {
-            //strange, no need to recover anything
-            return true;
-        }
-        if (!is_dir($contentdir)) {
-            if (!mkdir($contentdir, $this->dirpermissions, true)) {
-                return false;
-            }
-        }
-        return rename($trashfile, $contentfile);
-    }
-
-    /**
-     * Marks pool file as candidate for deleting.
-     *
-     * DO NOT call directly - reserved for core!!
-     *
-     * @param string $contenthash
-     */
-    public function deleted_file_cleanup($contenthash) {
-        global $DB;
-
-        if ($contenthash === sha1('')) {
-            // No need to delete empty content file with sha1('') content hash.
-            return;
-        }
+        debugging('The try_content_recovery function has been deprecated and should no longer be used.', DEBUG_DEVELOPER);
 
-        //Note: this section is critical - in theory file could be reused at the same
-        //      time, if this happens we can still recover the file from trash
-        if ($DB->record_exists('files', array('contenthash'=>$contenthash))) {
-            // file content is still used
-            return;
-        }
-        //move content file to trash
-        $contentfile = $this->path_from_hash($contenthash).'/'.$contenthash;
-        if (!file_exists($contentfile)) {
-            //weird, but no problem
-            return;
-        }
-        $trashpath = $this->trash_path_from_hash($contenthash);
-        $trashfile = $trashpath.'/'.$contenthash;
-        if (file_exists($trashfile)) {
-            // we already have this content in trash, no need to move it there
-            unlink($contentfile);
-            return;
-        }
-        if (!is_dir($trashpath)) {
-            mkdir($trashpath, $this->dirpermissions, true);
-        }
-        rename($contentfile, $trashfile);
-
-        // Fix permissions, only if needed.
-        $currentperms = octdec(substr(decoct(fileperms($trashfile)), -4));
-        if ((int)$this->filepermissions !== $currentperms) {
-            chmod($trashfile, $this->filepermissions);
-        }
+        return false;
     }
 
     /**
@@ -2509,27 +2266,46 @@ class file_storage {
     }
 
     /**
-     * Return mimetype by given file pathname
+     * Return mimetype by given file pathname.
      *
      * If file has a known extension, we return the mimetype based on extension.
      * Otherwise (when possible) we try to get the mimetype from file contents.
      *
-     * @param string $pathname full path to the file
-     * @param string $filename correct file name with extension, if omitted will be taken from $path
+     * @param string $fullpath Full path to the file on disk
+     * @param string $filename Correct file name with extension, if omitted will be taken from $path
      * @return string
      */
-    public static function mimetype($pathname, $filename = null) {
+    public static function mimetype($fullpath, $filename = null) {
         if (empty($filename)) {
-            $filename = $pathname;
+            $filename = $fullpath;
         }
+
+        // The mimeinfo function determines the mimetype purely based on the file extension.
         $type = mimeinfo('type', $filename);
-        if ($type === 'document/unknown' && class_exists('finfo') && file_exists($pathname)) {
-            $finfo = new finfo(FILEINFO_MIME_TYPE);
-            $type = mimeinfo_from_type('type', $finfo->file($pathname));
+
+        if ($type === 'document/unknown') {
+            // The type is unknown. Inspect the file now.
+            $type = self::mimetype_from_file($fullpath);
         }
         return $type;
     }
 
+    /**
+     * Inspect a file on disk for it's mimetype.
+     *
+     * @param string $fullpath Path to file on disk
+     * @return string The mimetype
+     */
+    public static function mimetype_from_file($fullpath) {
+        if (file_exists($fullpath)) {
+            // The type is unknown. Attempt to look up the file type now.
+            $finfo = new finfo(FILEINFO_MIME_TYPE);
+            return mimeinfo_from_type('type', $finfo->file($fullpath));
+        }
+
+        return 'document/unknown';
+    }
+
     /**
      * Cron cleanup job.
      */
@@ -2614,10 +2390,9 @@ class file_storage {
             $rs->close();
             mtrace('done.');
 
-            mtrace('Deleting trash files... ', '');
+            mtrace('Call filesystem cron tasks.', '');
             cron_trace_time_and_memory();
-            fulldelete($this->trashdir);
-            set_config('fileslastcleanup', time());
+            $this->filesystem->cron();
             mtrace('done.');
         }
     }
diff --git a/lib/filestorage/file_system.php b/lib/filestorage/file_system.php
new file mode 100644 (file)
index 0000000..bfee09c
--- /dev/null
@@ -0,0 +1,561 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Core file system class definition.
+ *
+ * @package   core_files
+ * @copyright 2017 Andrew Nicols <andrew@nicols.co.uk>
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * File system class used for low level access to real files in filedir.
+ *
+ * @package   core_files
+ * @category  files
+ * @copyright 2017 Andrew Nicols <andrew@nicols.co.uk>
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+abstract class file_system {
+
+    /**
+     * Private clone method to prevent cloning of the instance.
+     */
+    final protected function __clone() {
+        return;
+    }
+
+    /**
+     * Private wakeup method to prevent unserialising of the instance.
+     */
+    final protected function __wakeup() {
+        return;
+    }
+
+    /**
+     * Output the content of the specified stored file.
+     *
+     * Note, this is different to get_content() as it uses the built-in php
+     * readfile function which is more efficient.
+     *
+     * @param stored_file $file The file to serve.
+     * @return void
+     */
+    public function readfile(stored_file $file) {
+        if ($this->is_file_readable_locally_by_storedfile($file, false)) {
+            $path = $this->get_local_path_from_storedfile($file, false);
+        } else {
+            $path = $this->get_remote_path_from_storedfile($file);
+        }
+        readfile_allow_large($path, $file->get_filesize());
+    }
+
+    /**
+     * Get the full path on disk for the specified stored file.
+     *
+     * Note: This must return a consistent path for the file's contenthash
+     * and the path _will_ be in a standard local format.
+     * Streamable paths will not work.
+     * A local copy of the file _will_ be fetched if $fetchifnotfound is tree.
+     *
+     * The $fetchifnotfound allows you to determine the expected path of the file.
+     *
+     * @param stored_file $file The file to serve.
+     * @param bool $fetchifnotfound Whether to attempt to fetch from the remote path if not found.
+     * @return string full path to pool file with file content
+     */
+    protected function get_local_path_from_storedfile(stored_file $file, $fetchifnotfound = false) {
+        return $this->get_local_path_from_hash($file->get_contenthash(), $fetchifnotfound);
+    }
+
+    /**
+     * Get a remote filepath for the specified stored file.
+     *
+     * This is typically either the same as the local filepath, or it is a streamable resource.
+     *
+     * See https://secure.php.net/manual/en/wrappers.php for further information on valid wrappers.
+     *
+     * @param stored_file $file The file to serve.
+     * @return string full path to pool file with file content
+     */
+    protected function get_remote_path_from_storedfile(stored_file $file) {
+        return $this->get_remote_path_from_hash($file->get_contenthash(), false);
+    }
+
+    /**
+     * Get the full path for the specified hash, including the path to the filedir.
+     *
+     * Note: This must return a consistent path for the file's contenthash
+     * and the path _will_ be in a standard local format.
+     * Streamable paths will not work.
+     * A local copy of the file _will_ be fetched if $fetchifnotfound is tree.
+     *
+     * The $fetchifnotfound allows you to determine the expected path of the file.
+     *
+     * @param string $contenthash The content hash
+     * @param bool $fetchifnotfound Whether to attempt to fetch from the remote path if not found.
+     * @return string The full path to the content file
+     */
+    abstract protected function get_local_path_from_hash($contenthash, $fetchifnotfound = false);
+
+    /**
+     * Get the full path for the specified hash, including the path to the filedir.
+     *
+     * This is typically either the same as the local filepath, or it is a streamable resource.
+     *
+     * See https://secure.php.net/manual/en/wrappers.php for further information on valid wrappers.
+     *
+     * @param string $contenthash The content hash
+     * @return string The full path to the content file
+     */
+    abstract protected function get_remote_path_from_hash($contenthash);
+
+    /**
+     * Determine whether the file is present on the file system somewhere.
+     * A local copy of the file _will_ be fetched if $fetchifnotfound is tree.
+     *
+     * The $fetchifnotfound allows you to determine the expected path of the file.
+     *
+     * @param stored_file $file The file to ensure is available.
+     * @param bool $fetchifnotfound Whether to attempt to fetch from the remote path if not found.
+     * @return bool
+     */
+    public function is_file_readable_locally_by_storedfile(stored_file $file, $fetchifnotfound = false) {
+        if (!$file->get_filesize()) {
+            // Files with empty size are either directories or empty.
+            // We handle these virtually.
+            return true;
+        }
+
+        // Check to see if the file is currently readable.
+        $path = $this->get_local_path_from_storedfile($file, $fetchifnotfound);
+        if (is_readable($path)) {
+            return true;
+        }
+
+        return false;
+    }
+
+    /**
+     * Determine whether the file is present on the local file system somewhere.
+     *
+     * @param stored_file $file The file to ensure is available.
+     * @return bool
+     */
+    public function is_file_readable_remotely_by_storedfile(stored_file $file) {
+        if (!$file->get_filesize()) {
+            // Files with empty size are either directories or empty.
+            // We handle these virtually.
+            return true;
+        }
+
+        $path = $this->get_remote_path_from_storedfile($file, false);
+        if (is_readable($path)) {
+            return true;
+        }
+
+        return false;
+    }
+
+    /**
+     * Determine whether the file is present on the file system somewhere given
+     * the contenthash.
+     *
+     * @param string $contenthash The contenthash of the file to check.
+     * @param bool $fetchifnotfound Whether to attempt to fetch from the remote path if not found.
+     * @return bool
+     */
+    public function is_file_readable_locally_by_hash($contenthash, $fetchifnotfound = false) {
+        if ($contenthash === sha1('')) {
+            // Files with empty size are either directories or empty.
+            // We handle these virtually.
+            return true;
+        }
+
+        // This is called by file_storage::content_exists(), and in turn by the repository system.
+        $path = $this->get_local_path_from_hash($contenthash, $fetchifnotfound);
+
+        // Note - it is not possible to perform a content recovery safely from a hash alone.
+        return is_readable($path);
+    }
+
+    /**
+     * Determine whether the file is present locally on the file system somewhere given
+     * the contenthash.
+     *
+     * @param string $contenthash The contenthash of the file to check.
+     * @return bool
+     */
+    public function is_file_readable_remotely_by_hash($contenthash) {
+        if ($contenthash === sha1('')) {
+            // Files with empty size are either directories or empty.
+            // We handle these virtually.
+            return true;
+        }
+
+        $path = $this->get_remote_path_from_hash($contenthash, false);
+
+        // Note - it is not possible to perform a content recovery safely from a hash alone.
+        return is_readable($path);
+    }
+
+    /**
+     * Copy content of file to given pathname.
+     *
+     * @param stored_file $file The file to be copied
+     * @param string $target real path to the new file
+     * @return bool success
+     */
+    abstract public function copy_content_from_storedfile(stored_file $file, $target);
+
+    /**
+     * Remove the file with the specified contenthash.
+     *
+     * Note, if overriding this function, you _must_ check that the file is
+     * no longer in use - see {check_file_usage}.
+     *
+     * DO NOT call directly - reserved for core!!
+     *
+     * @param string $contenthash
+     */
+    abstract public function remove_file($contenthash);
+
+    /**
+     * Check whether a file is removable.
+     *
+     * This must be called prior to file removal.
+     *
+     * @param string $contenthash
+     * @return bool
+     */
+    protected static function is_file_removable($contenthash) {
+        global $DB;
+
+        if ($contenthash === sha1('')) {
+            // No need to delete empty content file with sha1('') content hash.
+            return false;
+        }
+
+        // Note: This section is critical - in theory file could be reused at the same time, if this
+        // happens we can still recover the file from trash.
+        // Technically this is the responsibility of the file_storage API, but as this method is public, we go belt-and-braces.
+        if ($DB->record_exists('files', array('contenthash' => $contenthash))) {
+            // File content is still used.
+            return false;
+        }
+
+        return true;
+    }
+
+    /**
+     * Get the content of the specified stored file.
+     *
+     * Generally you will probably want to use readfile() to serve content,
+     * and where possible you should see if you can use
+     * get_content_file_handle and work with the file stream instead.
+     *
+     * @param stored_file $file The file to retrieve
+     * @return string The full file content
+     */
+    public function get_content(stored_file $file) {
+        if (!$file->get_filesize()) {
+            // Directories are empty. Empty files are not worth fetching.
+            return '';
+        }
+
+        $source = $this->get_remote_path_from_storedfile($file);
+        return file_get_contents($source);
+    }
+
+    /**
+     * List contents of archive.
+     *
+     * @param stored_file $file The archive to inspect
+     * @param file_packer $packer file packer instance
+     * @return array of file infos
+     */
+    public function list_files($file, file_packer $packer) {
+        $archivefile = $this->get_local_path_from_storedfile($file, true);
+        return $packer->list_files($archivefile);
+    }
+
+    /**
+     * Extract file to given file path (real OS filesystem), existing files are overwritten.
+     *
+     * @param stored_file $file The archive to inspect
+     * @param file_packer $packer File packer instance
+     * @param string $pathname Target directory
+     * @param file_progress $progress progress indicator callback or null if not required
+     * @return array|bool List of processed files; false if error
+     */
+    public function extract_to_pathname(stored_file $file, file_packer $packer, $pathname, file_progress $progress = null) {
+        $archivefile = $this->get_local_path_from_storedfile($file, true);
+        return $packer->extract_to_pathname($archivefile, $pathname, null, $progress);
+    }
+
+    /**
+     * Extract file to given file path (real OS filesystem), existing files are overwritten.
+     *
+     * @param stored_file $file The archive to inspect
+     * @param file_packer $packer file packer instance
+     * @param int $contextid context ID
+     * @param string $component component
+     * @param string $filearea file area
+     * @param int $itemid item ID
+     * @param string $pathbase path base
+     * @param int $userid user ID
+     * @param file_progress $progress Progress indicator callback or null if not required
+     * @return array|bool list of processed files; false if error
+     */
+    public function extract_to_storage(stored_file $file, file_packer $packer, $contextid,
+            $component, $filearea, $itemid, $pathbase, $userid = null, file_progress $progress = null) {
+
+        // Since we do not know which extractor we have, and whether it supports remote paths, use a local path here.
+        $archivefile = $this->get_local_path_from_storedfile($file, true);
+        return $packer->extract_to_storage($archivefile, $contextid,
+                $component, $filearea, $itemid, $pathbase, $userid, $progress);
+    }
+
+    /**
+     * Add file/directory into archive.
+     *
+     * @param stored_file $file The file to archive
+     * @param file_archive $filearch file archive instance
+     * @param string $archivepath pathname in archive
+     * @return bool success
+     */
+    public function add_storedfile_to_archive(stored_file $file, file_archive $filearch, $archivepath) {
+        if ($file->is_directory()) {
+            return $filearch->add_directory($archivepath);
+        } else {
+            // Since we do not know which extractor we have, and whether it supports remote paths, use a local path here.
+            return $filearch->add_file_from_pathname($archivepath, $this->get_local_path_from_storedfile($file, true));
+        }
+    }
+
+    /**
+     * Adds this file path to a curl request (POST only).
+     *
+     * @param stored_file $file The file to add to the curl request
+     * @param curl $curlrequest The curl request object
+     * @param string $key What key to use in the POST request
+     * @return void
+     * This needs the fullpath for the storedfile :/
+     * Can this be achieved in some other fashion?
+     */
+    public function add_to_curl_request(stored_file $file, &$curlrequest, $key) {
+        // Note: curl_file_create does not work with remote paths.
+        $path = $this->get_local_path_from_storedfile($file, true);
+        $curlrequest->_tmp_file_post_params[$key] = curl_file_create($path);
+    }
+
+    /**
+     * Returns information about image.
+     * Information is determined from the file content
+     *
+     * @param stored_file $file The file to inspect
+     * @return mixed array with width, height and mimetype; false if not an image
+     */
+    public function get_imageinfo(stored_file $file) {
+        if (!$this->is_image_from_storedfile($file)) {
+            return false;
+        }
+
+        // Whilst get_imageinfo_from_path can use remote paths, it must download the entire file first.
+        // It is more efficient to use a local file when possible.
+        return $this->get_imageinfo_from_path($this->get_local_path_from_storedfile($file, true));
+    }
+
+    /**
+     * Attempt to determine whether the specified file is likely to be an
+     * image.
+     * Since this relies upon the mimetype stored in the files table, there
+     * may be times when this information is not 100% accurate.
+     *
+     * @param stored_file $file The file to check
+     * @return bool
+     */
+    public function is_image_from_storedfile(stored_file $file) {
+        if (!$file->get_filesize()) {
+            // An empty file cannot be an image.
+            return false;
+        }
+
+        $mimetype = $file->get_mimetype();
+        if (!preg_match('|^image/|', $mimetype)) {
+            // The mimetype does not include image.
+            return false;
+        }
+
+        // If it looks like an image, and it smells like an image, perhaps it's an image!
+        return true;
+    }
+
+    /**
+     * Returns image information relating to the specified path or URL.
+     *
+     * @param string $path The path to pass to getimagesize.
+     * @return array Containing width, height, and mimetype.
+     */
+    protected function get_imageinfo_from_path($path) {
+        $imageinfo = getimagesize($path);
+
+        $image = array(
+                'width'     => $imageinfo[0],
+                'height'    => $imageinfo[1],
+                'mimetype'  => image_type_to_mime_type($imageinfo[2]),
+            );
+        if (empty($image['width']) or empty($image['height']) or empty($image['mimetype'])) {
+            // GD can not parse it, sorry.
+            return false;
+        }
+        return $image;
+    }
+
+    /**
+     * Serve file content using X-Sendfile header.
+     * Please make sure that all headers are already sent and the all
+     * access control checks passed.
+     *
+     * @param string $contenthash The content hash of the file to be served
+     * @return bool success
+     */
+    public function xsendfile($contenthash) {
+        global $CFG;
+        require_once($CFG->libdir . "/xsendfilelib.php");
+
+        return xsendfile($this->get_remote_path_from_hash($contenthash));
+    }
+
+    /**
+     * Add the supplied file to the file system.
+     *
+     * Note: If overriding this function, it is advisable to store the file
+     * in the path returned by get_local_path_from_hash as there may be
+     * subsequent uses of the file in the same request.
+     *
+     * @param string $pathname Path to file currently on disk
+     * @param string $contenthash SHA1 hash of content if known (performance only)
+     * @return array (contenthash, filesize, newfile)
+     */
+    abstract public function add_file_from_path($pathname, $contenthash = null);
+
+    /**
+     * Add a file with the supplied content to the file system.
+     *
+     * Note: If overriding this function, it is advisable to store the file
+     * in the path returned by get_local_path_from_hash as there may be
+     * subsequent uses of the file in the same request.
+     *
+     * @param string $content file content - binary string
+     * @return array (contenthash, filesize, newfile)
+     */
+    abstract public function add_file_from_string($content);
+
+    /**
+     * Returns file handle - read only mode, no writing allowed into pool files!
+     *
+     * When you want to modify a file, create a new file and delete the old one.
+     *
+     * @param stored_file $file The file to retrieve a handle for
+     * @param int $type Type of file handle (FILE_HANDLE_xx constant)
+     * @return resource file handle
+     */
+    public function get_content_file_handle(stored_file $file, $type = stored_file::FILE_HANDLE_FOPEN) {
+        $path = $this->get_remote_path_from_storedfile($file);
+
+        return self::get_file_handle_for_path($path, $type);
+    }
+
+    /**
+     * Return a file handle for the specified path.
+     *
+     * This abstraction should be used when overriding get_content_file_handle in a new file system.
+     *
+     * @param string $path The path to the file. This shoudl be any type of path that fopen and gzopen accept.
+     * @param int $type Type of file handle (FILE_HANDLE_xx constant)
+     * @return resource
+     * @throws coding_exception When an unexpected type of file handle is requested
+     */
+    protected static function get_file_handle_for_path($path, $type = stored_file::FILE_HANDLE_FOPEN) {
+        switch ($type) {
+            case stored_file::FILE_HANDLE_FOPEN:
+                // Binary reading.
+                return fopen($path, 'rb');
+            case stored_file::FILE_HANDLE_GZOPEN:
+                // Binary reading of file in gz format.
+                return gzopen($path, 'rb');
+            default:
+                throw new coding_exception('Unexpected file handle type');
+        }
+    }
+
+    /**
+     * Retrieve the mime information for the specified stored file.
+     *
+     * @param string $contenthash
+     * @param string $filename
+     * @return string The MIME type.
+     */
+    public function mimetype_from_hash($contenthash, $filename) {
+        $pathname = $this->get_remote_path_from_hash($contenthash);
+        $mimetype = file_storage::mimetype($pathname, $filename);
+
+        if (!$this->is_file_readable_locally_by_hash($contenthash, false) && $mimetype === 'document/unknown') {
+            // The type is unknown, but the full checks weren't completed because the file isn't locally available.
+            // Ensure we have a local copy and try again.
+            $pathname = $this->get_local_path_from_hash($contenthash, true);
+
+            $mimetype = file_storage::mimetype_from_file($pathname);
+        }
+
+        return $mimetype;
+    }
+
+    /**
+     * Retrieve the mime information for the specified stored file.
+     *
+     * @param stored_file $file The stored file to retrieve mime information for
+     * @return string The MIME type.
+     */
+    public function mimetype_from_storedfile($file) {
+        if (!$file->get_filesize()) {
+            // Files with an empty filesize are treated as directories and have no mimetype.
+            return null;
+        }
+        $pathname = $this->get_remote_path_from_storedfile($file);
+        $mimetype = file_storage::mimetype($pathname, $file->get_filename());
+
+        if (!$this->is_file_readable_locally_by_storedfile($file) && $mimetype === 'document/unknown') {
+            // The type is unknown, but the full checks weren't completed because the file isn't locally available.
+            // Ensure we have a local copy and try again.
+            $pathname = $this->get_local_path_from_storedfile($file, true);
+
+            $mimetype = file_storage::mimetype_from_file($pathname);
+        }
+
+        return $mimetype;
+    }
+
+    /**
+     * Run any periodic tasks which must be performed.
+     */
+    public function cron() {
+    }
+}
diff --git a/lib/filestorage/file_system_filedir.php b/lib/filestorage/file_system_filedir.php
new file mode 100644 (file)
index 0000000..b307a5d
--- /dev/null
@@ -0,0 +1,515 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Core file system class definition.
+ *
+ * @package   core_files
+ * @copyright 2017 Andrew Nicols <andrew@nicols.co.uk>
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * File system class used for low level access to real files in filedir.
+ *
+ * @package   core_files
+ * @category  files
+ * @copyright 2017 Andrew Nicols <andrew@nicols.co.uk>
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class file_system_filedir extends file_system {
+
+    /**
+     * @var string The path to the local copy of the filedir.
+     */
+    protected $filedir = null;
+
+    /**
+     * @var string The path to the trashdir.
+     */
+    protected $trashdir = null;
+
+    /**
+     * @var string Default directory permissions for new dirs.
+     */
+    protected $dirpermissions = null;
+
+    /**
+     * @var string Default file permissions for new files.
+     */
+    protected $filepermissions = null;
+
+
+    /**
+     * Perform any custom setup for this type of file_system.
+     */
+    public function __construct() {
+        global $CFG;
+
+        if (isset($CFG->filedir)) {
+            $this->filedir = $CFG->filedir;
+        } else {
+            $this->filedir = $CFG->dataroot.'/filedir';
+        }
+
+        if (isset($CFG->trashdir)) {
+            $this->trashdir = $CFG->trashdir;
+        } else {
+            $this->trashdir = $CFG->dataroot.'/trashdir';
+        }
+
+        $this->dirpermissions = $CFG->directorypermissions;
+        $this->filepermissions = $CFG->filepermissions;
+
+        // Make sure the file pool directory exists.
+        if (!is_dir($this->filedir)) {
+            if (!mkdir($this->filedir, $this->dirpermissions, true)) {
+                // Permission trouble.
+                throw new file_exception('storedfilecannotcreatefiledirs');
+            }
+
+            // Place warning file in file pool root.
+            if (!file_exists($this->filedir.'/warning.txt')) {
+                file_put_contents($this->filedir.'/warning.txt',
+                        'This directory contains the content of uploaded files and is controlled by Moodle code. ' .
+                        'Do not manually move, change or rename any of the files and subdirectories here.');
+                chmod($this->filedir . '/warning.txt', $this->filepermissions);
+            }
+        }
+
+        // Make sure the trashdir directory exists too.
+        if (!is_dir($this->trashdir)) {
+            if (!mkdir($this->trashdir, $this->dirpermissions, true)) {
+                // Permission trouble.
+                throw new file_exception('storedfilecannotcreatefiledirs');
+            }
+        }
+    }
+
+    /**
+     * Get the full path for the specified hash, including the path to the filedir.
+     *
+     * @param string $contenthash The content hash
+     * @param bool $fetchifnotfound Whether to attempt to fetch from the remote path if not found.
+     * @return string The full path to the content file
+     */
+    protected function get_local_path_from_hash($contenthash, $fetchifnotfound = false) {
+        return $this->get_fulldir_from_hash($contenthash) . DIRECTORY_SEPARATOR . $contenthash;
+    }
+
+    /**
+     * Get a remote filepath for the specified stored file.
+     *
+     * @param stored_file $file The file to fetch the path for
+     * @param bool $fetchifnotfound Whether to attempt to fetch from the remote path if not found.
+     * @return string The full path to the content file
+     */
+    protected function get_local_path_from_storedfile(stored_file $file, $fetchifnotfound = false) {
+        $filepath = $this->get_local_path_from_hash($file->get_contenthash(), $fetchifnotfound);
+
+        // Try content recovery.
+        if ($fetchifnotfound && !is_readable($filepath)) {
+            $this->recover_file($file);
+        }
+
+        return $filepath;
+    }
+
+    /**
+     * Get a remote filepath for the specified stored file.
+     *
+     * @param stored_file $file The file to serve.
+     * @return string full path to pool file with file content
+     */
+    protected function get_remote_path_from_storedfile(stored_file $file) {
+        return $this->get_local_path_from_storedfile($file, false);
+    }
+
+    /**
+     * Get the full path for the specified hash, including the path to the filedir.
+     *
+     * @param string $contenthash The content hash
+     * @return string The full path to the content file
+     */
+    protected function get_remote_path_from_hash($contenthash) {
+        return $this->get_local_path_from_hash($contenthash, false);
+    }
+
+    /**
+     * Get the full directory to the stored file, including the path to the
+     * filedir, and the directory which the file is actually in.
+     *
+     * Note: This function does not ensure that the file is present on disk.
+     *
+     * @param stored_file $file The file to fetch details for.
+     * @return string The full path to the content directory
+     */
+    protected function get_fulldir_from_storedfile(stored_file $file) {
+        return $this->get_fulldir_from_hash($file->get_contenthash());
+    }
+
+    /**
+     * Get the full directory to the stored file, including the path to the
+     * filedir, and the directory which the file is actually in.
+     *
+     * @param string $contenthash The content hash
+     * @return string The full path to the content directory
+     */
+    protected function get_fulldir_from_hash($contenthash) {
+        return $this->filedir . DIRECTORY_SEPARATOR . $this->get_contentdir_from_hash($contenthash);
+    }
+
+    /**
+     * Get the content directory for the specified content hash.
+     * This is the directory that the file will be in, but without the
+     * fulldir.
+     *
+     * @param string $contenthash The content hash
+     * @return string The directory within filedir
+     */
+    protected function get_contentdir_from_hash($contenthash) {
+        $l1 = $contenthash[0] . $contenthash[1];
+        $l2 = $contenthash[2] . $contenthash[3];
+        return "$l1/$l2";
+    }
+
+    /**
+     * Get the content path for the specified content hash within filedir.
+     *
+     * This does not include the filedir, and is often used by file systems
+     * as the object key for storage and retrieval.
+     *
+     * @param string $contenthash The content hash
+     * @return string The filepath within filedir
+     */
+    protected function get_contentpath_from_hash($contenthash) {
+        return $this->get_contentdir_from_hash($contenthash) . "/$contenthash";
+    }
+
+    /**
+     * Get the full directory for the specified hash in the trash, including the path to the
+     * trashdir, and the directory which the file is actually in.
+     *
+     * @param string $contenthash The content hash
+     * @return string The full path to the trash directory
+     */
+    protected function get_trash_fulldir_from_hash($contenthash) {
+        return $this->trashdir . DIRECTORY_SEPARATOR . $this->get_contentdir_from_hash($contenthash);
+    }
+
+    /**
+     * Get the full path for the specified hash in the trash, including the path to the trashdir.
+     *
+     * @param string $contenthash The content hash
+     * @return string The full path to the trash file
+     */
+    protected function get_trash_fullpath_from_hash($contenthash) {
+        return $this->trashdir . DIRECTORY_SEPARATOR . $this->get_contentpath_from_hash($contenthash);
+    }
+
+    /**
+     * Copy content of file to given pathname.
+     *
+     * @param stored_file $file The file to be copied
+     * @param string $target real path to the new file
+     * @return bool success
+     */
+    public function copy_content_from_storedfile(stored_file $file, $target) {
+        $source = $this->get_local_path_from_storedfile($file, true);
+        return copy($source, $target);
+    }
+
+    /**
+     * Tries to recover missing content of file from trash.
+     *
+     * @param stored_file $file stored_file instance
+     * @return bool success
+     */
+    protected function recover_file(stored_file $file) {
+        $contentfile = $this->get_local_path_from_storedfile($file, false);
+
+        if (file_exists($contentfile)) {
+            // The file already exists on the file system. No need to recover.
+            return true;
+        }
+
+        $contenthash = $file->get_contenthash();
+        $contentdir = $this->get_fulldir_from_storedfile($file);
+        $trashfile = $this->get_trash_fullpath_from_hash($contenthash);
+        $alttrashfile = $this->trashdir . DIRECTORY_SEPARATOR . $contenthash;
+
+        if (!is_readable($trashfile)) {
+            // The trash file was not found. Check the alternative trash file too just in case.
+            if (!is_readable($alttrashfile)) {
+                return false;
+            }
+            // The alternative trash file in trash root exists.
+            $trashfile = $alttrashfile;
+        }
+
+        if (filesize($trashfile) != $file->get_filesize() or sha1_file($trashfile) != $contenthash) {
+            // The files are different. Leave this one in trash - something seems to be wrong with it.
+            return false;
+        }
+
+        if (!is_dir($contentdir)) {
+            if (!mkdir($contentdir, $this->dirpermissions, true)) {
+                // Unable to create the target directory.
+                return false;
+            }
+        }
+
+        // Perform a rename - these are generally atomic which gives us big
+        // performance wins, especially for large files.
+        return rename($trashfile, $contentfile);
+    }
+
+    /**
+     * Marks pool file as candidate for deleting.
+     *
+     * @param string $contenthash
+     */
+    public function remove_file($contenthash) {
+        if (!self::is_file_removable($contenthash)) {
+            // Don't remove the file - it's still in use.
+            return;
+        }
+
+        if (!$this->is_file_readable_remotely_by_hash($contenthash)) {
+            // The file wasn't found in the first place. Just ignore it.
+            return;
+        }
+
+        $trashpath  = $this->get_trash_fulldir_from_hash($contenthash);
+        $trashfile  = $this->get_trash_fullpath_from_hash($contenthash);
+        $contentfile = $this->get_local_path_from_hash($contenthash, true);
+
+        if (!is_dir($trashpath)) {
+            mkdir($trashpath, $this->dirpermissions, true);
+        }
+
+        if (file_exists($trashfile)) {
+            // A copy of this file is already in the trash.
+            // Remove the old version.
+            unlink($contentfile);
+            return;
+        }
+
+        // Move the contentfile to the trash, and fix permissions as required.
+        rename($contentfile, $trashfile);
+
+        // Fix permissions, only if needed.
+        $currentperms = octdec(substr(decoct(fileperms($trashfile)), -4));
+        if ((int)$this->filepermissions !== $currentperms) {
+            chmod($trashfile, $this->filepermissions);
+        }
+    }
+
+    /**
+     * Cleanup the trash directory.
+     */
+    public function cron() {
+        $this->empty_trash();
+    }
+
+    protected function empty_trash() {
+        fulldelete($this->trashdir);
+        set_config('fileslastcleanup', time());
+    }
+
+    /**
+     * Add the supplied file to the file system.
+     *
+     * Note: If overriding this function, it is advisable to store the file
+     * in the path returned by get_local_path_from_hash as there may be
+     * subsequent uses of the file in the same request.
+     *
+     * @param string $pathname Path to file currently on disk
+     * @param string $contenthash SHA1 hash of content if known (performance only)
+     * @return array (contenthash, filesize, newfile)
+     */
+    public function add_file_from_path($pathname, $contenthash = null) {
+        global $CFG;
+
+        if (!is_readable($pathname)) {
+            throw new file_exception('storedfilecannotread', '', $pathname);
+        }
+
+        $filesize = filesize($pathname);
+        if ($filesize === false) {
+            throw new file_exception('storedfilecannotread', '', $pathname);
+        }
+
+        if (is_null($contenthash)) {
+            $contenthash = sha1_file($pathname);
+        } else if ($CFG->debugdeveloper) {
+            $filehash = sha1_file($pathname);
+            if ($filehash === false) {
+                throw new file_exception('storedfilecannotread', '', $pathname);
+            }
+            if ($filehash !== $contenthash) {
+                // Hopefully this never happens, if yes we need to fix calling code.
+                debugging("Invalid contenthash submitted for file $pathname", DEBUG_DEVELOPER);
+                $contenthash = $filehash;
+            }
+        }
+        if ($contenthash === false) {
+            throw new file_exception('storedfilecannotread', '', $pathname);
+        }
+
+        if ($filesize > 0 and $contenthash === sha1('')) {
+            // Did the file change or is sha1_file() borked for this file?
+            clearstatcache();
+            $contenthash = sha1_file($pathname);
+            $filesize = filesize($pathname);
+
+            if ($contenthash === false or $filesize === false) {
+                throw new file_exception('storedfilecannotread', '', $pathname);
+            }
+            if ($filesize > 0 and $contenthash === sha1('')) {
+                // This is very weird...
+                throw new file_exception('storedfilecannotread', '', $pathname);
+            }
+        }
+
+        $hashpath = $this->get_fulldir_from_hash($contenthash);
+        $hashfile = $this->get_local_path_from_hash($contenthash, false);
+
+        $newfile = true;
+
+        if (file_exists($hashfile)) {
+            if (filesize($hashfile) === $filesize) {
+                return array($contenthash, $filesize, false);
+            }
+            if (sha1_file($hashfile) === $contenthash) {
+                // Jackpot! We have a sha1 collision.
+                mkdir("$this->filedir/jackpot/", $this->dirpermissions, true);
+                copy($pathname, "$this->filedir/jackpot/{$contenthash}_1");
+                copy($hashfile, "$this->filedir/jackpot/{$contenthash}_2");
+                throw new file_pool_content_exception($contenthash);
+            }
+            debugging("Replacing invalid content file $contenthash");
+            unlink($hashfile);
+            $newfile = false;
+        }
+
+        if (!is_dir($hashpath)) {
+            if (!mkdir($hashpath, $this->dirpermissions, true)) {
+                // Permission trouble.
+                throw new file_exception('storedfilecannotcreatefiledirs');
+            }
+        }
+
+        // Let's try to prevent some race conditions.
+
+        $prev = ignore_user_abort(true);
+        @unlink($hashfile.'.tmp');
+        if (!copy($pathname, $hashfile.'.tmp')) {
+            // Borked permissions or out of disk space.
+            ignore_user_abort($prev);
+            throw new file_exception('storedfilecannotcreatefile');
+        }
+        if (filesize($hashfile.'.tmp') !== $filesize) {
+            // This should not happen.
+            unlink($hashfile.'.tmp');
+            ignore_user_abort($prev);
+            throw new file_exception('storedfilecannotcreatefile');
+        }
+        rename($hashfile.'.tmp', $hashfile);
+        chmod($hashfile, $this->filepermissions); // Fix permissions if needed.
+        @unlink($hashfile.'.tmp'); // Just in case anything fails in a weird way.
+        ignore_user_abort($prev);
+
+        return array($contenthash, $filesize, $newfile);
+    }
+
+    /**
+     * Add a file with the supplied content to the file system.
+     *
+     * Note: If overriding this function, it is advisable to store the file
+     * in the path returned by get_local_path_from_hash as there may be
+     * subsequent uses of the file in the same request.
+     *
+     * @param string $content file content - binary string
+     * @return array (contenthash, filesize, newfile)
+     */
+    public function add_file_from_string($content) {
+        global $CFG;
+
+        $contenthash = sha1($content);
+        // Binary length.
+        $filesize = strlen($content);
+
+        $hashpath = $this->get_fulldir_from_hash($contenthash);
+        $hashfile = $this->get_local_path_from_hash($contenthash, false);
+
+        $newfile = true;
+
+        if (file_exists($hashfile)) {
+            if (filesize($hashfile) === $filesize) {
+                return array($contenthash, $filesize, false);
+            }
+            if (sha1_file($hashfile) === $contenthash) {
+                // Jackpot! We have a sha1 collision.
+                mkdir("$this->filedir/jackpot/", $this->dirpermissions, true);
+                copy($hashfile, "$this->filedir/jackpot/{$contenthash}_1");
+                file_put_contents("$this->filedir/jackpot/{$contenthash}_2", $content);
+                throw new file_pool_content_exception($contenthash);
+            }
+            debugging("Replacing invalid content file $contenthash");
+            unlink($hashfile);
+            $newfile = false;
+        }
+
+        if (!is_dir($hashpath)) {
+            if (!mkdir($hashpath, $this->dirpermissions, true)) {
+                // Permission trouble.
+                throw new file_exception('storedfilecannotcreatefiledirs');
+            }
+        }
+
+        // Hopefully this works around most potential race conditions.
+
+        $prev = ignore_user_abort(true);
+
+        if (!empty($CFG->preventfilelocking)) {
+            $newsize = file_put_contents($hashfile.'.tmp', $content);
+        } else {
+            $newsize = file_put_contents($hashfile.'.tmp', $content, LOCK_EX);
+        }
+
+        if ($newsize === false) {
+            // Borked permissions most likely.
+            ignore_user_abort($prev);
+            throw new file_exception('storedfilecannotcreatefile');
+        }
+        if (filesize($hashfile.'.tmp') !== $filesize) {
+            // Out of disk space?
+            unlink($hashfile.'.tmp');
+            ignore_user_abort($prev);
+            throw new file_exception('storedfilecannotcreatefile');
+        }
+        rename($hashfile.'.tmp', $hashfile);
+        chmod($hashfile, $this->filepermissions); // Fix permissions if needed.
+        @unlink($hashfile.'.tmp'); // Just in case anything fails in a weird way.
+        ignore_user_abort($prev);
+
+        return array($contenthash, $filesize, $newfile);
+    }
+
+}
index 7d3b64f..192e7b8 100644 (file)
@@ -26,6 +26,7 @@
 defined('MOODLE_INTERNAL') || die();
 
 require_once($CFG->dirroot . '/lib/filestorage/file_progress.php');
+require_once($CFG->dirroot . '/lib/filestorage/file_system.php');
 
 /**
  * Class representing local files stored in a sha1 file pool.
@@ -44,10 +45,10 @@ class stored_file {
     private $fs;
     /** @var stdClass record from the files table left join files_reference table */
     private $file_record;
-    /** @var string location of content files */
-    private $filedir;
     /** @var repository repository plugin instance */
     private $repository;
+    /** @var file_system filesystem instance */
+    private $filesystem;
 
     /**
      * @var int Indicates a file handle of the type returned by fopen.
@@ -65,13 +66,12 @@ class stored_file {
      *
      * @param file_storage $fs file  storage instance
      * @param stdClass $file_record description of file
-     * @param string $filedir location of file directory with sh1 named content files
+     * @param string $deprecated
      */
-    public function __construct(file_storage $fs, stdClass $file_record, $filedir) {
+    public function __construct(file_storage $fs, stdClass $file_record, $deprecated = null) {
         global $DB, $CFG;
         $this->fs          = $fs;
         $this->file_record = clone($file_record); // prevent modifications
-        $this->filedir     = $filedir; // keep secret, do not expose!
 
         if (!empty($file_record->repositoryid)) {
             require_once("$CFG->dirroot/repository/lib.php");
@@ -89,6 +89,8 @@ class stored_file {
                 $this->file_record->$key = null;
             }
         }
+
+        $this->filesystem = $fs->get_file_system();
     }
 
     /**
@@ -179,15 +181,7 @@ class stored_file {
             }
         }
         // Validate mimetype field
-        // we don't use {@link stored_file::get_content_file_location()} here becaues it will try to update file_record
-        $pathname = $this->get_pathname_by_contenthash();
-        // try to recover the content from trash
-        if (!is_readable($pathname)) {
-            if (!$this->fs->try_content_recovery($this) or !is_readable($pathname)) {
-                throw new file_exception('storedfilecannotread', '', $pathname);
-            }
-        }
-        $mimetype = $this->fs->mimetype($pathname, $this->file_record->filename);
+        $mimetype = $this->filesystem->mimetype_from_storedfile($this);
         $this->file_record->mimetype = $mimetype;
 
         $DB->update_record('files', $this->file_record);
@@ -255,8 +249,8 @@ class stored_file {
         }
 
         $filerecord = new stdClass;
-        $contenthash = $newfile->get_contenthash();
-        if ($this->fs->content_exists($contenthash)) {
+        if ($this->filesystem->is_file_readable_remotely_by_storedfile($newfile)) {
+            $contenthash = $newfile->get_contenthash();
             $filerecord->contenthash = $contenthash;
         } else {
             throw new file_exception('storedfileproblem', 'Invalid contenthash, content must be already in filepool', $contenthash);
@@ -357,40 +351,10 @@ class stored_file {
         }
 
         // Move pool file to trash if content not needed any more.
-        $this->fs->deleted_file_cleanup($this->file_record->contenthash);
+        $this->filesystem->remove_file($this->file_record->contenthash);
         return true; // BC only
     }
 
-    /**
-     * Get file pathname by contenthash
-     *
-     * NOTE, this function is not calling sync_external_file, it assume the contenthash is current
-     * Protected - developers must not gain direct access to this function.
-     *
-     * @return string full path to pool file with file content
-     */
-    protected function get_pathname_by_contenthash() {
-        // Detect is local file or not.
-        $contenthash = $this->file_record->contenthash;
-        $l1 = $contenthash[0].$contenthash[1];
-        $l2 = $contenthash[2].$contenthash[3];
-        return "$this->filedir/$l1/$l2/$contenthash";
-    }
-
-    /**
-     * Get file pathname by given contenthash, this method will try to sync files
-     *
-     * Protected - developers must not gain direct access to this function.
-     *
-     * NOTE: do not make this public, we must not modify or delete the pool files directly! ;-)
-     *
-     * @return string full path to pool file with file content
-     **/
-    protected function get_content_file_location() {
-        $this->sync_external_file();
-        return $this->get_pathname_by_contenthash();
-    }
-
     /**
     * adds this file path to a curl request (POST only)
     *
@@ -399,13 +363,7 @@ class stored_file {
     * @return void
     */
     public function add_to_curl_request(&$curlrequest, $key) {
-        if (function_exists('curl_file_create')) {
-            // As of PHP 5.5, the usage of the @filename API for file uploading is deprecated.
-            $value = curl_file_create($this->get_content_file_location());
-        } else {
-            $value = '@' . $this->get_content_file_location();
-        }
-        $curlrequest->_tmp_file_post_params[$key] = $value;
+        return $this->filesystem->add_to_curl_request($this, $curlrequest, $key);
     }
 
     /**
@@ -417,35 +375,14 @@ class stored_file {
      * @return resource file handle
      */
     public function get_content_file_handle($type = self::FILE_HANDLE_FOPEN) {
-        $path = $this->get_content_file_location();
-        if (!is_readable($path)) {
-            if (!$this->fs->try_content_recovery($this) or !is_readable($path)) {
-                throw new file_exception('storedfilecannotread', '', $path);
-            }
-        }
-        switch ($type) {
-            case self::FILE_HANDLE_FOPEN:
-                // Binary reading.
-                return fopen($path, 'rb');
-            case self::FILE_HANDLE_GZOPEN:
-                // Binary reading of file in gz format.
-                return gzopen($path, 'rb');
-            default:
-                throw new coding_exception('Unexpected file handle type');
-        }
+        return $this->filesystem->get_content_file_handle($this, $type);
     }
 
     /**
      * Dumps file content to page.
      */
     public function readfile() {
-        $path = $this->get_content_file_location();
-        if (!is_readable($path)) {
-            if (!$this->fs->try_content_recovery($this) or !is_readable($path)) {
-                throw new file_exception('storedfilecannotread', '', $path);
-            }
-        }
-        readfile_allow_large($path, $this->get_filesize());
+        return $this->filesystem->readfile($this);
     }
 
     /**
@@ -454,13 +391,7 @@ class stored_file {
      * @return string content
      */
     public function get_content() {
-        $path = $this->get_content_file_location();
-        if (!is_readable($path)) {
-            if (!$this->fs->try_content_recovery($this) or !is_readable($path)) {
-                throw new file_exception('storedfilecannotread', '', $path);
-            }
-        }
-        return file_get_contents($this->get_content_file_location());
+        return $this->filesystem->get_content($this);
     }
 
     /**
@@ -470,13 +401,7 @@ class stored_file {
      * @return bool success
      */
     public function copy_content_to($pathname) {
-        $path = $this->get_content_file_location();
-        if (!is_readable($path)) {
-            if (!$this->fs->try_content_recovery($this) or !is_readable($path)) {
-                throw new file_exception('storedfilecannotread', '', $path);
-            }
-        }
-        return copy($path, $pathname);
+        return $this->filesystem->copy_content_from_storedfile($this, $pathname);
     }
 
     /**
@@ -509,8 +434,7 @@ class stored_file {
      * @return array of file infos
      */
     public function list_files(file_packer $packer) {
-        $archivefile = $this->get_content_file_location();
-        return $packer->list_files($archivefile);
+        return $this->filesystem->list_files($this, $packer);
     }
 
     /**
@@ -523,8 +447,7 @@ class stored_file {
      */
     public function extract_to_pathname(file_packer $packer, $pathname,
             file_progress $progress = null) {
-        $archivefile = $this->get_content_file_location();
-        return $packer->extract_to_pathname($archivefile, $pathname, null, $progress);
+        return $this->filesystem->extract_to_pathname($this, $packer, $pathname, $progress);
     }
 
     /**
@@ -542,9 +465,9 @@ class stored_file {
      */
     public function extract_to_storage(file_packer $packer, $contextid,
             $component, $filearea, $itemid, $pathbase, $userid = null, file_progress $progress = null) {
-        $archivefile = $this->get_content_file_location();
-        return $packer->extract_to_storage($archivefile, $contextid,
-                $component, $filearea, $itemid, $pathbase, $userid, $progress);
+
+        return $this->filesystem->extract_to_storage($this, $packer, $contextid, $component, $filearea,
+                $itemid, $pathbase, $userid, $progress);
     }
 
     /**
@@ -555,15 +478,7 @@ class stored_file {
      * @return bool success
      */
     public function archive_file(file_archive $filearch, $archivepath) {
-        if ($this->is_directory()) {
-            return $filearch->add_directory($archivepath);
-        } else {
-            $path = $this->get_content_file_location();
-            if (!is_readable($path)) {
-                return false;
-            }
-            return $filearch->add_file_from_pathname($archivepath, $path);
-        }
+        return $this->filesystem->add_storedfile_to_archive($this, $filearch, $archivepath);
     }
 
     /**
@@ -573,22 +488,7 @@ class stored_file {
      * @return mixed array with width, height and mimetype; false if not an image
      */
     public function get_imageinfo() {
-        $path = $this->get_content_file_location();
-        if (!is_readable($path)) {
-            if (!$this->fs->try_content_recovery($this) or !is_readable($path)) {
-                throw new file_exception('storedfilecannotread', '', $path);
-            }
-        }
-        $mimetype = $this->get_mimetype();
-        if (!preg_match('|^image/|', $mimetype) || !filesize($path) || !($imageinfo = getimagesize($path))) {
-            return false;
-        }
-        $image = array('width'=>$imageinfo[0], 'height'=>$imageinfo[1], 'mimetype'=>image_type_to_mime_type($imageinfo[2]));
-        if (empty($image['width']) or empty($image['height']) or empty($image['mimetype'])) {
-            // gd can not parse it, sorry
-            return false;
-        }
-        return $image;
+        return $this->filesystem->get_imageinfo($this);
     }
 
     /**
@@ -985,7 +885,7 @@ class stored_file {
             $this->file_record->timemodified = $timemodified;
         }
         if (isset($oldcontenthash)) {
-            $this->fs->deleted_file_cleanup($oldcontenthash);
+            $this->filesystem->remove_file($oldcontenthash);
         }
     }
 
@@ -1051,14 +951,16 @@ class stored_file {
             return false;
         }
 
+        $content = $this->get_content();
+
         // Fetch the image information for this image.
-        $imageinfo = @getimagesizefromstring($this->get_content());
+        $imageinfo = @getimagesizefromstring($content);
         if (empty($imageinfo)) {
             return false;
         }
 
         // Create a new image from the file.
-        $original = @imagecreatefromstring($this->get_content());
+        $original = @imagecreatefromstring($content);
 
         // Generate the thumbnail.
         return generate_image_thumbnail_from_image($original, $imageinfo, $width, $height);
@@ -1075,14 +977,16 @@ class stored_file {
         global $CFG;
         require_once($CFG->libdir . '/gdlib.php');
 
+        $content = $this->get_content();
+
         // Fetch the image information for this image.
-        $imageinfo = @getimagesizefromstring($this->get_content());
+        $imageinfo = @getimagesizefromstring($content);
         if (empty($imageinfo)) {
             return false;
         }
 
         // Create a new image from the file.
-        $original = @imagecreatefromstring($this->get_content());
+        $original = @imagecreatefromstring($content);
 
         // Generate the resized image.
         return resize_image_from_image($original, $imageinfo, $width, $height);
index f16ada7..46e97fc 100644 (file)
@@ -64,7 +64,10 @@ class core_files_file_storage_testcase extends advanced_testcase {
 
         $this->assertTrue($DB->record_exists('files', array('pathnamehash'=>$pathhash)));
 
-        $location = test_stored_file_inspection::get_pretected_pathname($file);
+        $method = new ReflectionMethod('file_system', 'get_local_path_from_storedfile');
+        $method->setAccessible(true);
+        $filesystem = $fs->get_file_system();
+        $location = $method->invokeArgs($filesystem, array($file, true));
 
         $this->assertFileExists($location);
 
@@ -133,7 +136,10 @@ class core_files_file_storage_testcase extends advanced_testcase {
 
         $this->assertTrue($DB->record_exists('files', array('pathnamehash'=>$pathhash)));
 
-        $location = test_stored_file_inspection::get_pretected_pathname($file);
+        $method = new ReflectionMethod('file_system', 'get_local_path_from_storedfile');
+        $method->setAccessible(true);
+        $filesystem = $fs->get_file_system();
+        $location = $method->invokeArgs($filesystem, array($file, true));
 
         $this->assertFileExists($location);
 
@@ -1842,6 +1848,53 @@ class core_files_file_storage_testcase extends advanced_testcase {
         $this->expectException('coding_exception');
         $fs->get_unused_filename($contextid, $component, $filearea, $itemid, $filepath, '');
     }
+
+    /**
+     * Test that mimetype_from_file returns appropriate output when the
+     * file could not be found.
+     */
+    public function test_mimetype_not_found() {
+        $mimetype = file_storage::mimetype('/path/to/nonexistent/file');
+        $this->assertEquals('document/unknown', $mimetype);
+    }
+
+    /**
+     * Test that mimetype_from_file returns appropriate output for a known
+     * file.
+     *
+     * Note: this is not intended to check that functions outside of this
+     * file works. It is intended to validate the codepath contains no
+     * errors and behaves as expected.
+     */
+    public function test_mimetype_known() {
+        $filepath = __DIR__ . DIRECTORY_SEPARATOR . 'fixtures' . DIRECTORY_SEPARATOR . 'testimage.jpg';
+        $mimetype = file_storage::mimetype_from_file($filepath);
+        $this->assertEquals('image/jpeg', $mimetype);
+    }
+
+    /**
+     * Test that mimetype_from_file returns appropriate output when the
+     * file could not be found.
+     */
+    public function test_mimetype_from_file_not_found() {
+        $mimetype = file_storage::mimetype_from_file('/path/to/nonexistent/file');
+        $this->assertEquals('document/unknown', $mimetype);
+    }
+
+    /**
+     * Test that mimetype_from_file returns appropriate output for a known
+     * file.
+     *
+     * Note: this is not intended to check that functions outside of this
+     * file works. It is intended to validate the codepath contains no
+     * errors and behaves as expected.
+     */
+    public function test_mimetype_from_file_known() {
+        $filepath = __DIR__ . DIRECTORY_SEPARATOR . 'fixtures' . DIRECTORY_SEPARATOR . 'testimage.jpg';
+        $mimetype = file_storage::mimetype_from_file($filepath);
+        $this->assertEquals('image/jpeg', $mimetype);
+    }
+
 }
 
 class test_stored_file_inspection extends stored_file {
diff --git a/lib/filestorage/tests/file_system_filedir_test.php b/lib/filestorage/tests/file_system_filedir_test.php
new file mode 100644 (file)
index 0000000..1f65099
--- /dev/null
@@ -0,0 +1,1063 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Unit tests for file_system_filedir.
+ *
+ * @package   core_files
+ * @category  phpunit
+ * @copyright 2017 Andrew Nicols <andrew@nicols.co.uk>
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+global $CFG;
+require_once($CFG->libdir . '/filestorage/file_system.php');
+require_once($CFG->libdir . '/filestorage/file_system_filedir.php');
+
+/**
+ * Unit tests for file_system_filedir.
+ *
+ * @package   core_files
+ * @category  files
+ * @copyright 2017 Andrew Nicols <andrew@nicols.co.uk>
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class core_files_file_system_filedir_testcase extends advanced_testcase {
+
+    /**
+     * Shared test setUp.
+     */
+    public function setUp() {
+        // Reset the file storage so that subsequent fetches to get_file_storage are called after
+        // configuration is prepared.
+        get_file_storage(true);
+    }
+
+    /**
+     * Shared teset tearDown.
+     */
+    public function tearDown() {
+        // Reset the file storage so that subsequent tests will use the standard file storage.
+        get_file_storage(true);
+    }
+
+    /**
+     * Helper function to help setup and configure the virtual file system stream.
+     *
+     * @param   array $filedir Directory structure and content of the filedir
+     * @param   array $trashdir Directory structure and content of the sourcedir
+     * @param   array $sourcedir Directory structure and content of a directory used for source files for tests
+     * @return  \org\bovigo\vfs\vfsStream
+     */
+    protected function setup_vfile_root($filedir = [], $trashdir = [], $sourcedir = null) {
+        global $CFG;
+        $this->resetAfterTest();
+
+        $content = [];
+        if ($filedir !== null) {
+            $content['filedir'] = $filedir;
+        }
+
+        if ($trashdir !== null) {
+            $content['trashdir'] = $trashdir;
+        }
+
+        if ($sourcedir !== null) {
+            $content['sourcedir'] = $sourcedir;
+        }
+
+        $vfileroot = \org\bovigo\vfs\vfsStream::setup('root', null, $content);
+
+        $CFG->filedir = \org\bovigo\vfs\vfsStream::url('root/filedir');
+        $CFG->trashdir = \org\bovigo\vfs\vfsStream::url('root/trashdir');
+
+        return $vfileroot;
+    }
+
+    /**
+     * Helper to create a stored file objectw with the given supplied content.
+     *
+     * @param   string  $filecontent The content of the mocked file
+     * @param   string  $filename The file name to use in the stored_file
+     * @param   array   $mockedmethods A list of methods you intend to override
+     *                  If no methods are specified, only abstract functions are mocked.
+     * @return stored_file
+     */
+    protected function get_stored_file($filecontent, $filename = null, $mockedmethods = null) {
+        $contenthash = sha1($filecontent);
+        if (empty($filename)) {
+            $filename = $contenthash;
+        }
+
+        $file = $this->getMockBuilder(stored_file::class)
+            ->setMethods($mockedmethods)
+            ->setConstructorArgs([
+                get_file_storage(),
+                (object) [
+                    'contenthash' => $contenthash,
+                    'filesize' => strlen($filecontent),
+                    'filename' => $filename,
+                ]
+            ])
+            ->getMock();
+
+        return $file;
+    }
+
+    /**
+     * Get a testable mock of the file_system_filedir class.
+     *
+     * @param   array   $mockedmethods A list of methods you intend to override
+     *                  If no methods are specified, only abstract functions are mocked.
+     * @return file_system
+     */
+    protected function get_testable_mock($mockedmethods = []) {
+        $fs = $this->getMockBuilder(file_system_filedir::class)
+            ->setMethods($mockedmethods)
+            ->getMock();
+
+        return $fs;
+    }
+
+    /**
+     * Ensure that an appropriate error is shown when the filedir directory
+     * is not writable.
+     */
+    public function test_readonly_filesystem_filedir() {
+        $this->resetAfterTest();
+
+        // Setup the filedir but remove permissions.
+        $vfileroot = $this->setup_vfile_root(null);
+
+        // Make the target path readonly.
+        $vfileroot->chmod(0444)
+            ->chown(\org\bovigo\vfs\vfsStream::OWNER_USER_2);
+
+        // This should generate an exception.
+        $this->setExpectedExceptionRegexp('file_exception',
+            '/Can not create local file pool directories, please verify permissions in dataroot./');
+
+        new file_system_filedir();
+    }
+
+    /**
+     * Ensure that an appropriate error is shown when the trash directory
+     * is not writable.
+     */
+    public function test_readonly_filesystem_trashdir() {
+        $this->resetAfterTest();
+
+        // Setup the trashdir but remove permissions.
+        $vfileroot = $this->setup_vfile_root([], null);
+
+        // Make the target path readonly.
+        $vfileroot->chmod(0444)
+            ->chown(\org\bovigo\vfs\vfsStream::OWNER_USER_2);
+
+        // This should generate an exception.
+        $this->setExpectedExceptionRegexp('file_exception',
+            '/Can not create local file pool directories, please verify permissions in dataroot./');
+
+        new file_system_filedir();
+    }
+
+    /**
+     * Test that the standard Moodle warning message is put into the filedir.
+     */
+    public function test_warnings_put_in_place() {
+        $this->resetAfterTest();
+
+        $vfileroot = $this->setup_vfile_root(null);
+
+        new file_system_filedir();
+        $this->assertTrue($vfileroot->hasChild('filedir/warning.txt'));
+        $this->assertEquals(
+            'This directory contains the content of uploaded files and is controlled by Moodle code. ' .
+                'Do not manually move, change or rename any of the files and subdirectories here.',
+            $vfileroot->getChild('filedir/warning.txt')->getContent()
+        );
+    }
+
+    /**
+     * Ensure that the default implementation of get_remote_path_from_hash
+     * simply calls get_local_path_from_hash.
+     */
+    public function test_get_remote_path_from_hash() {
+        $filecontent = 'example content';
+        $contenthash = sha1($filecontent);
+        $expectedresult = (object) [];
+
+        $fs = $this->get_testable_mock([
+            'get_local_path_from_hash',
+        ]);
+
+        $fs->expects($this->once())
+            ->method('get_local_path_from_hash')
+            ->with($this->equalTo($contenthash), $this->equalTo(false))
+            ->willReturn($expectedresult);
+
+        $method = new ReflectionMethod(file_system_filedir::class, 'get_remote_path_from_hash');
+        $method->setAccessible(true);
+        $result = $method->invokeArgs($fs, [$contenthash]);
+
+        $this->assertEquals($expectedresult, $result);
+    }
+
+    /**
+     * Test the stock implementation of get_local_path_from_storedfile_with_recovery with no file found and
+     * a failed recovery.
+     */
+    public function test_get_local_path_from_storedfile_with_recovery() {
+        $filecontent = 'example content';
+        $file = $this->get_stored_file($filecontent);
+        $fs = $this->get_testable_mock([
+            'get_local_path_from_hash',
+            'recover_file',
+        ]);
+        $filepath = '/path/to/nonexistent/file';
+
+        $fs->method('get_local_path_from_hash')
+            ->willReturn($filepath);
+
+        $fs->expects($this->once())
+            ->method('recover_file')
+            ->with($this->equalTo($file));
+
+        $file = $this->get_stored_file('example content');
+        $method = new ReflectionMethod(file_system_filedir::class, 'get_local_path_from_storedfile');
+        $method->setAccessible(true);
+        $result = $method->invokeArgs($fs, array($file, true));
+
+        $this->assertEquals($filepath, $result);
+    }
+
+    /**
+     * Test the stock implementation of get_local_path_from_storedfile_with_recovery with no file found and
+     * a failed recovery.
+     */
+    public function test_get_local_path_from_storedfile_without_recovery() {
+        $filecontent = 'example content';
+        $file = $this->get_stored_file($filecontent);
+        $fs = $this->get_testable_mock([
+            'get_local_path_from_hash',
+            'recover_file',
+        ]);
+        $filepath = '/path/to/nonexistent/file';
+
+        $fs->method('get_local_path_from_hash')
+            ->willReturn($filepath);
+
+        $fs->expects($this->never())
+            ->method('recover_file');
+
+        $file = $this->get_stored_file('example content');
+        $method = new ReflectionMethod(file_system_filedir::class, 'get_local_path_from_storedfile');
+        $method->setAccessible(true);
+        $result = $method->invokeArgs($fs, array($file, false));
+
+        $this->assertEquals($filepath, $result);
+    }
+
+    /**
+     * Test that the correct path is generated for the supplied content
+     * hashes.
+     *
+     * @dataProvider contenthash_dataprovider
+     * @param   string  $hash contenthash to test
+     * @param   string  $hashdir Expected format of content directory
+     */
+    public function test_get_fulldir_from_hash($hash, $hashdir) {
+        global $CFG;
+
+        $fs = new file_system_filedir();
+        $method = new ReflectionMethod(file_system_filedir::class, 'get_fulldir_from_hash');
+        $method->setAccessible(true);
+        $result = $method->invokeArgs($fs, array($hash));
+
+        $expectedpath = sprintf('%s/filedir/%s', $CFG->dataroot, $hashdir);
+        $this->assertEquals($expectedpath, $result);
+    }
+
+    /**
+     * Test that the correct path is generated for the supplied content
+     * hashes when used with a stored_file.
+     *
+     * @dataProvider contenthash_dataprovider
+     * @param   string  $hash contenthash to test
+     * @param   string  $hashdir Expected format of content directory
+     */
+    public function test_get_fulldir_from_storedfile($hash, $hashdir) {
+        global $CFG;
+
+        $file = $this->getMockBuilder('stored_file')
+            ->disableOriginalConstructor()
+            ->setMethods([
+                'sync_external_file',
+                'get_contenthash',
+            ])
+            ->getMock();
+
+        $file->method('get_contenthash')->willReturn($hash);
+
+        $fs = new file_system_filedir();
+        $method = new ReflectionMethod('file_system_filedir', 'get_fulldir_from_storedfile');
+        $method->setAccessible(true);
+        $result = $method->invokeArgs($fs, array($file));
+
+        $expectedpath = sprintf('%s/filedir/%s', $CFG->dataroot, $hashdir);
+        $this->assertEquals($expectedpath, $result);
+    }
+
+    /**
+     * Test that the correct content directory is generated for the supplied
+     * content hashes.
+     *
+     * @dataProvider contenthash_dataprovider
+     * @param   string  $hash contenthash to test
+     * @param   string  $hashdir Expected format of content directory
+     */
+    public function test_get_contentdir_from_hash($hash, $hashdir) {
+        $method = new ReflectionMethod(file_system_filedir::class, 'get_contentdir_from_hash');
+        $method->setAccessible(true);
+
+        $fs = new file_system_filedir();
+        $result = $method->invokeArgs($fs, array($hash));
+
+        $this->assertEquals($hashdir, $result);
+    }
+
+    /**
+     * Test that the correct content path is generated for the supplied
+     * content hashes.
+     *
+     * @dataProvider contenthash_dataprovider
+     * @param   string  $hash contenthash to test
+     * @param   string  $hashdir Expected format of content directory
+     */
+    public function test_get_contentpath_from_hash($hash, $hashdir) {
+        $method = new ReflectionMethod(file_system_filedir::class, 'get_contentpath_from_hash');
+        $method->setAccessible(true);
+
+        $fs = new file_system_filedir();
+        $result = $method->invokeArgs($fs, array($hash));
+
+        $expectedpath = sprintf('%s/%s', $hashdir, $hash);
+        $this->assertEquals($expectedpath, $result);
+    }
+
+    /**
+     * Test that the correct trash path is generated for the supplied
+     * content hashes.
+     *
+     * @dataProvider contenthash_dataprovider
+     * @param   string  $hash contenthash to test
+     * @param   string  $hashdir Expected format of content directory
+     */
+    public function test_get_trash_fullpath_from_hash($hash, $hashdir) {
+        global $CFG;
+
+        $fs = new file_system_filedir();
+        $method = new ReflectionMethod(file_system_filedir::class, 'get_trash_fullpath_from_hash');
+        $method->setAccessible(true);
+        $result = $method->invokeArgs($fs, array($hash));
+
+        $expectedpath = sprintf('%s/trashdir/%s/%s', $CFG->dataroot, $hashdir, $hash);
+        $this->assertEquals($expectedpath, $result);
+    }
+
+    /**
+     * Test that the correct trash directory is generated for the supplied
+     * content hashes.
+     *
+     * @dataProvider contenthash_dataprovider
+     * @param   string  $hash contenthash to test
+     * @param   string  $hashdir Expected format of content directory
+     */
+    public function test_get_trash_fulldir_from_hash($hash, $hashdir) {
+        global $CFG;
+
+        $fs = new file_system_filedir();
+        $method = new ReflectionMethod(file_system_filedir::class, 'get_trash_fulldir_from_hash');
+        $method->setAccessible(true);
+        $result = $method->invokeArgs($fs, array($hash));
+
+        $expectedpath = sprintf('%s/trashdir/%s', $CFG->dataroot, $hashdir);
+        $this->assertEquals($expectedpath, $result);
+    }
+
+    /**
+     * Ensure that copying a file to a target from a stored_file works as anticipated.
+     */
+    public function test_copy_content_from_storedfile() {
+        $this->resetAfterTest();
+        global $CFG;
+
+        $filecontent = 'example content';
+        $contenthash = sha1($filecontent);
+        $filedircontent = [
+            $contenthash => $filecontent,
+        ];
+        $vfileroot = $this->setup_vfile_root($filedircontent, [], []);
+
+        $fs = $this->getMockBuilder(file_system_filedir::class)
+            ->disableOriginalConstructor()
+            ->setMethods([
+                'get_local_path_from_storedfile',
+            ])
+            ->getMock();
+
+        $file = $this->getMockBuilder(stored_file::class)
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $sourcefile = \org\bovigo\vfs\vfsStream::url('root/filedir/' . $contenthash);
+        $fs->method('get_local_path_from_storedfile')->willReturn($sourcefile);
+
+        $targetfile = \org\bovigo\vfs\vfsStream::url('root/targetfile');
+        $CFG->preventfilelocking = true;
+        $result = $fs->copy_content_from_storedfile($file, $targetfile);
+
+        $this->assertTrue($result);
+        $this->assertEquals($filecontent, $vfileroot->getChild('targetfile')->getContent());
+    }
+
+    /**
+     * Ensure that content recovery works.
+     */
+    public function test_recover_file() {
+        $this->resetAfterTest();
+
+        // Setup the filedir.
+        // This contains a virtual file which has a cache mismatch.
+        $filecontent = 'example content';
+        $contenthash = sha1($filecontent);
+
+        $trashdircontent = [
+            '0f' => [
+                'f3' => [
+                    $contenthash => $filecontent,
+                ],
+            ],
+        ];
+
+        $vfileroot = $this->setup_vfile_root([], $trashdircontent);
+
+        $file = new stored_file(get_file_storage(), (object) [
+            'contenthash' => $contenthash,
+            'filesize' => strlen($filecontent),
+        ]);
+
+        $fs = new file_system_filedir();
+        $method = new ReflectionMethod(file_system_filedir::class, 'recover_file');
+        $method->setAccessible(true);
+        $result = $method->invokeArgs($fs, array($file));
+
+        // Test the output.
+        $this->assertTrue($result);
+
+        $this->assertEquals($filecontent, $vfileroot->getChild('filedir/0f/f3/' . $contenthash)->getContent());
+
+    }
+
+    /**
+     * Ensure that content recovery works.
+     */
+    public function test_recover_file_already_present() {
+        $this->resetAfterTest();
+
+        // Setup the filedir.
+        // This contains a virtual file which has a cache mismatch.
+        $filecontent = 'example content';
+        $contenthash = sha1($filecontent);
+
+        $filedircontent = $trashdircontent = [
+            '0f' => [
+                'f3' => [
+                    $contenthash => $filecontent,
+                ],
+            ],
+        ];
+
+        $vfileroot = $this->setup_vfile_root($filedircontent, $trashdircontent);
+
+        $file = new stored_file(get_file_storage(), (object) [
+            'contenthash' => $contenthash,
+            'filesize' => strlen($filecontent),
+        ]);
+
+        $fs = new file_system_filedir();
+        $method = new ReflectionMethod(file_system_filedir::class, 'recover_file');
+        $method->setAccessible(true);
+        $result = $method->invokeArgs($fs, array($file));
+
+        // Test the output.
+        $this->assertTrue($result);
+
+        $this->assertEquals($filecontent, $vfileroot->getChild('filedir/0f/f3/' . $contenthash)->getContent());
+    }
+
+    /**
+     * Ensure that content recovery works.
+     */
+    public function test_recover_file_size_mismatch() {
+        $this->resetAfterTest();
+
+        // Setup the filedir.
+        // This contains a virtual file which has a cache mismatch.
+        $filecontent = 'example content';
+        $contenthash = sha1($filecontent);
+
+        $trashdircontent = [
+            '0f' => [
+                'f3' => [
+                    $contenthash => $filecontent,
+                ],
+            ],
+        ];
+        $vfileroot = $this->setup_vfile_root([], $trashdircontent);
+
+        $file = new stored_file(get_file_storage(), (object) [
+            'contenthash' => $contenthash,
+            'filesize' => strlen($filecontent) + 1,
+        ]);
+
+        $fs = new file_system_filedir();
+        $method = new ReflectionMethod(file_system_filedir::class, 'recover_file');
+        $method->setAccessible(true);
+        $result = $method->invokeArgs($fs, array($file));
+
+        // Test the output.
+        $this->assertFalse($result);
+        $this->assertFalse($vfileroot->hasChild('filedir/0f/f3/' . $contenthash));
+    }
+
+    /**
+     * Ensure that content recovery works.
+     */
+    public function test_recover_file_has_mismatch() {
+        $this->resetAfterTest();
+
+        // Setup the filedir.
+        // This contains a virtual file which has a cache mismatch.
+        $filecontent = 'example content';
+        $contenthash = sha1($filecontent);
+
+        $trashdircontent = [
+            '0f' => [
+                'f3' => [
+                    $contenthash => $filecontent,
+                ],
+            ],
+        ];
+        $vfileroot = $this->setup_vfile_root([], $trashdircontent);
+
+        $file = new stored_file(get_file_storage(), (object) [
+            'contenthash' => $contenthash . " different",
+            'filesize' => strlen($filecontent),
+        ]);
+
+        $fs = new file_system_filedir();
+        $method = new ReflectionMethod(file_system_filedir::class, 'recover_file');
+        $method->setAccessible(true);
+        $result = $method->invokeArgs($fs, array($file));
+
+        // Test the output.
+        $this->assertFalse($result);
+        $this->assertFalse($vfileroot->hasChild('filedir/0f/f3/' . $contenthash));
+    }
+
+    /**
+     * Ensure that content recovery works when the content file is in the
+     * alt trash directory.
+     */
+    public function test_recover_file_alttrash() {
+        $this->resetAfterTest();
+
+        // Setup the filedir.
+        // This contains a virtual file which has a cache mismatch.
+        $filecontent = 'example content';
+        $contenthash = sha1($filecontent);
+
+        $trashdircontent = [
+            $contenthash => $filecontent,
+        ];
+        $vfileroot = $this->setup_vfile_root([], $trashdircontent);
+
+        $file = new stored_file(get_file_storage(), (object) [
+            'contenthash' => $contenthash,
+            'filesize' => strlen($filecontent),
+        ]);
+
+        $fs = new file_system_filedir();
+        $method = new ReflectionMethod(file_system_filedir::class, 'recover_file');
+        $method->setAccessible(true);
+        $result = $method->invokeArgs($fs, array($file));
+
+        // Test the output.
+        $this->assertTrue($result);
+
+        $this->assertEquals($filecontent, $vfileroot->getChild('filedir/0f/f3/' . $contenthash)->getContent());
+    }
+
+    /**
+     * Test that an appropriate error message is generated when adding a
+     * file to the pool when the pool directory structure is not writable.
+     */
+    public function test_recover_file_contentdir_readonly() {
+        $this->resetAfterTest();
+
+        $filecontent = 'example content';
+        $contenthash = sha1($filecontent);
+        $filedircontent = [
+            '0f' => [],
+        ];
+        $trashdircontent = [
+            $contenthash => $filecontent,
+        ];
+        $vfileroot = $this->setup_vfile_root($filedircontent, $trashdircontent);
+
+        // Make the target path readonly.
+        $vfileroot->getChild('filedir/0f')
+            ->chmod(0444)
+            ->chown(\org\bovigo\vfs\vfsStream::OWNER_USER_2);
+
+        $file = new stored_file(get_file_storage(), (object) [
+            'contenthash' => $contenthash,
+            'filesize' => strlen($filecontent),
+        ]);
+
+        $fs = new file_system_filedir();
+        $method = new ReflectionMethod(file_system_filedir::class, 'recover_file');
+        $method->setAccessible(true);
+        $result = $method->invokeArgs($fs, array($file));
+
+        // Test the output.
+        $this->assertFalse($result);
+    }
+
+    /**
+     * Test adding a file to the pool.
+     */
+    public function test_add_file_from_path() {
+        $this->resetAfterTest();
+        global $CFG;
+
+        // Setup the filedir.
+        // This contains a virtual file which has a cache mismatch.
+        $filecontent = 'example content';
+        $contenthash = sha1($filecontent);
+        $sourcedircontent = [
+            'file' => $filecontent,
+        ];
+
+        $vfileroot = $this->setup_vfile_root([], [], $sourcedircontent);
+
+        // Note, the vfs file system does not support locks - prevent file locking here.
+        $CFG->preventfilelocking = true;
+
+        // Attempt to add the file to the file pool.
+        $fs = new file_system_filedir();
+        $sourcefile = \org\bovigo\vfs\vfsStream::url('root/sourcedir/file');
+        $result = $fs->add_file_from_path($sourcefile);
+
+        // Test the output.
+        $this->assertEquals($contenthash, $result[0]);
+        $this->assertEquals(core_text::strlen($filecontent), $result[1]);
+        $this->assertTrue($result[2]);
+
+        $this->assertEquals($filecontent, $vfileroot->getChild('filedir/0f/f3/' . $contenthash)->getContent());
+    }
+
+    /**
+     * Test that an appropriate error message is generated when adding an
+     * unavailable file to the pool is attempted.
+     */
+    public function test_add_file_from_path_file_unavailable() {
+        $this->resetAfterTest();
+
+        // Setup the filedir.
+        $vfileroot = $this->setup_vfile_root();
+
+        $this->setExpectedExceptionRegexp('file_exception',
+            '/Cannot read file\. Either the file does not exist or there is a permission problem\./');
+
+        $fs = new file_system_filedir();
+        $fs->add_file_from_path(\org\bovigo\vfs\vfsStream::url('filedir/file'));
+    }
+
+    /**
+     * Test that an appropriate error message is generated when specifying
+     * the wrong contenthash when adding a file to the pool.
+     */
+    public function test_add_file_from_path_mismatched_hash() {
+        $this->resetAfterTest();
+
+        $filecontent = 'example content';
+        $contenthash = sha1($filecontent);
+        $sourcedir = [
+            'file' => $filecontent,
+        ];
+        $vfileroot = $this->setup_vfile_root([], [], $sourcedir);
+
+        $fs = new file_system_filedir();
+        $filepath = \org\bovigo\vfs\vfsStream::url('root/sourcedir/file');
+        $fs->add_file_from_path($filepath, 'eee4943847a35a4b6942c6f96daafde06bcfdfab');
+        $this->assertDebuggingCalled("Invalid contenthash submitted for file $filepath");
+    }
+
+    /**
+     * Test that an appropriate error message is generated when an existing
+     * file in the pool has the wrong contenthash
+     */
+    public function test_add_file_from_path_existing_content_invalid() {
+        $this->resetAfterTest();
+
+        $filecontent = 'example content';
+        $contenthash = sha1($filecontent);
+        $filedircontent = [
+            '0f' => [
+                'f3' => [
+                    // This contains a virtual file which has a cache mismatch.
+                    '0ff30941ca5acd879fd809e8c937d9f9e6dd1615' => 'different example content',
+                ],
+            ],
+        ];
+        $sourcedir = [
+            'file' => $filecontent,
+        ];
+        $vfileroot = $this->setup_vfile_root($filedircontent, [], $sourcedir);
+
+        // Check that we hit the jackpot.
+        $fs = new file_system_filedir();
+        $filepath = \org\bovigo\vfs\vfsStream::url('root/sourcedir/file');
+        $result = $fs->add_file_from_path($filepath);
+
+        // We provided a bad hash. Check that the file was replaced.
+        $this->assertDebuggingCalled("Replacing invalid content file $contenthash");
+
+        // Test the output.
+        $this->assertEquals($contenthash, $result[0]);
+        $this->assertEquals(core_text::strlen($filecontent), $result[1]);
+        $this->assertFalse($result[2]);
+
+        // Fetch the new file structure.
+        $structure = \org\bovigo\vfs\vfsStream::inspect(
+            new \org\bovigo\vfs\visitor\vfsStreamStructureVisitor()
+        )->getStructure();
+
+        $this->assertEquals($filecontent, $structure['root']['filedir']['0f']['f3'][$contenthash]);
+    }
+
+    /**
+     * Test that an appropriate error message is generated when adding a
+     * file to the pool when the pool directory structure is not writable.
+     */
+    public function test_add_file_from_path_existing_cannot_write_hashpath() {
+        $this->resetAfterTest();
+
+        $filecontent = 'example content';
+        $contenthash = sha1($filecontent);
+        $filedircontent = [
+            '0f' => [],
+        ];
+        $sourcedir = [
+            'file' => $filecontent,
+        ];
+        $vfileroot = $this->setup_vfile_root($filedircontent, [], $sourcedir);
+
+        // Make the target path readonly.
+        $vfileroot->getChild('filedir/0f')
+            ->chmod(0444)
+            ->chown(\org\bovigo\vfs\vfsStream::OWNER_USER_2);
+
+        $this->setExpectedException(
+            'file_exception',
+            "Can not create local file pool directories, please verify permissions in dataroot."
+        );
+
+        // Attempt to add the file to the file pool.
+        $fs = new file_system_filedir();
+        $sourcefile = \org\bovigo\vfs\vfsStream::url('root/sourcedir/file');
+        $fs->add_file_from_path($sourcefile);
+    }
+
+    /**
+     * Test adding a string to the pool.
+     */
+    public function test_add_file_from_string() {
+        $this->resetAfterTest();
+        global $CFG;
+
+        $filecontent = 'example content';
+        $contenthash = sha1($filecontent);
+        $vfileroot = $this->setup_vfile_root();
+
+        // Note, the vfs file system does not support locks - prevent file locking here.
+        $CFG->preventfilelocking = true;
+
+        // Attempt to add the file to the file pool.
+        $fs = new file_system_filedir();
+        $result = $fs->add_file_from_string($filecontent);
+
+        // Test the output.
+        $this->assertEquals($contenthash, $result[0]);
+        $this->assertEquals(core_text::strlen($filecontent), $result[1]);
+        $this->assertTrue($result[2]);
+    }
+
+    /**
+     * Test that an appropriate error message is generated when adding a
+     * string to the pool when the pool directory structure is not writable.
+     */
+    public function test_add_file_from_string_existing_cannot_write_hashpath() {
+        $this->resetAfterTest();
+
+        $filecontent = 'example content';
+        $contenthash = sha1($filecontent);
+
+        $filedircontent = [
+            '0f' => [],
+        ];
+        $vfileroot = $this->setup_vfile_root($filedircontent);
+
+        // Make the target path readonly.
+        $vfileroot->getChild('filedir/0f')
+            ->chmod(0444)
+            ->chown(\org\bovigo\vfs\vfsStream::OWNER_USER_2);
+
+        $this->setExpectedException(
+            'file_exception',
+            "Can not create local file pool directories, please verify permissions in dataroot."
+        );
+
+        // Attempt to add the file to the file pool.
+        $fs = new file_system_filedir();
+        $fs->add_file_from_string($filecontent);
+    }
+
+    /**
+     * Test adding a string to the pool when an item with the same
+     * contenthash is already present.
+     */
+    public function test_add_file_from_string_existing_matches() {
+        $this->resetAfterTest();
+        global $CFG;
+
+        $filecontent = 'example content';
+        $contenthash = sha1($filecontent);
+        $filedircontent = [
+            '0f' => [
+                'f3' => [
+                    $contenthash => $filecontent,
+                ],
+            ],
+        ];
+
+        $vfileroot = $this->setup_vfile_root($filedircontent);
+
+        // Note, the vfs file system does not support locks - prevent file locking here.
+        $CFG->preventfilelocking = true;
+
+        // Attempt to add the file to the file pool.
+        $fs = new file_system_filedir();
+        $result = $fs->add_file_from_string($filecontent);
+
+        // Test the output.
+        $this->assertEquals($contenthash, $result[0]);
+        $this->assertEquals(core_text::strlen($filecontent), $result[1]);
+        $this->assertFalse($result[2]);
+    }
+
+    /**
+     * Test the cleanup of deleted files when there are no files to delete.
+     */
+    public function test_remove_file_missing() {
+        $this->resetAfterTest();
+
+        $filecontent = 'example content';
+        $contenthash = sha1($filecontent);
+        $vfileroot = $this->setup_vfile_root();
+
+        $fs = new file_system_filedir();
+        $fs->remove_file($contenthash);
+
+        $this->assertFalse($vfileroot->hasChild('filedir/0f/f3/' . $contenthash));
+        // No file to move to trash, so the trash path will also be empty.
+        $this->assertFalse($vfileroot->hasChild('trashdir/0f'));
+        $this->assertFalse($vfileroot->hasChild('trashdir/0f/f3'));
+        $this->assertFalse($vfileroot->hasChild('trashdir/0f/f3/' . $contenthash));
+    }
+
+    /**
+     * Test the cleanup of deleted files when a file already exists in the
+     * trash for that path.
+     */
+    public function test_remove_file_existing_trash() {
+        $this->resetAfterTest();
+
+        $filecontent = 'example content';
+        $contenthash = sha1($filecontent);
+
+        $filedircontent = $trashdircontent = [
+            '0f' => [
+                'f3' => [
+                    $contenthash => $filecontent,
+                ],
+            ],
+        ];
+        $trashdircontent['0f']['f3'][$contenthash] .= 'different';
+        $vfileroot = $this->setup_vfile_root($filedircontent, $trashdircontent);
+
+        $fs = new file_system_filedir();
+        $fs->remove_file($contenthash);
+
+        $this->assertFalse($vfileroot->hasChild('filedir/0f/f3/' . $contenthash));
+        $this->assertTrue($vfileroot->hasChild('trashdir/0f/f3/' . $contenthash));
+        $this->assertNotEquals($filecontent, $vfileroot->getChild('trashdir/0f/f3/' . $contenthash)->getContent());
+    }
+
+    /**
+     * Ensure that remove_file does nothing with an empty file.
+     */
+    public function test_remove_file_empty() {
+        $this->resetAfterTest();
+        global $DB;
+
+        $DB = $this->getMockBuilder(\moodle_database::class)
+            ->setMethods(['record_exists'])
+            ->getMockForAbstractClass();
+
+        $DB->expects($this->never())
+            ->method('record_exists');
+
+        $fs = new file_system_filedir();
+
+        $result = $fs->remove_file(sha1(''));
+        $this->assertNull($result);
+    }
+
+    /**
+     * Ensure that remove_file does nothing when a file is still
+     * in use.
+     */
+    public function test_remove_file_in_use() {
+        $this->resetAfterTest();
+        global $DB;
+
+        $filecontent = 'example content';
+        $contenthash = sha1($filecontent);
+        $filedircontent = [
+            '0f' => [
+                'f3' => [
+                    $contenthash => $filecontent,
+                ],
+            ],
+        ];
+        $vfileroot = $this->setup_vfile_root($filedircontent);
+
+        $DB = $this->getMockBuilder(\moodle_database::class)
+            ->setMethods(['record_exists'])
+            ->getMockForAbstractClass();
+
+        $DB->method('record_exists')->willReturn(true);
+
+        $fs = new file_system_filedir();
+        $result = $fs->remove_file($contenthash);
+        $this->assertTrue($vfileroot->hasChild('filedir/0f/f3/' . $contenthash));
+        $this->assertFalse($vfileroot->hasChild('trashdir/0f/f3/' . $contenthash));
+    }
+
+    /**
+     * Ensure that remove_file removes the file when it is no
+     * longer in use.
+     */
+    public function test_remove_file_expired() {
+        $this->resetAfterTest();
+        global $DB;
+
+        $filecontent = 'example content';
+        $contenthash = sha1($filecontent);
+        $filedircontent = [
+            '0f' => [
+                'f3' => [
+                    $contenthash => $filecontent,
+                ],
+            ],
+        ];
+        $vfileroot = $this->setup_vfile_root($filedircontent);
+
+        $DB = $this->getMockBuilder(\moodle_database::class)
+            ->setMethods(['record_exists'])
+            ->getMockForAbstractClass();
+
+        $DB->method('record_exists')->willReturn(false);
+
+        $fs = new file_system_filedir();
+        $result = $fs->remove_file($contenthash);
+        $this->assertFalse($vfileroot->hasChild('filedir/0f/f3/' . $contenthash));
+        $this->assertTrue($vfileroot->hasChild('trashdir/0f/f3/' . $contenthash));
+    }
+
+    /**
+     * Test purging the cache.
+     */
+    public function test_empty_trash() {
+        $this->resetAfterTest();
+
+        $filecontent = 'example content';
+        $contenthash = sha1($filecontent);
+
+        $filedircontent = $trashdircontent = [
+            '0f' => [
+                'f3' => [
+                    $contenthash => $filecontent,
+                ],
+            ],
+        ];
+        $vfileroot = $this->setup_vfile_root($filedircontent, $trashdircontent);
+
+        $fs = new file_system_filedir();
+        $method = new ReflectionMethod(file_system_filedir::class, 'empty_trash');
+        $method->setAccessible(true);
+        $result = $method->invoke($fs);
+
+        $this->assertTrue($vfileroot->hasChild('filedir/0f/f3/' . $contenthash));
+        $this->assertFalse($vfileroot->hasChild('trashdir'));
+        $this->assertFalse($vfileroot->hasChild('trashdir/0f'));
+        $this->assertFalse($vfileroot->hasChild('trashdir/0f/f3'));
+        $this->assertFalse($vfileroot->hasChild('trashdir/0f/f3/' . $contenthash));
+    }
+
+    /**
+     * Data Provider for contenthash to contendir conversion.
+     *
+     * @return  array
+     */
+    public function contenthash_dataprovider() {
+        return array(
+            array(
+                'contenthash'   => 'eee4943847a35a4b6942c6f96daafde06bcfdfab',
+                'contentdir'    => 'ee/e4',
+            ),
+            array(
+                'contenthash'   => 'aef05a62ae81ca0005d2569447779af062b7cda0',
+                'contentdir'    => 'ae/f0',
+            ),
+        );
+    }
+}
diff --git a/lib/filestorage/tests/file_system_test.php b/lib/filestorage/tests/file_system_test.php
new file mode 100644 (file)
index 0000000..2aeff76
--- /dev/null
@@ -0,0 +1,1091 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Unit tests for file_system.
+ *
+ * @package   core_files
+ * @category  phpunit
+ * @copyright 2017 Andrew Nicols <andrew@nicols.co.uk>
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+global $CFG;
+require_once($CFG->libdir . '/filestorage/file_system.php');
+
+/**
+ * Unit tests for file_system.
+ *
+ * @package   core_files
+ * @category  phpunit
+ * @copyright 2017 Andrew Nicols <andrew@nicols.co.uk>
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class core_files_file_system_testcase extends advanced_testcase {
+
+    public function setUp() {
+        get_file_storage(true);
+    }
+
+    public function tearDown() {
+        get_file_storage(true);
+    }
+
+    /**
+     * Helper function to help setup and configure the virtual file system stream.
+     *
+     * @param   array $filedir Directory structure and content of the filedir
+     * @param   array $trashdir Directory structure and content of the sourcedir
+     * @param   array $sourcedir Directory structure and content of a directory used for source files for tests
+     * @return  \org\bovigo\vfs\vfsStream
+     */
+    protected function setup_vfile_root($content = []) {
+        $vfileroot = \org\bovigo\vfs\vfsStream::setup('root', null, $content);
+
+        return $vfileroot;
+    }
+
+    /**
+     * Helper to create a stored file objectw with the given supplied content.
+     *
+     * @param   string  $filecontent The content of the mocked file
+     * @param   string  $filename The file name to use in the stored_file
+     * @param   array   $mockedmethods A list of methods you intend to override
+     *                  If no methods are specified, only abstract functions are mocked.
+     * @return stored_file
+     */
+    protected function get_stored_file($filecontent, $filename = null, $mockedmethods = null) {
+        $contenthash = sha1($filecontent);
+        if (empty($filename)) {
+            $filename = $contenthash;
+        }
+
+        $file = $this->getMockBuilder(stored_file::class)
+            ->setMethods($mockedmethods)
+            ->setConstructorArgs([
+                get_file_storage(),
+                (object) [
+                    'contenthash' => $contenthash,
+                    'filesize' => strlen($filecontent),
+                    'filename' => $filename,
+                ]
+            ])
+            ->getMock();
+
+        return $file;
+    }
+
+    /**
+     * Get a testable mock of the abstract file_system class.
+     *
+     * @param   array   $mockedmethods A list of methods you intend to override
+     *                  If no methods are specified, only abstract functions are mocked.
+     * @return file_system
+     */
+    protected function get_testable_mock($mockedmethods = []) {
+        $fs = $this->getMockBuilder(file_system::class)
+            ->setMethods($mockedmethods)
+            ->getMockForAbstractClass();
+
+        return $fs;
+    }
+
+    /**
+     * Ensure that the file system is not clonable.
+     */
+    public function test_not_cloneable() {
+        $reflection = new ReflectionClass('file_system');
+        $this->assertFalse($reflection->isCloneable());
+    }
+
+    /**
+     * Ensure that the filedir file_system extension is used by default.
+     */
+    public function test_default_class() {
+        $this->resetAfterTest();
+
+        // Ensure that the alternative_file_system_class is null.
+        global $CFG;
+        $CFG->alternative_file_system_class = null;
+
+        $storage = get_file_storage();
+        $fs = $storage->get_file_system();
+        $this->assertInstanceOf(file_system::class, $fs);
+        $this->assertEquals(file_system_filedir::class, get_class($fs));
+    }
+
+    /**
+     * Ensure that the specified file_system extension class is used.
+     */
+    public function test_supplied_class() {
+        global $CFG;
+        $this->resetAfterTest();
+
+        // Mock the file_system.
+        // Mocks create a new child of the mocked class which is perfect for this test.
+        $filesystem = $this->getMockBuilder('file_system')
+            ->disableOriginalConstructor()
+            ->getMock();
+        $CFG->alternative_file_system_class = get_class($filesystem);
+
+        $storage = get_file_storage();
+        $fs = $storage->get_file_system();
+        $this->assertInstanceOf(file_system::class, $fs);
+        $this->assertEquals(get_class($filesystem), get_class($fs));
+    }
+
+    /**
+     * Test that the readfile function outputs content to disk.
+     */
+    public function test_readfile_remote() {
+        global $CFG;
+
+        // Mock the filesystem.
+        $filecontent = 'example content';
+        $vfileroot = $this->setup_vfile_root(['sourcefile' => $filecontent]);
+        $filepath = \org\bovigo\vfs\vfsStream::url('root/sourcefile');
+
+        $file = $this->get_stored_file($filecontent);
+
+        // Mock the file_system class.
+        // We need to override the get_remote_path_from_storedfile function.
+        $fs = $this->get_testable_mock([
+            'get_remote_path_from_storedfile',
+            'is_file_readable_locally_by_storedfile',
+            'get_local_path_from_storedfile',
+        ]);
+        $fs->method('get_remote_path_from_storedfile')->willReturn($filepath);
+        $fs->method('is_file_readable_locally_by_storedfile')->willReturn(false);
+        $fs->expects($this->never())->method('get_local_path_from_storedfile');
+
+        // Note: It is currently not possible to mock readfile_allow_large
+        // because file_system is in the global namespace.
+        // We must therefore check for expected output. This is not ideal.
+        $this->expectOutputString($filecontent);
+        $fs->readfile($file);
+    }
+
+    /**
+     * Test that the readfile function outputs content to disk.
+     */
+    public function test_readfile_local() {
+        global $CFG;
+
+        // Mock the filesystem.
+        $filecontent = 'example content';
+        $vfileroot = $this->setup_vfile_root(['sourcefile' => $filecontent]);
+        $filepath = \org\bovigo\vfs\vfsStream::url('root/sourcefile');
+
+        $file = $this->get_stored_file($filecontent);
+
+        // Mock the file_system class.
+        // We need to override the get_remote_path_from_storedfile function.
+        $fs = $this->get_testable_mock([
+            'get_remote_path_from_storedfile',
+            'is_file_readable_locally_by_storedfile',
+            'get_local_path_from_storedfile',
+        ]);
+        $fs->method('is_file_readable_locally_by_storedfile')->willReturn(true);
+        $fs->expects($this->never())->method('get_remote_path_from_storedfile');
+        $fs->expects($this->once())->method('get_local_path_from_storedfile')->willReturn($filepath);
+
+        // Note: It is currently not possible to mock readfile_allow_large
+        // because file_system is in the global namespace.
+        // We must therefore check for expected output. This is not ideal.
+        $this->expectOutputString($filecontent);
+        $fs->readfile($file);
+    }
+
+    /**
+     * Test that the get_local_path_from_storedfile function functions
+     * correctly when called with various args.
+     *
+     * @dataProvider get_local_path_from_storedfile_provider
+     * @param   array   $args The additional args to pass to get_local_path_from_storedfile
+     * @param   bool    $fetch Whether the combination of args should have caused a fetch
+     */
+    public function test_get_local_path_from_storedfile($args, $fetch) {
+        $filepath = '/path/to/file';
+        $filecontent = 'example content';
+
+        // Get the filesystem mock.
+        $fs = $this->get_testable_mock([
+            'get_local_path_from_hash',
+        ]);
+        $fs->expects($this->once())
+            ->method('get_local_path_from_hash')
+            ->with($this->equalTo(sha1($filecontent)), $this->equalTo($fetch))
+            ->willReturn($filepath);
+
+        $file = $this->get_stored_file($filecontent);
+
+        $method = new ReflectionMethod(file_system::class, 'get_local_path_from_storedfile');
+        $method->setAccessible(true);
+        $result = $method->invokeArgs($fs, array_merge([$file], $args));
+
+        $this->assertEquals($filepath, $result);
+    }
+
+    /**
+     * Ensure that the default implementation of get_remote_path_from_storedfile
+     * simply calls get_local_path_from_storedfile without requiring a
+     * fetch.
+     */
+    public function test_get_remote_path_from_storedfile() {
+        $filepath = '/path/to/file';
+        $filecontent = 'example content';
+
+        $fs = $this->get_testable_mock([
+            'get_remote_path_from_hash',
+        ]);
+
+        $fs->expects($this->once())
+            ->method('get_remote_path_from_hash')
+            ->with($this->equalTo(sha1($filecontent)), $this->equalTo(false))
+            ->willReturn($filepath);
+
+        $file = $this->get_stored_file($filecontent);
+
+        $method = new ReflectionMethod(file_system::class, 'get_remote_path_from_storedfile');
+        $method->setAccessible(true);
+        $result = $method->invokeArgs($fs, [$file]);
+
+        $this->assertEquals($filepath, $result);
+    }
+
+    /**
+     * Test the stock implementation of is_file_readable_locally_by_hash with a valid file.
+     *
+     * This should call get_local_path_from_hash and check the readability
+     * of the file.
+     *
+     * Fetching the file is optional.
+     */
+    public function test_is_file_readable_locally_by_hash() {
+        $filecontent = 'example content';
+        $contenthash = sha1($filecontent);
+        $filepath = __FILE__;
+
+        $fs = $this->get_testable_mock([
+            'get_local_path_from_hash',
+        ]);
+
+        $fs->method('get_local_path_from_hash')
+            ->with($this->equalTo($contenthash), $this->equalTo(false))
+            ->willReturn($filepath);
+
+        $this->assertTrue($fs->is_file_readable_locally_by_hash($contenthash));
+    }
+
+    /**
+     * Test the stock implementation of is_file_readable_locally_by_hash with an empty file.
+     */
+    public function test_is_file_readable_locally_by_hash_empty() {
+        $filecontent = '';
+        $contenthash = sha1($filecontent);
+
+        $fs = $this->get_testable_mock([
+            'get_local_path_from_hash',
+        ]);
+
+        $fs->expects($this->never())
+            ->method('get_local_path_from_hash');
+
+        $this->assertTrue($fs->is_file_readable_locally_by_hash($contenthash));
+    }
+
+    /**
+     * Test the stock implementation of is_file_readable_remotely_by_storedfile with a valid file.
+     */
+    public function test_is_file_readable_remotely_by_hash() {
+        $filecontent = 'example content';
+        $contenthash = sha1($filecontent);
+
+        $fs = $this->get_testable_mock([
+            'get_remote_path_from_hash',
+        ]);
+
+        $fs->method('get_remote_path_from_hash')
+            ->with($this->equalTo($contenthash), $this->equalTo(false))
+            ->willReturn(__FILE__);
+
+        $this->assertTrue($fs->is_file_readable_remotely_by_hash($contenthash));
+    }
+
+    /**
+     * Test the stock implementation of is_file_readable_remotely_by_storedfile with a valid file.
+     */
+    public function test_is_file_readable_remotely_by_hash_empty() {
+        $filecontent = '';
+        $contenthash = sha1($filecontent);
+
+        $fs = $this->get_testable_mock([
+            'get_remote_path_from_hash',
+        ]);
+
+        $fs->expects($this->never())
+            ->method('get_remote_path_from_hash');
+
+        $this->assertTrue($fs->is_file_readable_remotely_by_hash($contenthash));
+    }
+
+    /**
+     * Test the stock implementation of is_file_readable_remotely_by_storedfile with a valid file.
+     */
+    public function test_is_file_readable_remotely_by_hash_not_found() {
+        $filecontent = 'example content';
+        $contenthash = sha1($filecontent);
+
+        $fs = $this->get_testable_mock([
+            'get_remote_path_from_hash',
+        ]);
+
+        $fs->method('get_remote_path_from_hash')
+            ->with($this->equalTo($contenthash), $this->equalTo(false))
+            ->willReturn('/path/to/nonexistent/file');
+
+        $this->assertFalse($fs->is_file_readable_remotely_by_hash($contenthash));
+    }
+
+    /**
+     * Test the stock implementation of is_file_readable_remotely_by_storedfile with a valid file.
+     */
+    public function test_is_file_readable_remotely_by_storedfile() {
+        $file = $this->get_stored_file('example content');
+
+        $fs = $this->get_testable_mock([
+            'get_remote_path_from_storedfile',
+        ]);
+
+        $fs->method('get_remote_path_from_storedfile')
+            ->willReturn(__FILE__);
+
+        $this->assertTrue($fs->is_file_readable_remotely_by_storedfile($file));
+    }
+
+    /**
+     * Test the stock implementation of is_file_readable_remotely_by_storedfile with a valid file.
+     */
+    public function test_is_file_readable_remotely_by_storedfile_empty() {
+        $fs = $this->get_testable_mock([
+            'get_remote_path_from_storedfile',
+        ]);
+
+        $fs->expects($this->never())
+            ->method('get_remote_path_from_storedfile');
+
+        $file = $this->get_stored_file('');
+        $this->assertTrue($fs->is_file_readable_remotely_by_storedfile($file));
+    }
+
+    /**
+     * Test the stock implementation of is_file_readable_locally_by_storedfile with an empty file.
+     */
+    public function test_is_file_readable_locally_by_storedfile_empty() {
+        $fs = $this->get_testable_mock([
+            'get_local_path_from_storedfile',
+        ]);
+
+        $fs->expects($this->never())
+            ->method('get_local_path_from_storedfile');
+
+        $file = $this->get_stored_file('');
+        $this->assertTrue($fs->is_file_readable_locally_by_storedfile($file));
+    }
+
+    /**
+     * Test the stock implementation of is_file_readable_remotely_by_storedfile with a valid file.
+     */
+    public function test_is_file_readable_remotely_by_storedfile_not_found() {
+        $file = $this->get_stored_file('example content');
+
+        $fs = $this->get_testable_mock([
+            'get_remote_path_from_storedfile',
+        ]);
+
+        $fs->method('get_remote_path_from_storedfile')
+            ->willReturn(__LINE__);
+
+        $this->assertFalse($fs->is_file_readable_remotely_by_storedfile($file));
+    }
+
+    /**
+     * Test the stock implementation of is_file_readable_locally_by_storedfile with a valid file.
+     */
+    public function test_is_file_readable_locally_by_storedfile_unreadable() {
+        $fs = $this->get_testable_mock([
+            'get_local_path_from_storedfile',
+        ]);
+        $file = $this->get_stored_file('example content');
+
+        $fs->method('get_local_path_from_storedfile')
+            ->with($this->equalTo($file), $this->equalTo(false))
+            ->willReturn('/path/to/nonexistent/file');
+
+        $this->assertFalse($fs->is_file_readable_locally_by_storedfile($file));
+    }
+
+    /**
+     * Test the stock implementation of is_file_readable_locally_by_storedfile with a valid file should pass fetch.
+     */
+    public function test_is_file_readable_locally_by_storedfile_passes_fetch() {
+        $fs = $this->get_testable_mock([
+            'get_local_path_from_storedfile',
+        ]);
+        $file = $this->get_stored_file('example content');
+
+        $fs->method('get_local_path_from_storedfile')
+            ->with($this->equalTo($file), $this->equalTo(true))
+            ->willReturn('/path/to/nonexistent/file');
+
+        $this->assertFalse($fs->is_file_readable_locally_by_storedfile($file, true));
+    }
+
+    /**
+     * Ensure that is_file_removable returns correctly for an empty file.
+     */
+    public function test_is_file_removable_empty() {
+        $filecontent = '';
+        $contenthash = sha1($filecontent);
+
+        $method = new ReflectionMethod(file_system::class, 'is_file_removable');
+        $method->setAccessible(true);
+        $result = $method->invokeArgs(null, [$contenthash]);
+        $this->assertFalse($result);
+    }
+
+    /**
+     * Ensure that is_file_removable returns false if the file is still in use.
+     */
+    public function test_is_file_removable_in_use() {
+        $this->resetAfterTest();
+        global $DB;
+
+        $filecontent = 'example content';
+        $contenthash = sha1($filecontent);
+
+        $DB = $this->getMockBuilder(\moodle_database::class)
+            ->setMethods(['record_exists'])
+            ->getMockForAbstractClass();
+        $DB->method('record_exists')->willReturn(true);
+
+        $method = new ReflectionMethod(file_system::class, 'is_file_removable');
+        $method->setAccessible(true);
+        $result = $method->invokeArgs(null, [$contenthash]);
+
+        $this->assertFalse($result);
+    }
+
+    /**
+     * Ensure that is_file_removable returns false if the file is not in use.
+     */
+    public function test_is_file_removable_not_in_use() {
+        $this->resetAfterTest();
+        global $DB;
+
+        $filecontent = 'example content';
+        $contenthash = sha1($filecontent);
+
+        $DB = $this->getMockBuilder(\moodle_database::class)
+            ->setMethods(['record_exists'])
+            ->getMockForAbstractClass();
+        $DB->method('record_exists')->willReturn(false);
+
+        $method = new ReflectionMethod(file_system::class, 'is_file_removable');
+        $method->setAccessible(true);
+        $result = $method->invokeArgs(null, [$contenthash]);
+
+        $this->assertTrue($result);
+    }
+
+    /**
+     * Test the stock implementation of get_content.
+     */
+    public function test_get_content() {
+        global $CFG;
+
+        // Mock the filesystem.
+        $filecontent = 'example content';
+        $vfileroot = $this->setup_vfile_root(['sourcefile' => $filecontent]);
+        $filepath = \org\bovigo\vfs\vfsStream::url('root/sourcefile');
+
+        $file = $this->get_stored_file($filecontent);
+
+        // Mock the file_system class.
+        // We need to override the get_remote_path_from_storedfile function.
+        $fs = $this->get_testable_mock(['get_remote_path_from_storedfile']);
+        $fs->method('get_remote_path_from_storedfile')->willReturn($filepath);
+
+        $result = $fs->get_content($file);
+
+        $this->assertEquals($filecontent, $result);
+    }
+
+    /**
+     * Test the stock implementation of get_content.
+     */
+    public function test_get_content_empty() {
+        global $CFG;
+
+        $filecontent = '';
+        $file = $this->get_stored_file($filecontent);
+
+        // Mock the file_system class.
+        // We need to override the get_remote_path_from_storedfile function.
+        $fs = $this->get_testable_mock(['get_remote_path_from_storedfile']);
+        $fs->expects($this->never())
+            ->method('get_remote_path_from_storedfile');
+
+        $result = $fs->get_content($file);
+
+        $this->assertEquals($filecontent, $result);
+    }
+
+    /**
+     * Ensure that the list_files function requires a local copy of the
+     * file, and passes the path to the packer.
+     */
+    public function test_list_files() {
+        $filecontent = 'example content';
+        $file = $this->get_stored_file($filecontent);
+        $filepath = __FILE__;
+        $expectedresult = (object) [];
+
+        // Mock the file_system class.
+        $fs = $this->get_testable_mock(['get_local_path_from_storedfile']);
+        $fs->method('get_local_path_from_storedfile')
+            ->with($this->equalTo($file), $this->equalTo(true))
+            ->willReturn(__FILE__);
+
+        $packer = $this->getMockBuilder(file_packer::class)
+            ->setMethods(['list_files'])
+            ->getMockForAbstractClass();
+
+        $packer->expects($this->once())
+            ->method('list_files')
+            ->with($this->equalTo($filepath))
+            ->willReturn($expectedresult);
+
+        $result = $fs->list_files($file, $packer);
+
+        $this->assertEquals($expectedresult, $result);
+    }
+
+    /**
+     * Ensure that the extract_to_pathname function requires a local copy of the
+     * file, and passes the path to the packer.
+     */
+    public function test_extract_to_pathname() {
+        $filecontent = 'example content';
+        $file = $this->get_stored_file($filecontent);
+        $filepath = __FILE__;
+        $expectedresult = (object) [];
+        $outputpath = '/path/to/output';
+
+        // Mock the file_system class.
+        $fs = $this->get_testable_mock(['get_local_path_from_storedfile']);
+        $fs->method('get_local_path_from_storedfile')
+            ->with($this->equalTo($file), $this->equalTo(true))
+            ->willReturn(__FILE__);
+
+        $packer = $this->getMockBuilder(file_packer::class)
+            ->setMethods(['extract_to_pathname'])
+            ->getMockForAbstractClass();
+
+        $packer->expects($this->once())
+            ->method('extract_to_pathname')
+            ->with($this->equalTo($filepath), $this->equalTo($outputpath), $this->equalTo(null), $this->equalTo(null))
+            ->willReturn($expectedresult);
+
+        $result = $fs->extract_to_pathname($file, $packer, $outputpath);
+
+        $this->assertEquals($expectedresult, $result);
+    }
+
+    /**
+     * Ensure that the extract_to_storage function requires a local copy of the
+     * file, and passes the path to the packer.
+     */
+    public function test_extract_to_storage() {
+        $filecontent = 'example content';
+        $file = $this->get_stored_file($filecontent);
+        $filepath = __FILE__;
+        $expectedresult = (object) [];
+        $outputpath = '/path/to/output';
+
+        // Mock the file_system class.
+        $fs = $this->get_testable_mock(['get_local_path_from_storedfile']);
+        $fs->method('get_local_path_from_storedfile')
+            ->with($this->equalTo($file), $this->equalTo(true))
+            ->willReturn(__FILE__);
+
+        $packer = $this->getMockBuilder(file_packer::class)
+            ->setMethods(['extract_to_storage'])
+            ->getMockForAbstractClass();
+
+        $packer->expects($this->once())
+            ->method('extract_to_storage')
+            ->with(
+                $this->equalTo($filepath),
+                $this->equalTo(42),
+                $this->equalTo('component'),
+                $this->equalTo('filearea'),
+                $this->equalTo('itemid'),
+                $this->equalTo('pathbase'),
+                $this->equalTo('userid'),
+                $this->equalTo(null)
+            )
+            ->willReturn($expectedresult);
+
+        $result = $fs->extract_to_storage($file, $packer, 42, 'component','filearea', 'itemid', 'pathbase', 'userid');
+
+        $this->assertEquals($expectedresult, $result);
+    }
+
+    /**
+     * Ensure that the add_storedfile_to_archive function requires a local copy of the
+     * file, and passes the path to the archive.
+     */
+    public function test_add_storedfile_to_archive_directory() {
+        $file = $this->get_stored_file('', '.');
+        $archivepath = 'example';
+        $expectedresult = (object) [];
+
+        // Mock the file_system class.
+        $fs = $this->get_testable_mock(['get_local_path_from_storedfile']);
+        $fs->method('get_local_path_from_storedfile')
+            ->with($this->equalTo($file), $this->equalTo(true))
+            ->willReturn(__FILE__);
+
+        $archive = $this->getMockBuilder(file_archive::class)
+            ->setMethods([
+                'add_directory',
+                'add_file_from_pathname',
+            ])
+            ->getMockForAbstractClass();
+
+        $archive->expects($this->once())
+            ->method('add_directory')
+            ->with($this->equalTo($archivepath))
+            ->willReturn($expectedresult);
+
+        $archive->expects($this->never())
+            ->method('add_file_from_pathname');
+
+        $result = $fs->add_storedfile_to_archive($file, $archive, $archivepath);
+
+        $this->assertEquals($expectedresult, $result);
+    }
+
+    /**
+     * Ensure that the add_storedfile_to_archive function requires a local copy of the
+     * file, and passes the path to the archive.
+     */
+    public function test_add_storedfile_to_archive_file() {
+        $file = $this->get_stored_file('example content');
+        $filepath = __LINE__;
+        $archivepath = 'example';
+        $expectedresult = (object) [];
+
+        // Mock the file_system class.
+        $fs = $this->get_testable_mock(['get_local_path_from_storedfile']);
+        $fs->method('get_local_path_from_storedfile')
+            ->with($this->equalTo($file), $this->equalTo(true))
+            ->willReturn($filepath);
+
+        $archive = $this->getMockBuilder(file_archive::class)
+            ->setMethods([
+                'add_directory',
+                'add_file_from_pathname',
+            ])
+            ->getMockForAbstractClass();
+
+        $archive->expects($this->never())
+            ->method('add_directory');
+
+        $archive->expects($this->once())
+            ->method('add_file_from_pathname')
+            ->with(
+                $this->equalTo($archivepath),
+                $this->equalTo($filepath)
+            )
+            ->willReturn($expectedresult);
+
+        $result = $fs->add_storedfile_to_archive($file, $archive, $archivepath);
+
+        $this->assertEquals($expectedresult, $result);
+    }
+
+    /**
+     * Ensure that the add_to_curl_request function requires a local copy of the
+     * file, and passes the path to curl_file_create.
+     */
+    public function test_add_to_curl_request() {
+        $file = $this->get_stored_file('example content');
+        $filepath = __FILE__;
+        $archivepath = 'example';
+        $key = 'myfile';
+
+        // Mock the file_system class.
+        $fs = $this->get_testable_mock(['get_local_path_from_storedfile']);
+        $fs->method('get_local_path_from_storedfile')
+            ->with($this->equalTo($file), $this->equalTo(true))
+            ->willReturn($filepath);
+
+        $request = (object) ['_tmp_file_post_params' => []];
+        $fs->add_to_curl_request($file, $request, $key);
+        $this->assertArrayHasKey($key, $request->_tmp_file_post_params);
+        $this->assertEquals($filepath, $request->_tmp_file_post_params[$key]->name);
+    }
+
+    /**
+     * Ensure that test_get_imageinfo_not_image returns false if the file
+     * passed was deemed to not be an image.
+     */
+    public function test_get_imageinfo_not_image() {
+        $filecontent = 'example content';
+        $file = $this->get_stored_file($filecontent);
+
+        $fs = $this->get_testable_mock([
+            'is_image_from_storedfile',
+        ]);
+
+        $fs->expects($this->once())
+            ->method('is_image_from_storedfile')
+            ->with($this->equalTo($file))
+            ->willReturn(false);
+
+        $this->assertFalse($fs->get_imageinfo($file));
+    }
+
+    /**
+     * Ensure that test_get_imageinfo_not_image returns imageinfo.
+     */
+    public function test_get_imageinfo() {
+        $filepath = '/path/to/file';
+        $filecontent = 'example content';
+        $expectedresult = (object) [];
+        $file = $this->get_stored_file($filecontent);
+
+        $fs = $this->get_testable_mock([
+            'is_image_from_storedfile',
+            'get_local_path_from_storedfile',
+            'get_imageinfo_from_path',
+        ]);
+
+        $fs->expects($this->once())
+            ->method('is_image_from_storedfile')
+            ->with($this->equalTo($file))
+            ->willReturn(true);
+
+        $fs->expects($this->once())
+            ->method('get_local_path_from_storedfile')
+            ->with($this->equalTo($file), $this->equalTo(true))
+            ->willReturn($filepath);
+
+        $fs->expects($this->once())
+            ->method('get_imageinfo_from_path')
+            ->with($this->equalTo($filepath))
+            ->willReturn($expectedresult);
+
+        $this->assertEquals($expectedresult, $fs->get_imageinfo($file));
+    }
+
+    /**
+     * Ensure that is_image_from_storedfile always returns false for an
+     * empty file size.
+     */
+    public function test_is_image_empty_filesize() {
+        $filecontent = 'example content';
+        $file = $this->get_stored_file($filecontent, null, ['get_filesize']);
+
+        $file->expects($this->once())
+            ->method('get_filesize')
+            ->willReturn(0);
+
+        $fs = $this->get_testable_mock();
+        $this->assertFalse($fs->is_image_from_storedfile($file));
+    }
+
+    /**
+     * Ensure that is_image_from_storedfile behaves correctly based on
+     * mimetype.
+     *
+     * @dataProvider is_image_from_storedfile_provider
+     * @param   string  $mimetype Mimetype to test
+     * @param   bool    $isimage Whether this mimetype should be detected as an image
+     */
+    public function test_is_image_from_storedfile_mimetype($mimetype, $isimage) {
+        $filecontent = 'example content';
+        $file = $this->get_stored_file($filecontent, null, ['get_mimetype']);
+
+        $file->expects($this->once())
+            ->method('get_mimetype')
+            ->willReturn($mimetype);
+
+        $fs = $this->get_testable_mock();
+        $this->assertEquals($isimage, $fs->is_image_from_storedfile($file));
+    }
+
+    /**
+     * Test that get_imageinfo_from_path returns an appropriate response
+     * for an image.
+     */
+    public function test_get_imageinfo_from_path() {
+        $filepath = __DIR__ . DIRECTORY_SEPARATOR . 'fixtures' . DIRECTORY_SEPARATOR . 'testimage.jpg';
+
+        // Get the filesystem mock.
+        $fs = $this->get_testable_mock();
+
+        $method = new ReflectionMethod(file_system::class, 'get_imageinfo_from_path');
+        $method->setAccessible(true);
+        $result = $method->invokeArgs($fs, [$filepath]);
+
+        $this->assertArrayHasKey('width', $result);
+        $this->assertArrayHasKey('height', $result);
+        $this->assertArrayHasKey('mimetype', $result);
+        $this->assertEquals('image/jpeg', $result['mimetype']);
+    }
+
+    /**
+     * Test that get_imageinfo_from_path returns an appropriate response
+     * for a file which is not an image.
+     */
+    public function test_get_imageinfo_from_path_no_image() {
+        $filepath = __FILE__;
+
+        // Get the filesystem mock.
+        $fs = $this->get_testable_mock();
+
+        $method = new ReflectionMethod(file_system::class, 'get_imageinfo_from_path');
+        $method->setAccessible(true);
+        $result = $method->invokeArgs($fs, [$filepath]);
+
+        $this->assertFalse($result);
+    }
+
+    /**
+     * Ensure that get_content_file_handle returns a valid file handle.
+     */
+    public function test_get_content_file_handle_default() {
+        $filecontent = 'example content';
+        $file = $this->get_stored_file($filecontent);
+
+        $fs = $this->get_testable_mock(['get_remote_path_from_storedfile']);
+        $fs->method('get_remote_path_from_storedfile')
+            ->willReturn(__FILE__);
+
+        // Note: We are unable to determine the mode in which the $fh was opened.
+        $fh = $fs->get_content_file_handle($file);
+        $this->assertTrue(is_resource($fh));
+        $this->assertEquals('stream', get_resource_type($fh));
+        fclose($fh);
+    }
+
+    /**
+     * Ensure that get_content_file_handle returns a valid file handle for a gz file.
+     */
+    public function test_get_content_file_handle_gz() {
+        $filecontent = 'example content';
+        $file = $this->get_stored_file($filecontent);
+
+        $fs = $this->get_testable_mock(['get_remote_path_from_storedfile']);
+        $fs->method('get_remote_path_from_storedfile')
+            ->willReturn(__DIR__ . DIRECTORY_SEPARATOR . 'fixtures' . DIRECTORY_SEPARATOR . 'test.tgz');
+
+        // Note: We are unable to determine the mode in which the $fh was opened.
+        $fh = $fs->get_content_file_handle($file, stored_file::FILE_HANDLE_GZOPEN);
+        $this->assertTrue(is_resource($fh));
+        gzclose($fh);
+    }
+
+    /**
+     * Ensure that get_content_file_handle returns an exception when calling for a invalid file handle type.
+     */
+    public function test_get_content_file_handle_invalid() {
+        $filecontent = 'example content';
+        $file = $this->get_stored_file($filecontent);
+
+        $fs = $this->get_testable_mock(['get_remote_path_from_storedfile']);
+        $fs->method('get_remote_path_from_storedfile')
+            ->willReturn(__FILE__);
+
+        $this->expectException('coding_exception', 'Unexpected file handle type');
+        $fs->get_content_file_handle($file, -1);
+    }
+
+    /**
+     * Test that mimetype_from_hash returns the correct mimetype with
+     * a file whose filename suggests mimetype.
+     */
+    public function test_mimetype_from_hash_using_filename() {
+        $filepath = '/path/to/file/not/currently/on/disk';
+        $filecontent = 'example content';
+        $filename = 'test.jpg';
+        $contenthash = sha1($filecontent);
+
+        $fs = $this->get_testable_mock(['get_remote_path_from_hash']);
+        $fs->method('get_remote_path_from_hash')->willReturn($filepath);
+
+        $result = $fs->mimetype_from_hash($contenthash, $filename);
+        $this->assertEquals('image/jpeg', $result);
+    }
+
+    /**
+     * Test that mimetype_from_hash returns the correct mimetype with
+     * a locally available file whose filename does not suggest mimetype.
+     */
+    public function test_mimetype_from_hash_using_file_content() {
+        $filepath = '/path/to/file/not/currently/on/disk';
+        $filecontent = 'example content';
+        $contenthash = sha1($filecontent);
+        $filename = 'example';
+
+        $filepath = __DIR__ . DIRECTORY_SEPARATOR . 'fixtures' . DIRECTORY_SEPARATOR . 'testimage.jpg';
+        $fs = $this->get_testable_mock(['get_remote_path_from_hash']);
+        $fs->method('get_remote_path_from_hash')->willReturn($filepath);
+
+        $result = $fs->mimetype_from_hash($contenthash, $filename);
+        $this->assertEquals('image/jpeg', $result);
+    }
+
+    /**
+     * Test that mimetype_from_hash returns the correct mimetype with
+     * a remotely available file whose filename does not suggest mimetype.
+     */
+    public function test_mimetype_from_hash_using_file_content_remote() {
+        $filepath = '/path/to/file/not/currently/on/disk';
+        $filecontent = 'example content';
+        $contenthash = sha1($filecontent);
+        $filename = 'example';
+
+        $filepath = __DIR__ . DIRECTORY_SEPARATOR . 'fixtures' . DIRECTORY_SEPARATOR . 'testimage.jpg';
+
+        $fs = $this->get_testable_mock([
+            'get_remote_path_from_hash',
+            'is_file_readable_locally_by_hash',
+            'get_local_path_from_hash',
+        ]);
+
+        $fs->method('get_remote_path_from_hash')->willReturn('/path/to/remote/file');
+        $fs->method('is_file_readable_locally_by_hash')->willReturn(false);
+        $fs->method('get_local_path_from_hash')->willReturn($filepath);
+
+        $result = $fs->mimetype_from_hash($contenthash, $filename);
+        $this->assertEquals('image/jpeg', $result);
+    }
+
+    /**
+     * Test that mimetype_from_storedfile returns the correct mimetype with
+     * a file whose filename suggests mimetype.
+     */
+    public function test_mimetype_from_storedfile_empty() {
+        $file = $this->get_stored_file('');
+
+        $fs = $this->get_testable_mock();
+        $result = $fs->mimetype_from_storedfile($file);
+        $this->assertNull($result);
+    }
+
+    /**
+     * Test that mimetype_from_storedfile returns the correct mimetype with
+     * a file whose filename suggests mimetype.
+     */
+    public function test_mimetype_from_storedfile_using_filename() {
+        $filepath = '/path/to/file/not/currently/on/disk';
+        $fs = $this->get_testable_mock(['get_remote_path_from_storedfile']);
+        $fs->method('get_remote_path_from_storedfile')->willReturn($filepath);
+
+        $file = $this->get_stored_file('example content', 'test.jpg');
+
+        $result = $fs->mimetype_from_storedfile($file);
+        $this->assertEquals('image/jpeg', $result);
+    }
+
+    /**
+     * Test that mimetype_from_storedfile returns the correct mimetype with
+     * a locally available file whose filename does not suggest mimetype.
+     */
+    public function test_mimetype_from_storedfile_using_file_content() {
+        $filepath = __DIR__ . DIRECTORY_SEPARATOR . 'fixtures' . DIRECTORY_SEPARATOR . 'testimage.jpg';
+        $fs = $this->get_testable_mock(['get_remote_path_from_storedfile']);
+        $fs->method('get_remote_path_from_storedfile')->willReturn($filepath);
+
+        $file = $this->get_stored_file('example content');
+
+        $result = $fs->mimetype_from_storedfile($file);
+        $this->assertEquals('image/jpeg', $result);
+    }
+
+    /**
+     * Test that mimetype_from_storedfile returns the correct mimetype with
+     * a remotely available file whose filename does not suggest mimetype.
+     */
+    public function test_mimetype_from_storedfile_using_file_content_remote() {
+        $filepath = __DIR__ . DIRECTORY_SEPARATOR . 'fixtures' . DIRECTORY_SEPARATOR . 'testimage.jpg';
+
+        $fs = $this->get_testable_mock([
+            'get_remote_path_from_storedfile',
+            'is_file_readable_locally_by_storedfile',
+            'get_local_path_from_storedfile',
+        ]);
+
+        $fs->method('get_remote_path_from_storedfile')->willReturn('/path/to/remote/file');
+        $fs->method('is_file_readable_locally_by_storedfile')->willReturn(false);
+        $fs->method('get_local_path_from_storedfile')->willReturn($filepath);
+
+        $file = $this->get_stored_file('example content');
+
+        $result = $fs->mimetype_from_storedfile($file);
+        $this->assertEquals('image/jpeg', $result);
+    }
+
+    /**
+     * Data Provider for is_image_from_storedfile tests.
+     *
+     * @return array
+     */
+    public function is_image_from_storedfile_provider() {
+        return array(
+            'Standard image'            => array('image/png', true),
+            'Made up document/image'    => array('document/image', false),
+        );
+    }
+
+    /**
+     * Data provider for get_local_path_from_storedfile tests.
+     *
+     * @return array
+     */
+    public function get_local_path_from_storedfile_provider() {
+        return [
+            'default args (nofetch)' => [
+                'args' => [],
+                'fetch' => 0,
+            ],
+            'explicit: nofetch' => [
+                'args' => [false],
+                'fetch' => 0,
+            ],
+            'explicit: fetch' => [
+                'args' => [true],
+                'fetch' => 1,
+            ],
+        ];
+    }
+}
diff --git a/lib/filestorage/tests/fixtures/test.tgz b/lib/filestorage/tests/fixtures/test.tgz
new file mode 100644 (file)
index 0000000..6c92dce
Binary files /dev/null and b/lib/filestorage/tests/fixtures/test.tgz differ
index 966112f..497fea4 100644 (file)
@@ -6369,30 +6369,23 @@ function email_is_not_allowed($email) {
  *
  * @return file_storage
  */
-function get_file_storage() {
+function get_file_storage($reset = false) {
     global $CFG;
 
     static $fs = null;
 
+    if ($reset) {
+        $fs = null;
+        return;
+    }
+
     if ($fs) {
         return $fs;
     }
 
     require_once("$CFG->libdir/filelib.php");
 
-    if (isset($CFG->filedir)) {
-        $filedir = $CFG->filedir;
-    } else {
-        $filedir = $CFG->dataroot.'/filedir';
-    }
-
-    if (isset($CFG->trashdir)) {
-        $trashdirdir = $CFG->trashdir;
-    } else {
-        $trashdirdir = $CFG->dataroot.'/trashdir';
-    }
-
-    $fs = new file_storage($filedir, $trashdirdir, "$CFG->tempdir/filestorage", $CFG->directorypermissions, $CFG->filepermissions);
+    $fs = new file_storage();
 
     return $fs;
 }
index be273d5..6b5d4ac 100644 (file)
@@ -18,6 +18,10 @@ information provided here is intended especially for developers.
 * The mcore YUI rollup which included various YUI modules such as moodle-core-notification is no longer included on every
   page. Missing YUI depdencies may be exposed by this change (e.g. missing a requirement on moodle-core-notification when
   using M.core.dialogue).
+* The following functions have been deprecated and should not be used any more:
+  - file_storage::try_content_recovery  - See MDL-46375 for more information
+  - file_storage::content_exists        - See MDL-46375 for more information
+  - file_storage::deleted_file_cleanup  - See MDL-46375 for more information
 
 === 3.2 ===
 
index 5033827..a2bf4fb 100644 (file)
@@ -47,7 +47,7 @@ class qformat_blackboard_six_base extends qformat_based_on_xml {
 
     /**
      * Check if the given file is capable of being imported by this plugin.
-     * As {@link file_storage::mimetype()} now uses finfo PHP extension if available,
+     * As {@link file_storage::mimetype()} may use finfo PHP extension if available,
      * the value returned by $file->get_mimetype for a .dat file is not the same on all servers.
      * So we must made 2 checks to verify if the plugin can import the file.
      * @param stored_file $file the file to check
index 5d0d905..5cfd4bf 100644 (file)
@@ -1717,12 +1717,13 @@ abstract class repository implements cacheable_object {
                                                                         'size' => $maxbytesdisplay));
             }
             $fs = get_file_storage();
-            $contentexists = $fs->content_exists($file->get_contenthash());
-            if ($contentexists && $file->get_filesize() && $file->get_contenthash() === sha1('')) {
-                // even when 'file_storage::content_exists()' returns true this may be an empty
-                // content for the file that was not actually downloaded
-                $contentexists = false;
-            }
+
+            // If a file has been downloaded, the file record should report both a positive file
+            // size, and a contenthash which does not related to empty content.
+            // If thereis no file size, or the contenthash is for an empty file, then the file has
+            // yet to be successfully downloaded.
+            $contentexists = $file->get_filesize() && $file->get_contenthash() !== sha1('');
+
             if (!$file->get_status() && $contentexists) {
                 // we already have the content in moodle filepool and it was synchronised recently.
                 // Repositories may overwrite it if they want to force synchronisation anyway!