MDL-59961 core_files: make content hash validation reusable
[moodle.git] / lib / filestorage / file_system_filedir.php
CommitLineData
16a34ae1
AN
1<?php
2// This file is part of Moodle - http://moodle.org/
3//
4// Moodle is free software: you can redistribute it and/or modify
5// it under the terms of the GNU General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8//
9// Moodle is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12// GNU General Public License for more details.
13//
14// You should have received a copy of the GNU General Public License
15// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
16
17/**
18 * Core file system class definition.
19 *
20 * @package core_files
21 * @copyright 2017 Andrew Nicols <andrew@nicols.co.uk>
22 * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
23 */
24
25defined('MOODLE_INTERNAL') || die();
26
27/**
28 * File system class used for low level access to real files in filedir.
29 *
30 * @package core_files
31 * @category files
32 * @copyright 2017 Andrew Nicols <andrew@nicols.co.uk>
33 * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
34 */
35class file_system_filedir extends file_system {
36
37 /**
38 * @var string The path to the local copy of the filedir.
39 */
40 protected $filedir = null;
41
42 /**
43 * @var string The path to the trashdir.
44 */
45 protected $trashdir = null;
46
47 /**
48 * @var string Default directory permissions for new dirs.
49 */
50 protected $dirpermissions = null;
51
52 /**
53 * @var string Default file permissions for new files.
54 */
55 protected $filepermissions = null;
56
57
58 /**
59 * Perform any custom setup for this type of file_system.
60 */
61 public function __construct() {
62 global $CFG;
63
64 if (isset($CFG->filedir)) {
65 $this->filedir = $CFG->filedir;
66 } else {
67 $this->filedir = $CFG->dataroot.'/filedir';
68 }
69
70 if (isset($CFG->trashdir)) {
71 $this->trashdir = $CFG->trashdir;
72 } else {
73 $this->trashdir = $CFG->dataroot.'/trashdir';
74 }
75
76 $this->dirpermissions = $CFG->directorypermissions;
77 $this->filepermissions = $CFG->filepermissions;
78
79 // Make sure the file pool directory exists.
80 if (!is_dir($this->filedir)) {
81 if (!mkdir($this->filedir, $this->dirpermissions, true)) {
82 // Permission trouble.
83 throw new file_exception('storedfilecannotcreatefiledirs');
84 }
85
86 // Place warning file in file pool root.
87 if (!file_exists($this->filedir.'/warning.txt')) {
88 file_put_contents($this->filedir.'/warning.txt',
89 'This directory contains the content of uploaded files and is controlled by Moodle code. ' .
90 'Do not manually move, change or rename any of the files and subdirectories here.');
91 chmod($this->filedir . '/warning.txt', $this->filepermissions);
92 }
93 }
94
95 // Make sure the trashdir directory exists too.
96 if (!is_dir($this->trashdir)) {
97 if (!mkdir($this->trashdir, $this->dirpermissions, true)) {
98 // Permission trouble.
99 throw new file_exception('storedfilecannotcreatefiledirs');
100 }
101 }
102 }
103
104 /**
105 * Get the full path for the specified hash, including the path to the filedir.
106 *
107 * @param string $contenthash The content hash
108 * @param bool $fetchifnotfound Whether to attempt to fetch from the remote path if not found.
109 * @return string The full path to the content file
110 */
111 protected function get_local_path_from_hash($contenthash, $fetchifnotfound = false) {
90202125 112 return $this->get_fulldir_from_hash($contenthash) . '/' .$contenthash;
16a34ae1
AN
113 }
114
115 /**
116 * Get a remote filepath for the specified stored file.
117 *
118 * @param stored_file $file The file to fetch the path for
119 * @param bool $fetchifnotfound Whether to attempt to fetch from the remote path if not found.
120 * @return string The full path to the content file
121 */
122 protected function get_local_path_from_storedfile(stored_file $file, $fetchifnotfound = false) {
123 $filepath = $this->get_local_path_from_hash($file->get_contenthash(), $fetchifnotfound);
124
125 // Try content recovery.
126 if ($fetchifnotfound && !is_readable($filepath)) {
127 $this->recover_file($file);
128 }
129
130 return $filepath;
131 }
132
133 /**
134 * Get a remote filepath for the specified stored file.
135 *
136 * @param stored_file $file The file to serve.
137 * @return string full path to pool file with file content
138 */
139 protected function get_remote_path_from_storedfile(stored_file $file) {
140 return $this->get_local_path_from_storedfile($file, false);
141 }
142
143 /**
144 * Get the full path for the specified hash, including the path to the filedir.
145 *
146 * @param string $contenthash The content hash
147 * @return string The full path to the content file
148 */
149 protected function get_remote_path_from_hash($contenthash) {
150 return $this->get_local_path_from_hash($contenthash, false);
151 }
152
153 /**
154 * Get the full directory to the stored file, including the path to the
155 * filedir, and the directory which the file is actually in.
156 *
157 * Note: This function does not ensure that the file is present on disk.
158 *
159 * @param stored_file $file The file to fetch details for.
160 * @return string The full path to the content directory
161 */
162 protected function get_fulldir_from_storedfile(stored_file $file) {
163 return $this->get_fulldir_from_hash($file->get_contenthash());
164 }
165
166 /**
167 * Get the full directory to the stored file, including the path to the
168 * filedir, and the directory which the file is actually in.
169 *
170 * @param string $contenthash The content hash
171 * @return string The full path to the content directory
172 */
173 protected function get_fulldir_from_hash($contenthash) {
90202125 174 return $this->filedir . '/' . $this->get_contentdir_from_hash($contenthash);
16a34ae1
AN
175 }
176
177 /**
178 * Get the content directory for the specified content hash.
179 * This is the directory that the file will be in, but without the
180 * fulldir.
181 *
182 * @param string $contenthash The content hash
183 * @return string The directory within filedir
184 */
185 protected function get_contentdir_from_hash($contenthash) {
186 $l1 = $contenthash[0] . $contenthash[1];
187 $l2 = $contenthash[2] . $contenthash[3];
188 return "$l1/$l2";
189 }
190
191 /**
192 * Get the content path for the specified content hash within filedir.
193 *
194 * This does not include the filedir, and is often used by file systems
195 * as the object key for storage and retrieval.
196 *
197 * @param string $contenthash The content hash
198 * @return string The filepath within filedir
199 */
200 protected function get_contentpath_from_hash($contenthash) {
90202125 201 return $this->get_contentdir_from_hash($contenthash) . '/' . $contenthash;
16a34ae1
AN
202 }
203
204 /**
205 * Get the full directory for the specified hash in the trash, including the path to the
206 * trashdir, and the directory which the file is actually in.
207 *
208 * @param string $contenthash The content hash
209 * @return string The full path to the trash directory
210 */
211 protected function get_trash_fulldir_from_hash($contenthash) {
90202125 212 return $this->trashdir . '/' . $this->get_contentdir_from_hash($contenthash);
16a34ae1
AN
213 }
214
215 /**
216 * Get the full path for the specified hash in the trash, including the path to the trashdir.
217 *
218 * @param string $contenthash The content hash
219 * @return string The full path to the trash file
220 */
221 protected function get_trash_fullpath_from_hash($contenthash) {
90202125 222 return $this->trashdir . '/' . $this->get_contentpath_from_hash($contenthash);
16a34ae1
AN
223 }
224
225 /**
226 * Copy content of file to given pathname.
227 *
228 * @param stored_file $file The file to be copied
229 * @param string $target real path to the new file
230 * @return bool success
231 */
232 public function copy_content_from_storedfile(stored_file $file, $target) {
233 $source = $this->get_local_path_from_storedfile($file, true);
234 return copy($source, $target);
235 }
236
237 /**
238 * Tries to recover missing content of file from trash.
239 *
240 * @param stored_file $file stored_file instance
241 * @return bool success
242 */
243 protected function recover_file(stored_file $file) {
244 $contentfile = $this->get_local_path_from_storedfile($file, false);
245
246 if (file_exists($contentfile)) {
247 // The file already exists on the file system. No need to recover.
248 return true;
249 }
250
251 $contenthash = $file->get_contenthash();
252 $contentdir = $this->get_fulldir_from_storedfile($file);
253 $trashfile = $this->get_trash_fullpath_from_hash($contenthash);
90202125 254 $alttrashfile = "{$this->trashdir}/{$contenthash}";
16a34ae1
AN
255
256 if (!is_readable($trashfile)) {
257 // The trash file was not found. Check the alternative trash file too just in case.
258 if (!is_readable($alttrashfile)) {
259 return false;
260 }
261 // The alternative trash file in trash root exists.
262 $trashfile = $alttrashfile;
263 }
264
a30a04fa 265 if (filesize($trashfile) != $file->get_filesize() or file_storage::hash_from_path($trashfile) != $contenthash) {
16a34ae1
AN
266 // The files are different. Leave this one in trash - something seems to be wrong with it.
267 return false;
268 }
269
270 if (!is_dir($contentdir)) {
271 if (!mkdir($contentdir, $this->dirpermissions, true)) {
272 // Unable to create the target directory.
273 return false;
274 }
275 }
276
277 // Perform a rename - these are generally atomic which gives us big
278 // performance wins, especially for large files.
279 return rename($trashfile, $contentfile);
280 }
281
282 /**
283 * Marks pool file as candidate for deleting.
284 *
285 * @param string $contenthash
286 */
287 public function remove_file($contenthash) {
288 if (!self::is_file_removable($contenthash)) {
289 // Don't remove the file - it's still in use.
290 return;
291 }
292
293 if (!$this->is_file_readable_remotely_by_hash($contenthash)) {
294 // The file wasn't found in the first place. Just ignore it.
295 return;
296 }
297
298 $trashpath = $this->get_trash_fulldir_from_hash($contenthash);
299 $trashfile = $this->get_trash_fullpath_from_hash($contenthash);
300 $contentfile = $this->get_local_path_from_hash($contenthash, true);
301
302 if (!is_dir($trashpath)) {
303 mkdir($trashpath, $this->dirpermissions, true);
304 }
305
306 if (file_exists($trashfile)) {
307 // A copy of this file is already in the trash.
308 // Remove the old version.
309 unlink($contentfile);
310 return;
311 }
312
313 // Move the contentfile to the trash, and fix permissions as required.
314 rename($contentfile, $trashfile);
315
316 // Fix permissions, only if needed.
317 $currentperms = octdec(substr(decoct(fileperms($trashfile)), -4));
318 if ((int)$this->filepermissions !== $currentperms) {
319 chmod($trashfile, $this->filepermissions);
320 }
321 }
322
323 /**
324 * Cleanup the trash directory.
325 */
326 public function cron() {
327 $this->empty_trash();
328 }
329
330 protected function empty_trash() {
331 fulldelete($this->trashdir);
332 set_config('fileslastcleanup', time());
333 }
334
335 /**
336 * Add the supplied file to the file system.
337 *
338 * Note: If overriding this function, it is advisable to store the file
339 * in the path returned by get_local_path_from_hash as there may be
340 * subsequent uses of the file in the same request.
341 *
342 * @param string $pathname Path to file currently on disk
343 * @param string $contenthash SHA1 hash of content if known (performance only)
344 * @return array (contenthash, filesize, newfile)
345 */
346 public function add_file_from_path($pathname, $contenthash = null) {
16a34ae1 347
9eb1a2c3 348 list($contenthash, $filesize) = $this->validate_hash_and_file_size($contenthash, $pathname);
16a34ae1
AN
349
350 $hashpath = $this->get_fulldir_from_hash($contenthash);
351 $hashfile = $this->get_local_path_from_hash($contenthash, false);
352
353 $newfile = true;
354
355 if (file_exists($hashfile)) {
356 if (filesize($hashfile) === $filesize) {
357 return array($contenthash, $filesize, false);
358 }
a30a04fa
AN
359 if (file_storage::hash_from_path($hashfile) === $contenthash) {
360 // Jackpot! We have a hash collision.
16a34ae1
AN
361 mkdir("$this->filedir/jackpot/", $this->dirpermissions, true);
362 copy($pathname, "$this->filedir/jackpot/{$contenthash}_1");
363 copy($hashfile, "$this->filedir/jackpot/{$contenthash}_2");
364 throw new file_pool_content_exception($contenthash);
365 }
366 debugging("Replacing invalid content file $contenthash");
367 unlink($hashfile);
368 $newfile = false;
369 }
370
371 if (!is_dir($hashpath)) {
372 if (!mkdir($hashpath, $this->dirpermissions, true)) {
373 // Permission trouble.
374 throw new file_exception('storedfilecannotcreatefiledirs');
375 }
376 }
377
378 // Let's try to prevent some race conditions.
379
380 $prev = ignore_user_abort(true);
381 @unlink($hashfile.'.tmp');
382 if (!copy($pathname, $hashfile.'.tmp')) {
383 // Borked permissions or out of disk space.
efa7d241 384 @unlink($hashfile.'.tmp');
16a34ae1
AN
385 ignore_user_abort($prev);
386 throw new file_exception('storedfilecannotcreatefile');
387 }
a30a04fa 388 if (file_storage::hash_from_path($hashfile.'.tmp') !== $contenthash) {
961323a0
TB
389 // Highly unlikely edge case, but this can happen on an NFS volume with no space remaining.
390 @unlink($hashfile.'.tmp');
16a34ae1
AN
391 ignore_user_abort($prev);
392 throw new file_exception('storedfilecannotcreatefile');
393 }
394 rename($hashfile.'.tmp', $hashfile);
395 chmod($hashfile, $this->filepermissions); // Fix permissions if needed.
396 @unlink($hashfile.'.tmp'); // Just in case anything fails in a weird way.
397 ignore_user_abort($prev);
398
399 return array($contenthash, $filesize, $newfile);
400 }
401
402 /**
403 * Add a file with the supplied content to the file system.
404 *
405 * Note: If overriding this function, it is advisable to store the file
406 * in the path returned by get_local_path_from_hash as there may be
407 * subsequent uses of the file in the same request.
408 *
409 * @param string $content file content - binary string
410 * @return array (contenthash, filesize, newfile)
411 */
412 public function add_file_from_string($content) {
413 global $CFG;
414
a30a04fa 415 $contenthash = file_storage::hash_from_string($content);
16a34ae1
AN
416 // Binary length.
417 $filesize = strlen($content);
418
419 $hashpath = $this->get_fulldir_from_hash($contenthash);
420 $hashfile = $this->get_local_path_from_hash($contenthash, false);
421
422 $newfile = true;
423
424 if (file_exists($hashfile)) {
425 if (filesize($hashfile) === $filesize) {
426 return array($contenthash, $filesize, false);
427 }
a30a04fa
AN
428 if (file_storage::hash_from_path($hashfile) === $contenthash) {
429 // Jackpot! We have a hash collision.
16a34ae1
AN
430 mkdir("$this->filedir/jackpot/", $this->dirpermissions, true);
431 copy($hashfile, "$this->filedir/jackpot/{$contenthash}_1");
432 file_put_contents("$this->filedir/jackpot/{$contenthash}_2", $content);
433 throw new file_pool_content_exception($contenthash);
434 }
435 debugging("Replacing invalid content file $contenthash");
436 unlink($hashfile);
437 $newfile = false;
438 }
439
440 if (!is_dir($hashpath)) {
441 if (!mkdir($hashpath, $this->dirpermissions, true)) {
442 // Permission trouble.
443 throw new file_exception('storedfilecannotcreatefiledirs');
444 }
445 }
446
447 // Hopefully this works around most potential race conditions.
448
449 $prev = ignore_user_abort(true);
450
451 if (!empty($CFG->preventfilelocking)) {
452 $newsize = file_put_contents($hashfile.'.tmp', $content);
453 } else {
454 $newsize = file_put_contents($hashfile.'.tmp', $content, LOCK_EX);
455 }
456
457 if ($newsize === false) {
458 // Borked permissions most likely.
459 ignore_user_abort($prev);
460 throw new file_exception('storedfilecannotcreatefile');
461 }
462 if (filesize($hashfile.'.tmp') !== $filesize) {
463 // Out of disk space?
464 unlink($hashfile.'.tmp');
465 ignore_user_abort($prev);
466 throw new file_exception('storedfilecannotcreatefile');
467 }
468 rename($hashfile.'.tmp', $hashfile);
469 chmod($hashfile, $this->filepermissions); // Fix permissions if needed.
470 @unlink($hashfile.'.tmp'); // Just in case anything fails in a weird way.
471 ignore_user_abort($prev);
472
473 return array($contenthash, $filesize, $newfile);
474 }
475
476}