From: Miklos Szeredi &lt;mszeredi(a)redhat.com&gt; commit 6433b8998a21dc597002731c4ceb4144e856edc4 upstream Add flag returned by OPENDIR request to allow kernel to cache directory contents in page cache. The effect of FOPEN_CACHE_DIR is twofold: a) if not already cached, it writes entries into the cache b) if already cached, it allows reading entries from the cache The FOPEN_KEEP_CACHE has the same effect as on regular files: unless this flag is given the cache is cleared upon completion of open. So FOPEN_KEEP_CACHE and FOPEN_KEEP_CACHE flags should be used together to make use of the directory caching facility introduced in the following patches. The FUSE_AUTO_INVAL_DATA flag returned in INIT reply also has the same affect on the directory cache as it has on data cache for regular files. Signed-off-by: Miklos Szeredi &lt;mszeredi(a)redhat.com&gt; Signed-off-by: Jeffle Xu &lt;jefflexu(a)linux.alibaba.com&gt; --- include/uapi/linux/fuse.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index 969dbd591e37..f008a61c5506 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -116,6 +116,9 @@ * * 7.27 * - add FUSE_ABORT_ERROR + * + * 7.28 + * - add FOPEN_CACHE_DIR */ #ifndef _LINUX_FUSE_H @@ -219,11 +222,13 @@ struct fuse_file_lock { * FOPEN_DIRECT_IO: bypass page cache for this open file * FOPEN_KEEP_CACHE: don't invalidate the data cache on open * FOPEN_NONSEEKABLE: the file is not seekable + * FOPEN_CACHE_DIR: allow caching this directory * FOPEN_STREAM: the file is stream-like (no file position at all) */ #define FOPEN_DIRECT_IO (1 << 0) #define FOPEN_KEEP_CACHE (1 << 1) #define FOPEN_NONSEEKABLE (1 << 2) +#define FOPEN_CACHE_DIR (1 << 3) #define FOPEN_STREAM (1 << 4) /** -- 2.27.0

From: Miklos Szeredi &lt;mszeredi(a)redhat.com&gt; commit 18172b10b674a7cd5340b2dd70202ce6622400bd upstream Prepare for cache filling by introducing a helper for emitting a single directory entry. Signed-off-by: Miklos Szeredi &lt;mszeredi(a)redhat.com&gt; Signed-off-by: Jeffle Xu &lt;jefflexu(a)linux.alibaba.com&gt; --- fs/fuse/readdir.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/fs/fuse/readdir.c b/fs/fuse/readdir.c index 499455687e37..a107291219a3 100644 --- a/fs/fuse/readdir.c +++ b/fs/fuse/readdir.c @@ -26,6 +26,13 @@ static bool fuse_use_readdirplus(struct inode *dir, struct dir_context *ctx) return false; } +static bool fuse_emit(struct file *file, struct dir_context *ctx, + struct fuse_dirent *dirent) +{ + return dir_emit(ctx, dirent->name, dirent->namelen, dirent->ino, + dirent->type); +} + static int parse_dirfile(char *buf, size_t nbytes, struct file *file, struct dir_context *ctx) { @@ -39,8 +46,7 @@ static int parse_dirfile(char *buf, size_t nbytes, struct file *file, if (memchr(dirent->name, '/', dirent->namelen) != NULL) return -EIO; - if (!dir_emit(ctx, dirent->name, dirent->namelen, - dirent->ino, dirent->type)) + if (!fuse_emit(file, ctx, dirent)) break; buf += reclen; @@ -183,8 +189,7 @@ static int parse_dirplusfile(char *buf, size_t nbytes, struct file *file, we need to send a FORGET for each of those which we did not link. */ - over = !dir_emit(ctx, dirent->name, dirent->namelen, - dirent->ino, dirent->type); + over = !fuse_emit(file, ctx, dirent); if (!over) ctx->pos = dirent->off; } -- 2.27.0

From: Miklos Szeredi &lt;mszeredi(a)redhat.com&gt; commit 69e34551152a286f827d54dcb5700da6aeaac1fb upstream This patch just adds the cache filling functions, which are invoked if FOPEN_CACHE_DIR flag is set in the OPENDIR reply. Cache reading and cache invalidation are added by subsequent patches. The directory cache uses the page cache. Directory entries are packed into a page in the same format as in the READDIR reply. A page only contains whole entries, the space at the end of the page is cleared. The page is locked while being modified. Multiple parallel readdirs on the same directory can fill the cache; the only constraint is that continuity must be maintained (d_off of last entry points to position of current entry). Signed-off-by: Miklos Szeredi &lt;mszeredi(a)redhat.com&gt; Signed-off-by: Jeffle Xu &lt;jefflexu(a)linux.alibaba.com&gt; --- fs/fuse/fuse_i.h | 15 ++++++++ fs/fuse/inode.c | 4 +++ fs/fuse/readdir.c | 91 ++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 109 insertions(+), 1 deletion(-) diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 83632816b581..7cdc5246bb24 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -141,6 +141,21 @@ struct fuse_inode { /** List of writepage requestst (pending or sent) */ struct list_head writepages; + /* readdir cache */ + struct { + /* true if fully cached */ + bool cached; + + /* size of cache */ + loff_t size; + + /* position at end of cache (position of next entry) */ + loff_t pos; + + /* protects above fields */ + spinlock_t lock; + } rdc; + /** Miscellaneous bits describing inode state */ unsigned long state; diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index e66d9b402668..477f6f40cf5a 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -85,6 +85,10 @@ static struct inode *fuse_alloc_inode(struct super_block *sb) INIT_LIST_HEAD(&fi->queued_writes); INIT_LIST_HEAD(&fi->writepages); init_waitqueue_head(&fi->page_waitq); + spin_lock_init(&fi->rdc.lock); + fi->rdc.cached = false; + fi->rdc.size = 0; + fi->rdc.pos = 0; mutex_init(&fi->mutex); init_rwsem(&fi->i_mmap_sem); spin_lock_init(&fi->lock); diff --git a/fs/fuse/readdir.c b/fs/fuse/readdir.c index a107291219a3..61e985606232 100644 --- a/fs/fuse/readdir.c +++ b/fs/fuse/readdir.c @@ -9,6 +9,8 @@ #include "fuse_i.h" #include <linux/posix_acl.h> +#include <linux/pagemap.h> +#include <linux/highmem.h> static bool fuse_use_readdirplus(struct inode *dir, struct dir_context *ctx) { @@ -26,9 +28,91 @@ static bool fuse_use_readdirplus(struct inode *dir, struct dir_context *ctx) return false; } +static void fuse_add_dirent_to_cache(struct file *file, + struct fuse_dirent *dirent, loff_t pos) +{ + struct fuse_inode *fi = get_fuse_inode(file_inode(file)); + size_t reclen = FUSE_DIRENT_SIZE(dirent); + pgoff_t index; + struct page *page; + loff_t size; + unsigned int offset; + void *addr; + + spin_lock(&fi->rdc.lock); + /* + * Is cache already completed? Or this entry does not go at the end of + * cache? + */ + if (fi->rdc.cached || pos != fi->rdc.pos) { + spin_unlock(&fi->rdc.lock); + return; + } + size = fi->rdc.size; + offset = size & ~PAGE_MASK; + index = size >> PAGE_SHIFT; + /* Dirent doesn't fit in current page? Jump to next page. */ + if (offset + reclen > PAGE_SIZE) { + index++; + offset = 0; + } + spin_unlock(&fi->rdc.lock); + + if (offset) { + page = find_lock_page(file->f_mapping, index); + } else { + page = find_or_create_page(file->f_mapping, index, + mapping_gfp_mask(file->f_mapping)); + } + if (!page) + return; + + spin_lock(&fi->rdc.lock); + /* Raced with another readdir */ + if (fi->rdc.size != size || WARN_ON(fi->rdc.pos != pos)) + goto unlock; + + addr = kmap_atomic(page); + if (!offset) + clear_page(addr); + memcpy(addr + offset, dirent, reclen); + kunmap_atomic(addr); + fi->rdc.size = (index << PAGE_SHIFT) + offset + reclen; + fi->rdc.pos = dirent->off; +unlock: + spin_unlock(&fi->rdc.lock); + unlock_page(page); + put_page(page); +} + +static void fuse_readdir_cache_end(struct file *file, loff_t pos) +{ + struct fuse_inode *fi = get_fuse_inode(file_inode(file)); + loff_t end; + + spin_lock(&fi->rdc.lock); + /* does cache end position match current position? */ + if (fi->rdc.pos != pos) { + spin_unlock(&fi->rdc.lock); + return; + } + + fi->rdc.cached = true; + end = ALIGN(fi->rdc.size, PAGE_SIZE); + spin_unlock(&fi->rdc.lock); + + /* truncate unused tail of cache */ + truncate_inode_pages(file->f_mapping, end); +} + static bool fuse_emit(struct file *file, struct dir_context *ctx, struct fuse_dirent *dirent) { + struct fuse_file *ff = file->private_data; + + if (ff->open_flags & FOPEN_CACHE_DIR) + fuse_add_dirent_to_cache(file, dirent, ctx->pos); + return dir_emit(ctx, dirent->name, dirent->namelen, dirent->ino, dirent->type); } @@ -249,7 +333,12 @@ int fuse_readdir(struct file *file, struct dir_context *ctx) err = req->out.h.error; fuse_put_request(fc, req); if (!err) { - if (plus) { + if (!nbytes) { + struct fuse_file *ff = file->private_data; + + if (ff->open_flags & FOPEN_CACHE_DIR) + fuse_readdir_cache_end(file, ctx->pos); + } else if (plus) { err = parse_dirplusfile(page_address(page), nbytes, file, ctx, attr_version); } else { -- 2.27.0

From: Miklos Szeredi &lt;mszeredi(a)redhat.com&gt; commit 5d7bc7e8680c7ca4c8a4f139ce2a54ccb8131ef0 upstream The cache is only used if it's completed, not while it's still being filled; this constraint could be lifted later, if it turns out to be useful. Introduce state in struct fuse_file that indicates the position within the cache. After a seek, reset the position to the beginning of the cache and search the cache for the current position. If the current position is not found in the cache, then fall back to uncached readdir. It can also happen that page(s) disappear from the cache, in which case we must also fall back to uncached readdir. Signed-off-by: Miklos Szeredi &lt;mszeredi(a)redhat.com&gt; Signed-off-by: Jeffle Xu &lt;jefflexu(a)linux.alibaba.com&gt; --- fs/fuse/file.c | 2 + fs/fuse/fuse_i.h | 15 +++++ fs/fuse/readdir.c | 148 ++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 161 insertions(+), 4 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 602a9990fa7b..80f991ac4a0a 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -57,6 +57,7 @@ struct fuse_file *fuse_file_alloc(struct fuse_conn *fc) } INIT_LIST_HEAD(&ff->write_entry); + mutex_init(&ff->readdir.lock); refcount_set(&ff->count, 1); RB_CLEAR_NODE(&ff->polled_node); init_waitqueue_head(&ff->poll_wait); @@ -71,6 +72,7 @@ struct fuse_file *fuse_file_alloc(struct fuse_conn *fc) void fuse_file_free(struct fuse_file *ff) { fuse_request_free(ff->reserved_req); + mutex_destroy(&ff->readdir.lock); kfree(ff); } diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 7cdc5246bb24..c310ddb5f7ab 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -221,6 +221,21 @@ struct fuse_file { /** Entry on inode's write_files list */ struct list_head write_entry; + /* Readdir related */ + struct { + /* + * Protects below fields against (crazy) parallel readdir on + * same open file. Uncontended in the normal case. + */ + struct mutex lock; + + /* Dir stream position */ + loff_t pos; + + /* Offset in cache */ + loff_t cache_off; + } readdir; + /** RB node to be linked on fuse_conn->polled_files */ struct rb_node polled_node; diff --git a/fs/fuse/readdir.c b/fs/fuse/readdir.c index 61e985606232..06702958724b 100644 --- a/fs/fuse/readdir.c +++ b/fs/fuse/readdir.c @@ -289,7 +289,7 @@ static int parse_dirplusfile(char *buf, size_t nbytes, struct file *file, return 0; } -int fuse_readdir(struct file *file, struct dir_context *ctx) +static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx) { int plus, err; size_t nbytes; @@ -300,9 +300,6 @@ int fuse_readdir(struct file *file, struct dir_context *ctx) u64 attr_version = 0; bool locked; - if (is_bad_inode(inode)) - return -EIO; - req = fuse_get_req(fc, 1); if (IS_ERR(req)) return PTR_ERR(req); @@ -351,3 +348,146 @@ int fuse_readdir(struct file *file, struct dir_context *ctx) fuse_invalidate_atime(inode); return err; } + +enum fuse_parse_result { + FOUND_ERR = -1, + FOUND_NONE = 0, + FOUND_SOME, + FOUND_ALL, +}; + +static enum fuse_parse_result fuse_parse_cache(struct fuse_file *ff, + void *addr, unsigned int size, + struct dir_context *ctx) +{ + unsigned int offset = ff->readdir.cache_off & ~PAGE_MASK; + enum fuse_parse_result res = FOUND_NONE; + + WARN_ON(offset >= size); + + for (;;) { + struct fuse_dirent *dirent = addr + offset; + unsigned int nbytes = size - offset; + size_t reclen = FUSE_DIRENT_SIZE(dirent); + + if (nbytes < FUSE_NAME_OFFSET || !dirent->namelen) + break; + + if (WARN_ON(dirent->namelen > FUSE_NAME_MAX)) + return FOUND_ERR; + if (WARN_ON(reclen > nbytes)) + return FOUND_ERR; + if (WARN_ON(memchr(dirent->name, '/', dirent->namelen) != NULL)) + return FOUND_ERR; + + if (ff->readdir.pos == ctx->pos) { + res = FOUND_SOME; + if (!dir_emit(ctx, dirent->name, dirent->namelen, + dirent->ino, dirent->type)) + return FOUND_ALL; + ctx->pos = dirent->off; + } + ff->readdir.pos = dirent->off; + ff->readdir.cache_off += reclen; + + offset += reclen; + } + + return res; +} + +#define UNCACHED 1 + +static int fuse_readdir_cached(struct file *file, struct dir_context *ctx) +{ + struct fuse_file *ff = file->private_data; + struct inode *inode = file_inode(file); + struct fuse_inode *fi = get_fuse_inode(inode); + enum fuse_parse_result res; + pgoff_t index; + unsigned int size; + struct page *page; + void *addr; + + /* Seeked? If so, reset the cache stream */ + if (ff->readdir.pos != ctx->pos) { + ff->readdir.pos = 0; + ff->readdir.cache_off = 0; + } + +retry: + spin_lock(&fi->rdc.lock); + if (!fi->rdc.cached) { + spin_unlock(&fi->rdc.lock); + return UNCACHED; + } + WARN_ON(fi->rdc.size < ff->readdir.cache_off); + + index = ff->readdir.cache_off >> PAGE_SHIFT; + + if (index == (fi->rdc.size >> PAGE_SHIFT)) + size = fi->rdc.size & ~PAGE_MASK; + else + size = PAGE_SIZE; + spin_unlock(&fi->rdc.lock); + + /* EOF? */ + if ((ff->readdir.cache_off & ~PAGE_MASK) == size) + return 0; + + page = find_get_page_flags(file->f_mapping, index, + FGP_ACCESSED | FGP_LOCK); + if (!page) { + /* + * Uh-oh: page gone missing, cache is useless + */ + return UNCACHED; + } + + addr = kmap(page); + res = fuse_parse_cache(ff, addr, size, ctx); + kunmap(page); + unlock_page(page); + put_page(page); + + if (res == FOUND_ERR) + return -EIO; + + if (res == FOUND_ALL) + return 0; + + if (size == PAGE_SIZE) { + /* We hit end of page: skip to next page. */ + ff->readdir.cache_off = ALIGN(ff->readdir.cache_off, PAGE_SIZE); + goto retry; + } + + /* + * End of cache reached. If found position, then we are done, otherwise + * need to fall back to uncached, since the position we were looking for + * wasn't in the cache. + */ + return res == FOUND_SOME ? 0 : UNCACHED; +} + +int fuse_readdir(struct file *file, struct dir_context *ctx) +{ + struct fuse_file *ff = file->private_data; + struct inode *inode = file_inode(file); + int err; + + if (is_bad_inode(inode)) + return -EIO; + + mutex_lock(&ff->readdir.lock); + + err = UNCACHED; + if (ff->open_flags & FOPEN_CACHE_DIR) + err = fuse_readdir_cached(file, ctx); + if (err == UNCACHED) + err = fuse_readdir_uncached(file, ctx); + + mutex_unlock(&ff->readdir.lock); + + return err; +} -- 2.27.0

From: Tejun Heo &lt;tj(a)kernel.org&gt; commit e5854b1cdf6cb48a20e01e3bdad0476a4c60a077 upstream With DEBUG_PAGEALLOC on, the following triggers. BUG: unable to handle page fault for address: ffff88859367c000 #PF: supervisor read access in kernel mode #PF: error_code(0x0000) - not-present page PGD 3001067 P4D 3001067 PUD 406d3a8067 PMD 406d30c067 PTE 800ffffa6c983060 Oops: 0000 [#1] SMP DEBUG_PAGEALLOC CPU: 38 PID: 3110657 Comm: python2.7 RIP: 0010:fuse_readdir+0x88f/0xe7a [fuse] Code: 49 8b 4d 08 49 39 4e 60 0f 84 44 04 00 00 48 8b 43 08 43 8d 1c 3c 4d 01 7e 68 49 89 dc 48 03 5c 24 38 49 89 46 60 8b 44 24 30 <8b> 4b 10 44 29 e0 48 89 ca 48 83 c1 1f 48 83 e1 f8 83 f8 17 49 89 RSP: 0018:ffffc90035edbde0 EFLAGS: 00010286 RAX: 0000000000001000 RBX: ffff88859367bff0 RCX: 0000000000000000 RDX: 0000000000000000 RSI: ffff88859367bfed RDI: 0000000000920907 RBP: ffffc90035edbe90 R08: 000000000000014b R09: 0000000000000004 R10: ffff88859367b000 R11: 0000000000000000 R12: 0000000000000ff0 R13: ffffc90035edbee0 R14: ffff889fb8546180 R15: 0000000000000020 FS: 00007f80b5f4a740(0000) GS:ffff889fffa00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: ffff88859367c000 CR3: 0000001c170c2001 CR4: 00000000003606e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 Call Trace: iterate_dir+0x122/0x180 __x64_sys_getdents+0xa6/0x140 do_syscall_64+0x42/0x100 entry_SYSCALL_64_after_hwframe+0x44/0xa9 It's in fuse_parse_cache(). %rbx (ffff88859367bff0) is fuse_dirent pointer - addr + offset. FUSE_DIRENT_SIZE() is trying to dereference namelen off of it but that derefs into the next page which is disabled by pagealloc debug causing a PF. This is caused by dirent->namelen being accessed before ensuring that there's enough bytes in the page for the dirent. Fix it by pushing down reclen calculation. Signed-off-by: Tejun Heo &lt;tj(a)kernel.org&gt; Fixes: 5d7bc7e8680c ("fuse: allow using readdir cache") Cc: stable(a)vger.kernel.org # v4.20+ Signed-off-by: Miklos Szeredi &lt;mszeredi(a)redhat.com&gt; Signed-off-by: Jeffle Xu &lt;jefflexu(a)linux.alibaba.com&gt; --- fs/fuse/readdir.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fs/fuse/readdir.c b/fs/fuse/readdir.c index 06702958724b..b19c67e003e7 100644 --- a/fs/fuse/readdir.c +++ b/fs/fuse/readdir.c @@ -368,11 +368,13 @@ static enum fuse_parse_result fuse_parse_cache(struct fuse_file *ff, for (;;) { struct fuse_dirent *dirent = addr + offset; unsigned int nbytes = size - offset; - size_t reclen = FUSE_DIRENT_SIZE(dirent); + size_t reclen; if (nbytes < FUSE_NAME_OFFSET || !dirent->namelen) break; + reclen = FUSE_DIRENT_SIZE(dirent); /* derefs ->namelen */ + if (WARN_ON(dirent->namelen > FUSE_NAME_MAX)) return FOUND_ERR; if (WARN_ON(reclen > nbytes)) -- 2.27.0

From: Miklos Szeredi &lt;mszeredi(a)redhat.com&gt; commit 3494927e090bf511e54eecaf33a8e56e5c0463db upstream Allow the cache to be invalidated when page(s) have gone missing. In this case increment the version of the cache and reset to an empty state. Add a version number to the directory stream in struct fuse_file as well, indicating the version of the cache it's supposed to be reading. If the cache version doesn't match the stream's version, then reset the stream to the beginning of the cache. Signed-off-by: Miklos Szeredi &lt;mszeredi(a)redhat.com&gt; Signed-off-by: Jeffle Xu &lt;jefflexu(a)linux.alibaba.com&gt; --- fs/fuse/fuse_i.h | 7 +++++++ fs/fuse/inode.c | 1 + fs/fuse/readdir.c | 45 ++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index c310ddb5f7ab..14e468eb0200 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -152,6 +152,9 @@ struct fuse_inode { /* position at end of cache (position of next entry) */ loff_t pos; + /* version of the cache */ + u64 version; + /* protects above fields */ spinlock_t lock; } rdc; @@ -234,6 +237,10 @@ struct fuse_file { /* Offset in cache */ loff_t cache_off; + + /* Version of cache we are reading */ + u64 version; + } readdir; /** RB node to be linked on fuse_conn->polled_files */ diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 477f6f40cf5a..1757fbc0d631 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -89,6 +89,7 @@ static struct inode *fuse_alloc_inode(struct super_block *sb) fi->rdc.cached = false; fi->rdc.size = 0; fi->rdc.pos = 0; + fi->rdc.version = 0; mutex_init(&fi->mutex); init_rwsem(&fi->i_mmap_sem); spin_lock_init(&fi->lock); diff --git a/fs/fuse/readdir.c b/fs/fuse/readdir.c index b19c67e003e7..f26dff1eacc3 100644 --- a/fs/fuse/readdir.c +++ b/fs/fuse/readdir.c @@ -36,6 +36,7 @@ static void fuse_add_dirent_to_cache(struct file *file, pgoff_t index; struct page *page; loff_t size; + u64 version; unsigned int offset; void *addr; @@ -48,6 +49,7 @@ static void fuse_add_dirent_to_cache(struct file *file, spin_unlock(&fi->rdc.lock); return; } + version = fi->rdc.version; size = fi->rdc.size; offset = size & ~PAGE_MASK; index = size >> PAGE_SHIFT; @@ -69,7 +71,8 @@ static void fuse_add_dirent_to_cache(struct file *file, spin_lock(&fi->rdc.lock); /* Raced with another readdir */ - if (fi->rdc.size != size || WARN_ON(fi->rdc.pos != pos)) + if (fi->rdc.version != version || fi->rdc.size != size || + WARN_ON(fi->rdc.pos != pos)) goto unlock; addr = kmap_atomic(page); @@ -398,6 +401,14 @@ static enum fuse_parse_result fuse_parse_cache(struct fuse_file *ff, return res; } +static void fuse_rdc_reset(struct fuse_inode *fi) +{ + fi->rdc.cached = false; + fi->rdc.version++; + fi->rdc.size = 0; + fi->rdc.pos = 0; +} + #define UNCACHED 1 static int fuse_readdir_cached(struct file *file, struct dir_context *ctx) @@ -423,6 +434,21 @@ static int fuse_readdir_cached(struct file *file, struct dir_context *ctx) spin_unlock(&fi->rdc.lock); return UNCACHED; } + /* + * If cache version changed since the last getdents() call, then reset + * the cache stream. + */ + if (ff->readdir.version != fi->rdc.version) { + ff->readdir.pos = 0; + ff->readdir.cache_off = 0; + } + /* + * If at the beginning of the cache, than reset version to + * current. + */ + if (ff->readdir.pos == 0) + ff->readdir.version = fi->rdc.version; + WARN_ON(fi->rdc.size < ff->readdir.cache_off); index = ff->readdir.cache_off >> PAGE_SHIFT; @@ -439,13 +465,30 @@ static int fuse_readdir_cached(struct file *file, struct dir_context *ctx) page = find_get_page_flags(file->f_mapping, index, FGP_ACCESSED | FGP_LOCK); + spin_lock(&fi->rdc.lock); if (!page) { /* * Uh-oh: page gone missing, cache is useless */ + if (fi->rdc.version == ff->readdir.version) + fuse_rdc_reset(fi); + spin_unlock(&fi->rdc.lock); return UNCACHED; } + /* Make sure it's still the same version after getting the page. */ + if (ff->readdir.version != fi->rdc.version) { + spin_unlock(&fi->rdc.lock); + unlock_page(page); + put_page(page); + goto retry; + } + spin_unlock(&fi->rdc.lock); + + /* + * Contents of the page are now protected against changing by holding + * the page lock. + */ addr = kmap(page); res = fuse_parse_cache(ff, addr, size, ctx); kunmap(page); -- 2.27.0

From: Miklos Szeredi &lt;mszeredi(a)redhat.com&gt; commit 7118883b44b8edfea732aadeb0d4424da3f152b2 upstream Store the modification time of the directory in the cache, obtained before starting to fill the cache. When reading the cache, verify that the directory hasn't changed, by checking if current modification time is the same as the one stored in the cache. This only needs to be done when the current file position is at the beginning of the directory, as mandated by POSIX. Signed-off-by: Miklos Szeredi &lt;mszeredi(a)redhat.com&gt; Signed-off-by: Jeffle Xu &lt;jefflexu(a)linux.alibaba.com&gt; --- fs/fuse/fuse_i.h | 3 +++ fs/fuse/readdir.c | 38 ++++++++++++++++++++++++++++++++++---- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 14e468eb0200..6f1e69a9d0ce 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -155,6 +155,9 @@ struct fuse_inode { /* version of the cache */ u64 version; + /* modification time of directory when cache was started */ + struct timespec64 mtime; + /* protects above fields */ spinlock_t lock; } rdc; diff --git a/fs/fuse/readdir.c b/fs/fuse/readdir.c index f26dff1eacc3..6e86bea9c4bf 100644 --- a/fs/fuse/readdir.c +++ b/fs/fuse/readdir.c @@ -401,8 +401,10 @@ static enum fuse_parse_result fuse_parse_cache(struct fuse_file *ff, return res; } -static void fuse_rdc_reset(struct fuse_inode *fi) +static void fuse_rdc_reset(struct inode *inode) { + struct fuse_inode *fi = get_fuse_inode(inode); + fi->rdc.cached = false; fi->rdc.version++; fi->rdc.size = 0; @@ -415,6 +417,7 @@ static int fuse_readdir_cached(struct file *file, struct dir_context *ctx) { struct fuse_file *ff = file->private_data; struct inode *inode = file_inode(file); + struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); enum fuse_parse_result res; pgoff_t index; @@ -428,12 +431,40 @@ static int fuse_readdir_cached(struct file *file, struct dir_context *ctx) ff->readdir.cache_off = 0; } + /* + * We're just about to start reading into the cache or reading the + * cache; both cases require an up-to-date mtime value. + */ + if (!ctx->pos && fc->auto_inval_data) { + int err = fuse_update_attributes(inode, file); + + if (err) + return err; + } + retry: spin_lock(&fi->rdc.lock); +retry_locked: if (!fi->rdc.cached) { + /* Starting cache? Set cache mtime. */ + if (!ctx->pos && !fi->rdc.size) { + fi->rdc.mtime = inode->i_mtime; + } spin_unlock(&fi->rdc.lock); return UNCACHED; } + /* + * When at the beginning of the directory (i.e. just after opendir(3) or + * rewinddir(3)), then need to check whether directory contents have + * changed, and reset the cache if so. + */ + if (!ctx->pos) { + if (!timespec64_equal(&fi->rdc.mtime, &inode->i_mtime)) { + fuse_rdc_reset(inode); + goto retry_locked; + } + } + /* * If cache version changed since the last getdents() call, then reset * the cache stream. @@ -471,9 +502,8 @@ static int fuse_readdir_cached(struct file *file, struct dir_context *ctx) * Uh-oh: page gone missing, cache is useless */ if (fi->rdc.version == ff->readdir.version) - fuse_rdc_reset(fi); - spin_unlock(&fi->rdc.lock); - return UNCACHED; + fuse_rdc_reset(inode); + goto retry_locked; } /* Make sure it's still the same version after getting the page. */ -- 2.27.0

From: Miklos Szeredi &lt;mszeredi(a)redhat.com&gt; commit 261aaba72fdba17b74a3a434d9f925b43d90e958 upstream Use the internal iversion counter to make sure modifications of the directory through this filesystem are not missed by the mtime check (due to mtime granularity). Signed-off-by: Miklos Szeredi &lt;mszeredi(a)redhat.com&gt; Signed-off-by: Jeffle Xu &lt;jefflexu(a)linux.alibaba.com&gt; --- fs/fuse/dir.c | 21 ++++++++++++++------- fs/fuse/fuse_i.h | 3 +++ fs/fuse/readdir.c | 5 ++++- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 7d76b615ee11..27814ce8b52a 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -14,6 +14,7 @@ #include <linux/namei.h> #include <linux/slab.h> #include <linux/xattr.h> +#include <linux/iversion.h> #include <linux/posix_acl.h> static void fuse_advise_use_readdirplus(struct inode *dir) @@ -105,6 +106,12 @@ void fuse_invalidate_attr(struct inode *inode) get_fuse_inode(inode)->i_time = 0; } +static void fuse_dir_changed(struct inode *dir) +{ + fuse_invalidate_attr(dir); + inode_maybe_inc_iversion(dir, false); +} + /** * Mark the attributes as stale due to an atime change. Avoid the invalidate if * atime is not used. @@ -463,7 +470,7 @@ static int fuse_create_open(struct inode *dir, struct dentry *entry, kfree(forget); d_instantiate(entry, inode); fuse_change_entry_timeout(entry, &outentry); - fuse_invalidate_attr(dir); + fuse_dir_changed(dir); err = finish_open(file, entry, generic_file_open); if (err) { fi = get_fuse_inode(inode); @@ -578,7 +585,7 @@ static int create_new_entry(struct fuse_conn *fc, struct fuse_args *args, } else { fuse_change_entry_timeout(entry, &outarg); } - fuse_invalidate_attr(dir); + fuse_dir_changed(dir); return 0; out_put_forget_req: @@ -688,7 +695,7 @@ static int fuse_unlink(struct inode *dir, struct dentry *entry) drop_nlink(inode); spin_unlock(&fi->lock); fuse_invalidate_attr(inode); - fuse_invalidate_attr(dir); + fuse_dir_changed(dir); fuse_invalidate_entry_cache(entry); fuse_update_ctime(inode); } else if (err == -EINTR) @@ -710,7 +717,7 @@ static int fuse_rmdir(struct inode *dir, struct dentry *entry) err = fuse_simple_request(fc, &args); if (!err) { clear_nlink(d_inode(entry)); - fuse_invalidate_attr(dir); + fuse_dir_changed(dir); fuse_invalidate_entry_cache(entry); } else if (err == -EINTR) fuse_invalidate_entry(entry); @@ -749,9 +756,9 @@ static int fuse_rename_common(struct inode *olddir, struct dentry *oldent, fuse_update_ctime(d_inode(newent)); } - fuse_invalidate_attr(olddir); + fuse_dir_changed(olddir); if (olddir != newdir) - fuse_invalidate_attr(newdir); + fuse_dir_changed(newdir); /* newent will end up negative */ if (!(flags & RENAME_EXCHANGE) && d_really_is_positive(newent)) { @@ -986,7 +993,7 @@ int fuse_reverse_inval_entry(struct super_block *sb, u64 parent_nodeid, if (!entry) goto unlock; - fuse_invalidate_attr(parent); + fuse_dir_changed(parent); fuse_invalidate_entry(entry); if (child_nodeid != 0 && d_really_is_positive(entry)) { diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 6f1e69a9d0ce..086b0c4c4a59 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -158,6 +158,9 @@ struct fuse_inode { /* modification time of directory when cache was started */ struct timespec64 mtime; + /* iversion of directory when cache was started */ + u64 iversion; + /* protects above fields */ spinlock_t lock; } rdc; diff --git a/fs/fuse/readdir.c b/fs/fuse/readdir.c index 6e86bea9c4bf..656afd035e21 100644 --- a/fs/fuse/readdir.c +++ b/fs/fuse/readdir.c @@ -8,6 +8,7 @@ #include "fuse_i.h" +#include <linux/iversion.h> #include <linux/posix_acl.h> #include <linux/pagemap.h> #include <linux/highmem.h> @@ -449,6 +450,7 @@ static int fuse_readdir_cached(struct file *file, struct dir_context *ctx) /* Starting cache? Set cache mtime. */ if (!ctx->pos && !fi->rdc.size) { fi->rdc.mtime = inode->i_mtime; + fi->rdc.iversion = inode_query_iversion(inode); } spin_unlock(&fi->rdc.lock); return UNCACHED; @@ -459,7 +461,8 @@ static int fuse_readdir_cached(struct file *file, struct dir_context *ctx) * changed, and reset the cache if so. */ if (!ctx->pos) { - if (!timespec64_equal(&fi->rdc.mtime, &inode->i_mtime)) { + if (inode_peek_iversion(inode) != fi->rdc.iversion || + !timespec64_equal(&fi->rdc.mtime, &inode->i_mtime)) { fuse_rdc_reset(inode); goto retry_locked; } -- 2.27.0

From: Dan Schatzberg &lt;dschatzberg(a)fb.com&gt; commit 5571f1e65486be025f73fa6aa30fb03725d362a2 upstream FUSE file reads are cached in the page cache, but symlink reads are not. This patch enables FUSE READLINK operations to be cached which can improve performance of some FUSE workloads. In particular, I'm working on a FUSE filesystem for access to source code and discovered that about a 10% improvement to build times is achieved with this patch (there are a lot of symlinks in the source tree). Signed-off-by: Miklos Szeredi &lt;mszeredi(a)redhat.com&gt; Signed-off-by: Jeffle Xu &lt;jefflexu(a)linux.alibaba.com&gt; --- fs/fuse/dir.c | 108 +++++++++++++++++++++++++++++--------- fs/fuse/fuse_i.h | 3 ++ fs/fuse/inode.c | 4 +- include/uapi/linux/fuse.h | 3 ++ 4 files changed, 92 insertions(+), 26 deletions(-) diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 27814ce8b52a..c3d14a2d5966 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -1169,38 +1169,78 @@ static int fuse_permission(struct inode *inode, int mask) return err; } -static const char *fuse_get_link(struct dentry *dentry, - struct inode *inode, - struct delayed_call *done) +static int fuse_readlink_page(struct inode *inode, struct page *page) { struct fuse_conn *fc = get_fuse_conn(inode); - FUSE_ARGS(args); - char *link; - ssize_t ret; + struct fuse_req *req; + int err; - if (!dentry) - return ERR_PTR(-ECHILD); + req = fuse_get_req(fc, 1); + if (IS_ERR(req)) + return PTR_ERR(req); + + req->out.page_zeroing = 1; + req->out.argpages = 1; + req->num_pages = 1; + req->pages[0] = page; + req->page_descs[0].length = PAGE_SIZE - 1; + req->in.h.opcode = FUSE_READLINK; + req->in.h.nodeid = get_node_id(inode); + req->out.argvar = 1; + req->out.numargs = 1; + req->out.args[0].size = PAGE_SIZE - 1; + fuse_request_send(fc, req); + err = req->out.h.error; - link = kmalloc(PAGE_SIZE, GFP_KERNEL); - if (!link) - return ERR_PTR(-ENOMEM); + if (!err) { + char *link = page_address(page); + size_t len = req->out.args[0].size; - args.in.h.opcode = FUSE_READLINK; - args.in.h.nodeid = get_node_id(inode); - args.out.argvar = 1; - args.out.numargs = 1; - args.out.args[0].size = PAGE_SIZE - 1; - args.out.args[0].value = link; - ret = fuse_simple_request(fc, &args); - if (ret < 0) { - kfree(link); - link = ERR_PTR(ret); - } else { - link[ret] = '\0'; - set_delayed_call(done, kfree_link, link); + BUG_ON(len >= PAGE_SIZE); + link[len] = '\0'; } + + fuse_put_request(fc, req); fuse_invalidate_atime(inode); - return link; + + return err; +} + +static const char *fuse_get_link(struct dentry *dentry, struct inode *inode, + struct delayed_call *callback) +{ + struct fuse_conn *fc = get_fuse_conn(inode); + struct page *page; + int err; + + err = -EIO; + if (is_bad_inode(inode)) + goto out_err; + + if (fc->cache_symlinks) + return page_get_link(dentry, inode, callback); + + err = -ECHILD; + if (!dentry) + goto out_err; + + page = alloc_page(GFP_KERNEL); + err = -ENOMEM; + if (!page) + goto out_err; + + err = fuse_readlink_page(inode, page); + if (err) { + __free_page(page); + goto out_err; + } + + set_delayed_call(callback, page_put_link, page); + + return page_address(page); + +out_err: + return ERR_PTR(err); } static int fuse_dir_open(struct inode *inode, struct file *file) @@ -1673,7 +1713,25 @@ void fuse_init_dir(struct inode *inode) inode->i_fop = &fuse_dir_operations; } +static int fuse_symlink_readpage(struct file *null, struct page *page) +{ + int err = fuse_readlink_page(page->mapping->host, page); + + if (!err) + SetPageUptodate(page); + + unlock_page(page); + + return err; +} + +static const struct address_space_operations fuse_symlink_aops = { + .readpage = fuse_symlink_readpage, +}; + void fuse_init_symlink(struct inode *inode) { inode->i_op = &fuse_symlink_inode_operations; + inode->i_data.a_ops = &fuse_symlink_aops; + inode_nohighmem(inode); } diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 086b0c4c4a59..ec608058f338 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -708,6 +708,9 @@ struct fuse_conn { /** handle fs handles killing suid/sgid/cap on write/chown/trunc */ unsigned handle_killpriv:1; + /** cache READLINK responses in page cache */ + unsigned cache_symlinks:1; + /* * The following bitfields are only for optimization purposes * and hence races in setting them will not cause malfunction diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 1757fbc0d631..b27a0ecec0ee 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -978,6 +978,8 @@ static void process_init_reply(struct fuse_conn *fc, struct fuse_req *req) fc->posix_acl = 1; fc->sb->s_xattr = fuse_acl_xattr_handlers; } + if (arg->flags & FUSE_CACHE_SYMLINKS) + fc->cache_symlinks = 1; if (arg->flags & FUSE_ABORT_ERROR) fc->abort_err = 1; if (IS_ENABLED(CONFIG_FUSE_DAX) && @@ -1022,7 +1024,7 @@ void fuse_send_init(struct fuse_conn *fc, struct fuse_req *req) FUSE_DO_READDIRPLUS | FUSE_READDIRPLUS_AUTO | FUSE_ASYNC_DIO | FUSE_WRITEBACK_CACHE | FUSE_NO_OPEN_SUPPORT | FUSE_PARALLEL_DIROPS | FUSE_HANDLE_KILLPRIV | FUSE_POSIX_ACL | - FUSE_ABORT_ERROR; + FUSE_ABORT_ERROR | FUSE_CACHE_SYMLINKS; #ifdef CONFIG_FUSE_DAX if (fc->dax) arg->flags |= FUSE_MAP_ALIGNMENT; diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index f008a61c5506..1c6956b32597 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -119,6 +119,7 @@ * * 7.28 * - add FOPEN_CACHE_DIR + * - add FUSE_CACHE_SYMLINKS */ #ifndef _LINUX_FUSE_H @@ -256,6 +257,7 @@ struct fuse_file_lock { * FUSE_HANDLE_KILLPRIV: fs handles killing suid/sgid/cap on write/chown/trunc * FUSE_POSIX_ACL: filesystem supports posix acls * FUSE_ABORT_ERROR: reading the device after abort returns ECONNABORTED + * FUSE_CACHE_SYMLINKS: cache READLINK responses * FUSE_MAP_ALIGNMENT: init_out.map_alignment contains log2(byte alignment) for * foffset and moffset fields in struct * fuse_setupmapping_out and fuse_removemapping_one. @@ -282,6 +284,7 @@ struct fuse_file_lock { #define FUSE_HANDLE_KILLPRIV (1 << 19) #define FUSE_POSIX_ACL (1 << 20) #define FUSE_ABORT_ERROR (1 << 21) +#define FUSE_CACHE_SYMLINKS (1 << 23) #define FUSE_MAP_ALIGNMENT (1 << 26) /** -- 2.27.0