From 6a3a68b01c506d3da2f7c9df0fb6cdaca802eec6 Mon Sep 17 00:00:00 2001 From: "mooredan@suncup.net" Date: Sat, 12 Feb 2011 15:02:44 +0000 Subject: [PATCH] Bound the size of stat_cache as described in issue #157 git-svn-id: http://s3fs.googlecode.com/svn/trunk@315 df820570-a93a-0410-bd06-b72b767a4274 --- configure.ac | 2 +- src/s3fs.cpp | 88 ++++++++++++++++++++++++++++++++++++---------------- src/s3fs.h | 12 ++++++- 3 files changed, 74 insertions(+), 28 deletions(-) diff --git a/configure.ac b/configure.ac index 78d0bcc..204f43b 100644 --- a/configure.ac +++ b/configure.ac @@ -1,7 +1,7 @@ dnl Process this file with autoconf to produce a configure script. AC_PREREQ(2.59) -AC_INIT(s3fs, 1.40) +AC_INIT(s3fs, 1.41) AC_CANONICAL_SYSTEM diff --git a/src/s3fs.cpp b/src/s3fs.cpp index 9092b88..9589049 100644 --- a/src/s3fs.cpp +++ b/src/s3fs.cpp @@ -2024,17 +2024,8 @@ static int s3fs_getattr(const char *path, struct stat *stbuf) { } { - pthread_mutex_lock( &stat_cache_lock ); - stat_cache_t::iterator iter = stat_cache.find(path); - if (iter != stat_cache.end()) { - if(foreground) - cout << "\tstat cache hit" << endl; - - *stbuf = (*iter).second; - pthread_mutex_unlock( &stat_cache_lock ); + if(get_stat_cache_entry(path, stbuf) == 0) return 0; - } - pthread_mutex_unlock( &stat_cache_lock ); } body.text = (char *)malloc(1); @@ -2109,9 +2100,7 @@ static int s3fs_getattr(const char *path, struct stat *stbuf) { destroy_curl_handle(curl); // update stat cache - pthread_mutex_lock(&stat_cache_lock); - stat_cache[path] = *stbuf; - pthread_mutex_unlock(&stat_cache_lock); + add_stat_cache_entry(path, stbuf); return 0; } @@ -3277,16 +3266,13 @@ static int s3fs_readdir( } bool in_stat_cache = false; - pthread_mutex_lock(&stat_cache_lock); - stat_cache_t::iterator iter = stat_cache.find("/" + Key); - if(iter != stat_cache.end()) { - // in stat cache, skip http request + string path = '/' + Key; + if(get_stat_cache_entry(path.c_str(), NULL) == 0) { if(filler(buf, mybasename(Key).c_str(), 0, 0)) break; in_stat_cache = true; } - pthread_mutex_unlock(&stat_cache_lock); if (Key.size() > 0 && !in_stat_cache) { if (filler(buf, mybasename(Key).c_str(), 0, 0)) { @@ -3458,10 +3444,6 @@ static int s3fs_readdir( st.st_uid = strtoul((*stuff.responseHeaders)["x-amz-meta-uid"].c_str(), (char **)NULL, 10); st.st_gid = strtoul((*stuff.responseHeaders)["x-amz-meta-gid"].c_str(), (char **)NULL, 10); - - pthread_mutex_lock( &stat_cache_lock ); - stat_cache[stuff.path] = st; - pthread_mutex_unlock( &stat_cache_lock ); } // if (code == 0) } // if (msg != NULL) { } // while (remaining_msgs) @@ -3477,16 +3459,70 @@ static int s3fs_readdir( return 0; } -static void delete_stat_cache_entry(const char *path) { +static int get_stat_cache_entry(const char *path, struct stat *buf) { pthread_mutex_lock(&stat_cache_lock); stat_cache_t::iterator iter = stat_cache.find(path); if(iter != stat_cache.end()) { if(foreground) - cout << "\tremoving \"" << path << "\" from stat cache" << endl; + cout << " stat cache hit [path=" << path << "]" + << " [hit count=" << (*iter).second.hit_count << "]" << endl; - stat_cache.erase(iter); + if(buf != NULL) + *buf = (*iter).second.stbuf; + + (*iter).second.hit_count++; + pthread_mutex_unlock(&stat_cache_lock); + return 0; } pthread_mutex_unlock(&stat_cache_lock); + + return -1; +} + +static void add_stat_cache_entry(const char *path, struct stat *st) { + if(foreground) + cout << " add_stat_cache_entry[path=" << path << "]" << endl; + + if(stat_cache.size() > max_stat_cache_size) + truncate_stat_cache(); + + pthread_mutex_lock(&stat_cache_lock); + stat_cache[path].stbuf = *st; + pthread_mutex_unlock(&stat_cache_lock); +} + +static void delete_stat_cache_entry(const char *path) { + if(foreground) + cout << " delete_stat_cache_entry[path=" << path << "]" << endl; + + pthread_mutex_lock(&stat_cache_lock); + stat_cache_t::iterator iter = stat_cache.find(path); + if(iter != stat_cache.end()) + stat_cache.erase(iter); + pthread_mutex_unlock(&stat_cache_lock); +} + +static void truncate_stat_cache() { + string path_to_delete; + unsigned int hit_count = 0; + unsigned int lowest_hit_count; + + pthread_mutex_lock(&stat_cache_lock); + stat_cache_t::iterator iter; + for(iter = stat_cache.begin(); iter != stat_cache.end(); iter++) { + hit_count = (* iter).second.hit_count; + + if(!lowest_hit_count) + lowest_hit_count = hit_count; + + if(lowest_hit_count > hit_count) + path_to_delete = (* iter).first; + } + + stat_cache.erase(path_to_delete); + pthread_mutex_unlock(&stat_cache_lock); + + cout << " purged " << path_to_delete << " from the stat cache" << endl; } /** @@ -4641,13 +4677,13 @@ static int my_fuse_opt_proc(void *data, const char *arg, int key, struct fuse_ar } } } + return 1; } int main(int argc, char *argv[]) { - int ch; int option_index = 0; diff --git a/src/s3fs.h b/src/s3fs.h index 26b2073..6c20247 100644 --- a/src/s3fs.h +++ b/src/s3fs.h @@ -49,6 +49,7 @@ static string passwd_file = ""; static bool debug = 0; static bool foreground = 0; static bool utility_mode = 0; +static unsigned long max_stat_cache_size = 10000; // if .size()==0 then local file cache is disabled static string use_cache; @@ -60,8 +61,14 @@ static string public_bucket; // private, public-read, public-read-write, authenticated-read static string default_acl("private"); +struct stat_cache_entry { + struct stat stbuf; + unsigned long hit_count; + + stat_cache_entry() : hit_count(0) {} +}; // key=path -typedef map stat_cache_t; +typedef map stat_cache_t; static stat_cache_t stat_cache; static pthread_mutex_t stat_cache_lock; @@ -84,7 +91,10 @@ static struct fuse_operations s3fs_oper; string urlEncode(const string &s); string lookupMimeType(string); +static int get_stat_cache_entry(const char *path, struct stat *buf); +static void add_stat_cache_entry(const char *path, struct stat *st); static void delete_stat_cache_entry(const char *path); +static void truncate_stat_cache(); static int s3fs_getattr(const char *path, struct stat *stbuf); static int s3fs_readlink(const char *path, char *buf, size_t size);