diff --git a/configure.ac b/configure.ac index 742725f..cd92777 100644 --- a/configure.ac +++ b/configure.ac @@ -1,7 +1,7 @@ dnl Process this file with autoconf to produce a configure script. AC_PREREQ(2.59) -AC_INIT(s3fs, 1.48) +AC_INIT(s3fs, 1.49) AC_CANONICAL_SYSTEM diff --git a/src/Makefile.am b/src/Makefile.am index 224324e..b0e7180 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -2,6 +2,6 @@ bin_PROGRAMS=s3fs AM_CPPFLAGS = $(DEPS_CFLAGS) -s3fs_SOURCES = s3fs.cpp s3fs.h string_util.cpp string_util.h +s3fs_SOURCES = s3fs.cpp cache.cpp string_util.cpp s3fs_LDADD = $(DEPS_LIBS) diff --git a/src/cache.cpp b/src/cache.cpp new file mode 100644 index 0000000..6fe28a6 --- /dev/null +++ b/src/cache.cpp @@ -0,0 +1,101 @@ +/* + * s3fs - FUSE-based file system backed by Amazon S3 + * + * Copyright 2007-2008 Randy Rizun + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "cache.h" + +using namespace std; + +typedef std::map stat_cache_t; // key=path +static stat_cache_t stat_cache; +pthread_mutex_t stat_cache_lock; + +int get_stat_cache_entry(const char *path, struct stat *buf) { + pthread_mutex_lock(&stat_cache_lock); + stat_cache_t::iterator iter = stat_cache.find(path); + if(iter != stat_cache.end()) { + if(foreground) + cout << " stat cache hit [path=" << path << "]" + << " [hit count=" << (*iter).second.hit_count << "]" << endl; + + if(buf != NULL) + *buf = (*iter).second.stbuf; + + (*iter).second.hit_count++; + pthread_mutex_unlock(&stat_cache_lock); + return 0; + } + pthread_mutex_unlock(&stat_cache_lock); + + return -1; +} + +void add_stat_cache_entry(const char *path, struct stat *st) { + if(foreground) + cout << " add_stat_cache_entry[path=" << path << "]" << endl; + + if(stat_cache.size() > max_stat_cache_size) + truncate_stat_cache(); + + pthread_mutex_lock(&stat_cache_lock); + stat_cache[path].stbuf = *st; + pthread_mutex_unlock(&stat_cache_lock); +} + +void delete_stat_cache_entry(const char *path) { + if(foreground) + cout << " delete_stat_cache_entry[path=" << path << "]" << endl; + + pthread_mutex_lock(&stat_cache_lock); + stat_cache_t::iterator iter = stat_cache.find(path); + if(iter != stat_cache.end()) + stat_cache.erase(iter); + pthread_mutex_unlock(&stat_cache_lock); +} + +void truncate_stat_cache() { + string path_to_delete; + unsigned int hit_count = 0; + unsigned int lowest_hit_count = NULL; + + pthread_mutex_lock(&stat_cache_lock); + stat_cache_t::iterator iter; + for(iter = stat_cache.begin(); iter != stat_cache.end(); iter++) { + hit_count = (* iter).second.hit_count; + + if(!lowest_hit_count) + lowest_hit_count = hit_count; + + if(lowest_hit_count > hit_count) + path_to_delete = (* iter).first; + } + + stat_cache.erase(path_to_delete); + pthread_mutex_unlock(&stat_cache_lock); + + cout << " purged " << path_to_delete << " from the stat cache" << endl; +} diff --git a/src/cache.h b/src/cache.h new file mode 100644 index 0000000..4aad645 --- /dev/null +++ b/src/cache.h @@ -0,0 +1,27 @@ +#ifndef S3FS_CACHE_H_ +#define S3FS_CACHE_H_ + +#include +#include +#include +#include +#include +#include +#include + +struct stat_cache_entry { + struct stat stbuf; + unsigned long hit_count; + + stat_cache_entry() : hit_count(0) {} +}; + +extern bool foreground; +extern unsigned long max_stat_cache_size; + +int get_stat_cache_entry(const char *path, struct stat *buf); +void add_stat_cache_entry(const char *path, struct stat *st); +void delete_stat_cache_entry(const char *path); +void truncate_stat_cache(); + +#endif // S3FS_CACHE_H_ diff --git a/src/s3fs.cpp b/src/s3fs.cpp index 0678cfa..27215b3 100644 --- a/src/s3fs.cpp +++ b/src/s3fs.cpp @@ -18,7 +18,6 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -#include "s3fs.h" #include #include @@ -41,8 +40,10 @@ #include #include #include -#include +#include +#include "s3fs.h" +#include "cache.h" #include "string_util.h" using namespace std; @@ -3296,72 +3297,6 @@ static int s3fs_readdir( return 0; } -static int get_stat_cache_entry(const char *path, struct stat *buf) { - pthread_mutex_lock(&stat_cache_lock); - stat_cache_t::iterator iter = stat_cache.find(path); - if(iter != stat_cache.end()) { - if(foreground) - cout << " stat cache hit [path=" << path << "]" - << " [hit count=" << (*iter).second.hit_count << "]" << endl; - - if(buf != NULL) - *buf = (*iter).second.stbuf; - - (*iter).second.hit_count++; - pthread_mutex_unlock(&stat_cache_lock); - return 0; - } - pthread_mutex_unlock(&stat_cache_lock); - - return -1; -} - -static void add_stat_cache_entry(const char *path, struct stat *st) { - if(foreground) - cout << " add_stat_cache_entry[path=" << path << "]" << endl; - - if(stat_cache.size() > max_stat_cache_size) - truncate_stat_cache(); - - pthread_mutex_lock(&stat_cache_lock); - stat_cache[path].stbuf = *st; - pthread_mutex_unlock(&stat_cache_lock); -} - -static void delete_stat_cache_entry(const char *path) { - if(foreground) - cout << " delete_stat_cache_entry[path=" << path << "]" << endl; - - pthread_mutex_lock(&stat_cache_lock); - stat_cache_t::iterator iter = stat_cache.find(path); - if(iter != stat_cache.end()) - stat_cache.erase(iter); - pthread_mutex_unlock(&stat_cache_lock); -} - -static void truncate_stat_cache() { - string path_to_delete; - unsigned int hit_count = 0; - unsigned int lowest_hit_count; - - pthread_mutex_lock(&stat_cache_lock); - stat_cache_t::iterator iter; - for(iter = stat_cache.begin(); iter != stat_cache.end(); iter++) { - hit_count = (* iter).second.hit_count; - - if(!lowest_hit_count) - lowest_hit_count = hit_count; - - if(lowest_hit_count > hit_count) - path_to_delete = (* iter).first; - } - - stat_cache.erase(path_to_delete); - pthread_mutex_unlock(&stat_cache_lock); - - cout << " purged " << path_to_delete << " from the stat cache" << endl; -} - /** * OpenSSL locking function. * diff --git a/src/s3fs.h b/src/s3fs.h index a52c39b..1df4abb 100644 --- a/src/s3fs.h +++ b/src/s3fs.h @@ -19,65 +19,53 @@ #include #include - -using namespace std; - #define YIKES(result) if (true) { \ syslog(LOG_ERR, "%d###result=%d", __LINE__, result); \ return result; \ } -typedef pair progress_t; +typedef std::pair progress_t; static long connect_timeout = 10; static time_t readwrite_timeout = 30; // static stack curl_handles; static pthread_mutex_t curl_handles_lock; -static map curl_times; -static map curl_progress; +static std::map curl_times; +static std::map curl_progress; static int retries = 2; -static string bucket; -static string mountpoint; -static string program_name; -static string AWSAccessKeyId; -static string AWSSecretAccessKey; -static string host = "http://s3.amazonaws.com"; -static string curl_ca_bundle; +static std::string bucket; +static std::string mountpoint; +static std::string program_name; +static std::string AWSAccessKeyId; +static std::string AWSSecretAccessKey; +static std::string host = "http://s3.amazonaws.com"; +static std::string curl_ca_bundle; static mode_t root_mode = 0; -static string service_path = "/"; -static string passwd_file = ""; +static std::string service_path = "/"; +static std::string passwd_file = ""; static bool debug = 0; -static bool foreground = 0; static bool utility_mode = 0; -static unsigned long max_stat_cache_size = 10000; +bool foreground = 0; +unsigned long max_stat_cache_size = 10000; // if .size()==0 then local file cache is disabled -static string use_cache; -static string use_rrs; -static string ssl_verify_hostname = "1"; -static string public_bucket; +static std::string use_cache; +static std::string use_rrs; +static std::string ssl_verify_hostname = "1"; +static std::string public_bucket; + +extern pthread_mutex_t stat_cache_lock; // TODO(apetresc): make this an enum // private, public-read, public-read-write, authenticated-read -static string default_acl("private"); - -struct stat_cache_entry { - struct stat stbuf; - unsigned long hit_count; - - stat_cache_entry() : hit_count(0) {} -}; -// key=path -typedef map stat_cache_t; -static stat_cache_t stat_cache; -static pthread_mutex_t stat_cache_lock; +static std::string default_acl("private"); struct file_part { char path[17]; - string etag; + std::string etag; bool uploaded; file_part() : uploaded(false) {} @@ -86,12 +74,12 @@ struct file_part { static const char hexAlphabet[] = "0123456789ABCDEF"; // http headers -typedef map headers_t; +typedef std::map headers_t; static const EVP_MD* evp_md = EVP_sha1(); // fd -> flags -typedef map s3fs_descriptors_t; +typedef std::map s3fs_descriptors_t; static s3fs_descriptors_t s3fs_descriptors; static pthread_mutex_t s3fs_descriptors_lock; @@ -99,17 +87,12 @@ static pthread_mutex_t *mutex_buf = NULL; static struct fuse_operations s3fs_oper; -string urlEncode(const string &s); -string lookupMimeType(string); -string initiate_multipart_upload(const char *path, off_t size, headers_t meta); -string upload_part(const char *path, const char *source, int part_number, string upload_id); -static int complete_multipart_upload(const char *path, string upload_id, vector parts); -string md5sum(int fd); - -static int get_stat_cache_entry(const char *path, struct stat *buf); -static void add_stat_cache_entry(const char *path, struct stat *st); -static void delete_stat_cache_entry(const char *path); -static void truncate_stat_cache(); +std::string urlEncode(const std::string &s); +std::string lookupMimeType(std::string); +std::string initiate_multipart_upload(const char *path, off_t size, headers_t meta); +std::string upload_part(const char *path, const char *source, int part_number, std::string upload_id); +static int complete_multipart_upload(const char *path, std::string upload_id, std::vector parts); +std::string md5sum(int fd); static int s3fs_getattr(const char *path, struct stat *stbuf); static int s3fs_readlink(const char *path, char *buf, size_t size); @@ -119,6 +102,7 @@ static int s3fs_unlink(const char *path); static int s3fs_rmdir(const char *path); static int s3fs_symlink(const char *from, const char *to); static int s3fs_rename(const char *from, const char *to); + static int s3fs_link(const char *from, const char *to); static int s3fs_chmod(const char *path, mode_t mode); static int s3fs_chown(const char *path, uid_t uid, gid_t gid);