octoleo/s3fs-fuse
1
0
Fork 0
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git synced 2024-06-06 11:00:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixed issue 291, and Adds "disable_noobj_cache" option.

Browse Source 
1) Man file has wrong permissions for passwd file(Issue 291)
    Fixes man page for wrong permissions of passwd file.

2) Fixes a bug and Strictly checks passwd file permission.
    * Fixes a bug about checking passwd file permission.
       A bug is that s3fs continues to run after s3fs finds invalid passwd 
      file permission.
    * Checks passwd file strictly.
       Before this revision, s3fs allows executable permission for a 
       passwd file and  allows group writable permission for a passwd 
       file(which is not "/etc/passwd-s3fs").
       New s3fs checks permission strictly, that is  /etc/passwd-s3fs is 
       allowed owner readable/writable and group readable, and  the 
       passwd file(which is not "/etc/passwd-s3fs") is allowed only owner 
       readable/writable.

3) Adds disable_noobj_cache option for no-existing object.
    s3fs v1.68 always has to check whether file(or sub directory) exists 
    under object(path)  when s3fs does some command, since s3fs has 
    recognized a directory which does not exist and has files or sub 
    directories under itself.
    It increases ListBucket request and makes performance bad.
    For performance if the disable_noobj_cache option is specified, s3fs 
    memorizes in stat cache that the object(file or directory) does not exist.




git-svn-id: http://s3fs.googlecode.com/svn/trunk@420 df820570-a93a-0410-bd06-b72b767a4274
This commit is contained in:
ggtakec@gmail.com 2013-05-08 07:51:22 +00:00
parent 5de37a6807
commit 6b78bfdf4b
5 changed files with 160 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
doc/man/s3fs.1
View File

@ -23,8 +23,8 @@ If you have more than one set of credentials, this syntax is also recognized:
.PP
Password files can be stored in two locations:
.RS 4
\fB/etc/passwd-s3fs\fP [0600]
\fB$HOME/.passwd-s3fs\fP [0640]
\fB/etc/passwd-s3fs\fP [0640]
\fB$HOME/.passwd-s3fs\fP [0600]
.RE
.SH OPTIONS
.SS "general options"
@ -80,6 +80,12 @@ maximum number of entries in the stat cache
\fB\-o\fR stat_cache_expire (default is no expire)
specify expire time(seconds) for entries in the stat cache
.TP
\fB\-o\fR enable_noobj_cache (default is disable)
enable cache entries for the object which does not exist.
s3fs always has to check whether file(or sub directory) exists under object(path) when s3fs does some command, since s3fs has recognized a directory which does not exist and has files or sub directories under itself.
It increases ListBucket request and makes performance bad.
You can specify this option for performance, s3fs memorizes in stat cache that the object(file or directory) does not exist.
.TP
\fB\-o\fR url (default="http://s3.amazonaws.com")
sets the url to use to access Amazon S3. If you want to use HTTPS, then you can set url=https://s3.amazonaws.com
.TP

96
src/cache.cpp
View File

@ -51,9 +51,10 @@ StatCache::StatCache()
}else{
assert(false);
}
CacheSize = 1000;
ExpireTime = 0;
IsExpireTime = false;
CacheSize = 1000;
ExpireTime = 0;
IsExpireTime = false;
IsCacheNoObject = false;
}
StatCache::~StatCache()
@ -101,6 +102,13 @@ time_t StatCache::UnsetExpireTime(void)
return old;
}
bool StatCache::SetCacheNoObject(bool flag)
{
bool old = IsCacheNoObject;
IsCacheNoObject = flag;
return old;
}
bool StatCache::GetStat(string& key, struct stat* pst, headers_t* meta, bool overcheck, const char* petag, bool* pisforce)
{
bool is_delete_cache = false;
@ -120,6 +128,16 @@ bool StatCache::GetStat(string& key, struct stat* pst, headers_t* meta, bool ove
if(iter != stat_cache.end()) {
if(!IsExpireTime|| ((*iter).second.cache_date + ExpireTime) >= time(NULL)){
if((*iter).second.noobjcache){
pthread_mutex_unlock(&StatCache::stat_cache_lock);
if(!IsCacheNoObject){
// need to delete this cache.
DelStat(strpath);
}else{
// noobjcache = true means no object.
}
return false;
}
// hit without checking etag
if(petag){
string stretag = (*iter).second.meta["ETag"];
@ -165,6 +183,47 @@ bool StatCache::GetStat(string& key, struct stat* pst, headers_t* meta, bool ove
return false;
}
bool StatCache::IsNoObjectCache(string& key, bool overcheck)
{
bool is_delete_cache = false;
string strpath = key;
if(!IsCacheNoObject){
return false;
}
pthread_mutex_lock(&StatCache::stat_cache_lock);
stat_cache_t::iterator iter = stat_cache.end();
if(overcheck && '/' != strpath[strpath.length() - 1]){
strpath += "/";
iter = stat_cache.find(strpath.c_str());
}
if(iter == stat_cache.end()){
strpath = key;
iter = stat_cache.find(strpath.c_str());
}
if(iter != stat_cache.end()) {
if(!IsExpireTime|| ((*iter).second.cache_date + ExpireTime) >= time(NULL)){
if((*iter).second.noobjcache){
// noobjcache = true means no object.
pthread_mutex_unlock(&StatCache::stat_cache_lock);
return true;
}
}else{
// timeout
is_delete_cache = true;
}
}
pthread_mutex_unlock(&StatCache::stat_cache_lock);
if(is_delete_cache){
DelStat(strpath);
}
return false;
}
bool StatCache::AddStat(std::string& key, headers_t& meta, bool forcedir)
{
if(CacheSize< 1){
@ -188,6 +247,7 @@ bool StatCache::AddStat(std::string& key, headers_t& meta, bool forcedir)
stat_cache[key].hit_count = 0;
stat_cache[key].cache_date = time(NULL); // Set time.
stat_cache[key].isforce = forcedir;
stat_cache[key].noobjcache = false;
//copy only some keys
for (headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter) {
@ -216,6 +276,36 @@ bool StatCache::AddStat(std::string& key, headers_t& meta, bool forcedir)
return true;
}
bool StatCache::AddNoObjectCache(string& key)
{
if(!IsCacheNoObject){
return true; // pretend successful
}
if(CacheSize < 1){
return true;
}
FGPRINT(" add_stat_cache_entry - noobjcache[path=%s]\n", key.c_str());
if(stat_cache.size() > CacheSize){
if(!TruncateCache()){
return false;
}
}
struct stat st;
memset(&st, 0, sizeof(struct stat));
pthread_mutex_lock(&StatCache::stat_cache_lock);
stat_cache[key].stbuf = st;
stat_cache[key].hit_count = 0;
stat_cache[key].cache_date = time(NULL); // Set time.
stat_cache[key].isforce = false;
stat_cache[key].noobjcache = true;
pthread_mutex_unlock(&StatCache::stat_cache_lock);
return true;
}
bool StatCache::TruncateCache(void)
{
string path_to_delete;

18
src/cache.h
View File

@ -12,8 +12,9 @@ struct stat_cache_entry {
time_t cache_date;
headers_t meta;
bool isforce;
bool noobjcache; // Flag: cache is no object for no listing.
stat_cache_entry() : hit_count(0), cache_date(0), isforce(false) {
stat_cache_entry() : hit_count(0), cache_date(0), isforce(false), noobjcache(false) {
memset(&stbuf, 0, sizeof(struct stat));
meta.clear();
}
@ -33,6 +34,7 @@ class StatCache
bool IsExpireTime;
time_t ExpireTime;
unsigned long CacheSize;
bool IsCacheNoObject;
private:
bool GetStat(std::string& key, struct stat* pst, headers_t* meta, bool overcheck, const char* petag, bool* pisforce);
@ -54,6 +56,16 @@ class StatCache
time_t GetExpireTime(void) const;
time_t SetExpireTime(time_t expire);
time_t UnsetExpireTime(void);
bool SetCacheNoObject(bool flag);
bool EnableCacheNoObject(void) {
return SetCacheNoObject(true);
}
bool DisableCacheNoObject(void) {
return SetCacheNoObject(false);
}
bool GetCacheNoObject(void) const {
return IsCacheNoObject;
}
// Get stat cache
bool GetStat(std::string& key, struct stat* pst, headers_t* meta, bool overcheck = true, bool* pisforce = NULL) {
@ -72,6 +84,10 @@ class StatCache
return GetStat(key, NULL, NULL, overcheck, etag, NULL);
}
// Cache For no object
bool IsNoObjectCache(std::string& key, bool overcheck = true);
bool AddNoObjectCache(std::string& key);
// Add stat cache
bool AddStat(std::string& key, headers_t& meta, bool forcedir = false);

36
src/s3fs.cpp
View File

@ -334,6 +334,10 @@ static int get_object_attribute(const char *path, struct stat *pstbuf, headers_t
if(StatCache::getStatCacheData()->GetStat(strpath, pstat, pheader, overcheck, pisforce)){
return 0;
}
if(StatCache::getStatCacheData()->IsNoObjectCache(strpath)){
// there is the path in the cache for no object, it is no object.
return -ENOENT;
}
// At first, check "object/".
strpath = path;
@ -374,6 +378,9 @@ static int get_object_attribute(const char *path, struct stat *pstbuf, headers_t
(*pisforce) = true;
}
}else{
// Add no object cache.
strpath = path; // reset original
StatCache::getStatCacheData()->AddNoObjectCache(strpath);
return result;
}
}
@ -1712,9 +1719,10 @@ static int s3fs_create(const char *path, mode_t mode, struct fuse_file_info *fi)
return result;
}
result = create_file_object(path, mode, pcxt->uid, pcxt->gid);
if(result != 0)
StatCache::getStatCacheData()->DelStat(path);
if(result != 0){
return result;
}
// object created, open it
if((fi->fh = get_local_fd(path)) <= 0)
@ -1804,7 +1812,9 @@ static int s3fs_mkdir(const char *path, mode_t mode)
return result;
}
return create_directory_object(path, mode, time(NULL), pcxt->uid, pcxt->gid);
result = create_directory_object(path, mode, time(NULL), pcxt->uid, pcxt->gid);
StatCache::getStatCacheData()->DelStat(path);
return result;
}
static int s3fs_unlink(const char *path) {
@ -3781,6 +3791,18 @@ static int check_passwd_file_perms (void) {
program_name.c_str(), passwd_file.c_str());
return EXIT_FAILURE;
}
}else{
// "/etc/passwd-s3fs" does not allow group write.
if((info.st_mode & S_IWGRP)){
fprintf (stderr, "%s: credentials file %s should not have group writable permissions\n",
program_name.c_str(), passwd_file.c_str());
return EXIT_FAILURE;
}
}
if((info.st_mode & S_IXUSR) || (info.st_mode & S_IXGRP)){
fprintf (stderr, "%s: credentials file %s should not have executable permissions\n",
program_name.c_str(), passwd_file.c_str());
return EXIT_FAILURE;
}
return EXIT_SUCCESS;
}
@ -3812,7 +3834,9 @@ static int read_passwd_file (void) {
// if you got here, the password file
// exists and is readable by the
// current user, check for permissions
check_passwd_file_perms();
if(EXIT_SUCCESS != check_passwd_file_perms()){
return EXIT_FAILURE;
}
aws_format = check_for_aws_format();
if(1 == aws_format){
@ -4151,6 +4175,10 @@ static int my_fuse_opt_proc(void *data, const char *arg, int key, struct fuse_ar
StatCache::getStatCacheData()->SetExpireTime(expr_time);
return 0;
}
if(strstr(arg, "enable_noobj_cache") != 0) {
StatCache::getStatCacheData()->EnableCacheNoObject();
return 0;
}
if(strstr(arg, "noxmlns") != 0) {
noxmlns = true;
return 0;

10
src/s3fs_util.cpp
View File

@ -594,6 +594,16 @@ void show_help (void)
" stat_cache_expire (default is no expire)\n"
" - specify expire time(seconds) for entries in the stat cache.\n"
"\n"
" enable_noobj_cache (default is disable)\n"
" - enable cache entries for the object which does not exist.\n"
" s3fs always has to check whether file(or sub directory) exists \n"
" under object(path) when s3fs does some command, since s3fs has \n"
" recognized a directory which does not exist and has files or \n"
" sub directories under itself. It increases ListBucket request \n"
" and makes performance bad.\n"
" You can specify this option for performance, s3fs memorizes \n"
" in stat cache that the object(file or directory) does not exist.\n"
"\n"
" url (default=\"http://s3.amazonaws.com\")\n"
" - sets the url to use to access amazon s3\n"
"\n"