octoleo/s3fs-fuse
1
0
Fork 0
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git synced 2024-11-04 20:07:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Supported extended attributes, initial commit

Browse Source
This commit is contained in:
Takeshi Nakatani 2015-04-20 17:24:57 +00:00
parent 3701f1c16b
commit f258a14070
4 changed files with 448 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/common.h
View File

@ -81,6 +81,13 @@
//
typedef std::map<std::string, std::string> headers_t;
//
// Header "x-amz-meta-xattr" is for extended attributes.
// This header is url encoded string which is json formated.
// x-amz-meta-xattr:urlencod({"xattr-1":"value-1","xattr-2":"value-2","xattr-3":"value-3"})
//
typedef std::map<std::string, std::string> xattrs_t;
//
// Global valiables
//

396
src/s3fs.cpp
View File

@ -34,6 +34,7 @@
#include <pwd.h>
#include <grp.h>
#include <getopt.h>
#include <sys/xattr.h>
#include <fstream>
#include <vector>
@ -65,6 +66,10 @@ using namespace std;
#define IS_REPLACEDIR(type) (DIRTYPE_OLD == type || DIRTYPE_FOLDER == type || DIRTYPE_NOOBJ == type)
#define IS_RMTYPEDIR(type) (DIRTYPE_OLD == type || DIRTYPE_FOLDER == type)
#if !defined(ENOATTR)
#define ENOATTR ENODATA
#endif
//-------------------------------------------------------------------
// Structs
//-------------------------------------------------------------------
@ -153,6 +158,9 @@ static xmlChar* get_exp_value_xml(xmlDocPtr doc, xmlXPathContextPtr ctx, const c
static void print_uncomp_mp_list(uncomp_mp_list_t& list);
static bool abort_uncomp_mp_list(uncomp_mp_list_t& list);
static bool get_uncomp_mp_list(xmlDocPtr doc, uncomp_mp_list_t& list);
static bool parse_xattr_keyval(const std::string& xattrpair, string& key, string& val);
static size_t parse_xattrs(const std::string& strxattrs, xattrs_t& xattrs);
static std::string build_xattrs(const xattrs_t& xattrs);
static int s3fs_utility_mode(void);
static int s3fs_check_service(void);
static int check_for_aws_format(void);
@ -192,6 +200,10 @@ static int s3fs_readdir(const char* path, void* buf, fuse_fill_dir_t filler, off
static int s3fs_access(const char* path, int mask);
static void* s3fs_init(struct fuse_conn_info* conn);
static void s3fs_destroy(void*);
static int s3fs_setxattr(const char* path, const char* name, const char* value, size_t size, int flags);
static int s3fs_getxattr(const char* path, const char* name, char* value, size_t size);
static int s3fs_listxattr(const char* path, char* list, size_t size);
static int s3fs_removexattr(const char* path, const char* name);
//-------------------------------------------------------------------
// Functions
@ -2708,6 +2720,385 @@ static int remote_mountpath_exists(const char* path)
return 0;
}
static bool parse_xattr_keyval(const std::string& xattrpair, string& key, string& val)
{
// parse key and value
size_t pos;
if(string::npos == (pos = xattrpair.find_first_of(":"))){
DPRNNN("one of xattr pair(%s) is wrong format.", xattrpair.c_str());
return false;
}
key = xattrpair.substr(0, pos);
val = xattrpair.substr(pos + 1);
if(!takeout_str_dquart(key) || !takeout_str_dquart(val)){
DPRNNN("one of xattr pair(%s) is wrong format.", xattrpair.c_str());
return false;
}
return true;
}
static size_t parse_xattrs(const std::string& strxattrs, xattrs_t& xattrs)
{
xattrs.clear();
// get from "{" to "}"
string restxattrs;
{
size_t startpos = string::npos;
size_t endpos = string::npos;
if(string::npos != (startpos = strxattrs.find_first_of("{"))){
endpos = strxattrs.find_last_of("}");
}
if(startpos == string::npos || endpos == string::npos || endpos <= startpos){
DPRNNN("xattr header(%s) is not json format.", strxattrs.c_str());
return 0;
}
restxattrs = strxattrs.substr(startpos + 1, endpos - (startpos + 1));
}
// parse each key:val
for(size_t pair_nextpos = restxattrs.find_first_of(","); 0 < restxattrs.length(); restxattrs = (pair_nextpos != string::npos ? restxattrs.substr(pair_nextpos + 1) : string("")), pair_nextpos = restxattrs.find_first_of(",")){
string pair = pair_nextpos != string::npos ? restxattrs.substr(0, pair_nextpos) : restxattrs;
string key = "";
string val = "";
if(!parse_xattr_keyval(pair, key, val)){
// something format error, so skip this.
continue;
}
xattrs[key] = val;
}
return xattrs.size();
}
static std::string build_xattrs(const xattrs_t& xattrs)
{
string strxattrs("{");
bool is_set = false;
for(xattrs_t::const_iterator iter = xattrs.begin(); iter != xattrs.end(); ++iter){
if(is_set){
strxattrs += ',';
}else{
is_set = true;
}
strxattrs += '\"';
strxattrs += iter->first;
strxattrs += "\":\"";
strxattrs += iter->second;
strxattrs += '\"';
}
strxattrs += '}';
return strxattrs;
}
static int set_xattrs_to_header(headers_t& meta, const char* name, const char* value, size_t size, int flags)
{
string strxattrs;
xattrs_t xattrs;
if(meta.end() == meta.find("x-amz-meta-xattr")){
if(XATTR_REPLACE == (flags & XATTR_REPLACE)){
// there is no xattr header but flags is replace, so failure.
return -ENOATTR;
}
}else{
if(XATTR_CREATE == (flags & XATTR_CREATE)){
// found xattr header but flags is only creating, so failure.
return -EEXIST;
}
strxattrs = urlDecode(meta["x-amz-meta-xattr"]);
}
// get map as xattrs_t
parse_xattrs(strxattrs, xattrs);
// add name(do not care overwrite and empty name/value)
xattrs[string(name)] = value ? string(value, size) : string("");
// build new strxattrs(not encoded) and set it to headers_t
meta["x-amz-meta-xattr"] = build_xattrs(xattrs);
return 0;
}
static int s3fs_setxattr(const char* path, const char* name, const char* value, size_t size, int flags)
{
FPRN("[path=%s][name=%s][value=%s][size=%zu][flags=%d]", path, name, value, size, flags);
int result;
string strpath;
string newpath;
string nowcache;
headers_t meta;
struct stat stbuf;
int nDirType = DIRTYPE_UNKNOWN;
if(0 == strcmp(path, "/")){
DPRNNN("Could not change mode for mount point.");
return -EIO;
}
if(0 != (result = check_parent_object_access(path, X_OK))){
return result;
}
if(0 != (result = check_object_owner(path, &stbuf))){
return result;
}
if(S_ISDIR(stbuf.st_mode)){
result = chk_dir_object_type(path, newpath, strpath, nowcache, &meta, &nDirType);
}else{
strpath = path;
nowcache = strpath;
result = get_object_attribute(strpath.c_str(), NULL, &meta);
}
if(0 != result){
return result;
}
// make new header_t
if(0 != (result = set_xattrs_to_header(meta, name, value, size, flags))){
return result;
}
if(S_ISDIR(stbuf.st_mode) && IS_REPLACEDIR(nDirType)){
// Should rebuild directory object(except new type)
// Need to remove old dir("dir" etc) and make new dir("dir/")
// At first, remove directory old object
if(IS_RMTYPEDIR(nDirType)){
S3fsCurl s3fscurl;
if(0 != (result = s3fscurl.DeleteRequest(strpath.c_str()))){
return result;
}
}
StatCache::getStatCacheData()->DelStat(nowcache);
// Make new directory object("dir/")
if(0 != (result = create_directory_object(newpath.c_str(), stbuf.st_mode, stbuf.st_mtime, stbuf.st_uid, stbuf.st_gid))){
return result;
}
// need to set xattr header for directory.
strpath = newpath;
nowcache = strpath;
}
// set xattr all object
meta["x-amz-copy-source"] = urlEncode(service_path + bucket + get_realpath(strpath.c_str()));
meta["x-amz-metadata-directive"] = "REPLACE";
if(0 != put_headers(strpath.c_str(), meta, true)){
return -EIO;
}
StatCache::getStatCacheData()->DelStat(nowcache);
return 0;
}
static int s3fs_getxattr(const char* path, const char* name, char* value, size_t size)
{
FPRN("[path=%s][name=%s][value=%p][size=%zu]", path, name, value, size);
if(!path || !name){
return -EIO;
}
int result;
headers_t meta;
xattrs_t xattrs;
// check parent directory attribute.
if(0 != (result = check_parent_object_access(path, X_OK))){
return result;
}
if(0 != (result = check_object_owner(path, NULL))){
return result;
}
// get headders
if(0 != (result = get_object_attribute(path, NULL, &meta))){
return result;
}
// get xattrs
if(meta.end() == meta.find("x-amz-meta-xattr")){
// object does not have xattrs
return -ENOATTR;
}
string strxattrs = urlDecode(meta["x-amz-meta-xattr"]);
parse_xattrs(strxattrs, xattrs);
// search name
string strname = name;
if(xattrs.end() == xattrs.find(strname)){
// not found name in xattrs
return -ENOATTR;
}
if(size <= 0){
return static_cast<int>(xattrs[strname].length() + 1);
}
if(!value || size <= xattrs[strname].length()){
// over buffer size
return -ERANGE;
}
strcpy(value, xattrs[strname].c_str());
return static_cast<int>(strlen(value) + 1);
}
static int s3fs_listxattr(const char* path, char* list, size_t size)
{
FPRN("[path=%s][list=%p][size=%zu]", path, list, size);
if(!path){
return -EIO;
}
int result;
headers_t meta;
xattrs_t xattrs;
// check parent directory attribute.
if(0 != (result = check_parent_object_access(path, X_OK))){
return result;
}
if(0 != (result = check_object_owner(path, NULL))){
return result;
}
// get headders
if(0 != (result = get_object_attribute(path, NULL, &meta))){
return result;
}
// get xattrs
if(meta.end() == meta.find("x-amz-meta-xattr")){
// object does not have xattrs
return -ENOATTR;
}
string strxattrs = urlDecode(meta["x-amz-meta-xattr"]);
parse_xattrs(strxattrs, xattrs);
// calculate total name length
size_t total = 0;
for(xattrs_t::const_iterator iter = xattrs.begin(); iter != xattrs.end(); ++iter){
total += iter->first.length() + 1;
}
// check parameters
if(size <= 0){
return total;
}
if(!list || size < total){
return -ERANGE;
}
// copy to list
char* setpos = list;
for(xattrs_t::const_iterator iter = xattrs.begin(); iter != xattrs.end(); ++iter){
strcpy(setpos, iter->first.c_str());
setpos = &setpos[strlen(setpos) + 1];
}
return total;
}
static int s3fs_removexattr(const char* path, const char* name)
{
FPRN("[path=%s][name=%s]", path, name);
if(!path || !name){
return -EIO;
}
int result;
string strpath;
string newpath;
string nowcache;
headers_t meta;
xattrs_t xattrs;
struct stat stbuf;
int nDirType = DIRTYPE_UNKNOWN;
if(0 == strcmp(path, "/")){
DPRNNN("Could not change mode for mount point.");
return -EIO;
}
if(0 != (result = check_parent_object_access(path, X_OK))){
return result;
}
if(0 != (result = check_object_owner(path, &stbuf))){
return result;
}
if(S_ISDIR(stbuf.st_mode)){
result = chk_dir_object_type(path, newpath, strpath, nowcache, &meta, &nDirType);
}else{
strpath = path;
nowcache = strpath;
result = get_object_attribute(strpath.c_str(), NULL, &meta);
}
if(0 != result){
return result;
}
// get xattrs
if(meta.end() == meta.find("x-amz-meta-xattr")){
// object does not have xattrs
return -ENOATTR;
}
string strxattrs = urlDecode(meta["x-amz-meta-xattr"]);
parse_xattrs(strxattrs, xattrs);
// check name xattrs
string strname = name;
if(xattrs.end() == xattrs.find(strname)){
return -ENOATTR;
}
// make new header_t after deleting name xattr
xattrs.erase(strname);
meta["x-amz-meta-xattr"] = build_xattrs(xattrs);
if(S_ISDIR(stbuf.st_mode) && IS_REPLACEDIR(nDirType)){
// Should rebuild directory object(except new type)
// Need to remove old dir("dir" etc) and make new dir("dir/")
// At first, remove directory old object
if(IS_RMTYPEDIR(nDirType)){
S3fsCurl s3fscurl;
if(0 != (result = s3fscurl.DeleteRequest(strpath.c_str()))){
return result;
}
}
StatCache::getStatCacheData()->DelStat(nowcache);
// Make new directory object("dir/")
if(0 != (result = create_directory_object(newpath.c_str(), stbuf.st_mode, stbuf.st_mtime, stbuf.st_uid, stbuf.st_gid))){
return result;
}
// need to set xattr header for directory.
strpath = newpath;
nowcache = strpath;
}
// set xattr all object
meta["x-amz-copy-source"] = urlEncode(service_path + bucket + get_realpath(strpath.c_str()));
meta["x-amz-metadata-directive"] = "REPLACE";
if(0 != put_headers(strpath.c_str(), meta, true)){
return -EIO;
}
StatCache::getStatCacheData()->DelStat(nowcache);
return 0;
}
static void* s3fs_init(struct fuse_conn_info* conn)
{
FPRN("init");
@ -4164,6 +4555,11 @@ int main(int argc, char* argv[])
s3fs_oper.destroy = s3fs_destroy;
s3fs_oper.access = s3fs_access;
s3fs_oper.create = s3fs_create;
// extended attributes
s3fs_oper.setxattr = s3fs_setxattr;
s3fs_oper.getxattr = s3fs_getxattr;
s3fs_oper.listxattr = s3fs_listxattr;
s3fs_oper.removexattr = s3fs_removexattr;
if(!s3fs_init_global_ssl()){
fprintf(stderr, "%s: could not initialize for ssl libraries.\n", program_name.c_str());

43
src/string_util.cpp
View File

@ -174,6 +174,49 @@ string urlEncode2(const string &s)
return result;
}
string urlDecode(const string& s)
{
string result;
for(unsigned i = 0; i < s.length(); ++i){
if(s[i] != '%'){
result += s[i];
}else{
char ch = 0;
if(s.length() <= ++i){
break; // wrong format.
}
ch += ('0' <= s[i] && s[i] <= '9') ? (s[i] - '0') : ('A' <= s[i] && s[i] <= 'F') ? (s[i] - 'A' + 0x10) : ('a' <= s[i] && s[i] <= 'f') ? (s[i] - 'a' + 0x10) : 0x00;
if(s.length() <= ++i){
break; // wrong format.
}
ch *= 16;
ch += ('0' <= s[i] && s[i] <= '9') ? (s[i] - '0') : ('A' <= s[i] && s[i] <= 'F') ? (s[i] - 'A' + 0x10) : ('a' <= s[i] && s[i] <= 'f') ? (s[i] - 'a' + 0x10) : 0x00;
result += ch;
}
}
return result;
}
bool takeout_str_dquart(string& str)
{
size_t pos;
// '"' for start
if(string::npos != (pos = str.find_first_of("\""))){
str = str.substr(pos + 1);
// '"' for end
if(string::npos == (pos = str.find_last_of("\""))){
return false;
}
str = str.substr(0, pos);
if(string::npos != str.find_first_of("\"")){
return false;
}
}
return true;
}
//
// ex. target="http://......?keyword=value&..."
//

2
src/string_util.h
View File

@ -52,6 +52,8 @@ std::string get_date_string(time_t tm);
std::string get_date_iso8601(time_t tm);
std::string urlEncode(const std::string &s);
std::string urlEncode2(const std::string &s);
std::string urlDecode(const std::string& s);
bool takeout_str_dquart(std::string& str);
bool get_keyword_value(std::string& target, const char* keyword, std::string& value);
#endif // S3FS_STRING_UTIL_H_