Merge pull request #199 from s3fs-fuse/xattr

Supported extended attributes(retry)

src/common.h

@@ -81,6 +81,26 @@
 //
 typedef std::map<std::string, std::string> headers_t;
 
+//
+// Header "x-amz-meta-xattr" is for extended attributes.
+// This header is url encoded string which is json formated.
+//   x-amz-meta-xattr:urlencod({"xattr-1":"base64(value-1)","xattr-2":"base64(value-2)","xattr-3":"base64(value-3)"})
+//
+typedef struct xattr_value{
+  unsigned char* pvalue;
+  size_t         length;
+
+  xattr_value(unsigned char* pval = NULL, size_t len = 0) : pvalue(pval), length(len) {}
+  ~xattr_value()
+  {
+    if(pvalue){
+      free(pvalue);
+    }
+  }
+}XATTRVAL, *PXATTRVAL;
+
+typedef std::map<std::string, PXATTRVAL> xattrs_t;
+
 //
 // Global valiables
 //

src/common_auth.cpp

@@ -30,7 +30,7 @@ using namespace std;
 //-------------------------------------------------------------------
 // Utility Function
 //-------------------------------------------------------------------
-char* s3fs_base64(unsigned char* input, size_t length)
+char* s3fs_base64(const unsigned char* input, size_t length)
 {
   static const char* base = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=";
   char* result;
@@ -61,6 +61,61 @@ char* s3fs_base64(unsigned char* input, size_t length)
   return result;
 }
 
+inline unsigned char char_decode64(const char ch)
+{
+  unsigned char by;
+  if('A' <= ch && ch <= 'Z'){                   // A - Z
+    by = static_cast<unsigned char>(ch - 'A');
+  }else if('a' <= ch && ch <= 'z'){             // a - z
+    by = static_cast<unsigned char>(ch - 'a' + 26);
+  }else if('0' <= ch && ch <= '9'){             // 0 - 9
+    by = static_cast<unsigned char>(ch - '0' + 52);
+  }else if('+' == ch){                         // +
+    by = 62;
+  }else if('/' == ch){                         // /
+    by = 63;
+  }else if('=' == ch){                         // =
+    by = 64;
+  }else{                                       // something wrong
+    by = 64;
+  }
+  return by;
+}
+
+unsigned char* s3fs_decode64(const char* input, size_t* plength)
+{
+  unsigned char* result;
+  if(!input || 0 == strlen(input) || !plength){
+    return NULL;
+  }
+  if(NULL == (result = (unsigned char*)malloc((strlen(input) + 1)))){
+    return NULL; // ENOMEM
+  }
+
+  unsigned char parts[4];
+  size_t input_len = strlen(input);
+  size_t rpos;
+  size_t wpos;
+  for(rpos = 0, wpos = 0; rpos < input_len; rpos += 4){
+    parts[0] = char_decode64(input[rpos]);
+    parts[1] = (rpos + 1) < input_len ? char_decode64(input[rpos + 1]) : 64;
+    parts[2] = (rpos + 2) < input_len ? char_decode64(input[rpos + 2]) : 64;
+    parts[3] = (rpos + 3) < input_len ? char_decode64(input[rpos + 3]) : 64;
+
+    result[wpos++] = ((parts[0] << 2) & 0xfc) | ((parts[1] >> 4) & 0x03);
+    if(64 == parts[2]){
+      break;
+    }
+    result[wpos++] = ((parts[1] << 4) & 0xf0) | ((parts[2] >> 2) & 0x0f);
+    if(64 == parts[3]){
+      break;
+    }
+    result[wpos++] = ((parts[2] << 6) & 0xc0) | (parts[3] & 0x3f);
+  }
+  *plength = wpos;
+  return result;
+}
+
 string s3fs_get_content_md5(int fd)
 {
   unsigned char* md5hex;

src/s3fs.cpp

@@ -34,6 +34,7 @@
 #include <pwd.h>
 #include <grp.h>
 #include <getopt.h>
+#include <sys/xattr.h>
 
 #include <fstream>
 #include <vector>
@@ -65,6 +66,10 @@ using namespace std;
 #define	IS_REPLACEDIR(type) (DIRTYPE_OLD == type || DIRTYPE_FOLDER == type || DIRTYPE_NOOBJ == type)
 #define	IS_RMTYPEDIR(type)  (DIRTYPE_OLD == type || DIRTYPE_FOLDER == type)
 
+#if !defined(ENOATTR)
+#define ENOATTR				ENODATA
+#endif
+
 //-------------------------------------------------------------------
 // Structs
 //-------------------------------------------------------------------
@@ -153,6 +158,10 @@ static xmlChar* get_exp_value_xml(xmlDocPtr doc, xmlXPathContextPtr ctx, const c
 static void print_uncomp_mp_list(uncomp_mp_list_t& list);
 static bool abort_uncomp_mp_list(uncomp_mp_list_t& list);
 static bool get_uncomp_mp_list(xmlDocPtr doc, uncomp_mp_list_t& list);
+static void free_xattrs(xattrs_t& xattrs);
+static bool parse_xattr_keyval(const std::string& xattrpair, string& key, PXATTRVAL& pval);
+static size_t parse_xattrs(const std::string& strxattrs, xattrs_t& xattrs);
+static std::string build_xattrs(const xattrs_t& xattrs);
 static int s3fs_utility_mode(void);
 static int s3fs_check_service(void);
 static int check_for_aws_format(void);
@@ -192,6 +201,10 @@ static int s3fs_readdir(const char* path, void* buf, fuse_fill_dir_t filler, off
 static int s3fs_access(const char* path, int mask);
 static void* s3fs_init(struct fuse_conn_info* conn);
 static void s3fs_destroy(void*);
+static int s3fs_setxattr(const char* path, const char* name, const char* value, size_t size, int flags);
+static int s3fs_getxattr(const char* path, const char* name, char* value, size_t size);
+static int s3fs_listxattr(const char* path, char* list, size_t size);
+static int s3fs_removexattr(const char* path, const char* name);
 
 //-------------------------------------------------------------------
 // Functions
@@ -2708,6 +2721,476 @@ static int remote_mountpath_exists(const char* path)
   return 0;
 }
 
+
+static void free_xattrs(xattrs_t& xattrs)
+{
+  for(xattrs_t::iterator iter = xattrs.begin(); iter != xattrs.end(); xattrs.erase(iter++)){
+    if(iter->second){
+      delete iter->second;
+    }
+  }
+}
+
+static bool parse_xattr_keyval(const std::string& xattrpair, string& key, PXATTRVAL& pval)
+{
+  // parse key and value
+  size_t pos;
+  string tmpval;
+  if(string::npos == (pos = xattrpair.find_first_of(":"))){
+    DPRNNN("one of xattr pair(%s) is wrong format.", xattrpair.c_str());
+    return false;
+  }
+  key    = xattrpair.substr(0, pos);
+  tmpval = xattrpair.substr(pos + 1);
+
+  if(!takeout_str_dquart(key) || !takeout_str_dquart(tmpval)){
+    DPRNNN("one of xattr pair(%s) is wrong format.", xattrpair.c_str());
+    return false;
+  }
+
+  pval = new XATTRVAL;
+  pval->length = 0;
+  pval->pvalue = s3fs_decode64(tmpval.c_str(), &pval->length);
+
+  return true;
+}
+
+static size_t parse_xattrs(const std::string& strxattrs, xattrs_t& xattrs)
+{
+  xattrs.clear();
+
+  // decode
+  string jsonxattrs = urlDecode(strxattrs);
+
+  // get from "{" to "}"
+  string restxattrs;
+  {
+    size_t startpos = string::npos;
+    size_t endpos   = string::npos;
+    if(string::npos != (startpos = jsonxattrs.find_first_of("{"))){
+      endpos = jsonxattrs.find_last_of("}");
+    }
+    if(startpos == string::npos || endpos == string::npos || endpos <= startpos){
+      DPRNNN("xattr header(%s) is not json format.", jsonxattrs.c_str());
+      return 0;
+    }
+    restxattrs = jsonxattrs.substr(startpos + 1, endpos - (startpos + 1));
+  }
+
+  // parse each key:val
+  for(size_t pair_nextpos = restxattrs.find_first_of(","); 0 < restxattrs.length(); restxattrs = (pair_nextpos != string::npos ? restxattrs.substr(pair_nextpos + 1) : string("")), pair_nextpos = restxattrs.find_first_of(",")){
+    string pair = pair_nextpos != string::npos ? restxattrs.substr(0, pair_nextpos) : restxattrs;
+    string    key  = "";
+    PXATTRVAL pval = NULL;
+    if(!parse_xattr_keyval(pair, key, pval)){
+      // something format error, so skip this.
+      continue;
+    }
+    xattrs[key] = pval;
+  }
+  return xattrs.size();
+}
+
+static std::string build_xattrs(const xattrs_t& xattrs)
+{
+  string strxattrs("{");
+
+  bool is_set = false;
+  for(xattrs_t::const_iterator iter = xattrs.begin(); iter != xattrs.end(); ++iter){
+    if(is_set){
+      strxattrs += ',';
+    }else{
+      is_set = true;
+    }
+    strxattrs += '\"';
+    strxattrs += iter->first;
+    strxattrs += "\":\"";
+
+    if(iter->second){
+      char* base64val = s3fs_base64((iter->second)->pvalue, (iter->second)->length);
+      if(base64val){
+        strxattrs += base64val;
+        free(base64val);
+      }
+    }
+    strxattrs += '\"';
+  }
+  strxattrs += '}';
+
+  strxattrs = urlEncode(strxattrs);
+
+  return strxattrs;
+}
+
+static int set_xattrs_to_header(headers_t& meta, const char* name, const char* value, size_t size, int flags)
+{
+  string   strxattrs;
+  xattrs_t xattrs;
+
+  if(meta.end() == meta.find("x-amz-meta-xattr")){
+    if(XATTR_REPLACE == (flags & XATTR_REPLACE)){
+      // there is no xattr header but flags is replace, so failure.
+      return -ENOATTR;
+    }
+  }else{
+    if(XATTR_CREATE == (flags & XATTR_CREATE)){
+      // found xattr header but flags is only creating, so failure.
+      return -EEXIST;
+    }
+    strxattrs = meta["x-amz-meta-xattr"];
+  }
+
+  // get map as xattrs_t
+  parse_xattrs(strxattrs, xattrs);
+
+  // add name(do not care overwrite and empty name/value)
+  if(xattrs.end() != xattrs.find(string(name))){
+    // found same head. free value.
+    delete xattrs[string(name)];
+  }
+
+  PXATTRVAL pval = new XATTRVAL;
+  pval->length = size;
+  if(0 < size){
+    if(NULL == (pval->pvalue = (unsigned char*)malloc(size))){
+      delete pval;
+      free_xattrs(xattrs);
+      return -ENOMEM;
+    }
+    memcpy(pval->pvalue, value, size);
+  }else{
+    pval->pvalue = NULL;
+  }
+  xattrs[string(name)] = pval;
+
+  // build new strxattrs(not encoded) and set it to headers_t
+  meta["x-amz-meta-xattr"] = build_xattrs(xattrs);
+
+  free_xattrs(xattrs);
+
+  return 0;
+}
+
+static int s3fs_setxattr(const char* path, const char* name, const char* value, size_t size, int flags)
+{
+  FPRN("[path=%s][name=%s][value=%p][size=%zu][flags=%d]", path, name, value, size, flags);
+
+  if((value && 0 == size) || (!value && 0 < size)){
+    DPRN("Wrong parameter: value(%p), size(%zu)", value, size);
+    return 0;
+  }
+
+  int         result;
+  string      strpath;
+  string      newpath;
+  string      nowcache;
+  headers_t   meta;
+  struct stat stbuf;
+  int         nDirType = DIRTYPE_UNKNOWN;
+
+  if(0 == strcmp(path, "/")){
+    DPRNNN("Could not change mode for mount point.");
+    return -EIO;
+  }
+  if(0 != (result = check_parent_object_access(path, X_OK))){
+    return result;
+  }
+  if(0 != (result = check_object_owner(path, &stbuf))){
+    return result;
+  }
+
+  if(S_ISDIR(stbuf.st_mode)){
+    result = chk_dir_object_type(path, newpath, strpath, nowcache, &meta, &nDirType);
+  }else{
+    strpath  = path;
+    nowcache = strpath;
+    result   = get_object_attribute(strpath.c_str(), NULL, &meta);
+  }
+  if(0 != result){
+    return result;
+  }
+
+  // make new header_t
+  if(0 != (result = set_xattrs_to_header(meta, name, value, size, flags))){
+    return result;
+  }
+
+  if(S_ISDIR(stbuf.st_mode) && IS_REPLACEDIR(nDirType)){
+    // Should rebuild directory object(except new type)
+    // Need to remove old dir("dir" etc) and make new dir("dir/")
+
+    // At first, remove directory old object
+    if(IS_RMTYPEDIR(nDirType)){
+      S3fsCurl s3fscurl;
+      if(0 != (result = s3fscurl.DeleteRequest(strpath.c_str()))){
+        return result;
+      }
+    }
+    StatCache::getStatCacheData()->DelStat(nowcache);
+
+    // Make new directory object("dir/")
+    if(0 != (result = create_directory_object(newpath.c_str(), stbuf.st_mode, stbuf.st_mtime, stbuf.st_uid, stbuf.st_gid))){
+      return result;
+    }
+
+    // need to set xattr header for directory.
+    strpath  = newpath;
+    nowcache = strpath;
+  }
+
+  // set xattr all object
+  meta["x-amz-copy-source"]        = urlEncode(service_path + bucket + get_realpath(strpath.c_str()));
+  meta["x-amz-metadata-directive"] = "REPLACE";
+
+  if(0 != put_headers(strpath.c_str(), meta, true)){
+    return -EIO;
+  }
+  StatCache::getStatCacheData()->DelStat(nowcache);
+
+  return 0;
+}
+
+static int s3fs_getxattr(const char* path, const char* name, char* value, size_t size)
+{
+  FPRN("[path=%s][name=%s][value=%p][size=%zu]", path, name, value, size);
+
+  if(!path || !name){
+    return -EIO;
+  }
+
+  int       result;
+  headers_t meta;
+  xattrs_t  xattrs;
+
+  // check parent directory attribute.
+  if(0 != (result = check_parent_object_access(path, X_OK))){
+    return result;
+  }
+
+  // get headders
+  if(0 != (result = get_object_attribute(path, NULL, &meta))){
+    return result;
+  }
+
+  // get xattrs
+  headers_t::iterator hiter = meta.find("x-amz-meta-xattr");
+  if(meta.end() == hiter){
+    // object does not have xattrs
+    return -ENOATTR;
+  }
+  string strxattrs = hiter->second;
+
+  parse_xattrs(strxattrs, xattrs);
+
+  // search name
+  string             strname = name;
+  xattrs_t::iterator xiter   = xattrs.find(strname);
+  if(xattrs.end() == xiter){
+    // not found name in xattrs
+    free_xattrs(xattrs);
+    return -ENOATTR;
+  }
+
+  // decode
+  size_t         length = 0;
+  unsigned char* pvalue = NULL;
+  if(NULL != xiter->second){
+    length = xiter->second->length;
+    pvalue = xiter->second->pvalue;
+  }
+
+  if(0 < size){
+    if(static_cast<size_t>(size) < length){
+      // over buffer size
+      free_xattrs(xattrs);
+      return -ERANGE;
+    }
+    if(pvalue){
+      memcpy(value, pvalue, length);
+    }
+  }
+  free_xattrs(xattrs);
+
+  return static_cast<int>(length);
+}
+
+static int s3fs_listxattr(const char* path, char* list, size_t size)
+{
+  FPRN("[path=%s][list=%p][size=%zu]", path, list, size);
+
+  if(!path){
+    return -EIO;
+  }
+
+  int       result;
+  headers_t meta;
+  xattrs_t  xattrs;
+
+  // check parent directory attribute.
+  if(0 != (result = check_parent_object_access(path, X_OK))){
+    return result;
+  }
+
+  // get headders
+  if(0 != (result = get_object_attribute(path, NULL, &meta))){
+    return result;
+  }
+
+  // get xattrs
+  if(meta.end() == meta.find("x-amz-meta-xattr")){
+    // object does not have xattrs
+    return 0;
+  }
+  string strxattrs = meta["x-amz-meta-xattr"];
+
+  parse_xattrs(strxattrs, xattrs);
+
+  // calculate total name length
+  size_t total = 0;
+  for(xattrs_t::const_iterator iter = xattrs.begin(); iter != xattrs.end(); ++iter){
+    if(0 < iter->first.length()){
+      total += iter->first.length() + 1;
+    }
+  }
+
+  if(0 == total){
+    free_xattrs(xattrs);
+    return 0;
+  }
+
+  // check parameters
+  if(size <= 0){
+    free_xattrs(xattrs);
+    return total;
+  }
+  if(!list || size < total){
+    free_xattrs(xattrs);
+    return -ERANGE;
+  }
+
+  // copy to list
+  char* setpos = list;
+  for(xattrs_t::const_iterator iter = xattrs.begin(); iter != xattrs.end(); ++iter){
+    if(0 < iter->first.length()){
+      strcpy(setpos, iter->first.c_str());
+      setpos = &setpos[strlen(setpos) + 1];
+    }
+  }
+  free_xattrs(xattrs);
+
+  return total;
+}
+
+static int s3fs_removexattr(const char* path, const char* name)
+{
+  FPRN("[path=%s][name=%s]", path, name);
+
+  if(!path || !name){
+    return -EIO;
+  }
+
+  int         result;
+  string      strpath;
+  string      newpath;
+  string      nowcache;
+  headers_t   meta;
+  xattrs_t    xattrs;
+  struct stat stbuf;
+  int         nDirType = DIRTYPE_UNKNOWN;
+
+  if(0 == strcmp(path, "/")){
+    DPRNNN("Could not change mode for mount point.");
+    return -EIO;
+  }
+  if(0 != (result = check_parent_object_access(path, X_OK))){
+    return result;
+  }
+  if(0 != (result = check_object_owner(path, &stbuf))){
+    return result;
+  }
+
+  if(S_ISDIR(stbuf.st_mode)){
+    result = chk_dir_object_type(path, newpath, strpath, nowcache, &meta, &nDirType);
+  }else{
+    strpath  = path;
+    nowcache = strpath;
+    result   = get_object_attribute(strpath.c_str(), NULL, &meta);
+  }
+  if(0 != result){
+    return result;
+  }
+
+  // get xattrs
+  headers_t::iterator hiter = meta.find("x-amz-meta-xattr");
+  if(meta.end() == hiter){
+    // object does not have xattrs
+    return -ENOATTR;
+  }
+  string strxattrs = hiter->second;
+
+  parse_xattrs(strxattrs, xattrs);
+
+  // check name xattrs
+  string             strname = name;
+  xattrs_t::iterator xiter   = xattrs.find(strname);
+  if(xattrs.end() == xiter){
+    free_xattrs(xattrs);
+    return -ENOATTR;
+  }
+
+  // make new header_t after deleting name xattr
+  if(xiter->second){
+    delete xiter->second;
+  }
+  xattrs.erase(strname);
+
+  // build new xattr
+  if(!xattrs.empty()){
+    meta["x-amz-meta-xattr"] = build_xattrs(xattrs);
+  }else{
+    meta.erase("x-amz-meta-xattr");
+  }
+
+  if(S_ISDIR(stbuf.st_mode) && IS_REPLACEDIR(nDirType)){
+    // Should rebuild directory object(except new type)
+    // Need to remove old dir("dir" etc) and make new dir("dir/")
+
+    // At first, remove directory old object
+    if(IS_RMTYPEDIR(nDirType)){
+      S3fsCurl s3fscurl;
+      if(0 != (result = s3fscurl.DeleteRequest(strpath.c_str()))){
+        free_xattrs(xattrs);
+        return result;
+      }
+    }
+    StatCache::getStatCacheData()->DelStat(nowcache);
+
+    // Make new directory object("dir/")
+    if(0 != (result = create_directory_object(newpath.c_str(), stbuf.st_mode, stbuf.st_mtime, stbuf.st_uid, stbuf.st_gid))){
+      free_xattrs(xattrs);
+      return result;
+    }
+
+    // need to set xattr header for directory.
+    strpath  = newpath;
+    nowcache = strpath;
+  }
+
+  // set xattr all object
+  meta["x-amz-copy-source"]        = urlEncode(service_path + bucket + get_realpath(strpath.c_str()));
+  meta["x-amz-metadata-directive"] = "REPLACE";
+
+  if(0 != put_headers(strpath.c_str(), meta, true)){
+    free_xattrs(xattrs);
+    return -EIO;
+  }
+  StatCache::getStatCacheData()->DelStat(nowcache);
+
+  free_xattrs(xattrs);
+
+  return 0;
+}
+
 static void* s3fs_init(struct fuse_conn_info* conn)
 {
   FPRN("init");
@@ -4170,6 +4653,11 @@ int main(int argc, char* argv[])
   s3fs_oper.destroy   = s3fs_destroy;
   s3fs_oper.access    = s3fs_access;
   s3fs_oper.create    = s3fs_create;
+  // extended attributes
+  s3fs_oper.setxattr    = s3fs_setxattr;
+  s3fs_oper.getxattr    = s3fs_getxattr;
+  s3fs_oper.listxattr   = s3fs_listxattr;
+  s3fs_oper.removexattr = s3fs_removexattr;
 
   if(!s3fs_init_global_ssl()){
     fprintf(stderr, "%s: could not initialize for ssl libraries.\n", program_name.c_str());

src/s3fs_auth.h

@@ -26,7 +26,8 @@
 //
 // in common_auth.cpp
 //
-char* s3fs_base64(unsigned char* input, size_t length);
+char* s3fs_base64(const unsigned char* input, size_t length);
+unsigned char* s3fs_decode64(const char* input, size_t* plength);
 std::string s3fs_get_content_md5(int fd);
 std::string s3fs_md5sum(int fd, off_t start, ssize_t size);
 std::string s3fs_sha256sum(int fd, off_t start, ssize_t size);

src/string_util.cpp

@@ -18,6 +18,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
 #include <syslog.h>
 
@@ -174,6 +175,49 @@ string urlEncode2(const string &s)
   return result;
 }
 
+string urlDecode(const string& s)
+{
+  string result;
+  for(unsigned i = 0; i < s.length(); ++i){
+    if(s[i] != '%'){
+      result += s[i];
+    }else{
+      char ch = 0;
+      if(s.length() <= ++i){
+        break;       // wrong format.
+      }
+      ch += ('0' <= s[i] && s[i] <= '9') ? (s[i] - '0') : ('A' <= s[i] && s[i] <= 'F') ? (s[i] - 'A' + 0x0a) : ('a' <= s[i] && s[i] <= 'f') ? (s[i] - 'a' + 0x0a) : 0x00;
+      if(s.length() <= ++i){
+        break;       // wrong format.
+      }
+      ch *= 16;
+      ch += ('0' <= s[i] && s[i] <= '9') ? (s[i] - '0') : ('A' <= s[i] && s[i] <= 'F') ? (s[i] - 'A' + 0x0a) : ('a' <= s[i] && s[i] <= 'f') ? (s[i] - 'a' + 0x0a) : 0x00;
+      result += ch;
+    }
+  }
+  return result;
+}
+
+bool takeout_str_dquart(string& str)
+{
+  size_t pos;
+
+  // '"' for start
+  if(string::npos != (pos = str.find_first_of("\""))){
+    str = str.substr(pos + 1);
+
+    // '"' for end
+    if(string::npos == (pos = str.find_last_of("\""))){
+      return false;
+    }
+    str = str.substr(0, pos);
+    if(string::npos != str.find_first_of("\"")){
+      return false;
+    }
+  }
+  return true;
+}
+
 //
 // ex. target="http://......?keyword=value&..."
 //

src/string_util.h

@@ -52,6 +52,8 @@ std::string get_date_string(time_t tm);
 std::string get_date_iso8601(time_t tm);
 std::string urlEncode(const std::string &s);
 std::string urlEncode2(const std::string &s);
+std::string urlDecode(const std::string& s);
+bool takeout_str_dquart(std::string& str);
 bool get_keyword_value(std::string& target, const char* keyword, std::string& value);
 
 #endif // S3FS_STRING_UTIL_H_