Added a non-interactive option to utility mode

Rebase for resolving conflicts

Deleted interactive mode on utility mode

Fixed minor mistakes

doc/man/s3fs.1

@@ -16,7 +16,9 @@ For root.
 For unprivileged user.
 .SS utility mode ( remove interrupted multipart uploading objects )
 .TP
-\fBs3fs \-u bucket
+\fBs3fs --incomplete-mpu-list(-u) bucket
+.TP
+\fBs3fs --incomplete-mpu-abort[=all | =<expire date format>] bucket
 .SH DESCRIPTION
 s3fs is a FUSE filesystem that allows you to mount an Amazon S3 bucket as a local filesystem. It stores files natively and transparently in S3 (i.e., you can use other programs to access the same files).
 .SH AUTHENTICATION
@@ -292,6 +294,18 @@ When s3fs catch the signal SIGUSR2, the debug level is bumpup.
 .TP
 \fB\-o\fR curldbg - put curl debug message
 Put the debug message from libcurl when this option is specified.
+.SS "utility mode options"
+.TP
+\fB\-u\fR or \fB\-\-incomplete\-mpu\-list\fR
+Lists multipart incomplete objects uploaded to the specified bucket.
+.TP
+\fB\-\-incomplete\-mpu\-abort\fR all or date format(default="24H")
+Delete the multipart incomplete object uploaded to the specified bucket.
+If "all" is specified for this option, all multipart incomplete objects will be deleted.
+If you specify no argument as an option, objects older than 24 hours(24H) will be deleted(This is the default value).
+You can specify an optional date format.
+It can be specified as year, month, day, hour, minute, second, and it is expressed as "Y", "M", "D", "h", "m", "s" respectively.
+For example, "1Y6M10D12h30m30s".
 .SH FUSE/MOUNT OPTIONS
 .TP
 Most of the generic mount options described in 'man mount' are supported (ro, rw, suid, nosuid, dev, nodev, exec, noexec, atime, noatime, sync async, dirsync).  Filesystems are mounted with '\-onodev,nosuid' by default, which can only be overridden by a privileged user.

src/s3fs.cpp

@@ -73,16 +73,25 @@ static bool IS_RMTYPEDIR(dirtype type) { return DIRTYPE_OLD == type || DIRTYPE_F
 #define ENOATTR				ENODATA
 #endif
 
+//
+// Type of utility process mode
+//
+enum utility_incomp_type{
+  NO_UTILITY_MODE = 0,      // not utility mode
+  INCOMP_TYPE_LIST,         // list of incomplete mpu
+  INCOMP_TYPE_ABORT         // delete incomplete mpu
+};
+
 //-------------------------------------------------------------------
 // Structs
 //-------------------------------------------------------------------
-typedef struct incomplete_multipart_info{
+typedef struct incomplete_multipart_upload_info{
   string key;
   string id;
   string date;
-}UNCOMP_MP_INFO;
+}INCOMP_MPU_INFO;
 
-typedef std::list<UNCOMP_MP_INFO>          uncomp_mp_list_t;
+typedef std::list<INCOMP_MPU_INFO>         incomp_mpu_list_t;
 typedef std::list<std::string>             readline_t;
 typedef std::map<std::string, std::string> kvmap_t;
 typedef std::map<std::string, kvmap_t>     bucketkvmap_t;
@@ -115,7 +124,7 @@ static mode_t mp_umask            = 0;    // umask for mount point
 static bool is_mp_umask           = false;// default does not set.
 static std::string mountpoint;
 static std::string passwd_file;
-static bool utility_mode          = false;
+static utility_incomp_type utility_mode = NO_UTILITY_MODE;
 static bool noxmlns               = false;
 static bool nocopyapi             = false;
 static bool norenameapi           = false;
@@ -183,14 +192,14 @@ static int clone_directory_object(const char* from, const char* to);
 static int rename_directory(const char* from, const char* to);
 static int remote_mountpath_exists(const char* path);
 static xmlChar* get_exp_value_xml(xmlDocPtr doc, xmlXPathContextPtr ctx, const char* exp_key);
-static void print_uncomp_mp_list(uncomp_mp_list_t& list);
+static void print_incomp_mpu_list(incomp_mpu_list_t& list);
-static bool abort_uncomp_mp_list(uncomp_mp_list_t& list);
+static bool abort_incomp_mpu_list(incomp_mpu_list_t& list, time_t abort_time);
-static bool get_uncomp_mp_list(xmlDocPtr doc, uncomp_mp_list_t& list);
+static bool get_incomp_mpu_list(xmlDocPtr doc, incomp_mpu_list_t& list);
 static void free_xattrs(xattrs_t& xattrs);
 static bool parse_xattr_keyval(const std::string& xattrpair, string& key, PXATTRVAL& pval);
 static size_t parse_xattrs(const std::string& strxattrs, xattrs_t& xattrs);
 static std::string build_xattrs(const xattrs_t& xattrs);
-static int s3fs_utility_mode();
+static int s3fs_utility_processing(time_t abort_time);
 static int s3fs_check_service();
 static int parse_passwd_file(bucketkvmap_t& resmap);
 static int check_for_aws_format(const kvmap_t& kvmap);
@@ -3493,7 +3502,7 @@ static xmlChar* get_exp_value_xml(xmlDocPtr doc, xmlXPathContextPtr ctx, const c
   return exp_value;
 }
 
-static void print_uncomp_mp_list(uncomp_mp_list_t& list)
+static void print_incomp_mpu_list(incomp_mpu_list_t& list)
 {
   printf("\n");
   printf("Lists the parts that have been uploaded for a specific multipart upload.\n");
@@ -3503,7 +3512,7 @@ static void print_uncomp_mp_list(uncomp_mp_list_t& list)
     printf("---------------------------------------------------------------\n");
 
     int cnt = 0;
-    for(uncomp_mp_list_t::iterator iter = list.begin(); iter != list.end(); ++iter, ++cnt){
+    for(incomp_mpu_list_t::iterator iter = list.begin(); iter != list.end(); ++iter, ++cnt){
       printf(" Path     : %s\n", (*iter).key.c_str());
       printf(" UploadId : %s\n", (*iter).id.c_str());
       printf(" Date     : %s\n", (*iter).date.c_str());
@@ -3516,35 +3525,31 @@ static void print_uncomp_mp_list(uncomp_mp_list_t& list)
   }
 }
 
-static bool abort_uncomp_mp_list(uncomp_mp_list_t& list)
+static bool abort_incomp_mpu_list(incomp_mpu_list_t& list, time_t abort_time)
 {
-  char buff[1024];
-
   if(list.empty()){
     return true;
   }
-  memset(buff, 0, sizeof(buff));
+  time_t now_time = time(NULL);
 
-  // confirm
+  // do removing.
-  while(true){
-    printf("Would you remove all objects? [Y/N]\n");
-    if(NULL != fgets(buff, sizeof(buff), stdin)){
-      if(0 == strcasecmp(buff, "Y\n") || 0 == strcasecmp(buff, "YES\n")){
-        break;
-      }else if(0 == strcasecmp(buff, "N\n") || 0 == strcasecmp(buff, "NO\n")){
-        return true;
-      }
-      printf("*** please put Y(yes) or N(no).\n");
-    }
-  }
-
-  // do removing their.
   S3fsCurl s3fscurl;
   bool     result = true;
-  for(uncomp_mp_list_t::iterator iter = list.begin(); iter != list.end(); ++iter){
+  for(incomp_mpu_list_t::iterator iter = list.begin(); iter != list.end(); ++iter){
     const char* tpath     = (*iter).key.c_str();
     string      upload_id = (*iter).id;
 
+    if(0 != abort_time){    // abort_time is 0, it means all.
+      time_t    date = 0;
+      if(!get_unixtime_from_iso8601((*iter).date.c_str(), date)){
+        S3FS_PRN_DBG("date format is not ISO 8601 for %s multipart uploading object, skip this.", tpath);
+        continue;
+      }
+      if(now_time <= (date + abort_time)){
+        continue;
+      }
+    }
+
     if(0 != s3fscurl.AbortMultipartUpload(tpath, upload_id)){
       S3FS_PRN_EXIT("Failed to remove %s multipart uploading object.", tpath);
       result = false;
@@ -3559,7 +3564,7 @@ static bool abort_uncomp_mp_list(uncomp_mp_list_t& list)
   return result;
 }
 
-static bool get_uncomp_mp_list(xmlDocPtr doc, uncomp_mp_list_t& list)
+static bool get_incomp_mpu_list(xmlDocPtr doc, incomp_mpu_list_t& list)
 {
   if(!doc){
     return false;
@@ -3605,7 +3610,7 @@ static bool get_uncomp_mp_list(xmlDocPtr doc, uncomp_mp_list_t& list)
   for(cnt = 0, upload_nodes = upload_xp->nodesetval; cnt < upload_nodes->nodeNr; cnt++){
     ctx->node = upload_nodes->nodeTab[cnt];
 
-    UNCOMP_MP_INFO  part;
+    INCOMP_MPU_INFO part;
     xmlChar*        ex_value;
 
     // search "Key" tag
@@ -3643,18 +3648,18 @@ static bool get_uncomp_mp_list(xmlDocPtr doc, uncomp_mp_list_t& list)
   return true;
 }
 
-static int s3fs_utility_mode()
+static int s3fs_utility_processing(time_t abort_time)
 {
-  if(!utility_mode){
+  if(NO_UTILITY_MODE == utility_mode){
     return EXIT_FAILURE;
   }
-  printf("Utility Mode\n");
+  printf("\n*** s3fs run as utility mode.\n\n");
 
   S3fsCurl s3fscurl;
   string   body;
   int      result = EXIT_SUCCESS;
   if(0 != s3fscurl.MultipartListRequest(body)){
-    S3FS_PRN_EXIT("Could not get list multipart upload.");
+    S3FS_PRN_EXIT("Could not get list multipart upload.\nThere is no incomplete multipart uploaded object in bucket.\n");
     result = EXIT_FAILURE;
   }else{
     // parse result(incomplete multipart upload information)
@@ -3666,19 +3671,22 @@ static int s3fs_utility_mode()
       result = EXIT_FAILURE;
 
     }else{
-      // make working uploads list
+      // make incomplete uploads list
-      uncomp_mp_list_t list;
+      incomp_mpu_list_t list;
-      if(!get_uncomp_mp_list(doc, list)){
+      if(!get_incomp_mpu_list(doc, list)){
-        S3FS_PRN_DBG("get_uncomp_mp_list exited with error.");
+        S3FS_PRN_DBG("get_incomp_mpu_list exited with error.");
         result = EXIT_FAILURE;
 
       }else{
-        // print list
+        if(INCOMP_TYPE_LIST == utility_mode){
-        print_uncomp_mp_list(list);
+          // print list
-        // remove
+          print_incomp_mpu_list(list);
-        if(!abort_uncomp_mp_list(list)){
+        }else if(INCOMP_TYPE_ABORT == utility_mode){
-          S3FS_PRN_DBG("an error occurred during removal process.");
+          // remove
-          result = EXIT_FAILURE;
+          if(!abort_incomp_mpu_list(list, abort_time)){
+            S3FS_PRN_DBG("an error occurred during removal process.");
+            result = EXIT_FAILURE;
+          }
         }
       }
       S3FS_XMLFREEDOC(doc);
@@ -4343,7 +4351,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
     }
 
     // the second NONOPT option is the mountpoint(not utility mode)
-    if(mountpoint.empty() && !utility_mode){
+    if(mountpoint.empty() && NO_UTILITY_MODE == utility_mode){
       // save the mountpoint and do some basic error checking
       mountpoint = arg;
       struct stat stbuf;
@@ -4381,7 +4389,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
     }
 
     // Unknown option
-    if(!utility_mode){
+    if(NO_UTILITY_MODE == utility_mode){
       S3FS_PRN_EXIT("specified unknown third option(%s).", arg);
     }else{
       S3FS_PRN_EXIT("specified unknown second option(%s). you don't need to specify second option(mountpoint) for utility mode(-u).", arg);
@@ -4946,12 +4954,15 @@ int main(int argc, char* argv[])
   int fuse_res;
   int option_index = 0; 
   struct fuse_operations s3fs_oper;
+  time_t incomp_abort_time = (24 * 60 * 60);
 
   static const struct option long_opts[] = {
-    {"help",    no_argument, NULL, 'h'},
+    {"help",                 no_argument,       NULL, 'h'},
-    {"version", no_argument, 0,     0},
+    {"version",              no_argument,       0,     0},
-    {"debug",   no_argument, NULL, 'd'},
+    {"debug",                no_argument,       NULL, 'd'},
-    {0, 0, 0, 0}
+    {"incomplete-mpu-list",  no_argument,       NULL, 'u'},
+    {"incomplete-mpu-abort", optional_argument, NULL, 'a'}, // 'a' is only identifier and is not option.
+    {NULL, 0, NULL, 0}
   };
 
   // init syslog(default CRIT)
@@ -4989,8 +5000,30 @@ int main(int argc, char* argv[])
       break;
     case 's':
       break;
-    case 'u':
+    case 'u':   // --incomplete-mpu-list
-      utility_mode = true;
+      if(NO_UTILITY_MODE != utility_mode){
+        S3FS_PRN_EXIT("already utility mode option is specified.");
+        exit(EXIT_FAILURE);
+      }
+      utility_mode = INCOMP_TYPE_LIST;
+      break;
+    case 'a':   // --incomplete-mpu-abort
+      if(NO_UTILITY_MODE != utility_mode){
+        S3FS_PRN_EXIT("already utility mode option is specified.");
+        exit(EXIT_FAILURE);
+      }
+      utility_mode = INCOMP_TYPE_ABORT;
+
+      // check expire argument
+      if(NULL != optarg && 0 == strcasecmp(optarg, "all")){ // all is 0s
+        incomp_abort_time = 0;
+      }else if(NULL != optarg){
+        if(!convert_unixtime_from_option_arg(optarg, incomp_abort_time)){
+          S3FS_PRN_EXIT("--incomplete-mpu-abort option argument is wrong.");
+          exit(EXIT_FAILURE);
+        }
+      }
+      // if optarg is null, incomp_abort_time is 24H(default)
       break;
     default:
       exit(EXIT_FAILURE);
@@ -5082,7 +5115,7 @@ int main(int argc, char* argv[])
   // if the option was given, we all ready checked for a
   // readable, non-empty directory, this checks determines
   // if the mountpoint option was ever supplied
-  if(!utility_mode){
+  if(NO_UTILITY_MODE == utility_mode){
     if(mountpoint.empty()){
       S3FS_PRN_EXIT("missing MOUNTPOINT argument.");
       show_usage();
@@ -5182,8 +5215,8 @@ int main(int argc, char* argv[])
   }
   */
 
-  if(utility_mode){
+  if(NO_UTILITY_MODE != utility_mode){
-    int exitcode = s3fs_utility_mode();
+    int exitcode = s3fs_utility_processing(incomp_abort_time);
 
     S3fsCurl::DestroyS3fsCurl();
     s3fs_destroy_global_ssl();

src/s3fs_util.cpp

@@ -1013,13 +1013,14 @@ void show_help ()
     "   umounting\n"
     "     umount mountpoint\n"
     "\n"
-    "   utility mode (remove interrupted multipart uploading objects)\n"
-    "     s3fs -u bucket\n"
-    "\n"
     "   General forms for s3fs and FUSE/mount options:\n"
     "      -o opt[,opt...]\n"
     "      -o opt [-o opt] ...\n"
     "\n"
+    "   utility mode (remove interrupted multipart uploading objects)\n"
+    "     s3fs --incomplete-mpu-list(-u) bucket\n"
+    "     s3fs --incomplete-mpu-abort[=all | =<date format>] bucket\n"
+    "\n"
     "s3fs Options:\n"
     "\n"
     "   Most s3fs options are given in the form where \"opt\" is:\n"
@@ -1348,6 +1349,22 @@ void show_help ()
     "   There are many FUSE specific mount options that can be specified.\n"
     "   e.g. allow_other  See the FUSE's README for the full set.\n"
     "\n"
+    "Utility mode Options:\n"
+    "\n"
+    " -u, --incomplete-mpu-list\n"
+    "        Lists multipart incomplete objects uploaded to the specified\n"
+    "        bucket.\n"
+    " --incomplete-mpu-abort(=all or =<date format>)\n"
+    "        Delete the multipart incomplete object uploaded to the specified\n"
+    "        bucket.\n"
+    "        If \"all\" is specified for this option, all multipart incomplete\n"
+    "        objects will be deleted. If you specify no argument as an option,\n"
+    "        objects older than 24 hours(24H) will be deleted(This is the\n"
+    "        default value). You can specify an optional date format. It can\n"
+    "        be specified as year, month, day, hour, minute, second, and it is\n"
+    "        expressed as \"Y\", \"M\", \"D\", \"h\", \"m\", \"s\" respectively.\n"
+    "        For example, \"1Y6M10D12h30m30s\".\n"
+    "\n"
     "Miscellaneous Options:\n"
     "\n"
     " -h, --help        Output this help.\n"

src/string_util.cpp

@@ -22,6 +22,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <syslog.h>
+#include <time.h>
 
 #include <sstream>
 #include <string>
@@ -283,6 +284,75 @@ string get_date_iso8601(time_t tm)
   return buf;
 }
 
+bool get_unixtime_from_iso8601(const char* pdate, time_t& unixtime)
+{
+  if(!pdate){
+    return false;
+  }
+
+  struct tm tm;
+  char*     prest = strptime(pdate, "%Y-%m-%dT%T", &tm);
+  if(prest == pdate){
+    // wrong format
+    return false;
+  }
+  unixtime = mktime(&tm);
+  return true;
+}
+
+//
+// Convert to unixtime from string which formatted by following:
+//   "12Y12M12D12h12m12s", "86400s", "9h30m", etc
+//
+bool convert_unixtime_from_option_arg(const char* argv, time_t& unixtime)
+{
+  if(!argv){
+    return false;
+  }
+  unixtime = 0;
+  const char* ptmp;
+  int         last_unit_type = 0;       // unit flag.
+  bool        is_last_number;
+  time_t      tmptime;
+  for(ptmp = argv, is_last_number = true, tmptime = 0; ptmp && *ptmp; ++ptmp){
+    if('0' <= *ptmp && *ptmp <= '9'){
+      tmptime        *= 10;
+      tmptime        += static_cast<time_t>(*ptmp - '0');
+      is_last_number  = true;
+    }else if(is_last_number){
+      if('Y' == *ptmp && 1 > last_unit_type){
+        unixtime      += (tmptime * (60 * 60 * 24 * 365));   // average 365 day / year
+        last_unit_type = 1;
+      }else if('M' == *ptmp && 2 > last_unit_type){
+        unixtime      += (tmptime * (60 * 60 * 24 * 30));    // average 30 day / month
+        last_unit_type = 2;
+      }else if('D' == *ptmp && 3 > last_unit_type){
+        unixtime      += (tmptime * (60 * 60 * 24));
+        last_unit_type = 3;
+      }else if('h' == *ptmp && 4 > last_unit_type){
+        unixtime      += (tmptime * (60 * 60));
+        last_unit_type = 4;
+      }else if('m' == *ptmp && 5 > last_unit_type){
+        unixtime      += (tmptime * 60);
+        last_unit_type = 5;
+      }else if('s' == *ptmp && 6 > last_unit_type){
+        unixtime      += tmptime;
+        last_unit_type = 6;
+      }else{
+        return false;
+      }
+      tmptime        = 0;
+      is_last_number = false;
+    }else{
+      return false;
+    }
+  }
+  if(is_last_number){
+    return false;
+  }
+  return true;
+}
+
 std::string s3fs_hex(const unsigned char* input, size_t length)
 {
   std::string hex;

src/string_util.h

@@ -46,6 +46,8 @@ std::string get_date_rfc850(void);
 void get_date_sigv3(std::string& date, std::string& date8601);
 std::string get_date_string(time_t tm);
 std::string get_date_iso8601(time_t tm);
+bool get_unixtime_from_iso8601(const char* pdate, time_t& unixtime);
+bool convert_unixtime_from_option_arg(const char* argv, time_t& unixtime);
 std::string urlEncode(const std::string &s);
 std::string urlEncode2(const std::string &s);
 std::string urlDecode(const std::string& s);