mirror of
https://github.com/s3fs-fuse/s3fs-fuse.git
synced 2024-12-22 08:48:55 +00:00
Added a non-interactive option to utility mode
Rebase for resolving conflicts Deleted interactive mode on utility mode Fixed minor mistakes
This commit is contained in:
Takeshi Nakatani
parent
e00afa8128
commit
231fd001d9
@ -16,7 +16,9 @@ For root.
|
||||
For unprivileged user.
|
||||
.SS utility mode ( remove interrupted multipart uploading objects )
|
||||
.TP
|
||||
\fBs3fs \-u bucket
|
||||
\fBs3fs --incomplete-mpu-list(-u) bucket
|
||||
.TP
|
||||
\fBs3fs --incomplete-mpu-abort[=all | =<expire date format>] bucket
|
||||
.SH DESCRIPTION
|
||||
s3fs is a FUSE filesystem that allows you to mount an Amazon S3 bucket as a local filesystem. It stores files natively and transparently in S3 (i.e., you can use other programs to access the same files).
|
||||
.SH AUTHENTICATION
|
||||
@ -292,6 +294,18 @@ When s3fs catch the signal SIGUSR2, the debug level is bumpup.
|
||||
.TP
|
||||
\fB\-o\fR curldbg - put curl debug message
|
||||
Put the debug message from libcurl when this option is specified.
|
||||
.SS "utility mode options"
|
||||
.TP
|
||||
\fB\-u\fR or \fB\-\-incomplete\-mpu\-list\fR
|
||||
Lists multipart incomplete objects uploaded to the specified bucket.
|
||||
.TP
|
||||
\fB\-\-incomplete\-mpu\-abort\fR all or date format(default="24H")
|
||||
Delete the multipart incomplete object uploaded to the specified bucket.
|
||||
If "all" is specified for this option, all multipart incomplete objects will be deleted.
|
||||
If you specify no argument as an option, objects older than 24 hours(24H) will be deleted(This is the default value).
|
||||
You can specify an optional date format.
|
||||
It can be specified as year, month, day, hour, minute, second, and it is expressed as "Y", "M", "D", "h", "m", "s" respectively.
|
||||
For example, "1Y6M10D12h30m30s".
|
||||
.SH FUSE/MOUNT OPTIONS
|
||||
.TP
|
||||
Most of the generic mount options described in 'man mount' are supported (ro, rw, suid, nosuid, dev, nodev, exec, noexec, atime, noatime, sync async, dirsync). Filesystems are mounted with '\-onodev,nosuid' by default, which can only be overridden by a privileged user.
|
||||
|
||||
145
src/s3fs.cpp
145
src/s3fs.cpp
@ -73,16 +73,25 @@ static bool IS_RMTYPEDIR(dirtype type) { return DIRTYPE_OLD == type || DIRTYPE_F
|
||||
#define ENOATTR ENODATA
|
||||
#endif
|
||||
|
||||
//
|
||||
// Type of utility process mode
|
||||
//
|
||||
enum utility_incomp_type{
|
||||
NO_UTILITY_MODE = 0, // not utility mode
|
||||
INCOMP_TYPE_LIST, // list of incomplete mpu
|
||||
INCOMP_TYPE_ABORT // delete incomplete mpu
|
||||
};
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Structs
|
||||
//-------------------------------------------------------------------
|
||||
typedef struct incomplete_multipart_info{
|
||||
typedef struct incomplete_multipart_upload_info{
|
||||
string key;
|
||||
string id;
|
||||
string date;
|
||||
}UNCOMP_MP_INFO;
|
||||
}INCOMP_MPU_INFO;
|
||||
|
||||
typedef std::list<UNCOMP_MP_INFO> uncomp_mp_list_t;
|
||||
typedef std::list<INCOMP_MPU_INFO> incomp_mpu_list_t;
|
||||
typedef std::list<std::string> readline_t;
|
||||
typedef std::map<std::string, std::string> kvmap_t;
|
||||
typedef std::map<std::string, kvmap_t> bucketkvmap_t;
|
||||
@ -115,7 +124,7 @@ static mode_t mp_umask = 0; // umask for mount point
|
||||
static bool is_mp_umask = false;// default does not set.
|
||||
static std::string mountpoint;
|
||||
static std::string passwd_file;
|
||||
static bool utility_mode = false;
|
||||
static utility_incomp_type utility_mode = NO_UTILITY_MODE;
|
||||
static bool noxmlns = false;
|
||||
static bool nocopyapi = false;
|
||||
static bool norenameapi = false;
|
||||
@ -183,14 +192,14 @@ static int clone_directory_object(const char* from, const char* to);
|
||||
static int rename_directory(const char* from, const char* to);
|
||||
static int remote_mountpath_exists(const char* path);
|
||||
static xmlChar* get_exp_value_xml(xmlDocPtr doc, xmlXPathContextPtr ctx, const char* exp_key);
|
||||
static void print_uncomp_mp_list(uncomp_mp_list_t& list);
|
||||
static bool abort_uncomp_mp_list(uncomp_mp_list_t& list);
|
||||
static bool get_uncomp_mp_list(xmlDocPtr doc, uncomp_mp_list_t& list);
|
||||
static void print_incomp_mpu_list(incomp_mpu_list_t& list);
|
||||
static bool abort_incomp_mpu_list(incomp_mpu_list_t& list, time_t abort_time);
|
||||
static bool get_incomp_mpu_list(xmlDocPtr doc, incomp_mpu_list_t& list);
|
||||
static void free_xattrs(xattrs_t& xattrs);
|
||||
static bool parse_xattr_keyval(const std::string& xattrpair, string& key, PXATTRVAL& pval);
|
||||
static size_t parse_xattrs(const std::string& strxattrs, xattrs_t& xattrs);
|
||||
static std::string build_xattrs(const xattrs_t& xattrs);
|
||||
static int s3fs_utility_mode();
|
||||
static int s3fs_utility_processing(time_t abort_time);
|
||||
static int s3fs_check_service();
|
||||
static int parse_passwd_file(bucketkvmap_t& resmap);
|
||||
static int check_for_aws_format(const kvmap_t& kvmap);
|
||||
@ -3493,7 +3502,7 @@ static xmlChar* get_exp_value_xml(xmlDocPtr doc, xmlXPathContextPtr ctx, const c
|
||||
return exp_value;
|
||||
}
|
||||
|
||||
static void print_uncomp_mp_list(uncomp_mp_list_t& list)
|
||||
static void print_incomp_mpu_list(incomp_mpu_list_t& list)
|
||||
{
|
||||
printf("\n");
|
||||
printf("Lists the parts that have been uploaded for a specific multipart upload.\n");
|
||||
@ -3503,7 +3512,7 @@ static void print_uncomp_mp_list(uncomp_mp_list_t& list)
|
||||
printf("---------------------------------------------------------------\n");
|
||||
|
||||
int cnt = 0;
|
||||
for(uncomp_mp_list_t::iterator iter = list.begin(); iter != list.end(); ++iter, ++cnt){
|
||||
for(incomp_mpu_list_t::iterator iter = list.begin(); iter != list.end(); ++iter, ++cnt){
|
||||
printf(" Path : %s\n", (*iter).key.c_str());
|
||||
printf(" UploadId : %s\n", (*iter).id.c_str());
|
||||
printf(" Date : %s\n", (*iter).date.c_str());
|
||||
@ -3516,35 +3525,31 @@ static void print_uncomp_mp_list(uncomp_mp_list_t& list)
|
||||
}
|
||||
}
|
||||
|
||||
static bool abort_uncomp_mp_list(uncomp_mp_list_t& list)
|
||||
static bool abort_incomp_mpu_list(incomp_mpu_list_t& list, time_t abort_time)
|
||||
{
|
||||
char buff[1024];
|
||||
|
||||
if(list.empty()){
|
||||
return true;
|
||||
}
|
||||
memset(buff, 0, sizeof(buff));
|
||||
time_t now_time = time(NULL);
|
||||
|
||||
// confirm
|
||||
while(true){
|
||||
printf("Would you remove all objects? [Y/N]\n");
|
||||
if(NULL != fgets(buff, sizeof(buff), stdin)){
|
||||
if(0 == strcasecmp(buff, "Y\n") || 0 == strcasecmp(buff, "YES\n")){
|
||||
break;
|
||||
}else if(0 == strcasecmp(buff, "N\n") || 0 == strcasecmp(buff, "NO\n")){
|
||||
return true;
|
||||
}
|
||||
printf("*** please put Y(yes) or N(no).\n");
|
||||
}
|
||||
}
|
||||
|
||||
// do removing their.
|
||||
// do removing.
|
||||
S3fsCurl s3fscurl;
|
||||
bool result = true;
|
||||
for(uncomp_mp_list_t::iterator iter = list.begin(); iter != list.end(); ++iter){
|
||||
for(incomp_mpu_list_t::iterator iter = list.begin(); iter != list.end(); ++iter){
|
||||
const char* tpath = (*iter).key.c_str();
|
||||
string upload_id = (*iter).id;
|
||||
|
||||
if(0 != abort_time){ // abort_time is 0, it means all.
|
||||
time_t date = 0;
|
||||
if(!get_unixtime_from_iso8601((*iter).date.c_str(), date)){
|
||||
S3FS_PRN_DBG("date format is not ISO 8601 for %s multipart uploading object, skip this.", tpath);
|
||||
continue;
|
||||
}
|
||||
if(now_time <= (date + abort_time)){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if(0 != s3fscurl.AbortMultipartUpload(tpath, upload_id)){
|
||||
S3FS_PRN_EXIT("Failed to remove %s multipart uploading object.", tpath);
|
||||
result = false;
|
||||
@ -3559,7 +3564,7 @@ static bool abort_uncomp_mp_list(uncomp_mp_list_t& list)
|
||||
return result;
|
||||
}
|
||||
|
||||
static bool get_uncomp_mp_list(xmlDocPtr doc, uncomp_mp_list_t& list)
|
||||
static bool get_incomp_mpu_list(xmlDocPtr doc, incomp_mpu_list_t& list)
|
||||
{
|
||||
if(!doc){
|
||||
return false;
|
||||
@ -3605,7 +3610,7 @@ static bool get_uncomp_mp_list(xmlDocPtr doc, uncomp_mp_list_t& list)
|
||||
for(cnt = 0, upload_nodes = upload_xp->nodesetval; cnt < upload_nodes->nodeNr; cnt++){
|
||||
ctx->node = upload_nodes->nodeTab[cnt];
|
||||
|
||||
UNCOMP_MP_INFO part;
|
||||
INCOMP_MPU_INFO part;
|
||||
xmlChar* ex_value;
|
||||
|
||||
// search "Key" tag
|
||||
@ -3643,18 +3648,18 @@ static bool get_uncomp_mp_list(xmlDocPtr doc, uncomp_mp_list_t& list)
|
||||
return true;
|
||||
}
|
||||
|
||||
static int s3fs_utility_mode()
|
||||
static int s3fs_utility_processing(time_t abort_time)
|
||||
{
|
||||
if(!utility_mode){
|
||||
if(NO_UTILITY_MODE == utility_mode){
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
printf("Utility Mode\n");
|
||||
printf("\n*** s3fs run as utility mode.\n\n");
|
||||
|
||||
S3fsCurl s3fscurl;
|
||||
string body;
|
||||
int result = EXIT_SUCCESS;
|
||||
if(0 != s3fscurl.MultipartListRequest(body)){
|
||||
S3FS_PRN_EXIT("Could not get list multipart upload.");
|
||||
S3FS_PRN_EXIT("Could not get list multipart upload.\nThere is no incomplete multipart uploaded object in bucket.\n");
|
||||
result = EXIT_FAILURE;
|
||||
}else{
|
||||
// parse result(incomplete multipart upload information)
|
||||
@ -3666,19 +3671,22 @@ static int s3fs_utility_mode()
|
||||
result = EXIT_FAILURE;
|
||||
|
||||
}else{
|
||||
// make working uploads list
|
||||
uncomp_mp_list_t list;
|
||||
if(!get_uncomp_mp_list(doc, list)){
|
||||
S3FS_PRN_DBG("get_uncomp_mp_list exited with error.");
|
||||
// make incomplete uploads list
|
||||
incomp_mpu_list_t list;
|
||||
if(!get_incomp_mpu_list(doc, list)){
|
||||
S3FS_PRN_DBG("get_incomp_mpu_list exited with error.");
|
||||
result = EXIT_FAILURE;
|
||||
|
||||
}else{
|
||||
// print list
|
||||
print_uncomp_mp_list(list);
|
||||
// remove
|
||||
if(!abort_uncomp_mp_list(list)){
|
||||
S3FS_PRN_DBG("an error occurred during removal process.");
|
||||
result = EXIT_FAILURE;
|
||||
if(INCOMP_TYPE_LIST == utility_mode){
|
||||
// print list
|
||||
print_incomp_mpu_list(list);
|
||||
}else if(INCOMP_TYPE_ABORT == utility_mode){
|
||||
// remove
|
||||
if(!abort_incomp_mpu_list(list, abort_time)){
|
||||
S3FS_PRN_DBG("an error occurred during removal process.");
|
||||
result = EXIT_FAILURE;
|
||||
}
|
||||
}
|
||||
}
|
||||
S3FS_XMLFREEDOC(doc);
|
||||
@ -4343,7 +4351,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
|
||||
}
|
||||
|
||||
// the second NONOPT option is the mountpoint(not utility mode)
|
||||
if(mountpoint.empty() && !utility_mode){
|
||||
if(mountpoint.empty() && NO_UTILITY_MODE == utility_mode){
|
||||
// save the mountpoint and do some basic error checking
|
||||
mountpoint = arg;
|
||||
struct stat stbuf;
|
||||
@ -4381,7 +4389,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
|
||||
}
|
||||
|
||||
// Unknown option
|
||||
if(!utility_mode){
|
||||
if(NO_UTILITY_MODE == utility_mode){
|
||||
S3FS_PRN_EXIT("specified unknown third option(%s).", arg);
|
||||
}else{
|
||||
S3FS_PRN_EXIT("specified unknown second option(%s). you don't need to specify second option(mountpoint) for utility mode(-u).", arg);
|
||||
@ -4946,12 +4954,15 @@ int main(int argc, char* argv[])
|
||||
int fuse_res;
|
||||
int option_index = 0;
|
||||
struct fuse_operations s3fs_oper;
|
||||
time_t incomp_abort_time = (24 * 60 * 60);
|
||||
|
||||
static const struct option long_opts[] = {
|
||||
{"help", no_argument, NULL, 'h'},
|
||||
{"version", no_argument, 0, 0},
|
||||
{"debug", no_argument, NULL, 'd'},
|
||||
{0, 0, 0, 0}
|
||||
{"help", no_argument, NULL, 'h'},
|
||||
{"version", no_argument, 0, 0},
|
||||
{"debug", no_argument, NULL, 'd'},
|
||||
{"incomplete-mpu-list", no_argument, NULL, 'u'},
|
||||
{"incomplete-mpu-abort", optional_argument, NULL, 'a'}, // 'a' is only identifier and is not option.
|
||||
{NULL, 0, NULL, 0}
|
||||
};
|
||||
|
||||
// init syslog(default CRIT)
|
||||
@ -4989,8 +5000,30 @@ int main(int argc, char* argv[])
|
||||
break;
|
||||
case 's':
|
||||
break;
|
||||
case 'u':
|
||||
utility_mode = true;
|
||||
case 'u': // --incomplete-mpu-list
|
||||
if(NO_UTILITY_MODE != utility_mode){
|
||||
S3FS_PRN_EXIT("already utility mode option is specified.");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
utility_mode = INCOMP_TYPE_LIST;
|
||||
break;
|
||||
case 'a': // --incomplete-mpu-abort
|
||||
if(NO_UTILITY_MODE != utility_mode){
|
||||
S3FS_PRN_EXIT("already utility mode option is specified.");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
utility_mode = INCOMP_TYPE_ABORT;
|
||||
|
||||
// check expire argument
|
||||
if(NULL != optarg && 0 == strcasecmp(optarg, "all")){ // all is 0s
|
||||
incomp_abort_time = 0;
|
||||
}else if(NULL != optarg){
|
||||
if(!convert_unixtime_from_option_arg(optarg, incomp_abort_time)){
|
||||
S3FS_PRN_EXIT("--incomplete-mpu-abort option argument is wrong.");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
// if optarg is null, incomp_abort_time is 24H(default)
|
||||
break;
|
||||
default:
|
||||
exit(EXIT_FAILURE);
|
||||
@ -5082,7 +5115,7 @@ int main(int argc, char* argv[])
|
||||
// if the option was given, we all ready checked for a
|
||||
// readable, non-empty directory, this checks determines
|
||||
// if the mountpoint option was ever supplied
|
||||
if(!utility_mode){
|
||||
if(NO_UTILITY_MODE == utility_mode){
|
||||
if(mountpoint.empty()){
|
||||
S3FS_PRN_EXIT("missing MOUNTPOINT argument.");
|
||||
show_usage();
|
||||
@ -5182,8 +5215,8 @@ int main(int argc, char* argv[])
|
||||
}
|
||||
*/
|
||||
|
||||
if(utility_mode){
|
||||
int exitcode = s3fs_utility_mode();
|
||||
if(NO_UTILITY_MODE != utility_mode){
|
||||
int exitcode = s3fs_utility_processing(incomp_abort_time);
|
||||
|
||||
S3fsCurl::DestroyS3fsCurl();
|
||||
s3fs_destroy_global_ssl();
|
||||
|
||||
@ -1013,13 +1013,14 @@ void show_help ()
|
||||
" umounting\n"
|
||||
" umount mountpoint\n"
|
||||
"\n"
|
||||
" utility mode (remove interrupted multipart uploading objects)\n"
|
||||
" s3fs -u bucket\n"
|
||||
"\n"
|
||||
" General forms for s3fs and FUSE/mount options:\n"
|
||||
" -o opt[,opt...]\n"
|
||||
" -o opt [-o opt] ...\n"
|
||||
"\n"
|
||||
" utility mode (remove interrupted multipart uploading objects)\n"
|
||||
" s3fs --incomplete-mpu-list(-u) bucket\n"
|
||||
" s3fs --incomplete-mpu-abort[=all | =<date format>] bucket\n"
|
||||
"\n"
|
||||
"s3fs Options:\n"
|
||||
"\n"
|
||||
" Most s3fs options are given in the form where \"opt\" is:\n"
|
||||
@ -1348,6 +1349,22 @@ void show_help ()
|
||||
" There are many FUSE specific mount options that can be specified.\n"
|
||||
" e.g. allow_other See the FUSE's README for the full set.\n"
|
||||
"\n"
|
||||
"Utility mode Options:\n"
|
||||
"\n"
|
||||
" -u, --incomplete-mpu-list\n"
|
||||
" Lists multipart incomplete objects uploaded to the specified\n"
|
||||
" bucket.\n"
|
||||
" --incomplete-mpu-abort(=all or =<date format>)\n"
|
||||
" Delete the multipart incomplete object uploaded to the specified\n"
|
||||
" bucket.\n"
|
||||
" If \"all\" is specified for this option, all multipart incomplete\n"
|
||||
" objects will be deleted. If you specify no argument as an option,\n"
|
||||
" objects older than 24 hours(24H) will be deleted(This is the\n"
|
||||
" default value). You can specify an optional date format. It can\n"
|
||||
" be specified as year, month, day, hour, minute, second, and it is\n"
|
||||
" expressed as \"Y\", \"M\", \"D\", \"h\", \"m\", \"s\" respectively.\n"
|
||||
" For example, \"1Y6M10D12h30m30s\".\n"
|
||||
"\n"
|
||||
"Miscellaneous Options:\n"
|
||||
"\n"
|
||||
" -h, --help Output this help.\n"
|
||||
|
||||
@ -22,6 +22,7 @@
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <syslog.h>
|
||||
#include <time.h>
|
||||
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
@ -283,6 +284,75 @@ string get_date_iso8601(time_t tm)
|
||||
return buf;
|
||||
}
|
||||
|
||||
bool get_unixtime_from_iso8601(const char* pdate, time_t& unixtime)
|
||||
{
|
||||
if(!pdate){
|
||||
return false;
|
||||
}
|
||||
|
||||
struct tm tm;
|
||||
char* prest = strptime(pdate, "%Y-%m-%dT%T", &tm);
|
||||
if(prest == pdate){
|
||||
// wrong format
|
||||
return false;
|
||||
}
|
||||
unixtime = mktime(&tm);
|
||||
return true;
|
||||
}
|
||||
|
||||
//
|
||||
// Convert to unixtime from string which formatted by following:
|
||||
// "12Y12M12D12h12m12s", "86400s", "9h30m", etc
|
||||
//
|
||||
bool convert_unixtime_from_option_arg(const char* argv, time_t& unixtime)
|
||||
{
|
||||
if(!argv){
|
||||
return false;
|
||||
}
|
||||
unixtime = 0;
|
||||
const char* ptmp;
|
||||
int last_unit_type = 0; // unit flag.
|
||||
bool is_last_number;
|
||||
time_t tmptime;
|
||||
for(ptmp = argv, is_last_number = true, tmptime = 0; ptmp && *ptmp; ++ptmp){
|
||||
if('0' <= *ptmp && *ptmp <= '9'){
|
||||
tmptime *= 10;
|
||||
tmptime += static_cast<time_t>(*ptmp - '0');
|
||||
is_last_number = true;
|
||||
}else if(is_last_number){
|
||||
if('Y' == *ptmp && 1 > last_unit_type){
|
||||
unixtime += (tmptime * (60 * 60 * 24 * 365)); // average 365 day / year
|
||||
last_unit_type = 1;
|
||||
}else if('M' == *ptmp && 2 > last_unit_type){
|
||||
unixtime += (tmptime * (60 * 60 * 24 * 30)); // average 30 day / month
|
||||
last_unit_type = 2;
|
||||
}else if('D' == *ptmp && 3 > last_unit_type){
|
||||
unixtime += (tmptime * (60 * 60 * 24));
|
||||
last_unit_type = 3;
|
||||
}else if('h' == *ptmp && 4 > last_unit_type){
|
||||
unixtime += (tmptime * (60 * 60));
|
||||
last_unit_type = 4;
|
||||
}else if('m' == *ptmp && 5 > last_unit_type){
|
||||
unixtime += (tmptime * 60);
|
||||
last_unit_type = 5;
|
||||
}else if('s' == *ptmp && 6 > last_unit_type){
|
||||
unixtime += tmptime;
|
||||
last_unit_type = 6;
|
||||
}else{
|
||||
return false;
|
||||
}
|
||||
tmptime = 0;
|
||||
is_last_number = false;
|
||||
}else{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if(is_last_number){
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string s3fs_hex(const unsigned char* input, size_t length)
|
||||
{
|
||||
std::string hex;
|
||||
|
||||
@ -46,6 +46,8 @@ std::string get_date_rfc850(void);
|
||||
void get_date_sigv3(std::string& date, std::string& date8601);
|
||||
std::string get_date_string(time_t tm);
|
||||
std::string get_date_iso8601(time_t tm);
|
||||
bool get_unixtime_from_iso8601(const char* pdate, time_t& unixtime);
|
||||
bool convert_unixtime_from_option_arg(const char* argv, time_t& unixtime);
|
||||
std::string urlEncode(const std::string &s);
|
||||
std::string urlEncode2(const std::string &s);
|
||||
std::string urlDecode(const std::string& s);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user