octoleo/s3fs-fuse
1
0
Fork 0
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git synced 2024-12-23 09:18:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Cleaned up source codes

Browse Source 
No changes for logic, only changes layout of functions and valiables between a file to a file.
    Adds s3fs_util.cpp/s3fs_util.h/common.h



git-svn-id: http://s3fs.googlecode.com/svn/trunk@396 df820570-a93a-0410-bd06-b72b767a4274
This commit is contained in:
ggtakec@gmail.com 2013-03-30 13:37:14 +00:00
parent 00b735beaa
commit 953aedd7ad
12 changed files with 1380 additions and 1347 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/Makefile.am
View File

@ -2,6 +2,6 @@ bin_PROGRAMS=s3fs
AM_CPPFLAGS = $(DEPS_CFLAGS) AM_CPPFLAGS = $(DEPS_CFLAGS)
s3fs_SOURCES = s3fs.cpp s3fs.h curl.cpp curl.h cache.cpp cache.h string_util.cpp string_util.h s3fs_SOURCES = s3fs.cpp s3fs.h curl.cpp curl.h cache.cpp cache.h string_util.cpp string_util.h s3fs_util.cpp s3fs_util.h common.h
s3fs_LDADD = $(DEPS_LIBS) s3fs_LDADD = $(DEPS_LIBS)

68
src/cache.cpp
View File

@ -27,15 +27,37 @@
#include <string> #include <string>
#include <map> #include <map>
#include "common.h"
#include "cache.h" #include "cache.h"
using namespace std; using namespace std;
//-------------------------------------------------------------------
// Typedef
//-------------------------------------------------------------------
typedef std::map<std::string, struct stat_cache_entry> stat_cache_t; // key=path typedef std::map<std::string, struct stat_cache_entry> stat_cache_t; // key=path
static stat_cache_t stat_cache;
pthread_mutex_t stat_cache_lock;
int get_stat_cache_entry(const char *path, struct stat *buf) { //-------------------------------------------------------------------
// Static valiables
//-------------------------------------------------------------------
static stat_cache_t stat_cache;
static pthread_mutex_t stat_cache_lock;
//-------------------------------------------------------------------
// Functions
//-------------------------------------------------------------------
int init_stat_cache_mutex(void)
{
return pthread_mutex_init(&stat_cache_lock, NULL);
}
int destroy_stat_cache_mutex(void)
{
return pthread_mutex_destroy(&stat_cache_lock);
}
int get_stat_cache_entry(const char *path, struct stat *buf)
{
int is_delete_cache = 0; int is_delete_cache = 0;
string strpath = path; string strpath = path;
@ -54,14 +76,12 @@ int get_stat_cache_entry(const char *path, struct stat *buf) {
if(iter != stat_cache.end()) { if(iter != stat_cache.end()) {
if(!is_stat_cache_expire_time || ((*iter).second.cache_date + stat_cache_expire_time) >= time(NULL)){ if(!is_stat_cache_expire_time || ((*iter).second.cache_date + stat_cache_expire_time) >= time(NULL)){
// hit // hit
if(foreground) FGPRINT(" stat cache hit [path=%s] [time=%ld] [hit count=%lu]\n",
cout << " stat cache hit [path=" << strpath << "]" strpath.c_str(), (*iter).second.cache_date, (*iter).second.hit_count);
<< " [time=" << (*iter).second.cache_date << "]"
<< " [hit count=" << (*iter).second.hit_count << "]" << endl;
if(buf != NULL) if(buf != NULL){
*buf = (*iter).second.stbuf; *buf = (*iter).second.stbuf;
}
(*iter).second.hit_count++; (*iter).second.hit_count++;
pthread_mutex_unlock(&stat_cache_lock); pthread_mutex_unlock(&stat_cache_lock);
return 0; return 0;
@ -79,30 +99,31 @@ int get_stat_cache_entry(const char *path, struct stat *buf) {
return -1; return -1;
} }
void add_stat_cache_entry(const char *path, struct stat *st) { void add_stat_cache_entry(const char *path, struct stat *st)
if(foreground) {
cout << " add_stat_cache_entry[path=" << path << "]" << endl; FGPRINT(" add_stat_cache_entry[path=%s]\n", path);
if(max_stat_cache_size < 1) if(max_stat_cache_size < 1){
return; return;
}
if(stat_cache.size() > max_stat_cache_size) if(stat_cache.size() > max_stat_cache_size){
truncate_stat_cache(); truncate_stat_cache();
}
pthread_mutex_lock(&stat_cache_lock); pthread_mutex_lock(&stat_cache_lock);
stat_cache[path].stbuf = *st; stat_cache[path].stbuf = *st;
stat_cache[path].cache_date = time(NULL); // Set time. stat_cache[path].cache_date = time(NULL); // Set time.
pthread_mutex_unlock(&stat_cache_lock); pthread_mutex_unlock(&stat_cache_lock);
} }
void delete_stat_cache_entry(const char *path) { void delete_stat_cache_entry(const char *path)
if(foreground) {
cout << " delete_stat_cache_entry[path=" << path << "]" << endl; FGPRINT(" delete_stat_cache_entry[path=%s]\n", path);
pthread_mutex_lock(&stat_cache_lock); pthread_mutex_lock(&stat_cache_lock);
stat_cache_t::iterator iter = stat_cache.find(path); stat_cache_t::iterator iter = stat_cache.find(path);
if(iter != stat_cache.end()) if(iter != stat_cache.end()){
stat_cache.erase(iter); stat_cache.erase(iter);
}
pthread_mutex_unlock(&stat_cache_lock); pthread_mutex_unlock(&stat_cache_lock);
} }
@ -120,13 +141,14 @@ void truncate_stat_cache() {
lowest_hit_count = hit_count; lowest_hit_count = hit_count;
path_to_delete = (* iter).first; path_to_delete = (* iter).first;
} }
if(lowest_hit_count > hit_count){
if(lowest_hit_count > hit_count)
path_to_delete = (* iter).first; path_to_delete = (* iter).first;
}
} }
stat_cache.erase(path_to_delete); stat_cache.erase(path_to_delete);
pthread_mutex_unlock(&stat_cache_lock); pthread_mutex_unlock(&stat_cache_lock);
printf(" purged %s from the stat cache\n", path_to_delete.c_str()); FGPRINT(" purged %s from the stat cache\n", path_to_delete.c_str());
} }

13
src/cache.h
View File

@ -5,6 +5,9 @@
#include <sys/types.h> #include <sys/types.h>
#include <unistd.h> #include <unistd.h>
//
// Struct
//
struct stat_cache_entry { struct stat_cache_entry {
struct stat stbuf; struct stat stbuf;
unsigned long hit_count; unsigned long hit_count;
@ -13,11 +16,11 @@ struct stat_cache_entry {
stat_cache_entry() : hit_count(0), cache_date(0) {} stat_cache_entry() : hit_count(0), cache_date(0) {}
}; };
extern bool foreground; //
extern unsigned long max_stat_cache_size; // Functions
extern time_t stat_cache_expire_time; //
extern int is_stat_cache_expire_time; int init_stat_cache_mutex(void);
int destroy_stat_cache_mutex(void);
int get_stat_cache_entry(const char *path, struct stat *buf); int get_stat_cache_entry(const char *path, struct stat *buf);
void add_stat_cache_entry(const char *path, struct stat *st); void add_stat_cache_entry(const char *path, struct stat *st);
void delete_stat_cache_entry(const char *path); void delete_stat_cache_entry(const char *path);

52
src/common.h Normal file
View File

@ -0,0 +1,52 @@
#ifndef S3FS_COMMON_H_
#define S3FS_COMMON_H_
//
// Macro
//
#define SYSLOGINFO(...) syslog(LOG_INFO, __VA_ARGS__);
#define SYSLOGERR(...) syslog(LOG_ERR, __VA_ARGS__);
#define SYSLOGCRIT(...) syslog(LOG_CRIT, __VA_ARGS__);
#define SYSLOGDBG(...) \
if(debug){ \
syslog(LOG_DEBUG, __VA_ARGS__); \
}
#define SYSLOGDBGERR(...) \
if(debug){ \
syslog(LOG_ERR, __VA_ARGS__); \
}
#define FGPRINT(...) \
if(foreground){ \
printf(__VA_ARGS__); \
}
//
// Typedef
//
typedef std::map<std::string, std::string> headers_t;
//
// Global valiables
//
extern bool debug;
extern bool foreground;
extern unsigned long max_stat_cache_size;
extern time_t stat_cache_expire_time;
extern int is_stat_cache_expire_time;
extern int retries;
extern long connect_timeout;
extern time_t readwrite_timeout;
extern std::string AWSAccessKeyId;
extern std::string AWSSecretAccessKey;
extern std::string program_name;
extern std::string ssl_verify_hostname;
extern std::string service_path;
extern std::string host;
extern std::string bucket;
extern std::string public_bucket;
extern std::string mount_prefix;
#endif // S3FS_COMMON_H_

363
src/curl.cpp
View File

@ -40,36 +40,53 @@
#include <map> #include <map>
#include <algorithm> #include <algorithm>
#include "common.h"
#include "curl.h" #include "curl.h"
#include "string_util.h" #include "string_util.h"
using namespace std; using namespace std;
pthread_mutex_t curl_handles_lock; //-------------------------------------------------------------------
std::map<CURL*, time_t> curl_times; // Typedef
std::map<CURL*, progress_t> curl_progress; //-------------------------------------------------------------------
std::string curl_ca_bundle; struct case_insensitive_compare_func {
bool operator ()(const string &a, const string &b) {
class auto_curl_slist { return strcasecmp(a.c_str(), b.c_str()) < 0;
public:
auto_curl_slist() : slist(0) { }
~auto_curl_slist() { curl_slist_free_all(slist); }
struct curl_slist* get() const { return slist; }
void append(const string& s) {
slist = curl_slist_append(slist, s.c_str());
} }
private:
struct curl_slist* slist;
}; };
typedef map<string, string, case_insensitive_compare_func> mimes_t;
typedef pair<double, double> progress_t;
size_t header_callback(void *data, size_t blockSize, size_t numBlocks, void *userPtr) { //-------------------------------------------------------------------
// Static valiables
//-------------------------------------------------------------------
static pthread_mutex_t curl_handles_lock;
static const EVP_MD* evp_md = EVP_sha1();
static map<CURL*, time_t> curl_times;
static map<CURL*, progress_t> curl_progress;
static string curl_ca_bundle;
static mimes_t mimeTypes;
//-------------------------------------------------------------------
// Functions
//-------------------------------------------------------------------
int init_curl_handles_mutex(void)
{
return pthread_mutex_init(&curl_handles_lock, NULL);
}
int destroy_curl_handles_mutex(void)
{
return pthread_mutex_destroy(&curl_handles_lock);
}
size_t header_callback(void *data, size_t blockSize, size_t numBlocks, void *userPtr)
{
headers_t* headers = reinterpret_cast<headers_t*>(userPtr); headers_t* headers = reinterpret_cast<headers_t*>(userPtr);
string header(reinterpret_cast<char*>(data), blockSize * numBlocks); string header(reinterpret_cast<char*>(data), blockSize * numBlocks);
string key; string key;
stringstream ss(header); stringstream ss(header);
if (getline(ss, key, ':')) { if (getline(ss, key, ':')) {
// Force to lower, only "x-amz" // Force to lower, only "x-amz"
string lkey = key; string lkey = key;
@ -84,7 +101,8 @@ size_t header_callback(void *data, size_t blockSize, size_t numBlocks, void *use
return blockSize * numBlocks; return blockSize * numBlocks;
} }
CURL *create_curl_handle(void) { CURL *create_curl_handle(void)
{
time_t now; time_t now;
CURL *curl_handle; CURL *curl_handle;
@ -99,11 +117,12 @@ CURL *create_curl_handle(void) {
curl_easy_setopt(curl_handle, CURLOPT_PROGRESSDATA, curl_handle); curl_easy_setopt(curl_handle, CURLOPT_PROGRESSDATA, curl_handle);
// curl_easy_setopt(curl_handle, CURLOPT_FORBID_REUSE, 1); // curl_easy_setopt(curl_handle, CURLOPT_FORBID_REUSE, 1);
if(ssl_verify_hostname.substr(0,1) == "0") if(ssl_verify_hostname.substr(0,1) == "0"){
curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYHOST, 0); curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYHOST, 0);
if(curl_ca_bundle.size() != 0) }
if(curl_ca_bundle.size() != 0){
curl_easy_setopt(curl_handle, CURLOPT_CAINFO, curl_ca_bundle.c_str()); curl_easy_setopt(curl_handle, CURLOPT_CAINFO, curl_ca_bundle.c_str());
}
now = time(0); now = time(0);
curl_times[curl_handle] = now; curl_times[curl_handle] = now;
curl_progress[curl_handle] = progress_t(-1, -1); curl_progress[curl_handle] = progress_t(-1, -1);
@ -112,7 +131,8 @@ CURL *create_curl_handle(void) {
return curl_handle; return curl_handle;
} }
void destroy_curl_handle(CURL *curl_handle) { void destroy_curl_handle(CURL *curl_handle)
{
if(curl_handle != NULL) { if(curl_handle != NULL) {
pthread_mutex_lock(&curl_handles_lock); pthread_mutex_lock(&curl_handles_lock);
curl_times.erase(curl_handle); curl_times.erase(curl_handle);
@ -124,7 +144,8 @@ void destroy_curl_handle(CURL *curl_handle) {
return; return;
} }
int curl_delete(const char *path) { int curl_delete(const char *path)
{
int result; int result;
string date; string date;
string url; string url;
@ -139,10 +160,10 @@ int curl_delete(const char *path) {
headers.append("Date: " + date); headers.append("Date: " + date);
headers.append("Content-Type: "); headers.append("Content-Type: ");
if(public_bucket.substr(0,1) != "1") if(public_bucket.substr(0,1) != "1"){
headers.append("Authorization: AWS " + AWSAccessKeyId + ":" + headers.append("Authorization: AWS " + AWSAccessKeyId + ":" +
calc_signature("DELETE", "", date, headers.get(), resource)); calc_signature("DELETE", "", date, headers.get(), resource));
}
my_url = prepare_url(url.c_str()); my_url = prepare_url(url.c_str());
curl = create_curl_handle(); curl = create_curl_handle();
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE");
@ -154,12 +175,12 @@ int curl_delete(const char *path) {
return result; return result;
} }
int curl_get_headers(const char *path, headers_t &meta) { int curl_get_headers(const char *path, headers_t &meta)
{
int result; int result;
CURL *curl; CURL *curl;
if(foreground) FGPRINT(" curl_headers[path=%s]\n", path);
printf(" curl_headers[path=%s]\n", path);
string resource(urlEncode(service_path + bucket + path)); string resource(urlEncode(service_path + bucket + path));
string url(host + resource); string url(host + resource);
@ -185,8 +206,9 @@ int curl_get_headers(const char *path, headers_t &meta) {
result = my_curl_easy_perform(curl); result = my_curl_easy_perform(curl);
destroy_curl_handle(curl); destroy_curl_handle(curl);
if(result != 0) if(result != 0){
return result; return result;
}
// file exists in s3 // file exists in s3
// fixme: clean this up. // fixme: clean this up.
@ -215,24 +237,61 @@ int curl_get_headers(const char *path, headers_t &meta) {
return 0; return 0;
} }
CURL *create_head_handle(head_data *request_data)
{
CURL *curl_handle = create_curl_handle();
string resource = urlEncode(service_path + bucket + request_data->path);
string url = host + resource;
// libcurl 7.17 does deep copy of url, deep copy "stable" url
string my_url = prepare_url(url.c_str());
request_data->url = new string(my_url.c_str());
request_data->requestHeaders = 0;
request_data->responseHeaders = new headers_t;
curl_easy_setopt(curl_handle, CURLOPT_URL, request_data->url->c_str());
curl_easy_setopt(curl_handle, CURLOPT_NOBODY, true); // HEAD
curl_easy_setopt(curl_handle, CURLOPT_FILETIME, true); // Last-Modified
// requestHeaders
string date = get_date();
request_data->requestHeaders = curl_slist_append(
request_data->requestHeaders, string("Date: " + date).c_str());
request_data->requestHeaders = curl_slist_append(
request_data->requestHeaders, string("Content-Type: ").c_str());
if(public_bucket.substr(0,1) != "1") {
request_data->requestHeaders = curl_slist_append(
request_data->requestHeaders, string("Authorization: AWS " + AWSAccessKeyId + ":" +
calc_signature("HEAD", "", date, request_data->requestHeaders, resource)).c_str());
}
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, request_data->requestHeaders);
// responseHeaders
curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, request_data->responseHeaders);
curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, header_callback);
return curl_handle;
}
/** /**
* @return fuse return code * @return fuse return code
*/ */
int my_curl_easy_perform(CURL* curl, BodyStruct* body, FILE* f) { int my_curl_easy_perform(CURL* curl, BodyStruct* body, FILE* f)
{
char url[256]; char url[256];
time_t now; time_t now;
char* ptr_url = url; char* ptr_url = url;
curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL , &ptr_url); curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL , &ptr_url);
if(debug) SYSLOGDBG("connecting to URL %s", ptr_url);
syslog(LOG_DEBUG, "connecting to URL %s", ptr_url);
// curl_easy_setopt(curl, CURLOPT_VERBOSE, true); // curl_easy_setopt(curl, CURLOPT_VERBOSE, true);
if(ssl_verify_hostname.substr(0,1) == "0") if(ssl_verify_hostname.substr(0,1) == "0"){
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
}
if(curl_ca_bundle.size() != 0) if(curl_ca_bundle.size() != 0){
curl_easy_setopt(curl, CURLOPT_CAINFO, curl_ca_bundle.c_str()); curl_easy_setopt(curl, CURLOPT_CAINFO, curl_ca_bundle.c_str());
}
long responseCode; long responseCode;
@ -249,19 +308,16 @@ int my_curl_easy_perform(CURL* curl, BodyStruct* body, FILE* f) {
// Need to look at the HTTP response code // Need to look at the HTTP response code
if (curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &responseCode) != 0) { if (curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &responseCode) != 0) {
syslog(LOG_ERR, "curl_easy_getinfo failed while trying to retrieve HTTP response code"); SYSLOGERR("curl_easy_getinfo failed while trying to retrieve HTTP response code");
return -EIO; return -EIO;
} }
SYSLOGDBG("HTTP response code %ld", responseCode);
if(debug)
syslog(LOG_DEBUG, "HTTP response code %ld", responseCode);
if (responseCode < 400) { if (responseCode < 400) {
return 0; return 0;
} }
if (responseCode >= 500) { if (responseCode >= 500) {
syslog(LOG_ERR, "###HTTP response=%ld", responseCode); SYSLOGERR("###HTTP response=%ld", responseCode);
sleep(4); sleep(4);
break; break;
} }
@ -269,88 +325,82 @@ int my_curl_easy_perform(CURL* curl, BodyStruct* body, FILE* f) {
// Service response codes which are >= 400 && < 500 // Service response codes which are >= 400 && < 500
switch(responseCode) { switch(responseCode) {
case 400: case 400:
if(debug) syslog(LOG_ERR, "HTTP response code 400 was returned"); SYSLOGDBGERR("HTTP response code 400 was returned");
if(body) { if(body && body->size){
if(body->size && debug) { SYSLOGDBGERR("Body Text: %s", body->text);
syslog(LOG_ERR, "Body Text: %s", body->text);
}
} }
if(debug) syslog(LOG_DEBUG, "Now returning EIO"); SYSLOGDBG("Now returning EIO");
return -EIO; return -EIO;
case 403: case 403:
if(debug) syslog(LOG_ERR, "HTTP response code 403 was returned"); SYSLOGDBGERR("HTTP response code 403 was returned");
if(body) if(body && body->size){
if(body->size && debug) SYSLOGDBGERR("Body Text: %s", body->text);
syslog(LOG_ERR, "Body Text: %s", body->text); }
return -EPERM; return -EPERM;
case 404: case 404:
if(debug) syslog(LOG_DEBUG, "HTTP response code 404 was returned"); SYSLOGDBG("HTTP response code 404 was returned");
if(body) { if(body && body->size){
if(body->size && debug) { SYSLOGDBG("Body Text: %s", body->text);
syslog(LOG_DEBUG, "Body Text: %s", body->text);
}
} }
if(debug) syslog(LOG_DEBUG, "Now returning ENOENT"); SYSLOGDBG("Now returning ENOENT");
return -ENOENT; return -ENOENT;
default: default:
syslog(LOG_ERR, "###response=%ld", responseCode); SYSLOGERR("###response=%ld", responseCode);
printf("responseCode %ld\n", responseCode); printf("responseCode %ld\n", responseCode);
if(body) { if(body && body->size){
if(body->size) { printf("Body Text %s\n", body->text);
printf("Body Text %s\n", body->text);
}
} }
return -EIO; return -EIO;
} }
break; break;
case CURLE_WRITE_ERROR: case CURLE_WRITE_ERROR:
syslog(LOG_ERR, "### CURLE_WRITE_ERROR"); SYSLOGERR("### CURLE_WRITE_ERROR");
sleep(2); sleep(2);
break; break;
case CURLE_OPERATION_TIMEDOUT: case CURLE_OPERATION_TIMEDOUT:
syslog(LOG_ERR, "### CURLE_OPERATION_TIMEDOUT"); SYSLOGERR("### CURLE_OPERATION_TIMEDOUT");
sleep(2); sleep(2);
break; break;
case CURLE_COULDNT_RESOLVE_HOST: case CURLE_COULDNT_RESOLVE_HOST:
syslog(LOG_ERR, "### CURLE_COULDNT_RESOLVE_HOST"); SYSLOGERR("### CURLE_COULDNT_RESOLVE_HOST");
sleep(2); sleep(2);
break; break;
case CURLE_COULDNT_CONNECT: case CURLE_COULDNT_CONNECT:
syslog(LOG_ERR, "### CURLE_COULDNT_CONNECT"); SYSLOGERR("### CURLE_COULDNT_CONNECT");
sleep(4); sleep(4);
break; break;
case CURLE_GOT_NOTHING: case CURLE_GOT_NOTHING:
syslog(LOG_ERR, "### CURLE_GOT_NOTHING"); SYSLOGERR("### CURLE_GOT_NOTHING");
sleep(4); sleep(4);
break; break;
case CURLE_ABORTED_BY_CALLBACK: case CURLE_ABORTED_BY_CALLBACK:
syslog(LOG_ERR, "### CURLE_ABORTED_BY_CALLBACK"); SYSLOGERR("### CURLE_ABORTED_BY_CALLBACK");
sleep(4); sleep(4);
now = time(0); now = time(0);
curl_times[curl] = now; curl_times[curl] = now;
break; break;
case CURLE_PARTIAL_FILE: case CURLE_PARTIAL_FILE:
syslog(LOG_ERR, "### CURLE_PARTIAL_FILE"); SYSLOGERR("### CURLE_PARTIAL_FILE");
sleep(4); sleep(4);
break; break;
case CURLE_SEND_ERROR: case CURLE_SEND_ERROR:
syslog(LOG_ERR, "### CURLE_SEND_ERROR"); SYSLOGERR("### CURLE_SEND_ERROR");
sleep(2); sleep(2);
break; break;
case CURLE_RECV_ERROR: case CURLE_RECV_ERROR:
syslog(LOG_ERR, "### CURLE_RECV_ERROR"); SYSLOGERR("### CURLE_RECV_ERROR");
sleep(2); sleep(2);
break; break;
@ -365,8 +415,7 @@ int my_curl_easy_perform(CURL* curl, BodyStruct* body, FILE* f) {
continue; continue;
} }
} }
syslog(LOG_ERR, "curlCode: %i msg: %s", curlCode, SYSLOGERR("curlCode: %i msg: %s", curlCode, curl_easy_strerror(curlCode));
curl_easy_strerror(curlCode));;
fprintf (stderr, "%s: curlCode: %i -- %s\n", fprintf (stderr, "%s: curlCode: %i -- %s\n",
program_name.c_str(), program_name.c_str(),
curlCode, curlCode,
@ -395,12 +444,12 @@ int my_curl_easy_perform(CURL* curl, BodyStruct* body, FILE* f) {
// This should be invalid since curl option HTTP FAILONERROR is now off // This should be invalid since curl option HTTP FAILONERROR is now off
case CURLE_HTTP_RETURNED_ERROR: case CURLE_HTTP_RETURNED_ERROR:
syslog(LOG_ERR, "### CURLE_HTTP_RETURNED_ERROR"); SYSLOGERR("### CURLE_HTTP_RETURNED_ERROR");
if (curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &responseCode) != 0) { if (curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &responseCode) != 0) {
return -EIO; return -EIO;
} }
syslog(LOG_ERR, "###response=%ld", responseCode); SYSLOGERR("###response=%ld", responseCode);
// Let's try to retrieve the // Let's try to retrieve the
@ -414,19 +463,19 @@ int my_curl_easy_perform(CURL* curl, BodyStruct* body, FILE* f) {
// Unknown CURL return code // Unknown CURL return code
default: default:
syslog(LOG_ERR, "###curlCode: %i msg: %s", curlCode, SYSLOGERR("###curlCode: %i msg: %s", curlCode, curl_easy_strerror(curlCode));
curl_easy_strerror(curlCode));;
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
break; break;
} }
syslog(LOG_ERR, "###retrying..."); SYSLOGERR("###retrying...");
} }
syslog(LOG_ERR, "###giving up"); SYSLOGERR("###giving up");
return -EIO; return -EIO;
} }
// libcurl callback // libcurl callback
size_t WriteMemoryCallback(void *ptr, size_t blockSize, size_t numBlocks, void *data) { size_t WriteMemoryCallback(void *ptr, size_t blockSize, size_t numBlocks, void *data)
{
size_t realsize = blockSize * numBlocks; size_t realsize = blockSize * numBlocks;
struct BodyStruct *mem = (struct BodyStruct *)data; struct BodyStruct *mem = (struct BodyStruct *)data;
@ -446,12 +495,13 @@ size_t WriteMemoryCallback(void *ptr, size_t blockSize, size_t numBlocks, void *
// read_callback // read_callback
// http://curl.haxx.se/libcurl/c/post-callback.html // http://curl.haxx.se/libcurl/c/post-callback.html
size_t read_callback(void *ptr, size_t size, size_t nmemb, void *userp) { size_t read_callback(void *ptr, size_t size, size_t nmemb, void *userp)
{
struct WriteThis *pooh = (struct WriteThis *)userp; struct WriteThis *pooh = (struct WriteThis *)userp;
if(size*nmemb < 1) if(size*nmemb < 1){
return 0; return 0;
}
if(pooh->sizeleft) { if(pooh->sizeleft) {
*(char *)ptr = pooh->readptr[0]; /* copy one single byte */ *(char *)ptr = pooh->readptr[0]; /* copy one single byte */
pooh->readptr++; /* advance pointer */ pooh->readptr++; /* advance pointer */
@ -463,8 +513,8 @@ size_t read_callback(void *ptr, size_t size, size_t nmemb, void *userp) {
} }
// homegrown timeout mechanism // homegrown timeout mechanism
int my_curl_progress( int my_curl_progress(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
void *clientp, double dltotal, double dlnow, double ultotal, double ulnow) { {
CURL* curl = static_cast<CURL*>(clientp); CURL* curl = static_cast<CURL*>(clientp);
time_t now = time(0); time_t now = time(0);
@ -482,7 +532,7 @@ int my_curl_progress(
if (now - curl_times[curl] > readwrite_timeout) { if (now - curl_times[curl] > readwrite_timeout) {
pthread_mutex_unlock( &curl_handles_lock ); pthread_mutex_unlock( &curl_handles_lock );
syslog(LOG_ERR, "timeout now: %li curl_times[curl]: %lil readwrite_timeout: %li", SYSLOGERR("timeout now: %li curl_times[curl]: %lil readwrite_timeout: %li",
(long int)now, curl_times[curl], (long int)readwrite_timeout); (long int)now, curl_times[curl], (long int)readwrite_timeout);
return CURLE_ABORTED_BY_CALLBACK; return CURLE_ABORTED_BY_CALLBACK;
@ -501,8 +551,8 @@ int my_curl_progress(
* @param date e.g., get_date() * @param date e.g., get_date()
* @param resource e.g., "/pub" * @param resource e.g., "/pub"
*/ */
string calc_signature( string calc_signature(string method, string content_type, string date, curl_slist* headers, string resource)
string method, string content_type, string date, curl_slist* headers, string resource) { {
int ret; int ret;
int bytes_written; int bytes_written;
int offset; int offset;
@ -556,13 +606,13 @@ string calc_signature(
continue; continue;
// Too many write attempts // Too many write attempts
syslog(LOG_ERR, "Failure during BIO_write, returning null String"); SYSLOGERR("Failure during BIO_write, returning null String");
BIO_free_all(b64); BIO_free_all(b64);
Signature.clear(); Signature.clear();
return Signature; return Signature;
} else { } else {
// If not a retry then it is an error // If not a retry then it is an error
syslog(LOG_ERR, "Failure during BIO_write, returning null String"); SYSLOGERR("Failure during BIO_write, returning null String");
BIO_free_all(b64); BIO_free_all(b64);
Signature.clear(); Signature.clear();
return Signature; return Signature;
@ -575,14 +625,15 @@ string calc_signature(
// If there is no more data to write, the request sending has been // If there is no more data to write, the request sending has been
// completed // completed
if(md_len <= 0) if(md_len <= 0){
break; break;
}
} }
// Flush the data // Flush the data
ret = BIO_flush(b64); ret = BIO_flush(b64);
if ( ret <= 0) { if ( ret <= 0) {
syslog(LOG_ERR, "Failure during BIO_flush, returning null String"); SYSLOGERR("Failure during BIO_flush, returning null String");
BIO_free_all(b64); BIO_free_all(b64);
Signature.clear(); Signature.clear();
return Signature; return Signature;
@ -600,7 +651,8 @@ string calc_signature(
return Signature; return Signature;
} }
void locate_bundle(void) { void locate_bundle(void)
{
// See if environment variable CURL_CA_BUNDLE is set // See if environment variable CURL_CA_BUNDLE is set
// if so, check it, if it is a good path, then set the // if so, check it, if it is a good path, then set the
// curl_ca_bundle variable to it // curl_ca_bundle variable to it
@ -619,7 +671,6 @@ void locate_bundle(void) {
program_name.c_str()); program_name.c_str());
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
return; return;
} }
} }
@ -649,3 +700,125 @@ void locate_bundle(void) {
return; return;
} }
string md5sum(int fd)
{
MD5_CTX c;
char buf[512];
char hexbuf[3];
ssize_t bytes;
char md5[2 * MD5_DIGEST_LENGTH + 1];
unsigned char *result = (unsigned char *) malloc(MD5_DIGEST_LENGTH);
memset(buf, 0, 512);
MD5_Init(&c);
while((bytes = read(fd, buf, 512)) > 0) {
MD5_Update(&c, buf, bytes);
memset(buf, 0, 512);
}
MD5_Final(result, &c);
memset(md5, 0, 2 * MD5_DIGEST_LENGTH + 1);
for(int i = 0; i < MD5_DIGEST_LENGTH; i++) {
snprintf(hexbuf, 3, "%02x", result[i]);
strncat(md5, hexbuf, 2);
}
free(result);
lseek(fd, 0, 0);
return string(md5);
}
bool InitMimeType(const char* file)
{
if(!file){
return false;
}
string line;
ifstream MT(file);
if (MT.good()) {
while (getline(MT, line)) {
if(line[0]=='#'){
continue;
}
if(line.size() == 0){
continue;
}
stringstream tmp(line);
string mimeType;
tmp >> mimeType;
while (tmp) {
string ext;
tmp >> ext;
if (ext.size() == 0){
continue;
}
mimeTypes[ext] = mimeType;
}
}
}
return true;
}
/**
* @param s e.g., "index.html"
* @return e.g., "text/html"
*/
string lookupMimeType(string s)
{
string result("application/octet-stream");
string::size_type last_pos = s.find_last_of('.');
string::size_type first_pos = s.find_first_of('.');
string prefix, ext, ext2;
// No dots in name, just return
if(last_pos == string::npos){
return result;
}
// extract the last extension
if(last_pos != string::npos){
ext = s.substr(1+last_pos, string::npos);
}
if (last_pos != string::npos) {
// one dot was found, now look for another
if (first_pos != string::npos && first_pos < last_pos) {
prefix = s.substr(0, last_pos);
// Now get the second to last file extension
string::size_type next_pos = prefix.find_last_of('.');
if (next_pos != string::npos) {
ext2 = prefix.substr(1+next_pos, string::npos);
}
}
}
// if we get here, then we have an extension (ext)
mimes_t::const_iterator iter = mimeTypes.find(ext);
// if the last extension matches a mimeType, then return
// that mime type
if (iter != mimeTypes.end()) {
result = (*iter).second;
return result;
}
// return with the default result if there isn't a second extension
if(first_pos == last_pos){
return result;
}
// Didn't find a mime-type for the first extension
// Look for second extension in mimeTypes, return if found
iter = mimeTypes.find(ext2);
if (iter != mimeTypes.end()) {
result = (*iter).second;
return result;
}
// neither the last extension nor the second-to-last extension
// matched a mimeType, return the default mime type
return result;
}

89
src/curl.h
View File

@ -13,29 +13,83 @@ struct WriteThis {
int sizeleft; int sizeleft;
}; };
typedef std::pair<double, double> progress_t; class auto_curl_slist {
typedef std::map<std::string, std::string> headers_t; public:
auto_curl_slist() : slist(0) { }
~auto_curl_slist() { curl_slist_free_all(slist); }
extern int retries; struct curl_slist* get() const { return slist; }
extern long connect_timeout;
extern time_t readwrite_timeout;
extern bool debug;
extern std::string program_name;
extern std::string ssl_verify_hostname;
extern std::string AWSAccessKeyId;
extern std::string AWSSecretAccessKey;
extern std::string service_path;
extern std::string host;
extern std::string bucket;
extern std::string public_bucket;
static const EVP_MD* evp_md = EVP_sha1(); void append(const std::string& s) {
slist = curl_slist_append(slist, s.c_str());
}
private:
struct curl_slist* slist;
};
// header data
struct head_data {
std::string path;
std::string *url;
struct curl_slist *requestHeaders;
headers_t *responseHeaders;
};
typedef std::map<CURL*, head_data> headMap_t;
void destroy_curl_handle(CURL *curl_handle);
struct cleanup_head_data {
void operator()(std::pair<CURL*, head_data> qqq) {
CURL* curl_handle = qqq.first;
head_data response = qqq.second;
delete response.url;
curl_slist_free_all(response.requestHeaders);
delete response.responseHeaders;
destroy_curl_handle(curl_handle);
}
};
class auto_head {
public:
auto_head() {}
~auto_head() {
for_each(headMap.begin(), headMap.end(), cleanup_head_data());
}
headMap_t& get() { return headMap; }
void remove(CURL* curl_handle) {
headMap_t::iterator iter = headMap.find(curl_handle);
if(iter == headMap.end()){
return;
}
head_data response = iter->second;
delete response.url;
curl_slist_free_all(response.requestHeaders);
delete response.responseHeaders;
destroy_curl_handle(curl_handle);
headMap.erase(iter);
}
private:
headMap_t headMap;
};
//
// Functions
//
int init_curl_handles_mutex(void);
int destroy_curl_handles_mutex(void);
size_t header_callback(void *data, size_t blockSize, size_t numBlocks, void *userPtr); size_t header_callback(void *data, size_t blockSize, size_t numBlocks, void *userPtr);
CURL *create_curl_handle(void); CURL *create_curl_handle(void);
void destroy_curl_handle(CURL *curl_handle);
int curl_delete(const char *path); int curl_delete(const char *path);
int curl_get_headers(const char *path, headers_t &meta); int curl_get_headers(const char *path, headers_t &meta);
CURL *create_head_handle(struct head_data *request);
int my_curl_easy_perform(CURL* curl, BodyStruct* body = NULL, FILE* f = 0); int my_curl_easy_perform(CURL* curl, BodyStruct* body = NULL, FILE* f = 0);
size_t WriteMemoryCallback(void *ptr, size_t blockSize, size_t numBlocks, void *data); size_t WriteMemoryCallback(void *ptr, size_t blockSize, size_t numBlocks, void *data);
size_t read_callback(void *ptr, size_t size, size_t nmemb, void *userp); size_t read_callback(void *ptr, size_t size, size_t nmemb, void *userp);
@ -44,5 +98,8 @@ int my_curl_progress(
std::string calc_signature( std::string calc_signature(
std::string method, std::string content_type, std::string date, curl_slist* headers, std::string resource); std::string method, std::string content_type, std::string date, curl_slist* headers, std::string resource);
void locate_bundle(void); void locate_bundle(void);
std::string md5sum(int fd);
bool InitMimeType(const char* file);
std::string lookupMimeType(std::string);
#endif // S3FS_CURL_H_ #endif // S3FS_CURL_H_

1348
src/s3fs.cpp
View File

File diff suppressed because it is too large Load Diff

152
src/s3fs.h
View File

@ -1,25 +1,13 @@
#ifndef S3FS_S3_H_ #ifndef S3FS_S3_H_
#define S3FS_S3_H_ #define S3FS_S3_H_
#define FUSE_USE_VERSION 26 #define FUSE_USE_VERSION 26
#define MULTIPART_SIZE 10485760 // 10MB #define MULTIPART_SIZE 10485760 // 10MB
#define MAX_REQUESTS 100 // max number of concurrent HTTP requests #define MAX_REQUESTS 100 // max number of concurrent HTTP requests
#define MAX_COPY_SOURCE_SIZE 524288000 // 500MB #define MAX_COPY_SOURCE_SIZE 524288000 // 500MB
#define FIVE_GB 5368709120LL #define FIVE_GB 5368709120LL
#include <map>
#include <string>
#include <vector>
#include <curl/curl.h>
#include <fuse.h> #include <fuse.h>
#include <openssl/bio.h>
#include <openssl/buffer.h>
#include <openssl/evp.h>
#include <openssl/hmac.h>
#include <openssl/md5.h>
#include <pthread.h>
#include <sys/time.h>
#define YIKES(result) if (true) { \ #define YIKES(result) if (true) { \
syslog(LOG_ERR, "%d###result=%d", __LINE__, result); \ syslog(LOG_ERR, "%d###result=%d", __LINE__, result); \
@ -33,134 +21,4 @@
} \ } \
} }
long connect_timeout = 10;
time_t readwrite_timeout = 30;
int retries = 2;
bool debug = 0;
bool foreground = 0;
bool nomultipart = false;
bool service_validated = false;
std::string host = "http://s3.amazonaws.com";
std::string service_path = "/";
std::string bucket = "";
std::string mount_prefix = "";
static std::string mountpoint;
std::string program_name;
std::string AWSAccessKeyId;
std::string AWSSecretAccessKey;
static mode_t root_mode = 0;
static std::string passwd_file = "";
static bool utility_mode = 0;
unsigned long max_stat_cache_size = 10000;
time_t stat_cache_expire_time = 0;
int is_stat_cache_expire_time = 0;
bool noxmlns = false;
bool nocopyapi = false;
bool norenameapi = false;
// if .size()==0 then local file cache is disabled
static std::string use_cache;
static std::string use_rrs;
std::string ssl_verify_hostname = "1";
std::string public_bucket;
extern pthread_mutex_t stat_cache_lock;
extern pthread_mutex_t curl_handles_lock;
extern std::string curl_ca_bundle;
// TODO(apetresc): make this an enum
// private, public-read, public-read-write, authenticated-read
static std::string default_acl("private");
struct file_part {
char path[17];
std::string etag;
bool uploaded;
file_part() : uploaded(false) {}
};
static const char hexAlphabet[] = "0123456789ABCDEF";
// http headers
typedef std::map<std::string, std::string> headers_t;
// fd -> flags
typedef std::map<int, int> s3fs_descriptors_t;
static s3fs_descriptors_t s3fs_descriptors;
static pthread_mutex_t s3fs_descriptors_lock;
// path -> fd
typedef std::map<std::string, int> s3fs_pathtofd_t;
static s3fs_pathtofd_t s3fs_pathtofd;
static pthread_mutex_t *mutex_buf = NULL;
static struct fuse_operations s3fs_oper;
std::string lookupMimeType(std::string);
std::string initiate_multipart_upload(const char *path, off_t size, headers_t meta);
std::string upload_part(const char *path, const char *source, int part_number, std::string upload_id);
std::string copy_part(const char *from, const char *to, int part_number, std::string upload_id, headers_t meta);
static int complete_multipart_upload(const char *path, std::string upload_id, std::vector <file_part> parts);
std::string md5sum(int fd);
std::string get_realpath(const char *path);
time_t get_mtime(const char *s);
off_t get_size(const char *s);
mode_t get_mode(const char *s);
uid_t get_uid(const char *s);
gid_t get_gid(const char *s);
blkcnt_t get_blocks(off_t size);
static int insert_object(const char *name, struct s3_object **head);
//static unsigned int count_object_list(struct s3_object *list);
static int free_object(struct s3_object *object);
static int free_object_list(struct s3_object *head);
static CURL *create_head_handle(struct head_data *request);
static int list_bucket(const char *path, struct s3_object **head, const char* delimiter);
static bool is_truncated(const char *xml);
static int append_objects_from_xml_ex(const char* path, xmlDocPtr doc, xmlXPathContextPtr ctx,
const char* ex_contents, const char* ex_key, int isCPrefix, struct s3_object **head);
static int append_objects_from_xml(const char* path, const char *xml, struct s3_object **head);
static xmlChar* get_base_exp(const char* xml, const char* exp);
static xmlChar* get_prefix(const char *xml);
static xmlChar* get_next_marker(const char *xml);
static char *get_object_name(xmlDocPtr doc, xmlNodePtr node, const char* path);
static int put_headers(const char *path, headers_t meta);
static int put_multipart_headers(const char *path, headers_t meta);
static int s3fs_getattr(const char *path, struct stat *stbuf);
static int s3fs_readlink(const char *path, char *buf, size_t size);
static int s3fs_mknod(const char* path, mode_t mode, dev_t rdev);
static int s3fs_mkdir(const char *path, mode_t mode);
static int s3fs_unlink(const char *path);
static int s3fs_rmdir(const char *path);
static int s3fs_symlink(const char *from, const char *to);
static int s3fs_rename(const char *from, const char *to);
static int s3fs_link(const char *from, const char *to);
static int s3fs_chmod(const char *path, mode_t mode);
static int s3fs_chown(const char *path, uid_t uid, gid_t gid);
static int s3fs_truncate(const char *path, off_t size);
static int s3fs_open(const char *path, struct fuse_file_info *fi);
static int s3fs_read(
const char *path, char *buf, size_t size, off_t offset, struct fuse_file_info *fi);
static int s3fs_write(
const char *path, const char *buf, size_t size, off_t offset, struct fuse_file_info *fi);
static int s3fs_statfs(const char *path, struct statvfs *stbuf);
static int s3fs_flush(const char *path, struct fuse_file_info *fi);
static int s3fs_release(const char *path, struct fuse_file_info *fi);
static int s3fs_opendir(const char *path, struct fuse_file_info *fi);
static int s3fs_readdir(
const char *path, void *buf, fuse_fill_dir_t filler, off_t offset, struct fuse_file_info *fi);
static int s3fs_access(const char *path, int mask);
static int s3fs_utimens(const char *path, const struct timespec ts[2]);
static int remote_mountpath_exists(const char *path);
static void* s3fs_init(struct fuse_conn_info *conn);
static void s3fs_destroy(void*);
#endif // S3FS_S3_H_ #endif // S3FS_S3_H_

525
src/s3fs_util.cpp Normal file
View File

@ -0,0 +1,525 @@
/*
* s3fs - FUSE-based file system backed by Amazon S3
*
* Copyright 2007-2013 Takeshi Nakatani <ggtakec.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <errno.h>
#include <libgen.h>
#include <pwd.h>
#include <grp.h>
#include <syslog.h>
#include <string>
#include <sstream>
#include <map>
#include "common.h"
#include "s3fs_util.h"
#include "s3fs.h"
using namespace std;
//-------------------------------------------------------------------
// Global valiables
//-------------------------------------------------------------------
std::string mount_prefix = "";
//-------------------------------------------------------------------
// Utility
//-------------------------------------------------------------------
string get_realpath(const char *path) {
string realpath = mount_prefix;
realpath += path;
return realpath;
}
//-------------------------------------------------------------------
// Utility for listing objects
//-------------------------------------------------------------------
// [Change]
// The s3_object's name member can be set "dir" or "dir/" as directory name.
// Both of names are same directory s3_object.
// "dir/" is given priority to over "dir".
//
// [Notice]
// If there are "dir" and "dir/" object on S3, s3fs only recognizes "dir/".
// On this case, user can not know the "dir" object.
//
int insert_object(const char *name, struct s3_object **head)
{
struct s3_object *cur_object;
struct s3_object *new_object;
int nLen = name ? strlen(name) : 0;
int is_have_ndelimiter = 0;
// search same name object
if(nLen && '/' == name[nLen - 1]){
// case of "dir/"
nLen--;
is_have_ndelimiter = 1;
}
for(cur_object = *head; nLen && cur_object; cur_object = cur_object->next){
if(0 == strncmp(cur_object->name, name, nLen)){
int cLen = strlen(cur_object->name);
int is_have_cdelimiter = 0;
if('/' == cur_object->name[cLen - 1]){
cLen--;
is_have_cdelimiter = 1;
}
if(cLen == nLen){
// same object
if(is_have_cdelimiter == is_have_ndelimiter){
// perfect same object
return 0;
}
if(is_have_cdelimiter){
// already set "dir/"
return 0;
}
// new object is "dir/", replace name.
free(cur_object->name);
if(NULL == (cur_object->name = strdup(name))){
printf("insert_object: could not allocate memory\n");
S3FS_FUSE_EXIT();
return -1;
}
return 0;
}
}
}
// Not found same object.
new_object = (struct s3_object *) malloc(sizeof(struct s3_object));
if(new_object == NULL) {
printf("insert_object: could not allocate memory\n");
S3FS_FUSE_EXIT();
return -1;
}
if(NULL == (new_object->name = strdup(name))){
free(new_object);
printf("insert_object: could not allocate memory\n");
S3FS_FUSE_EXIT();
return -1;
}
if((*head) == NULL)
new_object->next = NULL;
else
new_object->next = (*head);
*head = new_object;
return 0;
}
int free_object(struct s3_object *object)
{
free(object->name);
free(object);
object = NULL;
return 0;
}
int free_object_list(struct s3_object *head)
{
struct s3_object *tmp = NULL;
struct s3_object *current = head;
current = head;
while(current != NULL) {
tmp = current;
current = current->next;
free_object(tmp);
}
return 0;
}
//-------------------------------------------------------------------
// Utility functions for moving objects
//-------------------------------------------------------------------
MVNODE *create_mvnode(const char *old_path, const char *new_path, bool is_dir)
{
MVNODE *p;
char *p_old_path;
char *p_new_path;
p = (MVNODE *) malloc(sizeof(MVNODE));
if (p == NULL) {
printf("create_mvnode: could not allocation memory for p\n");
S3FS_FUSE_EXIT();
return NULL;
}
if(NULL == (p_old_path = strdup(old_path))){
free(p);
printf("create_mvnode: could not allocation memory for p_old_path\n");
S3FS_FUSE_EXIT();
return NULL;
}
if(NULL == (p_new_path = strdup(new_path))){
free(p);
free(p_old_path);
printf("create_mvnode: could not allocation memory for p_new_path\n");
S3FS_FUSE_EXIT();
return NULL;
}
p->old_path = p_old_path;
p->new_path = p_new_path;
p->is_dir = is_dir;
p->prev = NULL;
p->next = NULL;
return p;
}
//
// Add sorted MVNODE data(Ascending order)
//
MVNODE *add_mvnode(MVNODE** head, MVNODE** tail, const char *old_path, const char *new_path, bool is_dir)
{
if(!head || !tail){
return NULL;
}
MVNODE* cur;
MVNODE* mvnew;
for(cur = *head; cur; cur = cur->next){
if(cur->is_dir == is_dir){
int nResult = strcmp(cur->old_path, old_path);
if(0 == nResult){
// Found same old_path.
return cur;
}else if(0 > nResult){
// next check.
// ex: cur("abc"), mvnew("abcd")
// ex: cur("abc"), mvnew("abd")
continue;
}else{
// Add into before cur-pos.
// ex: cur("abc"), mvnew("ab")
// ex: cur("abc"), mvnew("abb")
if(NULL == (mvnew = create_mvnode(old_path, new_path, is_dir))){
return NULL;
}
if(cur->prev){
(cur->prev)->next = mvnew;
}else{
*head = mvnew;
}
mvnew->prev = cur->prev;
mvnew->next = cur;
cur->prev = mvnew;
return mvnew;
}
}
}
// Add into tail.
if(NULL == (mvnew = create_mvnode(old_path, new_path, is_dir))){
return NULL;
}
mvnew->prev = (*tail);
if(*tail){
(*tail)->next = mvnew;
}
(*tail) = mvnew;
if(!(*head)){
(*head) = mvnew;
}
return mvnew;
}
void free_mvnodes(MVNODE *head)
{
MVNODE *my_head;
MVNODE *next;
for(my_head = head, next = NULL; my_head; my_head = next){
next = my_head->next;
free(my_head->old_path);
free(my_head->new_path);
free(my_head);
}
return;
}
//-------------------------------------------------------------------
// Utility for UID/GID
//-------------------------------------------------------------------
// get user name from uid
string get_username(uid_t uid)
{
struct passwd* ppw;
if(NULL == (ppw = getpwuid(uid)) || NULL == ppw->pw_name){
FGPRINT(" could not get username(errno=%d).\n", (int)errno);
SYSLOGDBG("could not get username(errno=%d).\n", (int)errno);
return NULL;
}
return string(ppw->pw_name);
}
// check uid in group(gid)
int is_uid_inculde_group(uid_t uid, gid_t gid)
{
static size_t maxlen = 0; // set onece
int result;
char* pbuf;
struct group ginfo;
struct group* pginfo = NULL;
// make buffer
if(0 == maxlen){
if(0 > (maxlen = (size_t)sysconf(_SC_GETGR_R_SIZE_MAX))){
FGPRINT(" could not get max name length.\n");
SYSLOGDBG("could not get max name length.\n");
maxlen = 0;
return -ERANGE;
}
}
if(NULL == (pbuf = (char*)malloc(sizeof(char) * maxlen))){
FGPRINT(" failed to allocate memory.\n");
SYSLOGERR("failed to allocate memory.\n");
return -ENOMEM;
}
// get group infomation
if(0 != (result = getgrgid_r(gid, &ginfo, pbuf, maxlen, &pginfo))){
FGPRINT(" could not get group infomation.\n");
SYSLOGDBG("could not get group infomation.\n");
free(pbuf);
return -result;
}
// check group
if(NULL == pginfo){
// there is not gid in group.
free(pbuf);
return -EINVAL;
}
string username = get_username(uid);
char** ppgr_mem;
for(ppgr_mem = pginfo->gr_mem; ppgr_mem && *ppgr_mem; ppgr_mem++){
if(username == *ppgr_mem){
// Found username in group.
free(pbuf);
return 1;
}
}
free(pbuf);
return 0;
}
//-------------------------------------------------------------------
// Utility for file and directory
//-------------------------------------------------------------------
// safe variant of dirname
// dirname clobbers path so let it operate on a tmp copy
string mydirname(string path)
{
return string(dirname(&path[0]));
}
// safe variant of basename
// basename clobbers path so let it operate on a tmp copy
string mybasename(string path)
{
return string(basename(&path[0]));
}
// mkdir --parents
int mkdirp(const string& path, mode_t mode)
{
string base;
string component;
stringstream ss(path);
while (getline(ss, component, '/')) {
base += "/" + component;
mkdir(base.c_str(), mode);
}
return 0;
}
//-------------------------------------------------------------------
// Utility functions for convert
//-------------------------------------------------------------------
time_t get_mtime(const char *s)
{
return (time_t) strtoul(s, (char **) NULL, 10);
}
off_t get_size(const char *s)
{
return (off_t) strtoul(s, (char **) NULL, 10);
}
mode_t get_mode(const char *s)
{
return (mode_t) strtoul(s, (char **) NULL, 10);
}
uid_t get_uid(const char *s)
{
return (uid_t) strtoul(s, (char **) NULL, 10);
}
gid_t get_gid(const char *s)
{
return (gid_t) strtoul(s, (char **) NULL, 10);
}
blkcnt_t get_blocks(off_t size)
{
return size / 512 + 1;
}
//-------------------------------------------------------------------
// Help
//-------------------------------------------------------------------
void show_usage (void)
{
printf("Usage: %s BUCKET:[PATH] MOUNTPOINT [OPTION]...\n",
program_name.c_str());
}
void show_help (void)
{
show_usage();
printf(
"\n"
"Mount an Amazon S3 bucket as a file system.\n"
"\n"
" General forms for s3fs and FUSE/mount options:\n"
" -o opt[,opt...]\n"
" -o opt [-o opt] ...\n"
"\n"
"s3fs Options:\n"
"\n"
" Most s3fs options are given in the form where \"opt\" is:\n"
"\n"
" <option_name>=<option_value>\n"
"\n"
" default_acl (default=\"private\")\n"
" - the default canned acl to apply to all written s3 objects\n"
" see http://aws.amazon.com/documentation/s3/ for the \n"
" full list of canned acls\n"
"\n"
" retries (default=\"2\")\n"
" - number of times to retry a failed s3 transaction\n"
"\n"
" use_cache (default=\"\" which means disabled)\n"
" - local folder to use for local file cache\n"
"\n"
" use_rrs (default=\"\" which means diabled)\n"
" - use Amazon's Reduced Redundancy Storage when set to 1\n"
"\n"
" public_bucket (default=\"\" which means disabled)\n"
" - anonymously mount a public bucket when set to 1\n"
"\n"
" passwd_file (default=\"\")\n"
" - specify which s3fs password file to use\n"
"\n"
" connect_timeout (default=\"10\" seconds)\n"
" - time to wait for connection before giving up\n"
"\n"
" readwrite_timeout (default=\"30\" seconds)\n"
" - time to wait between read/write activity before giving up\n"
"\n"
" max_stat_cache_size (default=\"10000\" entries (about 4MB))\n"
" - maximum number of entries in the stat cache\n"
"\n"
" stat_cache_expire (default is no expire)\n"
" - specify expire time(seconds) for entries in the stat cache.\n"
"\n"
" url (default=\"http://s3.amazonaws.com\")\n"
" - sets the url to use to access amazon s3\n"
"\n"
" nomultipart - disable multipart uploads\n"
"\n"
" noxmlns - disable registing xml name space.\n"
" disable registing xml name space for response of \n"
" ListBucketResult and ListVersionsResult etc. Default name \n"
" space is looked up from \"http://s3.amazonaws.com/doc/2006-03-01\".\n"
"\n"
" nocopyapi - for other incomplete compatibility object storage.\n"
" For a distributed object storage which is compatibility S3\n"
" API without PUT(copy api).\n"
" If you set this option, s3fs do not use PUT with \n"
" \"x-amz-copy-source\"(copy api). Because traffic is increased\n"
" 2-3 times by this option, we do not recommend this.\n"
"\n"
" norenameapi - for other incomplete compatibility object storage.\n"
" For a distributed object storage which is compatibility S3\n"
" API without PUT(copy api).\n"
" This option is a subset of nocopyapi option. The nocopyapi\n"
" option does not use copy-api for all command(ex. chmod, chown,\n"
" touch, mv, etc), but this option does not use copy-api for\n"
" only rename command(ex. mv). If this option is specified with\n"
" nocopapi, the s3fs ignores it.\n"
"\n"
"FUSE/mount Options:\n"
"\n"
" Most of the generic mount options described in 'man mount' are\n"
" supported (ro, rw, suid, nosuid, dev, nodev, exec, noexec, atime,\n"
" noatime, sync async, dirsync). Filesystems are mounted with\n"
" '-onodev,nosuid' by default, which can only be overridden by a\n"
" privileged user.\n"
" \n"
" There are many FUSE specific mount options that can be specified.\n"
" e.g. allow_other See the FUSE's README for the full set.\n"
"\n"
"Miscellaneous Options:\n"
"\n"
" -h, --help Output this help.\n"
" --version Output version info.\n"
" -d --debug Turn on DEBUG messages to syslog. Specifying -d\n"
" twice turns on FUSE debug messages to STDOUT.\n"
" -f FUSE foreground option - do not run as daemon.\n"
" -s FUSE singlethread option\n"
" disable multi-threaded operation\n"
"\n"
"\n"
"Report bugs to <s3fs-devel@googlegroups.com>\n"
"s3fs home page: <http://code.google.com/p/s3fs/>\n"
);
return;
}
void show_version(void)
{
printf(
"Amazon Simple Storage Service File System %s\n"
"Copyright (C) 2010 Randy Rizun <rrizun@gmail.com>\n"
"License GPL2: GNU GPL version 2 <http://gnu.org/licenses/gpl.html>\n"
"This is free software: you are free to change and redistribute it.\n"
"There is NO WARRANTY, to the extent permitted by law.\n", VERSION );
return;
}

52
src/s3fs_util.h Normal file
View File

@ -0,0 +1,52 @@
#ifndef S3FS_S3FS_UTIL_H_
#define S3FS_S3FS_UTIL_H_
//-------------------------------------------------------------------
// Typedef
//-------------------------------------------------------------------
struct s3_object {
char *name;
struct s3_object *next;
};
typedef struct mvnode {
char *old_path;
char *new_path;
bool is_dir;
struct mvnode *prev;
struct mvnode *next;
} MVNODE;
//-------------------------------------------------------------------
// Functions
//-------------------------------------------------------------------
std::string get_realpath(const char *path);
int insert_object(const char *name, struct s3_object **head);
int free_object(struct s3_object *object);
int free_object_list(struct s3_object *head);
MVNODE *create_mvnode(const char *old_path, const char *new_path, bool is_dir);
MVNODE *add_mvnode(MVNODE** head, MVNODE** tail, const char *old_path, const char *new_path, bool is_dir);
void free_mvnodes(MVNODE *head);
std::string get_username(uid_t uid);
int is_uid_inculde_group(uid_t uid, gid_t gid);
std::string mydirname(std::string path);
std::string mybasename(std::string path);
int mkdirp(const std::string& path, mode_t mode);
time_t get_mtime(const char *s);
off_t get_size(const char *s);
mode_t get_mode(const char *s);
uid_t get_uid(const char *s);
gid_t get_gid(const char *s);
blkcnt_t get_blocks(off_t size);
void show_usage(void);
void show_help(void);
void show_version(void);
#endif // S3FS_S3FS_UTIL_H_

56
src/string_util.cpp
View File

@ -22,42 +22,50 @@
#include <sstream> #include <sstream>
#include <string> #include <string>
#include <map>
#include "common.h"
#include "string_util.h" #include "string_util.h"
using namespace std; using namespace std;
static const char hexAlphabet[] = "0123456789ABCDEF"; static const char hexAlphabet[] = "0123456789ABCDEF";
string lower(string s) { string lower(string s)
{
// change each character of the string to lower case // change each character of the string to lower case
for(unsigned int i = 0; i < s.length(); i++) for(unsigned int i = 0; i < s.length(); i++){
s[i] = tolower(s[i]); s[i] = tolower(s[i]);
}
return s; return s;
} }
string IntToStr(int n) { string IntToStr(int n)
{
stringstream result; stringstream result;
result << n; result << n;
return result.str(); return result.str();
} }
string trim_left(const string &s, const string &t /* = SPACES */) { string trim_left(const string &s, const string &t /* = SPACES */)
{
string d(s); string d(s);
return d.erase(0, s.find_first_not_of(t)); return d.erase(0, s.find_first_not_of(t));
} }
string trim_right(const string &s, const string &t /* = SPACES */) { string trim_right(const string &s, const string &t /* = SPACES */)
{
string d(s); string d(s);
string::size_type i(d.find_last_not_of(t)); string::size_type i(d.find_last_not_of(t));
if (i == string::npos) if(i == string::npos){
return ""; return "";
else }else{
return d.erase(d.find_last_not_of(t) + 1); return d.erase(d.find_last_not_of(t) + 1);
}
} }
string trim(const string &s, const string &t /* = SPACES */) { string trim(const string &s, const string &t /* = SPACES */)
{
string d(s); string d(s);
return trim_left(trim_right(d, t), t); return trim_left(trim_right(d, t), t);
} }
@ -67,18 +75,19 @@ string trim(const string &s, const string &t /* = SPACES */) {
* taking into special consideration "/", * taking into special consideration "/",
* otherwise regular urlEncode. * otherwise regular urlEncode.
*/ */
string urlEncode(const string &s) { string urlEncode(const string &s)
{
string result; string result;
for (unsigned i = 0; i < s.length(); ++i) { for (unsigned i = 0; i < s.length(); ++i) {
if (s[i] == '/') // Note- special case for fuse paths... if (s[i] == '/') { // Note- special case for fuse paths...
result += s[i]; result += s[i];
else if (isalnum(s[i])) } else if (isalnum(s[i])) {
result += s[i]; result += s[i];
else if (s[i] == '.' || s[i] == '-' || s[i] == '*' || s[i] == '_') } else if (s[i] == '.' || s[i] == '-' || s[i] == '*' || s[i] == '_') {
result += s[i]; result += s[i];
else if (s[i] == ' ') } else if (s[i] == ' ') {
result += '+'; result += '+';
else { } else {
result += "%"; result += "%";
result += hexAlphabet[static_cast<unsigned char>(s[i]) / 16]; result += hexAlphabet[static_cast<unsigned char>(s[i]) / 16];
result += hexAlphabet[static_cast<unsigned char>(s[i]) % 16]; result += hexAlphabet[static_cast<unsigned char>(s[i]) % 16];
@ -88,9 +97,9 @@ string urlEncode(const string &s) {
return result; return result;
} }
string prepare_url(const char* url) { string prepare_url(const char* url)
if(debug) {
syslog(LOG_DEBUG, "URL is %s", url); SYSLOGDBG("URL is %s", url);
string uri; string uri;
string host; string host;
@ -101,17 +110,16 @@ string prepare_url(const char* url) {
int bucket_length = token.size(); int bucket_length = token.size();
int uri_length = 7; int uri_length = 7;
if(!strncasecmp(url_str.c_str(), "https://", 8)) if(!strncasecmp(url_str.c_str(), "https://", 8)){
uri_length = 8; uri_length = 8;
}
uri = url_str.substr(0, uri_length); uri = url_str.substr(0, uri_length);
host = bucket + "." + url_str.substr(uri_length, bucket_pos - uri_length).c_str(); host = bucket + "." + url_str.substr(uri_length, bucket_pos - uri_length).c_str();
path = url_str.substr((bucket_pos + bucket_length)); path = url_str.substr((bucket_pos + bucket_length));
url_str = uri + host + path; url_str = uri + host + path;
if(debug) SYSLOGDBG("URL changed is %s", url_str.c_str());
syslog(LOG_DEBUG, "URL changed is %s", url_str.c_str());
return str(url_str); return str(url_str);
} }
@ -120,9 +128,11 @@ string prepare_url(const char* url) {
* Returns the current date * Returns the current date
* in a format suitable for a HTTP request header. * in a format suitable for a HTTP request header.
*/ */
string get_date() { string get_date()
{
char buf[100]; char buf[100];
time_t t = time(NULL); time_t t = time(NULL);
strftime(buf, sizeof(buf), "%a, %d %b %Y %H:%M:%S GMT", gmtime(&t)); strftime(buf, sizeof(buf), "%a, %d %b %Y %H:%M:%S GMT", gmtime(&t));
return buf; return buf;
} }

7
src/string_util.h
View File

@ -18,12 +18,6 @@ template<typename T> std::string str(T value) {
return s.str(); return s.str();
} }
extern bool debug;
extern bool foreground;
extern bool service_validated;
extern std::string bucket;
std::string trim_left(const std::string &s, const std::string &t = SPACES); std::string trim_left(const std::string &s, const std::string &t = SPACES);
std::string trim_right(const std::string &s, const std::string &t = SPACES); std::string trim_right(const std::string &s, const std::string &t = SPACES);
std::string trim(const std::string &s, const std::string &t = SPACES); std::string trim(const std::string &s, const std::string &t = SPACES);
@ -33,5 +27,4 @@ std::string get_date();
std::string urlEncode(const std::string &s); std::string urlEncode(const std::string &s);
std::string prepare_url(const char* url); std::string prepare_url(const char* url);
#endif // S3FS_STRING_UTIL_H_ #endif // S3FS_STRING_UTIL_H_