mirror of
https://github.com/s3fs-fuse/s3fs-fuse.git
synced 2024-12-23 09:18:55 +00:00
Cleaned up source codes
No changes for logic, only changes layout of functions and valiables between a file to a file. Adds s3fs_util.cpp/s3fs_util.h/common.h git-svn-id: http://s3fs.googlecode.com/svn/trunk@396 df820570-a93a-0410-bd06-b72b767a4274
This commit is contained in:
parent
00b735beaa
commit
953aedd7ad
@ -2,6 +2,6 @@ bin_PROGRAMS=s3fs
|
||||
|
||||
AM_CPPFLAGS = $(DEPS_CFLAGS)
|
||||
|
||||
s3fs_SOURCES = s3fs.cpp s3fs.h curl.cpp curl.h cache.cpp cache.h string_util.cpp string_util.h
|
||||
s3fs_SOURCES = s3fs.cpp s3fs.h curl.cpp curl.h cache.cpp cache.h string_util.cpp string_util.h s3fs_util.cpp s3fs_util.h common.h
|
||||
s3fs_LDADD = $(DEPS_LIBS)
|
||||
|
||||
|
@ -27,15 +27,37 @@
|
||||
#include <string>
|
||||
#include <map>
|
||||
|
||||
#include "common.h"
|
||||
#include "cache.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Typedef
|
||||
//-------------------------------------------------------------------
|
||||
typedef std::map<std::string, struct stat_cache_entry> stat_cache_t; // key=path
|
||||
static stat_cache_t stat_cache;
|
||||
pthread_mutex_t stat_cache_lock;
|
||||
|
||||
int get_stat_cache_entry(const char *path, struct stat *buf) {
|
||||
//-------------------------------------------------------------------
|
||||
// Static valiables
|
||||
//-------------------------------------------------------------------
|
||||
static stat_cache_t stat_cache;
|
||||
static pthread_mutex_t stat_cache_lock;
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Functions
|
||||
//-------------------------------------------------------------------
|
||||
int init_stat_cache_mutex(void)
|
||||
{
|
||||
return pthread_mutex_init(&stat_cache_lock, NULL);
|
||||
}
|
||||
|
||||
int destroy_stat_cache_mutex(void)
|
||||
{
|
||||
return pthread_mutex_destroy(&stat_cache_lock);
|
||||
}
|
||||
|
||||
int get_stat_cache_entry(const char *path, struct stat *buf)
|
||||
{
|
||||
int is_delete_cache = 0;
|
||||
string strpath = path;
|
||||
|
||||
@ -54,14 +76,12 @@ int get_stat_cache_entry(const char *path, struct stat *buf) {
|
||||
if(iter != stat_cache.end()) {
|
||||
if(!is_stat_cache_expire_time || ((*iter).second.cache_date + stat_cache_expire_time) >= time(NULL)){
|
||||
// hit
|
||||
if(foreground)
|
||||
cout << " stat cache hit [path=" << strpath << "]"
|
||||
<< " [time=" << (*iter).second.cache_date << "]"
|
||||
<< " [hit count=" << (*iter).second.hit_count << "]" << endl;
|
||||
FGPRINT(" stat cache hit [path=%s] [time=%ld] [hit count=%lu]\n",
|
||||
strpath.c_str(), (*iter).second.cache_date, (*iter).second.hit_count);
|
||||
|
||||
if(buf != NULL)
|
||||
if(buf != NULL){
|
||||
*buf = (*iter).second.stbuf;
|
||||
|
||||
}
|
||||
(*iter).second.hit_count++;
|
||||
pthread_mutex_unlock(&stat_cache_lock);
|
||||
return 0;
|
||||
@ -79,30 +99,31 @@ int get_stat_cache_entry(const char *path, struct stat *buf) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
void add_stat_cache_entry(const char *path, struct stat *st) {
|
||||
if(foreground)
|
||||
cout << " add_stat_cache_entry[path=" << path << "]" << endl;
|
||||
void add_stat_cache_entry(const char *path, struct stat *st)
|
||||
{
|
||||
FGPRINT(" add_stat_cache_entry[path=%s]\n", path);
|
||||
|
||||
if(max_stat_cache_size < 1)
|
||||
if(max_stat_cache_size < 1){
|
||||
return;
|
||||
|
||||
if(stat_cache.size() > max_stat_cache_size)
|
||||
}
|
||||
if(stat_cache.size() > max_stat_cache_size){
|
||||
truncate_stat_cache();
|
||||
|
||||
}
|
||||
pthread_mutex_lock(&stat_cache_lock);
|
||||
stat_cache[path].stbuf = *st;
|
||||
stat_cache[path].cache_date = time(NULL); // Set time.
|
||||
pthread_mutex_unlock(&stat_cache_lock);
|
||||
}
|
||||
|
||||
void delete_stat_cache_entry(const char *path) {
|
||||
if(foreground)
|
||||
cout << " delete_stat_cache_entry[path=" << path << "]" << endl;
|
||||
void delete_stat_cache_entry(const char *path)
|
||||
{
|
||||
FGPRINT(" delete_stat_cache_entry[path=%s]\n", path);
|
||||
|
||||
pthread_mutex_lock(&stat_cache_lock);
|
||||
stat_cache_t::iterator iter = stat_cache.find(path);
|
||||
if(iter != stat_cache.end())
|
||||
if(iter != stat_cache.end()){
|
||||
stat_cache.erase(iter);
|
||||
}
|
||||
pthread_mutex_unlock(&stat_cache_lock);
|
||||
}
|
||||
|
||||
@ -120,13 +141,14 @@ void truncate_stat_cache() {
|
||||
lowest_hit_count = hit_count;
|
||||
path_to_delete = (* iter).first;
|
||||
}
|
||||
|
||||
if(lowest_hit_count > hit_count)
|
||||
if(lowest_hit_count > hit_count){
|
||||
path_to_delete = (* iter).first;
|
||||
}
|
||||
}
|
||||
|
||||
stat_cache.erase(path_to_delete);
|
||||
pthread_mutex_unlock(&stat_cache_lock);
|
||||
|
||||
printf(" purged %s from the stat cache\n", path_to_delete.c_str());
|
||||
FGPRINT(" purged %s from the stat cache\n", path_to_delete.c_str());
|
||||
}
|
||||
|
||||
|
13
src/cache.h
13
src/cache.h
@ -5,6 +5,9 @@
|
||||
#include <sys/types.h>
|
||||
#include <unistd.h>
|
||||
|
||||
//
|
||||
// Struct
|
||||
//
|
||||
struct stat_cache_entry {
|
||||
struct stat stbuf;
|
||||
unsigned long hit_count;
|
||||
@ -13,11 +16,11 @@ struct stat_cache_entry {
|
||||
stat_cache_entry() : hit_count(0), cache_date(0) {}
|
||||
};
|
||||
|
||||
extern bool foreground;
|
||||
extern unsigned long max_stat_cache_size;
|
||||
extern time_t stat_cache_expire_time;
|
||||
extern int is_stat_cache_expire_time;
|
||||
|
||||
//
|
||||
// Functions
|
||||
//
|
||||
int init_stat_cache_mutex(void);
|
||||
int destroy_stat_cache_mutex(void);
|
||||
int get_stat_cache_entry(const char *path, struct stat *buf);
|
||||
void add_stat_cache_entry(const char *path, struct stat *st);
|
||||
void delete_stat_cache_entry(const char *path);
|
||||
|
52
src/common.h
Normal file
52
src/common.h
Normal file
@ -0,0 +1,52 @@
|
||||
#ifndef S3FS_COMMON_H_
|
||||
#define S3FS_COMMON_H_
|
||||
|
||||
//
|
||||
// Macro
|
||||
//
|
||||
#define SYSLOGINFO(...) syslog(LOG_INFO, __VA_ARGS__);
|
||||
#define SYSLOGERR(...) syslog(LOG_ERR, __VA_ARGS__);
|
||||
#define SYSLOGCRIT(...) syslog(LOG_CRIT, __VA_ARGS__);
|
||||
|
||||
#define SYSLOGDBG(...) \
|
||||
if(debug){ \
|
||||
syslog(LOG_DEBUG, __VA_ARGS__); \
|
||||
}
|
||||
|
||||
#define SYSLOGDBGERR(...) \
|
||||
if(debug){ \
|
||||
syslog(LOG_ERR, __VA_ARGS__); \
|
||||
}
|
||||
|
||||
#define FGPRINT(...) \
|
||||
if(foreground){ \
|
||||
printf(__VA_ARGS__); \
|
||||
}
|
||||
|
||||
//
|
||||
// Typedef
|
||||
//
|
||||
typedef std::map<std::string, std::string> headers_t;
|
||||
|
||||
//
|
||||
// Global valiables
|
||||
//
|
||||
extern bool debug;
|
||||
extern bool foreground;
|
||||
extern unsigned long max_stat_cache_size;
|
||||
extern time_t stat_cache_expire_time;
|
||||
extern int is_stat_cache_expire_time;
|
||||
extern int retries;
|
||||
extern long connect_timeout;
|
||||
extern time_t readwrite_timeout;
|
||||
extern std::string AWSAccessKeyId;
|
||||
extern std::string AWSSecretAccessKey;
|
||||
extern std::string program_name;
|
||||
extern std::string ssl_verify_hostname;
|
||||
extern std::string service_path;
|
||||
extern std::string host;
|
||||
extern std::string bucket;
|
||||
extern std::string public_bucket;
|
||||
extern std::string mount_prefix;
|
||||
|
||||
#endif // S3FS_COMMON_H_
|
361
src/curl.cpp
361
src/curl.cpp
@ -40,36 +40,53 @@
|
||||
#include <map>
|
||||
#include <algorithm>
|
||||
|
||||
#include "common.h"
|
||||
#include "curl.h"
|
||||
#include "string_util.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
pthread_mutex_t curl_handles_lock;
|
||||
std::map<CURL*, time_t> curl_times;
|
||||
std::map<CURL*, progress_t> curl_progress;
|
||||
std::string curl_ca_bundle;
|
||||
|
||||
class auto_curl_slist {
|
||||
public:
|
||||
auto_curl_slist() : slist(0) { }
|
||||
~auto_curl_slist() { curl_slist_free_all(slist); }
|
||||
|
||||
struct curl_slist* get() const { return slist; }
|
||||
|
||||
void append(const string& s) {
|
||||
slist = curl_slist_append(slist, s.c_str());
|
||||
//-------------------------------------------------------------------
|
||||
// Typedef
|
||||
//-------------------------------------------------------------------
|
||||
struct case_insensitive_compare_func {
|
||||
bool operator ()(const string &a, const string &b) {
|
||||
return strcasecmp(a.c_str(), b.c_str()) < 0;
|
||||
}
|
||||
|
||||
private:
|
||||
struct curl_slist* slist;
|
||||
};
|
||||
typedef map<string, string, case_insensitive_compare_func> mimes_t;
|
||||
typedef pair<double, double> progress_t;
|
||||
|
||||
size_t header_callback(void *data, size_t blockSize, size_t numBlocks, void *userPtr) {
|
||||
//-------------------------------------------------------------------
|
||||
// Static valiables
|
||||
//-------------------------------------------------------------------
|
||||
static pthread_mutex_t curl_handles_lock;
|
||||
static const EVP_MD* evp_md = EVP_sha1();
|
||||
static map<CURL*, time_t> curl_times;
|
||||
static map<CURL*, progress_t> curl_progress;
|
||||
static string curl_ca_bundle;
|
||||
static mimes_t mimeTypes;
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Functions
|
||||
//-------------------------------------------------------------------
|
||||
int init_curl_handles_mutex(void)
|
||||
{
|
||||
return pthread_mutex_init(&curl_handles_lock, NULL);
|
||||
}
|
||||
|
||||
int destroy_curl_handles_mutex(void)
|
||||
{
|
||||
return pthread_mutex_destroy(&curl_handles_lock);
|
||||
}
|
||||
|
||||
size_t header_callback(void *data, size_t blockSize, size_t numBlocks, void *userPtr)
|
||||
{
|
||||
headers_t* headers = reinterpret_cast<headers_t*>(userPtr);
|
||||
string header(reinterpret_cast<char*>(data), blockSize * numBlocks);
|
||||
string key;
|
||||
stringstream ss(header);
|
||||
|
||||
if (getline(ss, key, ':')) {
|
||||
// Force to lower, only "x-amz"
|
||||
string lkey = key;
|
||||
@ -84,7 +101,8 @@ size_t header_callback(void *data, size_t blockSize, size_t numBlocks, void *use
|
||||
return blockSize * numBlocks;
|
||||
}
|
||||
|
||||
CURL *create_curl_handle(void) {
|
||||
CURL *create_curl_handle(void)
|
||||
{
|
||||
time_t now;
|
||||
CURL *curl_handle;
|
||||
|
||||
@ -99,11 +117,12 @@ CURL *create_curl_handle(void) {
|
||||
curl_easy_setopt(curl_handle, CURLOPT_PROGRESSDATA, curl_handle);
|
||||
// curl_easy_setopt(curl_handle, CURLOPT_FORBID_REUSE, 1);
|
||||
|
||||
if(ssl_verify_hostname.substr(0,1) == "0")
|
||||
if(ssl_verify_hostname.substr(0,1) == "0"){
|
||||
curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYHOST, 0);
|
||||
if(curl_ca_bundle.size() != 0)
|
||||
}
|
||||
if(curl_ca_bundle.size() != 0){
|
||||
curl_easy_setopt(curl_handle, CURLOPT_CAINFO, curl_ca_bundle.c_str());
|
||||
|
||||
}
|
||||
now = time(0);
|
||||
curl_times[curl_handle] = now;
|
||||
curl_progress[curl_handle] = progress_t(-1, -1);
|
||||
@ -112,7 +131,8 @@ CURL *create_curl_handle(void) {
|
||||
return curl_handle;
|
||||
}
|
||||
|
||||
void destroy_curl_handle(CURL *curl_handle) {
|
||||
void destroy_curl_handle(CURL *curl_handle)
|
||||
{
|
||||
if(curl_handle != NULL) {
|
||||
pthread_mutex_lock(&curl_handles_lock);
|
||||
curl_times.erase(curl_handle);
|
||||
@ -124,7 +144,8 @@ void destroy_curl_handle(CURL *curl_handle) {
|
||||
return;
|
||||
}
|
||||
|
||||
int curl_delete(const char *path) {
|
||||
int curl_delete(const char *path)
|
||||
{
|
||||
int result;
|
||||
string date;
|
||||
string url;
|
||||
@ -139,10 +160,10 @@ int curl_delete(const char *path) {
|
||||
|
||||
headers.append("Date: " + date);
|
||||
headers.append("Content-Type: ");
|
||||
if(public_bucket.substr(0,1) != "1")
|
||||
if(public_bucket.substr(0,1) != "1"){
|
||||
headers.append("Authorization: AWS " + AWSAccessKeyId + ":" +
|
||||
calc_signature("DELETE", "", date, headers.get(), resource));
|
||||
|
||||
}
|
||||
my_url = prepare_url(url.c_str());
|
||||
curl = create_curl_handle();
|
||||
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE");
|
||||
@ -154,12 +175,12 @@ int curl_delete(const char *path) {
|
||||
return result;
|
||||
}
|
||||
|
||||
int curl_get_headers(const char *path, headers_t &meta) {
|
||||
int curl_get_headers(const char *path, headers_t &meta)
|
||||
{
|
||||
int result;
|
||||
CURL *curl;
|
||||
|
||||
if(foreground)
|
||||
printf(" curl_headers[path=%s]\n", path);
|
||||
FGPRINT(" curl_headers[path=%s]\n", path);
|
||||
|
||||
string resource(urlEncode(service_path + bucket + path));
|
||||
string url(host + resource);
|
||||
@ -185,8 +206,9 @@ int curl_get_headers(const char *path, headers_t &meta) {
|
||||
result = my_curl_easy_perform(curl);
|
||||
destroy_curl_handle(curl);
|
||||
|
||||
if(result != 0)
|
||||
if(result != 0){
|
||||
return result;
|
||||
}
|
||||
|
||||
// file exists in s3
|
||||
// fixme: clean this up.
|
||||
@ -215,24 +237,61 @@ int curl_get_headers(const char *path, headers_t &meta) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
CURL *create_head_handle(head_data *request_data)
|
||||
{
|
||||
CURL *curl_handle = create_curl_handle();
|
||||
string resource = urlEncode(service_path + bucket + request_data->path);
|
||||
string url = host + resource;
|
||||
|
||||
// libcurl 7.17 does deep copy of url, deep copy "stable" url
|
||||
string my_url = prepare_url(url.c_str());
|
||||
request_data->url = new string(my_url.c_str());
|
||||
request_data->requestHeaders = 0;
|
||||
request_data->responseHeaders = new headers_t;
|
||||
|
||||
curl_easy_setopt(curl_handle, CURLOPT_URL, request_data->url->c_str());
|
||||
curl_easy_setopt(curl_handle, CURLOPT_NOBODY, true); // HEAD
|
||||
curl_easy_setopt(curl_handle, CURLOPT_FILETIME, true); // Last-Modified
|
||||
|
||||
// requestHeaders
|
||||
string date = get_date();
|
||||
request_data->requestHeaders = curl_slist_append(
|
||||
request_data->requestHeaders, string("Date: " + date).c_str());
|
||||
request_data->requestHeaders = curl_slist_append(
|
||||
request_data->requestHeaders, string("Content-Type: ").c_str());
|
||||
if(public_bucket.substr(0,1) != "1") {
|
||||
request_data->requestHeaders = curl_slist_append(
|
||||
request_data->requestHeaders, string("Authorization: AWS " + AWSAccessKeyId + ":" +
|
||||
calc_signature("HEAD", "", date, request_data->requestHeaders, resource)).c_str());
|
||||
}
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, request_data->requestHeaders);
|
||||
|
||||
// responseHeaders
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, request_data->responseHeaders);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, header_callback);
|
||||
|
||||
return curl_handle;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return fuse return code
|
||||
*/
|
||||
int my_curl_easy_perform(CURL* curl, BodyStruct* body, FILE* f) {
|
||||
int my_curl_easy_perform(CURL* curl, BodyStruct* body, FILE* f)
|
||||
{
|
||||
char url[256];
|
||||
time_t now;
|
||||
char* ptr_url = url;
|
||||
curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL , &ptr_url);
|
||||
|
||||
if(debug)
|
||||
syslog(LOG_DEBUG, "connecting to URL %s", ptr_url);
|
||||
SYSLOGDBG("connecting to URL %s", ptr_url);
|
||||
|
||||
// curl_easy_setopt(curl, CURLOPT_VERBOSE, true);
|
||||
if(ssl_verify_hostname.substr(0,1) == "0")
|
||||
if(ssl_verify_hostname.substr(0,1) == "0"){
|
||||
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
|
||||
|
||||
if(curl_ca_bundle.size() != 0)
|
||||
}
|
||||
if(curl_ca_bundle.size() != 0){
|
||||
curl_easy_setopt(curl, CURLOPT_CAINFO, curl_ca_bundle.c_str());
|
||||
}
|
||||
|
||||
long responseCode;
|
||||
|
||||
@ -249,19 +308,16 @@ int my_curl_easy_perform(CURL* curl, BodyStruct* body, FILE* f) {
|
||||
// Need to look at the HTTP response code
|
||||
|
||||
if (curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &responseCode) != 0) {
|
||||
syslog(LOG_ERR, "curl_easy_getinfo failed while trying to retrieve HTTP response code");
|
||||
SYSLOGERR("curl_easy_getinfo failed while trying to retrieve HTTP response code");
|
||||
return -EIO;
|
||||
}
|
||||
|
||||
if(debug)
|
||||
syslog(LOG_DEBUG, "HTTP response code %ld", responseCode);
|
||||
SYSLOGDBG("HTTP response code %ld", responseCode);
|
||||
|
||||
if (responseCode < 400) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (responseCode >= 500) {
|
||||
syslog(LOG_ERR, "###HTTP response=%ld", responseCode);
|
||||
SYSLOGERR("###HTTP response=%ld", responseCode);
|
||||
sleep(4);
|
||||
break;
|
||||
}
|
||||
@ -269,88 +325,82 @@ int my_curl_easy_perform(CURL* curl, BodyStruct* body, FILE* f) {
|
||||
// Service response codes which are >= 400 && < 500
|
||||
switch(responseCode) {
|
||||
case 400:
|
||||
if(debug) syslog(LOG_ERR, "HTTP response code 400 was returned");
|
||||
if(body) {
|
||||
if(body->size && debug) {
|
||||
syslog(LOG_ERR, "Body Text: %s", body->text);
|
||||
SYSLOGDBGERR("HTTP response code 400 was returned");
|
||||
if(body && body->size){
|
||||
SYSLOGDBGERR("Body Text: %s", body->text);
|
||||
}
|
||||
}
|
||||
if(debug) syslog(LOG_DEBUG, "Now returning EIO");
|
||||
SYSLOGDBG("Now returning EIO");
|
||||
return -EIO;
|
||||
|
||||
case 403:
|
||||
if(debug) syslog(LOG_ERR, "HTTP response code 403 was returned");
|
||||
if(body)
|
||||
if(body->size && debug)
|
||||
syslog(LOG_ERR, "Body Text: %s", body->text);
|
||||
SYSLOGDBGERR("HTTP response code 403 was returned");
|
||||
if(body && body->size){
|
||||
SYSLOGDBGERR("Body Text: %s", body->text);
|
||||
}
|
||||
return -EPERM;
|
||||
|
||||
case 404:
|
||||
if(debug) syslog(LOG_DEBUG, "HTTP response code 404 was returned");
|
||||
if(body) {
|
||||
if(body->size && debug) {
|
||||
syslog(LOG_DEBUG, "Body Text: %s", body->text);
|
||||
SYSLOGDBG("HTTP response code 404 was returned");
|
||||
if(body && body->size){
|
||||
SYSLOGDBG("Body Text: %s", body->text);
|
||||
}
|
||||
}
|
||||
if(debug) syslog(LOG_DEBUG, "Now returning ENOENT");
|
||||
SYSLOGDBG("Now returning ENOENT");
|
||||
return -ENOENT;
|
||||
|
||||
default:
|
||||
syslog(LOG_ERR, "###response=%ld", responseCode);
|
||||
SYSLOGERR("###response=%ld", responseCode);
|
||||
printf("responseCode %ld\n", responseCode);
|
||||
if(body) {
|
||||
if(body->size) {
|
||||
if(body && body->size){
|
||||
printf("Body Text %s\n", body->text);
|
||||
}
|
||||
}
|
||||
return -EIO;
|
||||
}
|
||||
break;
|
||||
|
||||
case CURLE_WRITE_ERROR:
|
||||
syslog(LOG_ERR, "### CURLE_WRITE_ERROR");
|
||||
SYSLOGERR("### CURLE_WRITE_ERROR");
|
||||
sleep(2);
|
||||
break;
|
||||
|
||||
case CURLE_OPERATION_TIMEDOUT:
|
||||
syslog(LOG_ERR, "### CURLE_OPERATION_TIMEDOUT");
|
||||
SYSLOGERR("### CURLE_OPERATION_TIMEDOUT");
|
||||
sleep(2);
|
||||
break;
|
||||
|
||||
case CURLE_COULDNT_RESOLVE_HOST:
|
||||
syslog(LOG_ERR, "### CURLE_COULDNT_RESOLVE_HOST");
|
||||
SYSLOGERR("### CURLE_COULDNT_RESOLVE_HOST");
|
||||
sleep(2);
|
||||
break;
|
||||
|
||||
case CURLE_COULDNT_CONNECT:
|
||||
syslog(LOG_ERR, "### CURLE_COULDNT_CONNECT");
|
||||
SYSLOGERR("### CURLE_COULDNT_CONNECT");
|
||||
sleep(4);
|
||||
break;
|
||||
|
||||
case CURLE_GOT_NOTHING:
|
||||
syslog(LOG_ERR, "### CURLE_GOT_NOTHING");
|
||||
SYSLOGERR("### CURLE_GOT_NOTHING");
|
||||
sleep(4);
|
||||
break;
|
||||
|
||||
case CURLE_ABORTED_BY_CALLBACK:
|
||||
syslog(LOG_ERR, "### CURLE_ABORTED_BY_CALLBACK");
|
||||
SYSLOGERR("### CURLE_ABORTED_BY_CALLBACK");
|
||||
sleep(4);
|
||||
now = time(0);
|
||||
curl_times[curl] = now;
|
||||
break;
|
||||
|
||||
case CURLE_PARTIAL_FILE:
|
||||
syslog(LOG_ERR, "### CURLE_PARTIAL_FILE");
|
||||
SYSLOGERR("### CURLE_PARTIAL_FILE");
|
||||
sleep(4);
|
||||
break;
|
||||
|
||||
case CURLE_SEND_ERROR:
|
||||
syslog(LOG_ERR, "### CURLE_SEND_ERROR");
|
||||
SYSLOGERR("### CURLE_SEND_ERROR");
|
||||
sleep(2);
|
||||
break;
|
||||
|
||||
case CURLE_RECV_ERROR:
|
||||
syslog(LOG_ERR, "### CURLE_RECV_ERROR");
|
||||
SYSLOGERR("### CURLE_RECV_ERROR");
|
||||
sleep(2);
|
||||
break;
|
||||
|
||||
@ -365,8 +415,7 @@ int my_curl_easy_perform(CURL* curl, BodyStruct* body, FILE* f) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
syslog(LOG_ERR, "curlCode: %i msg: %s", curlCode,
|
||||
curl_easy_strerror(curlCode));;
|
||||
SYSLOGERR("curlCode: %i msg: %s", curlCode, curl_easy_strerror(curlCode));
|
||||
fprintf (stderr, "%s: curlCode: %i -- %s\n",
|
||||
program_name.c_str(),
|
||||
curlCode,
|
||||
@ -395,12 +444,12 @@ int my_curl_easy_perform(CURL* curl, BodyStruct* body, FILE* f) {
|
||||
|
||||
// This should be invalid since curl option HTTP FAILONERROR is now off
|
||||
case CURLE_HTTP_RETURNED_ERROR:
|
||||
syslog(LOG_ERR, "### CURLE_HTTP_RETURNED_ERROR");
|
||||
SYSLOGERR("### CURLE_HTTP_RETURNED_ERROR");
|
||||
|
||||
if (curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &responseCode) != 0) {
|
||||
return -EIO;
|
||||
}
|
||||
syslog(LOG_ERR, "###response=%ld", responseCode);
|
||||
SYSLOGERR("###response=%ld", responseCode);
|
||||
|
||||
// Let's try to retrieve the
|
||||
|
||||
@ -414,19 +463,19 @@ int my_curl_easy_perform(CURL* curl, BodyStruct* body, FILE* f) {
|
||||
|
||||
// Unknown CURL return code
|
||||
default:
|
||||
syslog(LOG_ERR, "###curlCode: %i msg: %s", curlCode,
|
||||
curl_easy_strerror(curlCode));;
|
||||
SYSLOGERR("###curlCode: %i msg: %s", curlCode, curl_easy_strerror(curlCode));
|
||||
exit(EXIT_FAILURE);
|
||||
break;
|
||||
}
|
||||
syslog(LOG_ERR, "###retrying...");
|
||||
SYSLOGERR("###retrying...");
|
||||
}
|
||||
syslog(LOG_ERR, "###giving up");
|
||||
SYSLOGERR("###giving up");
|
||||
return -EIO;
|
||||
}
|
||||
|
||||
// libcurl callback
|
||||
size_t WriteMemoryCallback(void *ptr, size_t blockSize, size_t numBlocks, void *data) {
|
||||
size_t WriteMemoryCallback(void *ptr, size_t blockSize, size_t numBlocks, void *data)
|
||||
{
|
||||
size_t realsize = blockSize * numBlocks;
|
||||
struct BodyStruct *mem = (struct BodyStruct *)data;
|
||||
|
||||
@ -446,12 +495,13 @@ size_t WriteMemoryCallback(void *ptr, size_t blockSize, size_t numBlocks, void *
|
||||
|
||||
// read_callback
|
||||
// http://curl.haxx.se/libcurl/c/post-callback.html
|
||||
size_t read_callback(void *ptr, size_t size, size_t nmemb, void *userp) {
|
||||
size_t read_callback(void *ptr, size_t size, size_t nmemb, void *userp)
|
||||
{
|
||||
struct WriteThis *pooh = (struct WriteThis *)userp;
|
||||
|
||||
if(size*nmemb < 1)
|
||||
if(size*nmemb < 1){
|
||||
return 0;
|
||||
|
||||
}
|
||||
if(pooh->sizeleft) {
|
||||
*(char *)ptr = pooh->readptr[0]; /* copy one single byte */
|
||||
pooh->readptr++; /* advance pointer */
|
||||
@ -463,8 +513,8 @@ size_t read_callback(void *ptr, size_t size, size_t nmemb, void *userp) {
|
||||
}
|
||||
|
||||
// homegrown timeout mechanism
|
||||
int my_curl_progress(
|
||||
void *clientp, double dltotal, double dlnow, double ultotal, double ulnow) {
|
||||
int my_curl_progress(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
|
||||
{
|
||||
CURL* curl = static_cast<CURL*>(clientp);
|
||||
|
||||
time_t now = time(0);
|
||||
@ -482,7 +532,7 @@ int my_curl_progress(
|
||||
if (now - curl_times[curl] > readwrite_timeout) {
|
||||
pthread_mutex_unlock( &curl_handles_lock );
|
||||
|
||||
syslog(LOG_ERR, "timeout now: %li curl_times[curl]: %lil readwrite_timeout: %li",
|
||||
SYSLOGERR("timeout now: %li curl_times[curl]: %lil readwrite_timeout: %li",
|
||||
(long int)now, curl_times[curl], (long int)readwrite_timeout);
|
||||
|
||||
return CURLE_ABORTED_BY_CALLBACK;
|
||||
@ -501,8 +551,8 @@ int my_curl_progress(
|
||||
* @param date e.g., get_date()
|
||||
* @param resource e.g., "/pub"
|
||||
*/
|
||||
string calc_signature(
|
||||
string method, string content_type, string date, curl_slist* headers, string resource) {
|
||||
string calc_signature(string method, string content_type, string date, curl_slist* headers, string resource)
|
||||
{
|
||||
int ret;
|
||||
int bytes_written;
|
||||
int offset;
|
||||
@ -556,13 +606,13 @@ string calc_signature(
|
||||
continue;
|
||||
|
||||
// Too many write attempts
|
||||
syslog(LOG_ERR, "Failure during BIO_write, returning null String");
|
||||
SYSLOGERR("Failure during BIO_write, returning null String");
|
||||
BIO_free_all(b64);
|
||||
Signature.clear();
|
||||
return Signature;
|
||||
} else {
|
||||
// If not a retry then it is an error
|
||||
syslog(LOG_ERR, "Failure during BIO_write, returning null String");
|
||||
SYSLOGERR("Failure during BIO_write, returning null String");
|
||||
BIO_free_all(b64);
|
||||
Signature.clear();
|
||||
return Signature;
|
||||
@ -575,14 +625,15 @@ string calc_signature(
|
||||
|
||||
// If there is no more data to write, the request sending has been
|
||||
// completed
|
||||
if(md_len <= 0)
|
||||
if(md_len <= 0){
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Flush the data
|
||||
ret = BIO_flush(b64);
|
||||
if ( ret <= 0) {
|
||||
syslog(LOG_ERR, "Failure during BIO_flush, returning null String");
|
||||
SYSLOGERR("Failure during BIO_flush, returning null String");
|
||||
BIO_free_all(b64);
|
||||
Signature.clear();
|
||||
return Signature;
|
||||
@ -600,7 +651,8 @@ string calc_signature(
|
||||
return Signature;
|
||||
}
|
||||
|
||||
void locate_bundle(void) {
|
||||
void locate_bundle(void)
|
||||
{
|
||||
// See if environment variable CURL_CA_BUNDLE is set
|
||||
// if so, check it, if it is a good path, then set the
|
||||
// curl_ca_bundle variable to it
|
||||
@ -619,7 +671,6 @@ void locate_bundle(void) {
|
||||
program_name.c_str());
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
@ -649,3 +700,125 @@ void locate_bundle(void) {
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
string md5sum(int fd)
|
||||
{
|
||||
MD5_CTX c;
|
||||
char buf[512];
|
||||
char hexbuf[3];
|
||||
ssize_t bytes;
|
||||
char md5[2 * MD5_DIGEST_LENGTH + 1];
|
||||
unsigned char *result = (unsigned char *) malloc(MD5_DIGEST_LENGTH);
|
||||
|
||||
memset(buf, 0, 512);
|
||||
MD5_Init(&c);
|
||||
while((bytes = read(fd, buf, 512)) > 0) {
|
||||
MD5_Update(&c, buf, bytes);
|
||||
memset(buf, 0, 512);
|
||||
}
|
||||
|
||||
MD5_Final(result, &c);
|
||||
|
||||
memset(md5, 0, 2 * MD5_DIGEST_LENGTH + 1);
|
||||
for(int i = 0; i < MD5_DIGEST_LENGTH; i++) {
|
||||
snprintf(hexbuf, 3, "%02x", result[i]);
|
||||
strncat(md5, hexbuf, 2);
|
||||
}
|
||||
|
||||
free(result);
|
||||
lseek(fd, 0, 0);
|
||||
|
||||
return string(md5);
|
||||
}
|
||||
|
||||
bool InitMimeType(const char* file)
|
||||
{
|
||||
if(!file){
|
||||
return false;
|
||||
}
|
||||
|
||||
string line;
|
||||
ifstream MT(file);
|
||||
if (MT.good()) {
|
||||
while (getline(MT, line)) {
|
||||
if(line[0]=='#'){
|
||||
continue;
|
||||
}
|
||||
if(line.size() == 0){
|
||||
continue;
|
||||
}
|
||||
|
||||
stringstream tmp(line);
|
||||
string mimeType;
|
||||
tmp >> mimeType;
|
||||
while (tmp) {
|
||||
string ext;
|
||||
tmp >> ext;
|
||||
if (ext.size() == 0){
|
||||
continue;
|
||||
}
|
||||
mimeTypes[ext] = mimeType;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param s e.g., "index.html"
|
||||
* @return e.g., "text/html"
|
||||
*/
|
||||
string lookupMimeType(string s)
|
||||
{
|
||||
string result("application/octet-stream");
|
||||
string::size_type last_pos = s.find_last_of('.');
|
||||
string::size_type first_pos = s.find_first_of('.');
|
||||
string prefix, ext, ext2;
|
||||
|
||||
// No dots in name, just return
|
||||
if(last_pos == string::npos){
|
||||
return result;
|
||||
}
|
||||
// extract the last extension
|
||||
if(last_pos != string::npos){
|
||||
ext = s.substr(1+last_pos, string::npos);
|
||||
}
|
||||
if (last_pos != string::npos) {
|
||||
// one dot was found, now look for another
|
||||
if (first_pos != string::npos && first_pos < last_pos) {
|
||||
prefix = s.substr(0, last_pos);
|
||||
// Now get the second to last file extension
|
||||
string::size_type next_pos = prefix.find_last_of('.');
|
||||
if (next_pos != string::npos) {
|
||||
ext2 = prefix.substr(1+next_pos, string::npos);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// if we get here, then we have an extension (ext)
|
||||
mimes_t::const_iterator iter = mimeTypes.find(ext);
|
||||
// if the last extension matches a mimeType, then return
|
||||
// that mime type
|
||||
if (iter != mimeTypes.end()) {
|
||||
result = (*iter).second;
|
||||
return result;
|
||||
}
|
||||
|
||||
// return with the default result if there isn't a second extension
|
||||
if(first_pos == last_pos){
|
||||
return result;
|
||||
}
|
||||
|
||||
// Didn't find a mime-type for the first extension
|
||||
// Look for second extension in mimeTypes, return if found
|
||||
iter = mimeTypes.find(ext2);
|
||||
if (iter != mimeTypes.end()) {
|
||||
result = (*iter).second;
|
||||
return result;
|
||||
}
|
||||
|
||||
// neither the last extension nor the second-to-last extension
|
||||
// matched a mimeType, return the default mime type
|
||||
return result;
|
||||
}
|
||||
|
||||
|
89
src/curl.h
89
src/curl.h
@ -13,29 +13,83 @@ struct WriteThis {
|
||||
int sizeleft;
|
||||
};
|
||||
|
||||
typedef std::pair<double, double> progress_t;
|
||||
typedef std::map<std::string, std::string> headers_t;
|
||||
class auto_curl_slist {
|
||||
public:
|
||||
auto_curl_slist() : slist(0) { }
|
||||
~auto_curl_slist() { curl_slist_free_all(slist); }
|
||||
|
||||
extern int retries;
|
||||
extern long connect_timeout;
|
||||
extern time_t readwrite_timeout;
|
||||
extern bool debug;
|
||||
extern std::string program_name;
|
||||
extern std::string ssl_verify_hostname;
|
||||
extern std::string AWSAccessKeyId;
|
||||
extern std::string AWSSecretAccessKey;
|
||||
extern std::string service_path;
|
||||
extern std::string host;
|
||||
extern std::string bucket;
|
||||
extern std::string public_bucket;
|
||||
struct curl_slist* get() const { return slist; }
|
||||
|
||||
static const EVP_MD* evp_md = EVP_sha1();
|
||||
void append(const std::string& s) {
|
||||
slist = curl_slist_append(slist, s.c_str());
|
||||
}
|
||||
|
||||
private:
|
||||
struct curl_slist* slist;
|
||||
};
|
||||
|
||||
// header data
|
||||
struct head_data {
|
||||
std::string path;
|
||||
std::string *url;
|
||||
struct curl_slist *requestHeaders;
|
||||
headers_t *responseHeaders;
|
||||
};
|
||||
|
||||
typedef std::map<CURL*, head_data> headMap_t;
|
||||
|
||||
void destroy_curl_handle(CURL *curl_handle);
|
||||
|
||||
struct cleanup_head_data {
|
||||
void operator()(std::pair<CURL*, head_data> qqq) {
|
||||
CURL* curl_handle = qqq.first;
|
||||
|
||||
head_data response = qqq.second;
|
||||
delete response.url;
|
||||
curl_slist_free_all(response.requestHeaders);
|
||||
delete response.responseHeaders;
|
||||
destroy_curl_handle(curl_handle);
|
||||
}
|
||||
};
|
||||
|
||||
class auto_head {
|
||||
public:
|
||||
auto_head() {}
|
||||
~auto_head() {
|
||||
for_each(headMap.begin(), headMap.end(), cleanup_head_data());
|
||||
}
|
||||
|
||||
headMap_t& get() { return headMap; }
|
||||
|
||||
void remove(CURL* curl_handle) {
|
||||
headMap_t::iterator iter = headMap.find(curl_handle);
|
||||
if(iter == headMap.end()){
|
||||
return;
|
||||
}
|
||||
|
||||
head_data response = iter->second;
|
||||
delete response.url;
|
||||
curl_slist_free_all(response.requestHeaders);
|
||||
delete response.responseHeaders;
|
||||
destroy_curl_handle(curl_handle);
|
||||
|
||||
headMap.erase(iter);
|
||||
}
|
||||
|
||||
private:
|
||||
headMap_t headMap;
|
||||
};
|
||||
|
||||
//
|
||||
// Functions
|
||||
//
|
||||
int init_curl_handles_mutex(void);
|
||||
int destroy_curl_handles_mutex(void);
|
||||
size_t header_callback(void *data, size_t blockSize, size_t numBlocks, void *userPtr);
|
||||
CURL *create_curl_handle(void);
|
||||
void destroy_curl_handle(CURL *curl_handle);
|
||||
int curl_delete(const char *path);
|
||||
int curl_get_headers(const char *path, headers_t &meta);
|
||||
CURL *create_head_handle(struct head_data *request);
|
||||
int my_curl_easy_perform(CURL* curl, BodyStruct* body = NULL, FILE* f = 0);
|
||||
size_t WriteMemoryCallback(void *ptr, size_t blockSize, size_t numBlocks, void *data);
|
||||
size_t read_callback(void *ptr, size_t size, size_t nmemb, void *userp);
|
||||
@ -44,5 +98,8 @@ int my_curl_progress(
|
||||
std::string calc_signature(
|
||||
std::string method, std::string content_type, std::string date, curl_slist* headers, std::string resource);
|
||||
void locate_bundle(void);
|
||||
std::string md5sum(int fd);
|
||||
bool InitMimeType(const char* file);
|
||||
std::string lookupMimeType(std::string);
|
||||
|
||||
#endif // S3FS_CURL_H_
|
||||
|
1346
src/s3fs.cpp
1346
src/s3fs.cpp
File diff suppressed because it is too large
Load Diff
142
src/s3fs.h
142
src/s3fs.h
@ -7,19 +7,7 @@
|
||||
#define MAX_COPY_SOURCE_SIZE 524288000 // 500MB
|
||||
#define FIVE_GB 5368709120LL
|
||||
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include <curl/curl.h>
|
||||
#include <fuse.h>
|
||||
#include <openssl/bio.h>
|
||||
#include <openssl/buffer.h>
|
||||
#include <openssl/evp.h>
|
||||
#include <openssl/hmac.h>
|
||||
#include <openssl/md5.h>
|
||||
#include <pthread.h>
|
||||
#include <sys/time.h>
|
||||
|
||||
#define YIKES(result) if (true) { \
|
||||
syslog(LOG_ERR, "%d###result=%d", __LINE__, result); \
|
||||
@ -33,134 +21,4 @@
|
||||
} \
|
||||
}
|
||||
|
||||
long connect_timeout = 10;
|
||||
time_t readwrite_timeout = 30;
|
||||
|
||||
int retries = 2;
|
||||
|
||||
bool debug = 0;
|
||||
bool foreground = 0;
|
||||
bool nomultipart = false;
|
||||
bool service_validated = false;
|
||||
std::string host = "http://s3.amazonaws.com";
|
||||
std::string service_path = "/";
|
||||
std::string bucket = "";
|
||||
std::string mount_prefix = "";
|
||||
static std::string mountpoint;
|
||||
std::string program_name;
|
||||
std::string AWSAccessKeyId;
|
||||
std::string AWSSecretAccessKey;
|
||||
static mode_t root_mode = 0;
|
||||
static std::string passwd_file = "";
|
||||
static bool utility_mode = 0;
|
||||
unsigned long max_stat_cache_size = 10000;
|
||||
time_t stat_cache_expire_time = 0;
|
||||
int is_stat_cache_expire_time = 0;
|
||||
bool noxmlns = false;
|
||||
bool nocopyapi = false;
|
||||
bool norenameapi = false;
|
||||
|
||||
// if .size()==0 then local file cache is disabled
|
||||
static std::string use_cache;
|
||||
static std::string use_rrs;
|
||||
std::string ssl_verify_hostname = "1";
|
||||
std::string public_bucket;
|
||||
|
||||
extern pthread_mutex_t stat_cache_lock;
|
||||
extern pthread_mutex_t curl_handles_lock;
|
||||
extern std::string curl_ca_bundle;
|
||||
|
||||
// TODO(apetresc): make this an enum
|
||||
// private, public-read, public-read-write, authenticated-read
|
||||
static std::string default_acl("private");
|
||||
|
||||
struct file_part {
|
||||
char path[17];
|
||||
std::string etag;
|
||||
bool uploaded;
|
||||
|
||||
file_part() : uploaded(false) {}
|
||||
};
|
||||
|
||||
static const char hexAlphabet[] = "0123456789ABCDEF";
|
||||
|
||||
// http headers
|
||||
typedef std::map<std::string, std::string> headers_t;
|
||||
|
||||
// fd -> flags
|
||||
typedef std::map<int, int> s3fs_descriptors_t;
|
||||
static s3fs_descriptors_t s3fs_descriptors;
|
||||
static pthread_mutex_t s3fs_descriptors_lock;
|
||||
// path -> fd
|
||||
typedef std::map<std::string, int> s3fs_pathtofd_t;
|
||||
static s3fs_pathtofd_t s3fs_pathtofd;
|
||||
|
||||
static pthread_mutex_t *mutex_buf = NULL;
|
||||
|
||||
static struct fuse_operations s3fs_oper;
|
||||
|
||||
std::string lookupMimeType(std::string);
|
||||
std::string initiate_multipart_upload(const char *path, off_t size, headers_t meta);
|
||||
std::string upload_part(const char *path, const char *source, int part_number, std::string upload_id);
|
||||
std::string copy_part(const char *from, const char *to, int part_number, std::string upload_id, headers_t meta);
|
||||
static int complete_multipart_upload(const char *path, std::string upload_id, std::vector <file_part> parts);
|
||||
std::string md5sum(int fd);
|
||||
std::string get_realpath(const char *path);
|
||||
|
||||
time_t get_mtime(const char *s);
|
||||
off_t get_size(const char *s);
|
||||
mode_t get_mode(const char *s);
|
||||
uid_t get_uid(const char *s);
|
||||
gid_t get_gid(const char *s);
|
||||
blkcnt_t get_blocks(off_t size);
|
||||
|
||||
static int insert_object(const char *name, struct s3_object **head);
|
||||
//static unsigned int count_object_list(struct s3_object *list);
|
||||
static int free_object(struct s3_object *object);
|
||||
static int free_object_list(struct s3_object *head);
|
||||
|
||||
static CURL *create_head_handle(struct head_data *request);
|
||||
static int list_bucket(const char *path, struct s3_object **head, const char* delimiter);
|
||||
static bool is_truncated(const char *xml);
|
||||
static int append_objects_from_xml_ex(const char* path, xmlDocPtr doc, xmlXPathContextPtr ctx,
|
||||
const char* ex_contents, const char* ex_key, int isCPrefix, struct s3_object **head);
|
||||
static int append_objects_from_xml(const char* path, const char *xml, struct s3_object **head);
|
||||
static xmlChar* get_base_exp(const char* xml, const char* exp);
|
||||
static xmlChar* get_prefix(const char *xml);
|
||||
static xmlChar* get_next_marker(const char *xml);
|
||||
static char *get_object_name(xmlDocPtr doc, xmlNodePtr node, const char* path);
|
||||
|
||||
static int put_headers(const char *path, headers_t meta);
|
||||
static int put_multipart_headers(const char *path, headers_t meta);
|
||||
|
||||
static int s3fs_getattr(const char *path, struct stat *stbuf);
|
||||
static int s3fs_readlink(const char *path, char *buf, size_t size);
|
||||
static int s3fs_mknod(const char* path, mode_t mode, dev_t rdev);
|
||||
static int s3fs_mkdir(const char *path, mode_t mode);
|
||||
static int s3fs_unlink(const char *path);
|
||||
static int s3fs_rmdir(const char *path);
|
||||
static int s3fs_symlink(const char *from, const char *to);
|
||||
static int s3fs_rename(const char *from, const char *to);
|
||||
static int s3fs_link(const char *from, const char *to);
|
||||
static int s3fs_chmod(const char *path, mode_t mode);
|
||||
static int s3fs_chown(const char *path, uid_t uid, gid_t gid);
|
||||
static int s3fs_truncate(const char *path, off_t size);
|
||||
static int s3fs_open(const char *path, struct fuse_file_info *fi);
|
||||
static int s3fs_read(
|
||||
const char *path, char *buf, size_t size, off_t offset, struct fuse_file_info *fi);
|
||||
static int s3fs_write(
|
||||
const char *path, const char *buf, size_t size, off_t offset, struct fuse_file_info *fi);
|
||||
static int s3fs_statfs(const char *path, struct statvfs *stbuf);
|
||||
static int s3fs_flush(const char *path, struct fuse_file_info *fi);
|
||||
static int s3fs_release(const char *path, struct fuse_file_info *fi);
|
||||
static int s3fs_opendir(const char *path, struct fuse_file_info *fi);
|
||||
static int s3fs_readdir(
|
||||
const char *path, void *buf, fuse_fill_dir_t filler, off_t offset, struct fuse_file_info *fi);
|
||||
static int s3fs_access(const char *path, int mask);
|
||||
static int s3fs_utimens(const char *path, const struct timespec ts[2]);
|
||||
static int remote_mountpath_exists(const char *path);
|
||||
static void* s3fs_init(struct fuse_conn_info *conn);
|
||||
static void s3fs_destroy(void*);
|
||||
|
||||
#endif // S3FS_S3_H_
|
||||
|
||||
|
525
src/s3fs_util.cpp
Normal file
525
src/s3fs_util.cpp
Normal file
@ -0,0 +1,525 @@
|
||||
/*
|
||||
* s3fs - FUSE-based file system backed by Amazon S3
|
||||
*
|
||||
* Copyright 2007-2013 Takeshi Nakatani <ggtakec.com>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
#include <errno.h>
|
||||
#include <libgen.h>
|
||||
#include <pwd.h>
|
||||
#include <grp.h>
|
||||
#include <syslog.h>
|
||||
|
||||
#include <string>
|
||||
#include <sstream>
|
||||
#include <map>
|
||||
|
||||
#include "common.h"
|
||||
#include "s3fs_util.h"
|
||||
#include "s3fs.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Global valiables
|
||||
//-------------------------------------------------------------------
|
||||
std::string mount_prefix = "";
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Utility
|
||||
//-------------------------------------------------------------------
|
||||
string get_realpath(const char *path) {
|
||||
string realpath = mount_prefix;
|
||||
realpath += path;
|
||||
|
||||
return realpath;
|
||||
}
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Utility for listing objects
|
||||
//-------------------------------------------------------------------
|
||||
// [Change]
|
||||
// The s3_object's name member can be set "dir" or "dir/" as directory name.
|
||||
// Both of names are same directory s3_object.
|
||||
// "dir/" is given priority to over "dir".
|
||||
//
|
||||
// [Notice]
|
||||
// If there are "dir" and "dir/" object on S3, s3fs only recognizes "dir/".
|
||||
// On this case, user can not know the "dir" object.
|
||||
//
|
||||
int insert_object(const char *name, struct s3_object **head)
|
||||
{
|
||||
struct s3_object *cur_object;
|
||||
struct s3_object *new_object;
|
||||
int nLen = name ? strlen(name) : 0;
|
||||
int is_have_ndelimiter = 0;
|
||||
|
||||
// search same name object
|
||||
if(nLen && '/' == name[nLen - 1]){
|
||||
// case of "dir/"
|
||||
nLen--;
|
||||
is_have_ndelimiter = 1;
|
||||
}
|
||||
for(cur_object = *head; nLen && cur_object; cur_object = cur_object->next){
|
||||
if(0 == strncmp(cur_object->name, name, nLen)){
|
||||
int cLen = strlen(cur_object->name);
|
||||
int is_have_cdelimiter = 0;
|
||||
|
||||
if('/' == cur_object->name[cLen - 1]){
|
||||
cLen--;
|
||||
is_have_cdelimiter = 1;
|
||||
}
|
||||
if(cLen == nLen){
|
||||
// same object
|
||||
if(is_have_cdelimiter == is_have_ndelimiter){
|
||||
// perfect same object
|
||||
return 0;
|
||||
}
|
||||
if(is_have_cdelimiter){
|
||||
// already set "dir/"
|
||||
return 0;
|
||||
}
|
||||
// new object is "dir/", replace name.
|
||||
free(cur_object->name);
|
||||
if(NULL == (cur_object->name = strdup(name))){
|
||||
printf("insert_object: could not allocate memory\n");
|
||||
S3FS_FUSE_EXIT();
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Not found same object.
|
||||
new_object = (struct s3_object *) malloc(sizeof(struct s3_object));
|
||||
if(new_object == NULL) {
|
||||
printf("insert_object: could not allocate memory\n");
|
||||
S3FS_FUSE_EXIT();
|
||||
return -1;
|
||||
}
|
||||
|
||||
if(NULL == (new_object->name = strdup(name))){
|
||||
free(new_object);
|
||||
printf("insert_object: could not allocate memory\n");
|
||||
S3FS_FUSE_EXIT();
|
||||
return -1;
|
||||
}
|
||||
|
||||
if((*head) == NULL)
|
||||
new_object->next = NULL;
|
||||
else
|
||||
new_object->next = (*head);
|
||||
|
||||
*head = new_object;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int free_object(struct s3_object *object)
|
||||
{
|
||||
free(object->name);
|
||||
free(object);
|
||||
object = NULL;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int free_object_list(struct s3_object *head)
|
||||
{
|
||||
struct s3_object *tmp = NULL;
|
||||
struct s3_object *current = head;
|
||||
|
||||
current = head;
|
||||
while(current != NULL) {
|
||||
tmp = current;
|
||||
current = current->next;
|
||||
free_object(tmp);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Utility functions for moving objects
|
||||
//-------------------------------------------------------------------
|
||||
MVNODE *create_mvnode(const char *old_path, const char *new_path, bool is_dir)
|
||||
{
|
||||
MVNODE *p;
|
||||
char *p_old_path;
|
||||
char *p_new_path;
|
||||
|
||||
p = (MVNODE *) malloc(sizeof(MVNODE));
|
||||
if (p == NULL) {
|
||||
printf("create_mvnode: could not allocation memory for p\n");
|
||||
S3FS_FUSE_EXIT();
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if(NULL == (p_old_path = strdup(old_path))){
|
||||
free(p);
|
||||
printf("create_mvnode: could not allocation memory for p_old_path\n");
|
||||
S3FS_FUSE_EXIT();
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if(NULL == (p_new_path = strdup(new_path))){
|
||||
free(p);
|
||||
free(p_old_path);
|
||||
printf("create_mvnode: could not allocation memory for p_new_path\n");
|
||||
S3FS_FUSE_EXIT();
|
||||
return NULL;
|
||||
}
|
||||
|
||||
p->old_path = p_old_path;
|
||||
p->new_path = p_new_path;
|
||||
p->is_dir = is_dir;
|
||||
p->prev = NULL;
|
||||
p->next = NULL;
|
||||
return p;
|
||||
}
|
||||
|
||||
//
|
||||
// Add sorted MVNODE data(Ascending order)
|
||||
//
|
||||
MVNODE *add_mvnode(MVNODE** head, MVNODE** tail, const char *old_path, const char *new_path, bool is_dir)
|
||||
{
|
||||
if(!head || !tail){
|
||||
return NULL;
|
||||
}
|
||||
|
||||
MVNODE* cur;
|
||||
MVNODE* mvnew;
|
||||
for(cur = *head; cur; cur = cur->next){
|
||||
if(cur->is_dir == is_dir){
|
||||
int nResult = strcmp(cur->old_path, old_path);
|
||||
if(0 == nResult){
|
||||
// Found same old_path.
|
||||
return cur;
|
||||
|
||||
}else if(0 > nResult){
|
||||
// next check.
|
||||
// ex: cur("abc"), mvnew("abcd")
|
||||
// ex: cur("abc"), mvnew("abd")
|
||||
continue;
|
||||
|
||||
}else{
|
||||
// Add into before cur-pos.
|
||||
// ex: cur("abc"), mvnew("ab")
|
||||
// ex: cur("abc"), mvnew("abb")
|
||||
if(NULL == (mvnew = create_mvnode(old_path, new_path, is_dir))){
|
||||
return NULL;
|
||||
}
|
||||
if(cur->prev){
|
||||
(cur->prev)->next = mvnew;
|
||||
}else{
|
||||
*head = mvnew;
|
||||
}
|
||||
mvnew->prev = cur->prev;
|
||||
mvnew->next = cur;
|
||||
cur->prev = mvnew;
|
||||
|
||||
return mvnew;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Add into tail.
|
||||
if(NULL == (mvnew = create_mvnode(old_path, new_path, is_dir))){
|
||||
return NULL;
|
||||
}
|
||||
mvnew->prev = (*tail);
|
||||
if(*tail){
|
||||
(*tail)->next = mvnew;
|
||||
}
|
||||
(*tail) = mvnew;
|
||||
if(!(*head)){
|
||||
(*head) = mvnew;
|
||||
}
|
||||
return mvnew;
|
||||
}
|
||||
|
||||
void free_mvnodes(MVNODE *head)
|
||||
{
|
||||
MVNODE *my_head;
|
||||
MVNODE *next;
|
||||
|
||||
for(my_head = head, next = NULL; my_head; my_head = next){
|
||||
next = my_head->next;
|
||||
free(my_head->old_path);
|
||||
free(my_head->new_path);
|
||||
free(my_head);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Utility for UID/GID
|
||||
//-------------------------------------------------------------------
|
||||
// get user name from uid
|
||||
string get_username(uid_t uid)
|
||||
{
|
||||
struct passwd* ppw;
|
||||
if(NULL == (ppw = getpwuid(uid)) || NULL == ppw->pw_name){
|
||||
FGPRINT(" could not get username(errno=%d).\n", (int)errno);
|
||||
SYSLOGDBG("could not get username(errno=%d).\n", (int)errno);
|
||||
return NULL;
|
||||
}
|
||||
return string(ppw->pw_name);
|
||||
}
|
||||
|
||||
// check uid in group(gid)
|
||||
int is_uid_inculde_group(uid_t uid, gid_t gid)
|
||||
{
|
||||
static size_t maxlen = 0; // set onece
|
||||
int result;
|
||||
char* pbuf;
|
||||
struct group ginfo;
|
||||
struct group* pginfo = NULL;
|
||||
|
||||
// make buffer
|
||||
if(0 == maxlen){
|
||||
if(0 > (maxlen = (size_t)sysconf(_SC_GETGR_R_SIZE_MAX))){
|
||||
FGPRINT(" could not get max name length.\n");
|
||||
SYSLOGDBG("could not get max name length.\n");
|
||||
maxlen = 0;
|
||||
return -ERANGE;
|
||||
}
|
||||
}
|
||||
if(NULL == (pbuf = (char*)malloc(sizeof(char) * maxlen))){
|
||||
FGPRINT(" failed to allocate memory.\n");
|
||||
SYSLOGERR("failed to allocate memory.\n");
|
||||
return -ENOMEM;
|
||||
}
|
||||
// get group infomation
|
||||
if(0 != (result = getgrgid_r(gid, &ginfo, pbuf, maxlen, &pginfo))){
|
||||
FGPRINT(" could not get group infomation.\n");
|
||||
SYSLOGDBG("could not get group infomation.\n");
|
||||
free(pbuf);
|
||||
return -result;
|
||||
}
|
||||
|
||||
// check group
|
||||
if(NULL == pginfo){
|
||||
// there is not gid in group.
|
||||
free(pbuf);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
string username = get_username(uid);
|
||||
|
||||
char** ppgr_mem;
|
||||
for(ppgr_mem = pginfo->gr_mem; ppgr_mem && *ppgr_mem; ppgr_mem++){
|
||||
if(username == *ppgr_mem){
|
||||
// Found username in group.
|
||||
free(pbuf);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
free(pbuf);
|
||||
return 0;
|
||||
}
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Utility for file and directory
|
||||
//-------------------------------------------------------------------
|
||||
// safe variant of dirname
|
||||
// dirname clobbers path so let it operate on a tmp copy
|
||||
string mydirname(string path)
|
||||
{
|
||||
return string(dirname(&path[0]));
|
||||
}
|
||||
|
||||
// safe variant of basename
|
||||
// basename clobbers path so let it operate on a tmp copy
|
||||
string mybasename(string path)
|
||||
{
|
||||
return string(basename(&path[0]));
|
||||
}
|
||||
|
||||
// mkdir --parents
|
||||
int mkdirp(const string& path, mode_t mode)
|
||||
{
|
||||
string base;
|
||||
string component;
|
||||
stringstream ss(path);
|
||||
while (getline(ss, component, '/')) {
|
||||
base += "/" + component;
|
||||
mkdir(base.c_str(), mode);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Utility functions for convert
|
||||
//-------------------------------------------------------------------
|
||||
time_t get_mtime(const char *s)
|
||||
{
|
||||
return (time_t) strtoul(s, (char **) NULL, 10);
|
||||
}
|
||||
|
||||
off_t get_size(const char *s)
|
||||
{
|
||||
return (off_t) strtoul(s, (char **) NULL, 10);
|
||||
}
|
||||
|
||||
mode_t get_mode(const char *s)
|
||||
{
|
||||
return (mode_t) strtoul(s, (char **) NULL, 10);
|
||||
}
|
||||
|
||||
uid_t get_uid(const char *s)
|
||||
{
|
||||
return (uid_t) strtoul(s, (char **) NULL, 10);
|
||||
}
|
||||
|
||||
gid_t get_gid(const char *s)
|
||||
{
|
||||
return (gid_t) strtoul(s, (char **) NULL, 10);
|
||||
}
|
||||
|
||||
blkcnt_t get_blocks(off_t size)
|
||||
{
|
||||
return size / 512 + 1;
|
||||
}
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Help
|
||||
//-------------------------------------------------------------------
|
||||
void show_usage (void)
|
||||
{
|
||||
printf("Usage: %s BUCKET:[PATH] MOUNTPOINT [OPTION]...\n",
|
||||
program_name.c_str());
|
||||
}
|
||||
|
||||
void show_help (void)
|
||||
{
|
||||
show_usage();
|
||||
printf(
|
||||
"\n"
|
||||
"Mount an Amazon S3 bucket as a file system.\n"
|
||||
"\n"
|
||||
" General forms for s3fs and FUSE/mount options:\n"
|
||||
" -o opt[,opt...]\n"
|
||||
" -o opt [-o opt] ...\n"
|
||||
"\n"
|
||||
"s3fs Options:\n"
|
||||
"\n"
|
||||
" Most s3fs options are given in the form where \"opt\" is:\n"
|
||||
"\n"
|
||||
" <option_name>=<option_value>\n"
|
||||
"\n"
|
||||
" default_acl (default=\"private\")\n"
|
||||
" - the default canned acl to apply to all written s3 objects\n"
|
||||
" see http://aws.amazon.com/documentation/s3/ for the \n"
|
||||
" full list of canned acls\n"
|
||||
"\n"
|
||||
" retries (default=\"2\")\n"
|
||||
" - number of times to retry a failed s3 transaction\n"
|
||||
"\n"
|
||||
" use_cache (default=\"\" which means disabled)\n"
|
||||
" - local folder to use for local file cache\n"
|
||||
"\n"
|
||||
" use_rrs (default=\"\" which means diabled)\n"
|
||||
" - use Amazon's Reduced Redundancy Storage when set to 1\n"
|
||||
"\n"
|
||||
" public_bucket (default=\"\" which means disabled)\n"
|
||||
" - anonymously mount a public bucket when set to 1\n"
|
||||
"\n"
|
||||
" passwd_file (default=\"\")\n"
|
||||
" - specify which s3fs password file to use\n"
|
||||
"\n"
|
||||
" connect_timeout (default=\"10\" seconds)\n"
|
||||
" - time to wait for connection before giving up\n"
|
||||
"\n"
|
||||
" readwrite_timeout (default=\"30\" seconds)\n"
|
||||
" - time to wait between read/write activity before giving up\n"
|
||||
"\n"
|
||||
" max_stat_cache_size (default=\"10000\" entries (about 4MB))\n"
|
||||
" - maximum number of entries in the stat cache\n"
|
||||
"\n"
|
||||
" stat_cache_expire (default is no expire)\n"
|
||||
" - specify expire time(seconds) for entries in the stat cache.\n"
|
||||
"\n"
|
||||
" url (default=\"http://s3.amazonaws.com\")\n"
|
||||
" - sets the url to use to access amazon s3\n"
|
||||
"\n"
|
||||
" nomultipart - disable multipart uploads\n"
|
||||
"\n"
|
||||
" noxmlns - disable registing xml name space.\n"
|
||||
" disable registing xml name space for response of \n"
|
||||
" ListBucketResult and ListVersionsResult etc. Default name \n"
|
||||
" space is looked up from \"http://s3.amazonaws.com/doc/2006-03-01\".\n"
|
||||
"\n"
|
||||
" nocopyapi - for other incomplete compatibility object storage.\n"
|
||||
" For a distributed object storage which is compatibility S3\n"
|
||||
" API without PUT(copy api).\n"
|
||||
" If you set this option, s3fs do not use PUT with \n"
|
||||
" \"x-amz-copy-source\"(copy api). Because traffic is increased\n"
|
||||
" 2-3 times by this option, we do not recommend this.\n"
|
||||
"\n"
|
||||
" norenameapi - for other incomplete compatibility object storage.\n"
|
||||
" For a distributed object storage which is compatibility S3\n"
|
||||
" API without PUT(copy api).\n"
|
||||
" This option is a subset of nocopyapi option. The nocopyapi\n"
|
||||
" option does not use copy-api for all command(ex. chmod, chown,\n"
|
||||
" touch, mv, etc), but this option does not use copy-api for\n"
|
||||
" only rename command(ex. mv). If this option is specified with\n"
|
||||
" nocopapi, the s3fs ignores it.\n"
|
||||
"\n"
|
||||
"FUSE/mount Options:\n"
|
||||
"\n"
|
||||
" Most of the generic mount options described in 'man mount' are\n"
|
||||
" supported (ro, rw, suid, nosuid, dev, nodev, exec, noexec, atime,\n"
|
||||
" noatime, sync async, dirsync). Filesystems are mounted with\n"
|
||||
" '-onodev,nosuid' by default, which can only be overridden by a\n"
|
||||
" privileged user.\n"
|
||||
" \n"
|
||||
" There are many FUSE specific mount options that can be specified.\n"
|
||||
" e.g. allow_other See the FUSE's README for the full set.\n"
|
||||
"\n"
|
||||
"Miscellaneous Options:\n"
|
||||
"\n"
|
||||
" -h, --help Output this help.\n"
|
||||
" --version Output version info.\n"
|
||||
" -d --debug Turn on DEBUG messages to syslog. Specifying -d\n"
|
||||
" twice turns on FUSE debug messages to STDOUT.\n"
|
||||
" -f FUSE foreground option - do not run as daemon.\n"
|
||||
" -s FUSE singlethread option\n"
|
||||
" disable multi-threaded operation\n"
|
||||
"\n"
|
||||
"\n"
|
||||
"Report bugs to <s3fs-devel@googlegroups.com>\n"
|
||||
"s3fs home page: <http://code.google.com/p/s3fs/>\n"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
void show_version(void)
|
||||
{
|
||||
printf(
|
||||
"Amazon Simple Storage Service File System %s\n"
|
||||
"Copyright (C) 2010 Randy Rizun <rrizun@gmail.com>\n"
|
||||
"License GPL2: GNU GPL version 2 <http://gnu.org/licenses/gpl.html>\n"
|
||||
"This is free software: you are free to change and redistribute it.\n"
|
||||
"There is NO WARRANTY, to the extent permitted by law.\n", VERSION );
|
||||
return;
|
||||
}
|
||||
|
52
src/s3fs_util.h
Normal file
52
src/s3fs_util.h
Normal file
@ -0,0 +1,52 @@
|
||||
#ifndef S3FS_S3FS_UTIL_H_
|
||||
#define S3FS_S3FS_UTIL_H_
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Typedef
|
||||
//-------------------------------------------------------------------
|
||||
struct s3_object {
|
||||
char *name;
|
||||
struct s3_object *next;
|
||||
};
|
||||
|
||||
typedef struct mvnode {
|
||||
char *old_path;
|
||||
char *new_path;
|
||||
bool is_dir;
|
||||
struct mvnode *prev;
|
||||
struct mvnode *next;
|
||||
} MVNODE;
|
||||
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Functions
|
||||
//-------------------------------------------------------------------
|
||||
std::string get_realpath(const char *path);
|
||||
|
||||
int insert_object(const char *name, struct s3_object **head);
|
||||
int free_object(struct s3_object *object);
|
||||
int free_object_list(struct s3_object *head);
|
||||
|
||||
MVNODE *create_mvnode(const char *old_path, const char *new_path, bool is_dir);
|
||||
MVNODE *add_mvnode(MVNODE** head, MVNODE** tail, const char *old_path, const char *new_path, bool is_dir);
|
||||
void free_mvnodes(MVNODE *head);
|
||||
|
||||
std::string get_username(uid_t uid);
|
||||
int is_uid_inculde_group(uid_t uid, gid_t gid);
|
||||
|
||||
std::string mydirname(std::string path);
|
||||
std::string mybasename(std::string path);
|
||||
int mkdirp(const std::string& path, mode_t mode);
|
||||
|
||||
time_t get_mtime(const char *s);
|
||||
off_t get_size(const char *s);
|
||||
mode_t get_mode(const char *s);
|
||||
uid_t get_uid(const char *s);
|
||||
gid_t get_gid(const char *s);
|
||||
blkcnt_t get_blocks(off_t size);
|
||||
|
||||
void show_usage(void);
|
||||
void show_help(void);
|
||||
void show_version(void);
|
||||
|
||||
#endif // S3FS_S3FS_UTIL_H_
|
@ -22,42 +22,50 @@
|
||||
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <map>
|
||||
|
||||
#include "common.h"
|
||||
#include "string_util.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
static const char hexAlphabet[] = "0123456789ABCDEF";
|
||||
|
||||
string lower(string s) {
|
||||
string lower(string s)
|
||||
{
|
||||
// change each character of the string to lower case
|
||||
for(unsigned int i = 0; i < s.length(); i++)
|
||||
for(unsigned int i = 0; i < s.length(); i++){
|
||||
s[i] = tolower(s[i]);
|
||||
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
string IntToStr(int n) {
|
||||
string IntToStr(int n)
|
||||
{
|
||||
stringstream result;
|
||||
result << n;
|
||||
return result.str();
|
||||
}
|
||||
|
||||
string trim_left(const string &s, const string &t /* = SPACES */) {
|
||||
string trim_left(const string &s, const string &t /* = SPACES */)
|
||||
{
|
||||
string d(s);
|
||||
return d.erase(0, s.find_first_not_of(t));
|
||||
}
|
||||
|
||||
string trim_right(const string &s, const string &t /* = SPACES */) {
|
||||
string trim_right(const string &s, const string &t /* = SPACES */)
|
||||
{
|
||||
string d(s);
|
||||
string::size_type i(d.find_last_not_of(t));
|
||||
if (i == string::npos)
|
||||
if(i == string::npos){
|
||||
return "";
|
||||
else
|
||||
}else{
|
||||
return d.erase(d.find_last_not_of(t) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
string trim(const string &s, const string &t /* = SPACES */) {
|
||||
string trim(const string &s, const string &t /* = SPACES */)
|
||||
{
|
||||
string d(s);
|
||||
return trim_left(trim_right(d, t), t);
|
||||
}
|
||||
@ -67,18 +75,19 @@ string trim(const string &s, const string &t /* = SPACES */) {
|
||||
* taking into special consideration "/",
|
||||
* otherwise regular urlEncode.
|
||||
*/
|
||||
string urlEncode(const string &s) {
|
||||
string urlEncode(const string &s)
|
||||
{
|
||||
string result;
|
||||
for (unsigned i = 0; i < s.length(); ++i) {
|
||||
if (s[i] == '/') // Note- special case for fuse paths...
|
||||
if (s[i] == '/') { // Note- special case for fuse paths...
|
||||
result += s[i];
|
||||
else if (isalnum(s[i]))
|
||||
} else if (isalnum(s[i])) {
|
||||
result += s[i];
|
||||
else if (s[i] == '.' || s[i] == '-' || s[i] == '*' || s[i] == '_')
|
||||
} else if (s[i] == '.' || s[i] == '-' || s[i] == '*' || s[i] == '_') {
|
||||
result += s[i];
|
||||
else if (s[i] == ' ')
|
||||
} else if (s[i] == ' ') {
|
||||
result += '+';
|
||||
else {
|
||||
} else {
|
||||
result += "%";
|
||||
result += hexAlphabet[static_cast<unsigned char>(s[i]) / 16];
|
||||
result += hexAlphabet[static_cast<unsigned char>(s[i]) % 16];
|
||||
@ -88,9 +97,9 @@ string urlEncode(const string &s) {
|
||||
return result;
|
||||
}
|
||||
|
||||
string prepare_url(const char* url) {
|
||||
if(debug)
|
||||
syslog(LOG_DEBUG, "URL is %s", url);
|
||||
string prepare_url(const char* url)
|
||||
{
|
||||
SYSLOGDBG("URL is %s", url);
|
||||
|
||||
string uri;
|
||||
string host;
|
||||
@ -101,17 +110,16 @@ string prepare_url(const char* url) {
|
||||
int bucket_length = token.size();
|
||||
int uri_length = 7;
|
||||
|
||||
if(!strncasecmp(url_str.c_str(), "https://", 8))
|
||||
if(!strncasecmp(url_str.c_str(), "https://", 8)){
|
||||
uri_length = 8;
|
||||
|
||||
}
|
||||
uri = url_str.substr(0, uri_length);
|
||||
host = bucket + "." + url_str.substr(uri_length, bucket_pos - uri_length).c_str();
|
||||
path = url_str.substr((bucket_pos + bucket_length));
|
||||
|
||||
url_str = uri + host + path;
|
||||
|
||||
if(debug)
|
||||
syslog(LOG_DEBUG, "URL changed is %s", url_str.c_str());
|
||||
SYSLOGDBG("URL changed is %s", url_str.c_str());
|
||||
|
||||
return str(url_str);
|
||||
}
|
||||
@ -120,9 +128,11 @@ string prepare_url(const char* url) {
|
||||
* Returns the current date
|
||||
* in a format suitable for a HTTP request header.
|
||||
*/
|
||||
string get_date() {
|
||||
string get_date()
|
||||
{
|
||||
char buf[100];
|
||||
time_t t = time(NULL);
|
||||
strftime(buf, sizeof(buf), "%a, %d %b %Y %H:%M:%S GMT", gmtime(&t));
|
||||
return buf;
|
||||
}
|
||||
|
||||
|
@ -18,12 +18,6 @@ template<typename T> std::string str(T value) {
|
||||
return s.str();
|
||||
}
|
||||
|
||||
extern bool debug;
|
||||
extern bool foreground;
|
||||
extern bool service_validated;
|
||||
|
||||
extern std::string bucket;
|
||||
|
||||
std::string trim_left(const std::string &s, const std::string &t = SPACES);
|
||||
std::string trim_right(const std::string &s, const std::string &t = SPACES);
|
||||
std::string trim(const std::string &s, const std::string &t = SPACES);
|
||||
@ -33,5 +27,4 @@ std::string get_date();
|
||||
std::string urlEncode(const std::string &s);
|
||||
std::string prepare_url(const char* url);
|
||||
|
||||
|
||||
#endif // S3FS_STRING_UTIL_H_
|
||||
|
Loading…
Reference in New Issue
Block a user