mirror of
https://github.com/s3fs-fuse/s3fs-fuse.git
synced 2024-11-10 06:40:59 +00:00
merged in eu bucket changes from http://github.com/tractis/s3fs-fork
git-svn-id: http://s3fs.googlecode.com/svn/trunk@191 df820570-a93a-0410-bd06-b72b767a4274
This commit is contained in:
rrizun
parent
e6c737a567
commit
abab090301
@ -51,6 +51,9 @@ using namespace std;
|
||||
static long connect_timeout = 2;
|
||||
static time_t readwrite_timeout = 10;
|
||||
|
||||
string
|
||||
urlEncode(const string &s);
|
||||
|
||||
#define VERIFY(s) if (true) { \
|
||||
int result = (s); \
|
||||
if (result != 0) \
|
||||
@ -282,11 +285,34 @@ public:
|
||||
// -oretries=2
|
||||
static int retries = 2;
|
||||
|
||||
static string bucket;
|
||||
|
||||
static string prepare_url(const char* url){
|
||||
|
||||
syslog(LOG_DEBUG, "URL is %s", url);
|
||||
|
||||
string url_str = str(url);
|
||||
string token = str("/" + bucket);
|
||||
int bucket_pos = url_str.find(token);
|
||||
int bucket_size = token.size();
|
||||
|
||||
url_str = url_str.substr(0,7) + bucket + "." + url_str.substr(7,bucket_pos - 7) + url_str.substr((bucket_pos + bucket_size));
|
||||
|
||||
|
||||
syslog(LOG_DEBUG, "URL changed is %s", url_str.c_str());
|
||||
|
||||
return str(url_str);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return fuse return code
|
||||
*/
|
||||
static int
|
||||
my_curl_easy_perform(CURL* curl, FILE* f = 0) {
|
||||
char* url = new char[128];
|
||||
curl_easy_getinfo( curl, CURLINFO_EFFECTIVE_URL , &url );
|
||||
syslog(LOG_DEBUG, "connecting to URL %s", url);
|
||||
|
||||
// 1 attempt + retries...
|
||||
int t = 1+retries;
|
||||
while (t-- > 0) {
|
||||
@ -304,6 +330,7 @@ my_curl_easy_perform(CURL* curl, FILE* f = 0) {
|
||||
if (responseCode == 404)
|
||||
return -ENOENT;
|
||||
syslog(LOG_ERR, "###response=%ld", responseCode);
|
||||
|
||||
if (responseCode < 500)
|
||||
return -EIO;
|
||||
} else
|
||||
@ -314,7 +341,7 @@ my_curl_easy_perform(CURL* curl, FILE* f = 0) {
|
||||
return -EIO;
|
||||
}
|
||||
|
||||
static string bucket;
|
||||
|
||||
static string AWSAccessKeyId;
|
||||
static string AWSSecretAccessKey;
|
||||
static string host = "http://s3.amazonaws.com";
|
||||
@ -410,7 +437,6 @@ calc_signature(string method, string content_type, string date, curl_slist* head
|
||||
} while ((headers = headers->next) != 0);
|
||||
}
|
||||
StringToSign += resource;
|
||||
|
||||
const void* key = AWSSecretAccessKey.data();
|
||||
int key_len = AWSSecretAccessKey.size();
|
||||
const unsigned char* d = reinterpret_cast<const unsigned char*>(StringToSign.data());
|
||||
@ -511,7 +537,6 @@ get_headers(const char* path, headers_t& meta) {
|
||||
string url(host + resource);
|
||||
|
||||
auto_curl curl;
|
||||
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
|
||||
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true);
|
||||
curl_easy_setopt(curl, CURLOPT_NOBODY, true); // HEAD
|
||||
@ -524,9 +549,13 @@ get_headers(const char* path, headers_t& meta) {
|
||||
auto_curl_slist headers;
|
||||
string date = get_date();
|
||||
headers.append("Date: "+date);
|
||||
headers.append("Content-Type: ");
|
||||
headers.append("Authorization: AWS "+AWSAccessKeyId+":"+calc_signature("HEAD", "", date, headers.get(), resource));
|
||||
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
|
||||
|
||||
string my_url = prepare_url(url.c_str());
|
||||
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
||||
|
||||
VERIFY(my_curl_easy_perform((curl.get())));
|
||||
|
||||
// at this point we know the file exists in s3
|
||||
@ -618,7 +647,6 @@ get_local_fd(const char* path) {
|
||||
Yikes(-errno);
|
||||
|
||||
auto_curl curl;
|
||||
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
|
||||
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true);
|
||||
|
||||
@ -629,12 +657,17 @@ get_local_fd(const char* path) {
|
||||
|
||||
auto_curl_slist headers;
|
||||
string date = get_date();
|
||||
syslog(LOG_INFO, "LOCAL FD");
|
||||
headers.append("Date: "+date);
|
||||
headers.append("Content-Type: ");
|
||||
headers.append("Authorization: AWS "+AWSAccessKeyId+":"+calc_signature("GET", "", date, headers.get(), resource));
|
||||
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
|
||||
|
||||
cout << "downloading[path=" << path << "][fd=" << fd << "]" << endl;
|
||||
|
||||
string my_url = prepare_url(url.c_str());
|
||||
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
||||
|
||||
VERIFY(my_curl_easy_perform(curl.get(), f));
|
||||
|
||||
//only one of these is needed...
|
||||
@ -658,7 +691,6 @@ put_headers(const char* path, headers_t meta) {
|
||||
string url = host + resource;
|
||||
|
||||
auto_curl curl;
|
||||
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
|
||||
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true);
|
||||
|
||||
@ -698,6 +730,9 @@ put_headers(const char* path, headers_t meta) {
|
||||
syslog(LOG_INFO, "copy path=%s", path);
|
||||
cout << "copying[path=" << path << "]" << endl;
|
||||
|
||||
string my_url = prepare_url(url.c_str());
|
||||
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
||||
|
||||
VERIFY(my_curl_easy_perform(curl.get()));
|
||||
|
||||
return 0;
|
||||
@ -717,7 +752,6 @@ put_local_fd(const char* path, headers_t meta, int fd) {
|
||||
Yikes(-errno);
|
||||
|
||||
auto_curl curl;
|
||||
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
|
||||
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true);
|
||||
|
||||
@ -760,6 +794,9 @@ put_local_fd(const char* path, headers_t meta, int fd) {
|
||||
syslog(LOG_INFO, "upload path=%s size=%llu", path, st.st_size);
|
||||
cout << "uploading[path=" << path << "][fd=" << fd << "][size="<<st.st_size <<"]" << endl;
|
||||
|
||||
string my_url = prepare_url(url.c_str());
|
||||
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
||||
|
||||
VERIFY(my_curl_easy_perform(curl.get(), f));
|
||||
|
||||
return 0;
|
||||
@ -789,7 +826,6 @@ s3fs_getattr(const char *path, struct stat *stbuf) {
|
||||
string url = host + resource;
|
||||
|
||||
auto_curl curl;
|
||||
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
|
||||
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true);
|
||||
curl_easy_setopt(curl, CURLOPT_NOBODY, true); // HEAD
|
||||
@ -802,8 +838,11 @@ s3fs_getattr(const char *path, struct stat *stbuf) {
|
||||
auto_curl_slist headers;
|
||||
string date = get_date();
|
||||
headers.append("Date: "+date);
|
||||
headers.append("Content-Type: ");
|
||||
headers.append("Authorization: AWS "+AWSAccessKeyId+":"+calc_signature("HEAD", "", date, headers.get(), resource));
|
||||
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
|
||||
string my_url = prepare_url(url.c_str());
|
||||
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
||||
|
||||
VERIFY(my_curl_easy_perform(curl.get()));
|
||||
|
||||
@ -899,7 +938,6 @@ s3fs_mknod(const char *path, mode_t mode, dev_t rdev) {
|
||||
string url = host + resource;
|
||||
|
||||
auto_curl curl;
|
||||
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
|
||||
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true);
|
||||
curl_easy_setopt(curl, CURLOPT_UPLOAD, true); // HTTP PUT
|
||||
@ -919,6 +957,9 @@ s3fs_mknod(const char *path, mode_t mode, dev_t rdev) {
|
||||
headers.append("Authorization: AWS "+AWSAccessKeyId+":"+calc_signature("PUT", contentType, date, headers.get(), resource));
|
||||
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
|
||||
|
||||
string my_url = prepare_url(url.c_str());
|
||||
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
||||
|
||||
VERIFY(my_curl_easy_perform(curl.get()));
|
||||
|
||||
return 0;
|
||||
@ -932,7 +973,6 @@ s3fs_mkdir(const char *path, mode_t mode) {
|
||||
string url = host + resource;
|
||||
|
||||
auto_curl curl;
|
||||
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
|
||||
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true);
|
||||
curl_easy_setopt(curl, CURLOPT_UPLOAD, true); // HTTP PUT
|
||||
@ -951,6 +991,9 @@ s3fs_mkdir(const char *path, mode_t mode) {
|
||||
headers.append("Authorization: AWS "+AWSAccessKeyId+":"+calc_signature("PUT", "application/x-directory", date, headers.get(), resource));
|
||||
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
|
||||
|
||||
string my_url = prepare_url(url.c_str());
|
||||
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
||||
|
||||
VERIFY(my_curl_easy_perform(curl.get()));
|
||||
|
||||
return 0;
|
||||
@ -965,7 +1008,6 @@ s3fs_unlink(const char *path) {
|
||||
string url = host + resource;
|
||||
|
||||
auto_curl curl;
|
||||
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
|
||||
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true);
|
||||
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE");
|
||||
@ -973,9 +1015,13 @@ s3fs_unlink(const char *path) {
|
||||
auto_curl_slist headers;
|
||||
string date = get_date();
|
||||
headers.append("Date: "+date);
|
||||
headers.append("Content-Type: ");
|
||||
headers.append("Authorization: AWS "+AWSAccessKeyId+":"+calc_signature("DELETE", "", date, headers.get(), resource));
|
||||
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
|
||||
|
||||
string my_url = prepare_url(url.c_str());
|
||||
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
||||
|
||||
VERIFY(my_curl_easy_perform(curl.get()));
|
||||
|
||||
return 0;
|
||||
@ -989,7 +1035,6 @@ s3fs_rmdir(const char *path) {
|
||||
string url = host + resource;
|
||||
|
||||
auto_curl curl;
|
||||
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
|
||||
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true);
|
||||
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE");
|
||||
@ -997,9 +1042,13 @@ s3fs_rmdir(const char *path) {
|
||||
auto_curl_slist headers;
|
||||
string date = get_date();
|
||||
headers.append("Date: "+date);
|
||||
headers.append("Content-Type: ");
|
||||
headers.append("Authorization: AWS "+AWSAccessKeyId+":"+calc_signature("DELETE", "", date, headers.get(), resource));
|
||||
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
|
||||
|
||||
string my_url = prepare_url(url.c_str());
|
||||
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
||||
|
||||
VERIFY(my_curl_easy_perform(curl.get()));
|
||||
|
||||
return 0;
|
||||
@ -1249,7 +1298,9 @@ s3fs_readdir(const char *path, void *buf, fuse_fill_dir_t filler, off_t offset,
|
||||
|
||||
{
|
||||
auto_curl curl;
|
||||
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
|
||||
string my_url = prepare_url(url.c_str());
|
||||
|
||||
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
||||
curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
|
||||
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true);
|
||||
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &responseText);
|
||||
@ -1262,10 +1313,12 @@ s3fs_readdir(const char *path, void *buf, fuse_fill_dir_t filler, off_t offset,
|
||||
auto_curl_slist headers;
|
||||
string date = get_date();
|
||||
headers.append("Date: "+date);
|
||||
headers.append("Authorization: AWS "+AWSAccessKeyId+":"+calc_signature("GET", "", date, headers.get(), resource));
|
||||
headers.append("ContentType: ");
|
||||
headers.append("Authorization: AWS "+AWSAccessKeyId+":"+calc_signature("GET", "", date, headers.get(), resource + "/"));
|
||||
|
||||
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
|
||||
|
||||
|
||||
VERIFY(my_curl_easy_perform(curl.get()));
|
||||
}
|
||||
|
||||
@ -1304,6 +1357,7 @@ s3fs_readdir(const char *path, void *buf, fuse_fill_dir_t filler, off_t offset,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (Key.size() > 0) {
|
||||
if (filler(buf, mybasename(Key).c_str(), 0, 0))
|
||||
break;
|
||||
@ -1317,7 +1371,8 @@ s3fs_readdir(const char *path, void *buf, fuse_fill_dir_t filler, off_t offset,
|
||||
stuff.path = "/"+Key;
|
||||
|
||||
// libcurl 7.17 does deep copy of url... e.g., fc7 has libcurl 7.16... therefore, must deep copy "stable" url...
|
||||
stuff.url = new string(url);
|
||||
string my_url = prepare_url(url.c_str());
|
||||
stuff.url = new string(my_url.c_str());
|
||||
stuff.requestHeaders = 0;
|
||||
stuff.responseHeaders = new headers_t;
|
||||
|
||||
@ -1330,6 +1385,7 @@ s3fs_readdir(const char *path, void *buf, fuse_fill_dir_t filler, off_t offset,
|
||||
// requestHeaders
|
||||
string date = get_date();
|
||||
stuff.requestHeaders = curl_slist_append(stuff.requestHeaders, string("Date: "+date).c_str());
|
||||
stuff.requestHeaders = curl_slist_append(stuff.requestHeaders, string("Content-Type: ").c_str());
|
||||
stuff.requestHeaders = curl_slist_append(stuff.requestHeaders, string("Authorization: AWS "+AWSAccessKeyId+":"+calc_signature("HEAD", "", date, stuff.requestHeaders, resource)).c_str());
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, stuff.requestHeaders);
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user