git-svn-id: http://s3fs.googlecode.com/svn/trunk@81 df820570-a93a-0410-bd06-b72b767a4274
This commit is contained in:
rrizun 2008-02-20 04:37:20 +00:00
parent a7a99366c4
commit 52d6be7e20

View File

@ -22,6 +22,7 @@
#include <fuse.h> #include <fuse.h>
#include <stdio.h> #include <stdio.h>
#include <stdlib.h>
#include <string.h> #include <string.h>
#include <unistd.h> #include <unistd.h>
#include <fcntl.h> #include <fcntl.h>
@ -41,8 +42,15 @@
#include <stack> #include <stack>
#include <string> #include <string>
#include <vector> #include <vector>
using namespace std; using namespace std;
#define VERIFY(s) if (true) { \
int result = (s); \
if (result != 0) \
return result; \
}
#define Yikes(result) if (true) { \ #define Yikes(result) if (true) { \
cout << __LINE__ << "###result=" << result << endl; \ cout << __LINE__ << "###result=" << result << endl; \
return result; \ return result; \
@ -121,6 +129,7 @@ public:
} }
curl_easy_reset(curl); curl_easy_reset(curl);
long seconds = 10; long seconds = 10;
curl_easy_setopt(curl, CURLOPT_TIMEOUT, seconds);
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, seconds); curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, seconds);
} }
~auto_curl() { ~auto_curl() {
@ -165,41 +174,73 @@ public:
} }
}; };
// ### TODO replace this with a plain 'ol function (or a method on auto_curl) /**
#define MY_CURL_EASY_PERFORM(curl) \ * @return fuse return code
if (true) { \ */
CURLcode curlCode = curl_easy_perform(curl.get()); \ static int
if (curlCode == CURLE_OPERATION_TIMEDOUT) \ my_curl_easy_perform(CURL* curl) {
curlCode = curl_easy_perform(curl.get()); \ int retries = 3;
if (curlCode != 0) { \ while (retries-- > 0) {
long responseCode; \ CURLcode curlCode = curl_easy_perform(curl);
if (curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &responseCode) != 0) \ if (curlCode == 0)
Yikes(-EIO); \ return 0;
if (responseCode == 500) { \ if (curlCode == CURLE_OPERATION_TIMEDOUT)
cout << __LINE__ << "###curlCode=" << curlCode << "(" << curl_easy_strerror(curlCode) << ")" << "###responseCode=" << responseCode << endl; \ cout << "###timeout" << endl;
cout << "retrying..." << endl; \ else if (curlCode == CURLE_HTTP_RETURNED_ERROR) {
curlCode = curl_easy_perform(curl.get()); \ long responseCode;
if (curlCode == CURLE_OPERATION_TIMEDOUT) \ if (curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &responseCode) != 0)
curlCode = curl_easy_perform(curl.get()); \ return -EIO;
} \ if (responseCode == 404)
if (curlCode != 0) { \ return -ENOENT;
if (curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &responseCode) != 0) \ cout << "###" << responseCode << endl;
Yikes(-EIO); \ if (responseCode < 500)
if (responseCode == 404) \ return -EIO;
return -ENOENT; \ } else
cout << __LINE__ << "###curlCode=" << curlCode << "(" << curl_easy_strerror(curlCode) << ")" << "###responseCode=" << responseCode << endl; \ cout << "###" << curl_easy_strerror(curlCode) << endl;
cout << "giving up..." << endl; \ cout << "###retrying..." << endl;
Yikes(-EIO); \ }
} \ cout << "###giving up..." << endl;
} \ return -EIO;
} }
//// ### TODO replace this with a plain 'ol function (or a method on auto_curl)
//#define MY_CURL_EASY_PERFORM(curl) \
//if (true) { \
// CURLcode curlCode = curl_easy_perform(curl.get()); \
// if (curlCode == CURLE_OPERATION_TIMEDOUT) \
// curlCode = curl_easy_perform(curl.get()); \
// if (curlCode != 0) { \
// long responseCode; \
// if (curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &responseCode) != 0) \
// Yikes(-EIO); \
// if (responseCode == 500) { \
// cout << __LINE__ << "###curlCode=" << curlCode << "(" << curl_easy_strerror(curlCode) << ")" << "###responseCode=" << responseCode << endl; \
// cout << "retrying..." << endl; \
// curlCode = curl_easy_perform(curl.get()); \
// if (curlCode == CURLE_OPERATION_TIMEDOUT) \
// curlCode = curl_easy_perform(curl.get()); \
// } \
// if (curlCode != 0) { \
// if (curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &responseCode) != 0) \
// Yikes(-EIO); \
// if (responseCode == 404) \
// return -ENOENT; \
// cout << __LINE__ << "###curlCode=" << curlCode << "(" << curl_easy_strerror(curlCode) << ")" << "###responseCode=" << responseCode << endl; \
// cout << "giving up..." << endl; \
// Yikes(-EIO); \
// } \
// } \
//}
static string bucket; static string bucket;
static string AWSAccessKeyId; static string AWSAccessKeyId;
static string AWSSecretAccessKey; static string AWSSecretAccessKey;
static const string host = "http://s3.amazonaws.com"; static const string host = "http://s3.amazonaws.com";
static mode_t root_mode = 0; static mode_t root_mode = 0;
// if .size()==0 then local file cache is disabled
static string use_cache;
static const char hexAlphabet[] = "0123456789ABCDEF"; static const char hexAlphabet[] = "0123456789ABCDEF";
/** /**
@ -366,48 +407,14 @@ mkdirp(const string& path, mode_t mode) {
return 0; return 0;
} }
//#include <pwd.h>
//
//string
//expand_path(const string& path) {
// if (path.length() == 0 || path[0] != '~')
// return path;
// const char *pfx= NULL;
// string::size_type pos = path.find_first_of('/');
// if (path.length() == 1 || pos == 1) {
// pfx = getenv("HOME");
// if (!pfx) {
// // Punt. We're trying to expand ~/, but HOME isn't set
// struct passwd *pw = getpwuid(getuid());
// if (pw)
// pfx = pw->pw_dir;
// }
// } else {
// string user(path, 1, (pos==string::npos) ? string::npos : pos-1);
// struct passwd *pw = getpwnam(user.c_str());
// if (pw)
// pfx = pw->pw_dir;
// }
// // if we failed to find an expansion, return the path unchanged.
// if (!pfx)
// return path;
// string result(pfx);
// if (pos == string::npos)
// return result;
// if (result.length() == 0 || result[result.length()-1] != '/')
// result += '/';
// result += path.substr(pos+1);
// return result;
//}
#include <openssl/md5.h> #include <openssl/md5.h>
/** /**
* @return 0 if ok * @return fuse return code
* TODO make this return pair<int, headers_t> * TODO return pair<int, headers_t>?!?
*/ */
int int
get_headers(const char* path, headers_t& meta) { get_headers(const char* path, headers_t& meta) {
string resource(urlEncode("/"+bucket + path)); string resource(urlEncode("/"+bucket + path));
string url(host + resource); string url(host + resource);
@ -429,7 +436,7 @@ get_headers(const char* path, headers_t& meta) {
headers.append("Authorization: AWS "+AWSAccessKeyId+":"+calc_signature("HEAD", "", date, headers.get(), resource)); headers.append("Authorization: AWS "+AWSAccessKeyId+":"+calc_signature("HEAD", "", date, headers.get(), resource));
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get()); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
MY_CURL_EASY_PERFORM((curl)); VERIFY(my_curl_easy_perform((curl.get())));
// at this point we know the file exists in s3 // at this point we know the file exists in s3
@ -447,8 +454,6 @@ get_headers(const char* path, headers_t& meta) {
return 0; return 0;
} }
static string use_cache;
/** /**
* get_local_fd * get_local_fd
*/ */
@ -468,8 +473,7 @@ get_local_fd(const char* path) {
if (use_cache.size() > 0) { if (use_cache.size() > 0) {
if (get_headers(path, responseHeaders) != 0) VERIFY(get_headers(path, responseHeaders));
Yikes(-ENOENT);
fd = open(cache_path.c_str(), O_RDWR); // ### TODO should really somehow obey flags here fd = open(cache_path.c_str(), O_RDWR); // ### TODO should really somehow obey flags here
@ -509,7 +513,8 @@ get_local_fd(const char* path) {
if (use_cache.size() > 0) { if (use_cache.size() > 0) {
/*if (*/mkdirp(resolved_path + mydirname(path), 0777)/* == -1) /*if (*/mkdirp(resolved_path + mydirname(path), 0777)/* == -1)
return -errno*/; return -errno*/;
mode_t mode = atoi(responseHeaders["x-amz-meta-mode"].c_str()); //###mode_t mode = atoi(responseHeaders["x-amz-meta-mode"].c_str());
mode_t mode = strtol(responseHeaders["x-amz-meta-mode"].c_str(), (char **)NULL, 10);
fd = open(cache_path.c_str(), O_CREAT|O_RDWR|O_TRUNC, mode); fd = open(cache_path.c_str(), O_CREAT|O_RDWR|O_TRUNC, mode);
} else { } else {
fd = fileno(tmpfile()); fd = fileno(tmpfile());
@ -534,7 +539,8 @@ get_local_fd(const char* path) {
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get()); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
cout << "downloading[path=" << path << "][fd=" << fd << "]" << endl; cout << "downloading[path=" << path << "][fd=" << fd << "]" << endl;
MY_CURL_EASY_PERFORM((curl));
VERIFY(my_curl_easy_perform(curl.get()));
//only one of these is needed... //only one of these is needed...
fflush(f); fflush(f);
@ -549,63 +555,58 @@ get_local_fd(const char* path) {
/** /**
* create or update s3 object * create or update s3 object
* @return fuse return code
*/ */
static int static int
put_local_fd(const char* path, headers_t meta, int fd) { put_local_fd(const char* path, headers_t meta, int fd) {
string resource = urlEncode("/"+bucket + path);
string url = host + resource;
struct stat st; struct stat st;
if (fstat(fd, &st) == -1) if (fstat(fd, &st) == -1)
Yikes(-errno);
auto_curl curl;
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true);
string responseText;
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &responseText);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback);
curl_easy_setopt(curl, CURLOPT_UPLOAD, true); // HTTP PUT
curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, static_cast<curl_off_t>(st.st_size)); // Content-Length
FILE* f = fdopen(fd, "rb");
if (f == 0)
Yikes(-errno); Yikes(-errno);
curl_easy_setopt(curl, CURLOPT_INFILE, f);
{ string ContentType = meta["Content-Type"];
string resource = urlEncode("/"+bucket + path);
string url = host + resource; auto_curl_slist headers;
string date = get_date();
headers.append("Date: "+date);
auto_curl curl; for (headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter) {
curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); string key = (*iter).first;
curl_easy_setopt(curl, CURLOPT_FAILONERROR, true); string value = (*iter).second;
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true); if (key == "Content-Type")
headers.append(key+":"+value);
string responseText; if (key.substr(0,10) == "x-amz-meta")
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &responseText); headers.append(key+":"+value);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback);
curl_easy_setopt(curl, CURLOPT_UPLOAD, true); // HTTP PUT
curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, static_cast<curl_off_t>(st.st_size)); // Content-Length
FILE* f = fdopen(fd, "rb");
if (f == 0)
Yikes(-errno);
curl_easy_setopt(curl, CURLOPT_INFILE, f);
string ContentType = meta["Content-Type"];
auto_curl_slist headers;
string date = get_date();
headers.append("Date: "+date);
for (headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter) {
string key = (*iter).first;
string value = (*iter).second;
if (key == "Content-Type")
headers.append(key+":"+value);
if (key.substr(0,10) == "x-amz-meta")
headers.append(key+":"+value);
///////////////headers.append((*iter).first+":"+(*iter).second);
}
headers.append("Authorization: AWS "+AWSAccessKeyId+":"+calc_signature("PUT", ContentType, date, headers.get(), resource));
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
rewind(f);
cout << "uploading[path=" << path << "][fd=" << fd << "][size="<<st.st_size <<"]" << endl;
MY_CURL_EASY_PERFORM((curl));
return 0;
} }
headers.append("Authorization: AWS "+AWSAccessKeyId+":"+calc_signature("PUT", ContentType, date, headers.get(), resource));
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
rewind(f);
cout << "uploading[path=" << path << "][fd=" << fd << "][size="<<st.st_size <<"]" << endl;
VERIFY(my_curl_easy_perform(curl.get()));
return 0; return 0;
} }
@ -639,19 +640,20 @@ s3fs_getattr(const char *path, struct stat *stbuf) {
headers.append("Authorization: AWS "+AWSAccessKeyId+":"+calc_signature("HEAD", "", date, headers.get(), resource)); headers.append("Authorization: AWS "+AWSAccessKeyId+":"+calc_signature("HEAD", "", date, headers.get(), resource));
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get()); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
MY_CURL_EASY_PERFORM((curl)); VERIFY(my_curl_easy_perform(curl.get()));
stbuf->st_nlink = 1; // see fuse faq stbuf->st_nlink = 1; // see fuse faq
stbuf->st_mtime = atol(responseHeaders["x-amz-meta-mtime"].c_str()); //stbuf->st_mtime = atol(responseHeaders["x-amz-meta-mtime"].c_str());
stbuf->st_mtime = strtol(responseHeaders["x-amz-meta-mtime"].c_str(), (char **)NULL, 10);
if (stbuf->st_mtime == 0) { if (stbuf->st_mtime == 0) {
long LastModified; long LastModified;
if (curl_easy_getinfo(curl, CURLINFO_FILETIME, &LastModified) == 0) if (curl_easy_getinfo(curl, CURLINFO_FILETIME, &LastModified) == 0)
stbuf->st_mtime = LastModified; stbuf->st_mtime = LastModified;
} }
stbuf->st_mode = atoi(responseHeaders["x-amz-meta-mode"].c_str()); //stbuf->st_mode = atoi(responseHeaders["x-amz-meta-mode"].c_str());
stbuf->st_mode = strtol(responseHeaders["x-amz-meta-mode"].c_str(), (char **)NULL, 10);
char* ContentType = 0; char* ContentType = 0;
if (curl_easy_getinfo(curl, CURLINFO_CONTENT_TYPE, &ContentType) == 0) { if (curl_easy_getinfo(curl, CURLINFO_CONTENT_TYPE, &ContentType) == 0) {
if (ContentType) if (ContentType)
@ -699,7 +701,7 @@ s3fs_mknod(const char *path, mode_t mode, dev_t rdev) {
headers.append("Authorization: AWS "+AWSAccessKeyId+":"+calc_signature("PUT", "application/octet-stream", date, headers.get(), resource)); headers.append("Authorization: AWS "+AWSAccessKeyId+":"+calc_signature("PUT", "application/octet-stream", date, headers.get(), resource));
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get()); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
MY_CURL_EASY_PERFORM((curl)); VERIFY(my_curl_easy_perform(curl.get()));
return 0; return 0;
} }
@ -727,7 +729,7 @@ s3fs_mkdir(const char *path, mode_t mode) {
headers.append("Authorization: AWS "+AWSAccessKeyId+":"+calc_signature("PUT", "application/x-directory", date, headers.get(), resource)); headers.append("Authorization: AWS "+AWSAccessKeyId+":"+calc_signature("PUT", "application/x-directory", date, headers.get(), resource));
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get()); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
MY_CURL_EASY_PERFORM((curl)); VERIFY(my_curl_easy_perform(curl.get()));
return 0; return 0;
} }
@ -752,7 +754,7 @@ s3fs_unlink(const char *path) {
headers.append("Authorization: AWS "+AWSAccessKeyId+":"+calc_signature("DELETE", "", date, headers.get(), resource)); headers.append("Authorization: AWS "+AWSAccessKeyId+":"+calc_signature("DELETE", "", date, headers.get(), resource));
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get()); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
MY_CURL_EASY_PERFORM((curl)); VERIFY(my_curl_easy_perform(curl.get()));
return 0; return 0;
} }
@ -776,7 +778,7 @@ s3fs_rmdir(const char *path) {
headers.append("Authorization: AWS "+AWSAccessKeyId+":"+calc_signature("DELETE", "", date, headers.get(), resource)); headers.append("Authorization: AWS "+AWSAccessKeyId+":"+calc_signature("DELETE", "", date, headers.get(), resource));
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get()); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
MY_CURL_EASY_PERFORM((curl)); VERIFY(my_curl_easy_perform(curl.get()));
return 0; return 0;
} }
@ -797,9 +799,7 @@ s3fs_rename(const char *from, const char *to) {
// preserve meta headers across rename // preserve meta headers across rename
headers_t meta; headers_t meta;
result = get_headers(from, meta); VERIFY(get_headers(from, meta));
if (result != 0)
return result;
result = put_local_fd(to, meta, fd.get()); result = put_local_fd(to, meta, fd.get());
if (result != 0) if (result != 0)
@ -820,9 +820,7 @@ s3fs_chmod(const char *path, mode_t mode) {
cout << "chmod[path=" << path << "][mode=" << mode << "]" << endl; cout << "chmod[path=" << path << "][mode=" << mode << "]" << endl;
auto_fd fd(get_local_fd(path)); auto_fd fd(get_local_fd(path));
headers_t meta; headers_t meta;
int result = get_headers(path, meta); VERIFY(get_headers(path, meta));
if (result != 0)
return result;
meta["x-amz-meta-mode"] = stringificationizer(mode); meta["x-amz-meta-mode"] = stringificationizer(mode);
return put_local_fd(path, meta, fd.get()); return put_local_fd(path, meta, fd.get());
} }
@ -834,18 +832,11 @@ s3fs_chown(const char *path, uid_t uid, gid_t gid) {
return 0; return 0;
} }
#define VERIFY(s) if (true) { \
int result = (s); \
if (result != 0) \
return result; \
}
static int static int
s3fs_truncate(const char *path, off_t size) { s3fs_truncate(const char *path, off_t size) {
//###TODO honor size?!? //###TODO honor size?!?
cout << "truncate[path=" << path << "][size=" << size << "]" << endl; cout << "truncate[path=" << path << "][size=" << size << "]" << endl;
// preserve headers across truncate // preserve headers across truncate
headers_t meta; headers_t meta;
@ -867,10 +858,9 @@ s3fs_open(const char *path, struct fuse_file_info *fi) {
cout << "open[path=" << path << "][flags=" << fi->flags << "]" << endl; cout << "open[path=" << path << "][flags=" << fi->flags << "]" << endl;
headers_t meta; headers_t meta;
//###TODO check fi->fh here...
fi->fh = get_local_fd(path); fi->fh = get_local_fd(path);
//###TODO check fi->fh here...
// remember flags and headers... // remember flags and headers...
auto_lock lock(s3fs_descriptors_lock); auto_lock lock(s3fs_descriptors_lock);
@ -992,7 +982,7 @@ s3fs_readdir(const char *path, void *buf, fuse_fill_dir_t filler, off_t offset,
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get()); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
MY_CURL_EASY_PERFORM((curl)); VERIFY(my_curl_easy_perform(curl.get()));
{ {
xmlDocPtr doc = xmlReadMemory(responseText.c_str(), responseText.size(), "", NULL, 0); xmlDocPtr doc = xmlReadMemory(responseText.c_str(), responseText.size(), "", NULL, 0);
@ -1065,12 +1055,9 @@ s3fs_utimens(const char *path, const struct timespec ts[2]) {
cout << "utimens[path=" << path << "][mtime=" << stringificationizer(ts[1].tv_sec) << "]" << endl; cout << "utimens[path=" << path << "][mtime=" << stringificationizer(ts[1].tv_sec) << "]" << endl;
auto_fd fd(get_local_fd(path)); auto_fd fd(get_local_fd(path));
headers_t meta; headers_t meta;
int result = get_headers(path, meta); VERIFY(get_headers(path, meta));
if (result != 0)
return result;
meta["x-amz-meta-mtime"] = stringificationizer(ts[1].tv_sec); meta["x-amz-meta-mtime"] = stringificationizer(ts[1].tv_sec);
return put_local_fd(path, meta, fd.get()); return put_local_fd(path, meta, fd.get());
//return -EPERM;
} }
static int static int