octoleo/s3fs-fuse
1
0
Fork 0
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git synced 2024-12-23 09:18:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Further multipart cleanup/error checking, in preparation for multi-threaded uploads.

Browse Source 
"Last-Modified" is now returned with get_headers() data


git-svn-id: http://s3fs.googlecode.com/svn/trunk@320 df820570-a93a-0410-bd06-b72b767a4274
This commit is contained in:
ben.lemasurier@gmail.com 2011-02-17 17:31:43 +00:00
parent cfa0fd2992
commit 1496f6a81e
3 changed files with 52 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
configure.ac
View File

@ -1,7 +1,7 @@
dnl Process this file with autoconf to produce a configure script. dnl Process this file with autoconf to produce a configure script.
AC_PREREQ(2.59) AC_PREREQ(2.59)
AC_INIT(s3fs, 1.45) AC_INIT(s3fs, 1.46)
AC_CANONICAL_SYSTEM AC_CANONICAL_SYSTEM

131
src/s3fs.cpp
View File

@ -63,8 +63,6 @@ class auto_curl_slist {
struct curl_slist* slist; struct curl_slist* slist;
}; };
// Memory structure for the alternate // Memory structure for the alternate
// write memory callback used with curl_easy_perform // write memory callback used with curl_easy_perform
struct BodyStruct { struct BodyStruct {
@ -78,9 +76,6 @@ struct WriteThis {
int sizeleft; int sizeleft;
}; };
typedef struct curlhll { typedef struct curlhll {
CURL *handle; CURL *handle;
struct curlhll *next; struct curlhll *next;
@ -309,9 +304,6 @@ void free_mvnodes(MVNODE *head) {
return; return;
} }
///////////////////////////////////////////////// /////////////////////////////////////////////////
// Multi CURL stuff // Multi CURL stuff
///////////////////////////////////////////////// /////////////////////////////////////////////////
@ -341,7 +333,6 @@ void add_h_element(CURLHLL *head, CURL *handle) {
return; return;
} }
CURLMHLL *create_mh_element(CURLM *handle) { CURLMHLL *create_mh_element(CURLM *handle) {
CURLMHLL *p; CURLMHLL *p;
p = (CURLMHLL *) malloc(sizeof(CURLMHLL)); p = (CURLMHLL *) malloc(sizeof(CURLMHLL));
@ -622,7 +613,6 @@ static int my_curl_easy_perform(CURL* curl, BodyStruct* body = NULL, FILE* f = 0
} }
// Service response codes which are >= 400 && < 500 // Service response codes which are >= 400 && < 500
switch(responseCode) { switch(responseCode) {
case 400: case 400:
if(debug) syslog(LOG_ERR, "HTTP response code 400 was returned"); if(debug) syslog(LOG_ERR, "HTTP response code 400 was returned");
@ -769,7 +759,6 @@ static int my_curl_easy_perform(CURL* curl, BodyStruct* body = NULL, FILE* f = 0
return -EIO; return -EIO;
} }
/** /**
* urlEncode a fuse path, * urlEncode a fuse path,
* taking into special consideration "/", * taking into special consideration "/",
@ -1058,6 +1047,8 @@ int get_headers(const char* path, headers_t& meta) {
meta[key] = value; meta[key] = value;
if (key == "ETag") if (key == "ETag")
meta[key] = value; meta[key] = value;
if(key == "Last-Modified")
meta[key] = value;
if (key.substr(0, 5) == "x-amz") if (key.substr(0, 5) == "x-amz")
meta[key] = value; meta[key] = value;
} }
@ -1087,9 +1078,8 @@ int get_local_fd(const char* path) {
if (use_cache.size() > 0) { if (use_cache.size() > 0) {
result = get_headers(path, responseHeaders); result = get_headers(path, responseHeaders);
if(result != 0) { if(result != 0)
return -result; return -result;
}
fd = open(cache_path.c_str(), O_RDWR); // ### TODO should really somehow obey flags here fd = open(cache_path.c_str(), O_RDWR); // ### TODO should really somehow obey flags here
@ -1358,7 +1348,6 @@ static int put_local_fd_small_file(const char* path, headers_t meta, int fd) {
static int put_local_fd_big_file(const char* path, headers_t meta, int fd) { static int put_local_fd_big_file(const char* path, headers_t meta, int fd) {
struct stat st; struct stat st;
int result;
off_t lSize; off_t lSize;
int partfd = -1; int partfd = -1;
FILE* pSourceFile; FILE* pSourceFile;
@ -1367,17 +1356,12 @@ static int put_local_fd_big_file(const char* path, headers_t meta, int fd) {
unsigned long lBufferSize = 0; unsigned long lBufferSize = 0;
size_t bytesRead; size_t bytesRead;
size_t bytesWritten; size_t bytesWritten;
unsigned int partNumber;
string uploadId; string uploadId;
vector <string> etags; vector <file_part> parts;
char partFileName[17] = "";
if(foreground) if(foreground)
cout << " put_local_fd_big_file[path=" << path << "][fd=" << fd << "]" << endl; cout << " put_local_fd_big_file[path=" << path << "][fd=" << fd << "]" << endl;
// Initialization of variables
etags.clear();
if(fstat(fd, &st) == -1) if(fstat(fd, &st) == -1)
YIKES(-errno); YIKES(-errno);
@ -1399,20 +1383,19 @@ static int put_local_fd_big_file(const char* path, headers_t meta, int fd) {
lSize = st.st_size; lSize = st.st_size;
lBufferSize = 0; lBufferSize = 0;
partNumber = 0;
// cycle through open fd, pulling off 10MB chunks at a time // cycle through open fd, pulling off 10MB chunks at a time
while(lSize > 0) { while(lSize > 0) {
partNumber++; file_part part;
if(lSize >= 10485760) // 10MB
lBufferSize = 10485760; if(lSize >= MULTIPART_SIZE)
lBufferSize = MULTIPART_SIZE;
else else
lBufferSize = lSize; lBufferSize = lSize;
lSize = lSize - lBufferSize; lSize = lSize - lBufferSize;
buffer = (char*) malloc(sizeof(char) * lBufferSize); if((buffer = (char *) malloc(sizeof(char) * lBufferSize)) == NULL) {
if(buffer == NULL) {
syslog(LOG_CRIT, "Could not allocate memory for buffer\n"); syslog(LOG_CRIT, "Could not allocate memory for buffer\n");
exit(1); exit(1);
} }
@ -1429,22 +1412,17 @@ static int put_local_fd_big_file(const char* path, headers_t meta, int fd) {
return(-EIO); return(-EIO);
} }
// printf("bytesRead: %i\n", bytesRead); // create uniq temporary file
strncpy(part.path, "/tmp/s3fs.XXXXXX", sizeof part.path);
// source file portion is now in memory if((partfd = mkstemp(part.path)) == -1) {
// open a temporary file for upload
strncpy(partFileName, "/tmp/s3fs.XXXXXX", sizeof partFileName);
partfd = mkstemp(partFileName);
if(partfd == -1) {
syslog(LOG_ERR, "%d ### Could not create temporary file\n", __LINE__);
if(buffer) if(buffer)
free(buffer); free(buffer);
return(-errno); YIKES(-errno);
} }
pPartFile = fdopen(partfd, "wb"); // open a temporary file for upload
if(pPartFile == NULL) { if((pPartFile = fdopen(partfd, "wb")) == NULL) {
syslog(LOG_ERR, "%d ### Could not open temporary file: errno %i\n", syslog(LOG_ERR, "%d ### Could not open temporary file: errno %i\n",
__LINE__, errno); __LINE__, errno);
if(buffer) if(buffer)
@ -1453,13 +1431,8 @@ static int put_local_fd_big_file(const char* path, headers_t meta, int fd) {
return(-errno); return(-errno);
} }
// printf ("Tempfile opened: %s\n", partFileName); // copy buffer to temporary file
// Copy buffer to tmpfile
// printf("bufferSize: %lu\n", lBufferSize);
bytesWritten = fwrite(buffer, 1, (size_t)lBufferSize, pPartFile); bytesWritten = fwrite(buffer, 1, (size_t)lBufferSize, pPartFile);
// printf("bytesWritten: %u\n", bytesWritten);
if(bytesWritten != lBufferSize) { if(bytesWritten != lBufferSize) {
syslog(LOG_ERR, "%d ### bytesWritten:%zu does not match lBufferSize: %lu\n", syslog(LOG_ERR, "%d ### bytesWritten:%zu does not match lBufferSize: %lu\n",
__LINE__, bytesWritten, lBufferSize); __LINE__, bytesWritten, lBufferSize);
@ -1471,24 +1444,20 @@ static int put_local_fd_big_file(const char* path, headers_t meta, int fd) {
return(-EIO); return(-EIO);
} }
// printf ("Tempfile written: %s\n", partFileName);
fclose(pPartFile); fclose(pPartFile);
if(buffer) if(buffer)
free(buffer); free(buffer);
etags.push_back(upload_part(path, partFileName, partNumber, uploadId)); part.etag = upload_part(path, part.path, parts.size() + 1, uploadId);
// delete temporary part file // delete temporary part file
result = remove(partFileName); if(remove(part.path) != 0)
if(result != 0) { YIKES(-errno);
syslog(LOG_ERR, "Could not remove temporary file: %s errno: %i\n",
partFileName, errno);
return(-errno); parts.push_back(part);
}
} // while(lSize > 0) } // while(lSize > 0)
return complete_multipart_upload(path, uploadId, partNumber, etags); return complete_multipart_upload(path, uploadId, parts);
} }
/** /**
@ -1677,11 +1646,11 @@ string initiate_multipart_upload(const char *path, off_t size, headers_t meta) {
} }
static int complete_multipart_upload(const char *path, string upload_id, static int complete_multipart_upload(const char *path, string upload_id,
int n_parts, vector <string> etags) { vector <file_part> parts) {
CURL *curl = NULL; CURL *curl = NULL;
char *pData; char *pData;
int result; int result;
int i; int i, j;
string auth; string auth;
string date; string date;
string raw_date; string raw_date;
@ -1700,24 +1669,23 @@ static int complete_multipart_upload(const char *path, string upload_id,
body.text = (char *)malloc(1); body.text = (char *)malloc(1);
body.size = 0; body.size = 0;
curl = NULL; curl = NULL;
etags.clear();
postContent.clear(); postContent.clear();
postContent.append("<CompleteMultipartUpload>\n"); postContent.append("<CompleteMultipartUpload>\n");
for(i = 0; i < n_parts; i++) { for(i = 0, j = parts.size(); i < j; i++) {
postContent.append(" <Part>\n"); postContent.append(" <Part>\n");
postContent.append(" <PartNumber>"); postContent.append(" <PartNumber>");
postContent.append(IntToStr(i+1)); postContent.append(IntToStr(i+1));
postContent.append("</PartNumber>\n"); postContent.append("</PartNumber>\n");
postContent.append(" <ETag>"); postContent.append(" <ETag>");
postContent.append(etags[i]); postContent.append(parts[i].etag.insert(0, "\"").append("\""));
postContent.append("</ETag>\n"); postContent.append("</ETag>\n");
postContent.append(" </Part>\n"); postContent.append(" </Part>\n");
} }
postContent.append("</CompleteMultipartUpload>\n"); postContent.append("</CompleteMultipartUpload>\n");
// error check pData if((pData = (char *)malloc(postContent.size() + 1)) == NULL)
pData = (char *)malloc( postContent.size() + 1); YIKES(-errno)
pooh.readptr = pData; pooh.readptr = pData;
pooh.sizeleft = postContent.size(); pooh.sizeleft = postContent.size();
@ -1737,6 +1705,7 @@ static int complete_multipart_upload(const char *path, string upload_id,
curl = create_curl_handle(); curl = create_curl_handle();
// curl_easy_setopt(curl, CURLOPT_VERBOSE, true);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&body); curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&body);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
@ -1884,7 +1853,7 @@ string upload_part(const char *path, const char *source, int part_number, string
// calculate local md5sum, if it matches the header // calculate local md5sum, if it matches the header
// ETag value, the upload was successful. // ETag value, the upload was successful.
string md5 = md5sum(source).insert(0, "\"").append("\""); string md5 = md5sum(source);
if(!md5.empty() && strstr(header.text, md5.c_str())) { if(!md5.empty() && strstr(header.text, md5.c_str())) {
ETag.assign(md5); ETag.assign(md5);
} else { } else {
@ -3589,52 +3558,30 @@ static int s3fs_utimens(const char *path, const struct timespec ts[2]) {
/////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////
static int list_multipart_uploads(void) { static int list_multipart_uploads(void) {
CURL *curl = NULL; CURL *curl = NULL;
printf("List Multipart Uploads\n");
// struct stat st;
string resource; string resource;
string url; string url;
struct BodyStruct body; struct BodyStruct body;
// struct BodyStruct header;
// string ETag;
int result; int result;
// unsigned long lSize;
// FILE* pSourceFile;
// FILE* pPartFile;
// char * buffer;
// unsigned long lBufferSize = 0;
// size_t bytesRead;
// size_t bytesWritten;
// char * partFileName;
// int partNumber;
string date; string date;
// string expires;
string raw_date; string raw_date;
string auth; string auth;
string my_url; string my_url;
// const char * UploadId;
// string uploadId;
struct curl_slist *slist=NULL; struct curl_slist *slist=NULL;
// vector <string> etags;
// int i;
// Initialization of variables // Initialization of variables
body.text = (char *)malloc(1); body.text = (char *)malloc(1);
body.size = 0; body.size = 0;
curl = NULL; printf("List Multipart Uploads\n");
////////////////////////////////////////// //////////////////////////////////////////
// Syntax: // Syntax:
// //
// GET /?uploads HTTP/1.1 // GET /?uploads HTTP/1.1
// Host: BucketName.s3.amazonaws.com // Host: BucketName.s3.amazonaws.com
// Date: Date // Date: Date
// Authorization: Signature // Authorization: Signature
////////////////////////////////////////// //////////////////////////////////////////
// printf("service_path: %s\n", service_path.c_str()); // printf("service_path: %s\n", service_path.c_str());
@ -3648,8 +3595,6 @@ static int list_multipart_uploads(void) {
url = host + resource; url = host + resource;
// printf("url: %s\n", url.c_str()); // printf("url: %s\n", url.c_str());
my_url = prepare_url(url.c_str()); my_url = prepare_url(url.c_str());
// printf("my_url: %s\n", my_url.c_str()); // printf("my_url: %s\n", my_url.c_str());

12
src/s3fs.h
View File

@ -3,6 +3,8 @@
#define FUSE_USE_VERSION 26 #define FUSE_USE_VERSION 26
#define MULTIPART_SIZE 10485760 // 10MB
#include <map> #include <map>
#include <string> #include <string>
#include <vector> #include <vector>
@ -73,6 +75,14 @@ typedef map<string, struct stat_cache_entry> stat_cache_t;
static stat_cache_t stat_cache; static stat_cache_t stat_cache;
static pthread_mutex_t stat_cache_lock; static pthread_mutex_t stat_cache_lock;
struct file_part {
char path[17];
string etag;
bool uploaded;
file_part() : uploaded(false) {}
};
static const char hexAlphabet[] = "0123456789ABCDEF"; static const char hexAlphabet[] = "0123456789ABCDEF";
// http headers // http headers
@ -93,7 +103,7 @@ string urlEncode(const string &s);
string lookupMimeType(string); string lookupMimeType(string);
string initiate_multipart_upload(const char *path, off_t size, headers_t meta); string initiate_multipart_upload(const char *path, off_t size, headers_t meta);
string upload_part(const char *path, const char *source, int part_number, string upload_id); string upload_part(const char *path, const char *source, int part_number, string upload_id);
static int complete_multipart_upload(const char *path, string upload_id, int n_parts, vector <string> etags); static int complete_multipart_upload(const char *path, string upload_id, vector <file_part> parts);
string md5sum(const char *path); string md5sum(const char *path);
static int get_stat_cache_entry(const char *path, struct stat *buf); static int get_stat_cache_entry(const char *path, struct stat *buf);