mirror of
https://github.com/s3fs-fuse/s3fs-fuse.git
synced 2024-12-23 09:18:55 +00:00
Fixed issue: 326
1) Changes for fixing a bug(r326) The my_curl_easy_perform() function is not clearing the buffer(struct BodyStruct body) before retrying the request. 2) Other changes In conjunction with this issue, the "struct BodyStruct" is changed to "Class BodyData". New class is same as BodyStruct, but handling memory is automatically. And added a argument for my_curl_easy_perform(). This function is needed the buffer pointer, but the arguments is only for body buffer. Then I added the buffer pointer for header buffer. 3) Fixed memory leak In get_object_name() function, there was a memory leak. git-svn-id: http://s3fs.googlecode.com/svn/trunk@403 df820570-a93a-0410-bd06-b72b767a4274
This commit is contained in:
parent
8bd1483374
commit
f002cdb9b2
120
src/curl.cpp
120
src/curl.cpp
@ -43,6 +43,7 @@
|
|||||||
#include "common.h"
|
#include "common.h"
|
||||||
#include "curl.h"
|
#include "curl.h"
|
||||||
#include "string_util.h"
|
#include "string_util.h"
|
||||||
|
#include "s3fs.h"
|
||||||
|
|
||||||
using namespace std;
|
using namespace std;
|
||||||
|
|
||||||
@ -67,6 +68,76 @@ static map<CURL*, progress_t> curl_progress;
|
|||||||
static string curl_ca_bundle;
|
static string curl_ca_bundle;
|
||||||
static mimes_t mimeTypes;
|
static mimes_t mimeTypes;
|
||||||
|
|
||||||
|
//-------------------------------------------------------------------
|
||||||
|
// Class BodyData
|
||||||
|
//-------------------------------------------------------------------
|
||||||
|
#define BODYDATA_RESIZE_APPEND_MIN (1 * 1024) // 1KB
|
||||||
|
#define BODYDATA_RESIZE_APPEND_MID (1 * 1024 * 1024) // 1MB
|
||||||
|
#define BODYDATA_RESIZE_APPEND_MAX (10 * 1024 * 1024) // 10MB
|
||||||
|
|
||||||
|
bool BodyData::Resize(size_t addbytes)
|
||||||
|
{
|
||||||
|
if(IsSafeSize(addbytes)){
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
// New size
|
||||||
|
size_t need_size = (lastpos + addbytes + 1) - bufsize;
|
||||||
|
if(BODYDATA_RESIZE_APPEND_MAX < bufsize){
|
||||||
|
need_size = (BODYDATA_RESIZE_APPEND_MAX < need_size ? need_size : BODYDATA_RESIZE_APPEND_MAX);
|
||||||
|
}else if(BODYDATA_RESIZE_APPEND_MID < bufsize){
|
||||||
|
need_size = (BODYDATA_RESIZE_APPEND_MID < need_size ? need_size : BODYDATA_RESIZE_APPEND_MID);
|
||||||
|
}else if(BODYDATA_RESIZE_APPEND_MIN < bufsize){
|
||||||
|
need_size = ((bufsize * 2) < need_size ? need_size : (bufsize * 2));
|
||||||
|
}else{
|
||||||
|
need_size = (BODYDATA_RESIZE_APPEND_MIN < need_size ? need_size : BODYDATA_RESIZE_APPEND_MIN);
|
||||||
|
}
|
||||||
|
// realloc
|
||||||
|
if(NULL == (text = (char*)realloc(text, (bufsize + need_size)))){
|
||||||
|
FGPRINT("BodyData::Resize() not enough memory (realloc returned NULL)\n");
|
||||||
|
SYSLOGDBGERR("not enough memory (realloc returned NULL)\n");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
bufsize += need_size;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
void BodyData::Clear(void)
|
||||||
|
{
|
||||||
|
if(text){
|
||||||
|
free(text);
|
||||||
|
text = NULL;
|
||||||
|
}
|
||||||
|
lastpos = 0;
|
||||||
|
bufsize = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool BodyData::Append(void* ptr, size_t bytes)
|
||||||
|
{
|
||||||
|
if(!ptr){
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if(0 == bytes){
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if(!Resize(bytes)){
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
memcpy(&text[lastpos], ptr, bytes);
|
||||||
|
lastpos += bytes;
|
||||||
|
text[lastpos] = '\0';
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
const char* BodyData::str(void) const
|
||||||
|
{
|
||||||
|
static const char* strnull = "";
|
||||||
|
if(!text){
|
||||||
|
return strnull;
|
||||||
|
}
|
||||||
|
return text;
|
||||||
|
}
|
||||||
|
|
||||||
//-------------------------------------------------------------------
|
//-------------------------------------------------------------------
|
||||||
// Functions
|
// Functions
|
||||||
//-------------------------------------------------------------------
|
//-------------------------------------------------------------------
|
||||||
@ -277,7 +348,7 @@ CURL *create_head_handle(head_data *request_data)
|
|||||||
/**
|
/**
|
||||||
* @return fuse return code
|
* @return fuse return code
|
||||||
*/
|
*/
|
||||||
int my_curl_easy_perform(CURL* curl, BodyStruct* body, FILE* f)
|
int my_curl_easy_perform(CURL* curl, BodyData* body, BodyData* head, FILE* f)
|
||||||
{
|
{
|
||||||
char url[256];
|
char url[256];
|
||||||
time_t now;
|
time_t now;
|
||||||
@ -327,33 +398,26 @@ int my_curl_easy_perform(CURL* curl, BodyStruct* body, FILE* f)
|
|||||||
switch(responseCode) {
|
switch(responseCode) {
|
||||||
case 400:
|
case 400:
|
||||||
SYSLOGDBGERR("HTTP response code 400 was returned");
|
SYSLOGDBGERR("HTTP response code 400 was returned");
|
||||||
if(body && body->size){
|
SYSLOGDBGERR("Body Text: %s", (body ? body->str() : ""));
|
||||||
SYSLOGDBGERR("Body Text: %s", body->text);
|
|
||||||
}
|
|
||||||
SYSLOGDBG("Now returning EIO");
|
SYSLOGDBG("Now returning EIO");
|
||||||
return -EIO;
|
return -EIO;
|
||||||
|
|
||||||
case 403:
|
case 403:
|
||||||
SYSLOGDBGERR("HTTP response code 403 was returned");
|
SYSLOGDBGERR("HTTP response code 403 was returned");
|
||||||
if(body && body->size){
|
SYSLOGDBGERR("Body Text: %s", (body ? body->str() : ""));
|
||||||
SYSLOGDBGERR("Body Text: %s", body->text);
|
|
||||||
}
|
|
||||||
return -EPERM;
|
return -EPERM;
|
||||||
|
|
||||||
case 404:
|
case 404:
|
||||||
SYSLOGDBG("HTTP response code 404 was returned");
|
SYSLOGDBG("HTTP response code 404 was returned");
|
||||||
if(body && body->size){
|
SYSLOGDBG("Body Text: %s", (body ? body->str() : ""));
|
||||||
SYSLOGDBG("Body Text: %s", body->text);
|
|
||||||
}
|
|
||||||
SYSLOGDBG("Now returning ENOENT");
|
SYSLOGDBG("Now returning ENOENT");
|
||||||
return -ENOENT;
|
return -ENOENT;
|
||||||
|
|
||||||
default:
|
default:
|
||||||
SYSLOGERR("###response=%ld", responseCode);
|
SYSLOGERR("###response=%ld", responseCode);
|
||||||
printf("responseCode %ld\n", responseCode);
|
SYSLOGDBG("Body Text: %s", (body ? body->str() : ""));
|
||||||
if(body && body->size){
|
FGPRINT("responseCode %ld\n", responseCode);
|
||||||
printf("Body Text %s\n", body->text);
|
FGPRINT("Body Text: %s", (body ? body->str() : ""));
|
||||||
}
|
|
||||||
return -EIO;
|
return -EIO;
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
@ -413,7 +477,8 @@ int my_curl_easy_perform(CURL* curl, BodyStruct* body, FILE* f)
|
|||||||
if (curl_ca_bundle.size() != 0) {
|
if (curl_ca_bundle.size() != 0) {
|
||||||
t++;
|
t++;
|
||||||
curl_easy_setopt(curl, CURLOPT_CAINFO, curl_ca_bundle.c_str());
|
curl_easy_setopt(curl, CURLOPT_CAINFO, curl_ca_bundle.c_str());
|
||||||
continue;
|
// break for switch-case, and continue loop.
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
SYSLOGERR("curlCode: %i msg: %s", curlCode, curl_easy_strerror(curlCode));
|
SYSLOGERR("curlCode: %i msg: %s", curlCode, curl_easy_strerror(curlCode));
|
||||||
@ -468,6 +533,12 @@ int my_curl_easy_perform(CURL* curl, BodyStruct* body, FILE* f)
|
|||||||
exit(EXIT_FAILURE);
|
exit(EXIT_FAILURE);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
if(body){
|
||||||
|
body->Clear();
|
||||||
|
}
|
||||||
|
if(head){
|
||||||
|
head->Clear();
|
||||||
|
}
|
||||||
SYSLOGERR("###retrying...");
|
SYSLOGERR("###retrying...");
|
||||||
}
|
}
|
||||||
SYSLOGERR("###giving up");
|
SYSLOGERR("###giving up");
|
||||||
@ -477,21 +548,14 @@ int my_curl_easy_perform(CURL* curl, BodyStruct* body, FILE* f)
|
|||||||
// libcurl callback
|
// libcurl callback
|
||||||
size_t WriteMemoryCallback(void *ptr, size_t blockSize, size_t numBlocks, void *data)
|
size_t WriteMemoryCallback(void *ptr, size_t blockSize, size_t numBlocks, void *data)
|
||||||
{
|
{
|
||||||
size_t realsize = blockSize * numBlocks;
|
BodyData* body = (BodyData*)data;
|
||||||
struct BodyStruct *mem = (struct BodyStruct *)data;
|
|
||||||
|
|
||||||
mem->text = (char *)realloc(mem->text, mem->size + realsize + 1);
|
if(!body->Append(ptr, blockSize, numBlocks)){
|
||||||
if(mem->text == NULL) {
|
FGPRINT("WriteMemoryCallback(): BodyData.Append() returned false.\n");
|
||||||
/* out of memory! */
|
S3FS_FUSE_EXIT();
|
||||||
fprintf(stderr, "not enough memory (realloc returned NULL)\n");
|
return -1;
|
||||||
exit(EXIT_FAILURE);
|
|
||||||
}
|
}
|
||||||
|
return (blockSize * numBlocks);
|
||||||
memcpy(&(mem->text[mem->size]), ptr, realsize);
|
|
||||||
mem->size += realsize;
|
|
||||||
mem->text[mem->size] = 0;
|
|
||||||
|
|
||||||
return realsize;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// read_callback
|
// read_callback
|
||||||
|
35
src/curl.h
35
src/curl.h
@ -1,10 +1,35 @@
|
|||||||
#ifndef S3FS_CURL_H_
|
#ifndef S3FS_CURL_H_
|
||||||
#define S3FS_CURL_H_
|
#define S3FS_CURL_H_
|
||||||
|
|
||||||
// memory structure for curl write memory callback
|
// memory class for curl write memory callback
|
||||||
struct BodyStruct {
|
class BodyData
|
||||||
char *text;
|
{
|
||||||
size_t size;
|
private:
|
||||||
|
char* text;
|
||||||
|
size_t lastpos;
|
||||||
|
size_t bufsize;
|
||||||
|
|
||||||
|
private:
|
||||||
|
bool IsSafeSize(size_t addbytes) const {
|
||||||
|
return ((lastpos + addbytes + 1) > bufsize ? false : true);
|
||||||
|
}
|
||||||
|
bool Resize(size_t addbytes);
|
||||||
|
|
||||||
|
public:
|
||||||
|
BodyData() : text(NULL), lastpos(0), bufsize(0) {}
|
||||||
|
~BodyData() {
|
||||||
|
Clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
void Clear(void);
|
||||||
|
bool Append(void* ptr, size_t bytes);
|
||||||
|
bool Append(void* ptr, size_t blockSize, size_t numBlocks) {
|
||||||
|
return Append(ptr, (blockSize * numBlocks));
|
||||||
|
}
|
||||||
|
const char* str() const;
|
||||||
|
size_t size() const {
|
||||||
|
return lastpos;
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// memory structure for POST
|
// memory structure for POST
|
||||||
@ -90,7 +115,7 @@ CURL *create_curl_handle(void);
|
|||||||
int curl_delete(const char *path);
|
int curl_delete(const char *path);
|
||||||
int curl_get_headers(const char *path, headers_t &meta);
|
int curl_get_headers(const char *path, headers_t &meta);
|
||||||
CURL *create_head_handle(struct head_data *request);
|
CURL *create_head_handle(struct head_data *request);
|
||||||
int my_curl_easy_perform(CURL* curl, BodyStruct* body = NULL, FILE* f = 0);
|
int my_curl_easy_perform(CURL* curl, BodyData* body = NULL, BodyData* head = NULL, FILE* f = 0);
|
||||||
size_t WriteMemoryCallback(void *ptr, size_t blockSize, size_t numBlocks, void *data);
|
size_t WriteMemoryCallback(void *ptr, size_t blockSize, size_t numBlocks, void *data);
|
||||||
size_t read_callback(void *ptr, size_t size, size_t nmemb, void *userp);
|
size_t read_callback(void *ptr, size_t size, size_t nmemb, void *userp);
|
||||||
int my_curl_progress(
|
int my_curl_progress(
|
||||||
|
191
src/s3fs.cpp
191
src/s3fs.cpp
@ -454,7 +454,7 @@ static int get_local_fd(const char* path) {
|
|||||||
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
|
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
|
||||||
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
||||||
|
|
||||||
result = my_curl_easy_perform(curl, NULL, f);
|
result = my_curl_easy_perform(curl, NULL, NULL, f);
|
||||||
if(result != 0) {
|
if(result != 0) {
|
||||||
destroy_curl_handle(curl);
|
destroy_curl_handle(curl);
|
||||||
fclose(f);
|
fclose(f);
|
||||||
@ -497,7 +497,7 @@ static int put_headers(const char *path, headers_t meta) {
|
|||||||
string url;
|
string url;
|
||||||
string resource;
|
string resource;
|
||||||
struct stat buf;
|
struct stat buf;
|
||||||
struct BodyStruct body;
|
BodyData body;
|
||||||
CURL *curl = NULL;
|
CURL *curl = NULL;
|
||||||
|
|
||||||
FGPRINT(" put_headers[path=%s]\n", path);
|
FGPRINT(" put_headers[path=%s]\n", path);
|
||||||
@ -515,9 +515,6 @@ static int put_headers(const char *path, headers_t meta) {
|
|||||||
resource = urlEncode(service_path + bucket + s3_realpath);
|
resource = urlEncode(service_path + bucket + s3_realpath);
|
||||||
url = host + resource;
|
url = host + resource;
|
||||||
|
|
||||||
body.text = (char *)malloc(1);
|
|
||||||
body.size = 0;
|
|
||||||
|
|
||||||
auto_curl_slist headers;
|
auto_curl_slist headers;
|
||||||
string date = get_date();
|
string date = get_date();
|
||||||
headers.append("Date: " + date);
|
headers.append("Date: " + date);
|
||||||
@ -560,11 +557,7 @@ static int put_headers(const char *path, headers_t meta) {
|
|||||||
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
||||||
|
|
||||||
result = my_curl_easy_perform(curl, &body);
|
result = my_curl_easy_perform(curl, &body);
|
||||||
|
|
||||||
destroy_curl_handle(curl);
|
destroy_curl_handle(curl);
|
||||||
if(body.text)
|
|
||||||
free(body.text);
|
|
||||||
|
|
||||||
if(result != 0)
|
if(result != 0)
|
||||||
return result;
|
return result;
|
||||||
|
|
||||||
@ -605,7 +598,7 @@ static int put_multipart_headers(const char *path, headers_t meta) {
|
|||||||
string resource;
|
string resource;
|
||||||
string upload_id;
|
string upload_id;
|
||||||
struct stat buf;
|
struct stat buf;
|
||||||
struct BodyStruct body;
|
BodyData body;
|
||||||
vector <file_part> parts;
|
vector <file_part> parts;
|
||||||
|
|
||||||
FGPRINT(" put_multipart_headers[path=%s]\n", path);
|
FGPRINT(" put_multipart_headers[path=%s]\n", path);
|
||||||
@ -614,9 +607,6 @@ static int put_multipart_headers(const char *path, headers_t meta) {
|
|||||||
resource = urlEncode(service_path + bucket + s3_realpath);
|
resource = urlEncode(service_path + bucket + s3_realpath);
|
||||||
url = host + resource;
|
url = host + resource;
|
||||||
|
|
||||||
body.text = (char *)malloc(1);
|
|
||||||
body.size = 0;
|
|
||||||
|
|
||||||
// already checked by check_object_access(), so only get attr.
|
// already checked by check_object_access(), so only get attr.
|
||||||
if(0 != (result = get_object_attribute(path, &buf))){
|
if(0 != (result = get_object_attribute(path, &buf))){
|
||||||
return result;
|
return result;
|
||||||
@ -624,8 +614,6 @@ static int put_multipart_headers(const char *path, headers_t meta) {
|
|||||||
|
|
||||||
upload_id = initiate_multipart_upload(path, buf.st_size, meta);
|
upload_id = initiate_multipart_upload(path, buf.st_size, meta);
|
||||||
if(upload_id.size() == 0){
|
if(upload_id.size() == 0){
|
||||||
if(body.text)
|
|
||||||
free(body.text);
|
|
||||||
return(-EIO);
|
return(-EIO);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -653,8 +641,6 @@ static int put_multipart_headers(const char *path, headers_t meta) {
|
|||||||
|
|
||||||
result = complete_multipart_upload(path, upload_id, parts);
|
result = complete_multipart_upload(path, upload_id, parts);
|
||||||
if(result != 0) {
|
if(result != 0) {
|
||||||
if(body.text)
|
|
||||||
free(body.text);
|
|
||||||
return -EIO;
|
return -EIO;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -668,16 +654,11 @@ static int put_multipart_headers(const char *path, headers_t meta) {
|
|||||||
n_mtime.modtime = get_mtime(meta["x-amz-meta-mtime"].c_str());
|
n_mtime.modtime = get_mtime(meta["x-amz-meta-mtime"].c_str());
|
||||||
n_mtime.actime = n_mtime.modtime;
|
n_mtime.actime = n_mtime.modtime;
|
||||||
if((utime(cache_path.c_str(), &n_mtime)) == -1) {
|
if((utime(cache_path.c_str(), &n_mtime)) == -1) {
|
||||||
if(body.text)
|
|
||||||
free(body.text);
|
|
||||||
YIKES(-errno);
|
YIKES(-errno);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if(body.text)
|
|
||||||
free(body.text);
|
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -698,13 +679,10 @@ static int put_local_fd_small_file(const char* path, headers_t meta, int fd) {
|
|||||||
url = host + resource;
|
url = host + resource;
|
||||||
|
|
||||||
int result;
|
int result;
|
||||||
struct BodyStruct body;
|
BodyData body;
|
||||||
auto_curl_slist headers;
|
auto_curl_slist headers;
|
||||||
string date = get_date();
|
string date = get_date();
|
||||||
|
|
||||||
body.text = (char *) malloc(1);
|
|
||||||
body.size = 0;
|
|
||||||
|
|
||||||
curl = create_curl_handle();
|
curl = create_curl_handle();
|
||||||
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&body);
|
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&body);
|
||||||
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
|
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
|
||||||
@ -713,8 +691,6 @@ static int put_local_fd_small_file(const char* path, headers_t meta, int fd) {
|
|||||||
|
|
||||||
FILE* f = fdopen(fd, "rb");
|
FILE* f = fdopen(fd, "rb");
|
||||||
if(f == 0){
|
if(f == 0){
|
||||||
if(body.text)
|
|
||||||
free(body.text);
|
|
||||||
YIKES(-errno);
|
YIKES(-errno);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -749,12 +725,8 @@ static int put_local_fd_small_file(const char* path, headers_t meta, int fd) {
|
|||||||
string my_url = prepare_url(url.c_str());
|
string my_url = prepare_url(url.c_str());
|
||||||
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
||||||
|
|
||||||
result = my_curl_easy_perform(curl, &body, f);
|
result = my_curl_easy_perform(curl, &body, NULL, f);
|
||||||
|
|
||||||
if(body.text)
|
|
||||||
free(body.text);
|
|
||||||
destroy_curl_handle(curl);
|
destroy_curl_handle(curl);
|
||||||
|
|
||||||
if(result != 0)
|
if(result != 0)
|
||||||
return result;
|
return result;
|
||||||
|
|
||||||
@ -941,14 +913,11 @@ static string initiate_multipart_upload(const char *path, off_t size, headers_t
|
|||||||
string upload_id = "";
|
string upload_id = "";
|
||||||
string ContentType;
|
string ContentType;
|
||||||
string s3_realpath;
|
string s3_realpath;
|
||||||
struct BodyStruct body;
|
BodyData body;
|
||||||
struct curl_slist *slist=NULL;
|
struct curl_slist *slist=NULL;
|
||||||
|
|
||||||
FGPRINT(" initiate_multipart_upload [path=%s][size=%lu]\n", path, size);
|
FGPRINT(" initiate_multipart_upload [path=%s][size=%lu]\n", path, size);
|
||||||
|
|
||||||
body.text = (char *)malloc(1);
|
|
||||||
body.size = 0;
|
|
||||||
|
|
||||||
s3_realpath = get_realpath(path);
|
s3_realpath = get_realpath(path);
|
||||||
resource = urlEncode(service_path + bucket + s3_realpath);
|
resource = urlEncode(service_path + bucket + s3_realpath);
|
||||||
resource.append("?uploads");
|
resource.append("?uploads");
|
||||||
@ -1012,15 +981,13 @@ static string initiate_multipart_upload(const char *path, off_t size, headers_t
|
|||||||
destroy_curl_handle(curl);
|
destroy_curl_handle(curl);
|
||||||
|
|
||||||
if(result != 0) {
|
if(result != 0) {
|
||||||
if(body.text)
|
|
||||||
free(body.text);
|
|
||||||
return upload_id;
|
return upload_id;
|
||||||
}
|
}
|
||||||
|
|
||||||
// XML returns UploadId
|
// XML returns UploadId
|
||||||
// Parse XML body for UploadId
|
// Parse XML body for UploadId
|
||||||
upload_id.clear();
|
upload_id.clear();
|
||||||
xmlDocPtr doc = xmlReadMemory(body.text, body.size, "", NULL, 0);
|
xmlDocPtr doc = xmlReadMemory(body.str(), body.size(), "", NULL, 0);
|
||||||
if(doc != NULL && doc->children != NULL) {
|
if(doc != NULL && doc->children != NULL) {
|
||||||
for(xmlNodePtr cur_node = doc->children->children;
|
for(xmlNodePtr cur_node = doc->children->children;
|
||||||
cur_node != NULL;
|
cur_node != NULL;
|
||||||
@ -1044,11 +1011,6 @@ static string initiate_multipart_upload(const char *path, off_t size, headers_t
|
|||||||
} // if (doc != NULL && doc->children != NULL)
|
} // if (doc != NULL && doc->children != NULL)
|
||||||
xmlFreeDoc(doc);
|
xmlFreeDoc(doc);
|
||||||
|
|
||||||
// clean up
|
|
||||||
if(body.text)
|
|
||||||
free(body.text);
|
|
||||||
body.size = 0;
|
|
||||||
|
|
||||||
return upload_id;
|
return upload_id;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1066,7 +1028,7 @@ static int complete_multipart_upload(const char *path, string upload_id,
|
|||||||
string resource;
|
string resource;
|
||||||
string postContent;
|
string postContent;
|
||||||
string s3_realpath;
|
string s3_realpath;
|
||||||
struct BodyStruct body;
|
BodyData body;
|
||||||
struct WriteThis pooh;
|
struct WriteThis pooh;
|
||||||
struct curl_slist *slist = NULL;
|
struct curl_slist *slist = NULL;
|
||||||
|
|
||||||
@ -1106,9 +1068,6 @@ static int complete_multipart_upload(const char *path, string upload_id,
|
|||||||
url = host + resource;
|
url = host + resource;
|
||||||
my_url = prepare_url(url.c_str());
|
my_url = prepare_url(url.c_str());
|
||||||
|
|
||||||
body.text = (char *)malloc(1);
|
|
||||||
body.size = 0;
|
|
||||||
|
|
||||||
curl = create_curl_handle();
|
curl = create_curl_handle();
|
||||||
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&body);
|
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&body);
|
||||||
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
|
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
|
||||||
@ -1140,9 +1099,6 @@ static int complete_multipart_upload(const char *path, string upload_id,
|
|||||||
|
|
||||||
curl_slist_free_all(slist);
|
curl_slist_free_all(slist);
|
||||||
destroy_curl_handle(curl);
|
destroy_curl_handle(curl);
|
||||||
|
|
||||||
if(body.text)
|
|
||||||
free(body.text);
|
|
||||||
free(pData);
|
free(pData);
|
||||||
|
|
||||||
return result;
|
return result;
|
||||||
@ -1162,8 +1118,8 @@ static string upload_part(const char *path, const char *source, int part_number,
|
|||||||
string ETag;
|
string ETag;
|
||||||
string s3_realpath;
|
string s3_realpath;
|
||||||
struct stat st;
|
struct stat st;
|
||||||
struct BodyStruct body;
|
BodyData body;
|
||||||
struct BodyStruct header;
|
BodyData header;
|
||||||
struct curl_slist *slist = NULL;
|
struct curl_slist *slist = NULL;
|
||||||
|
|
||||||
// Now upload the file as the nth part
|
// Now upload the file as the nth part
|
||||||
@ -1204,12 +1160,6 @@ static string upload_part(const char *path, const char *source, int part_number,
|
|||||||
url = host + resource;
|
url = host + resource;
|
||||||
my_url = prepare_url(url.c_str());
|
my_url = prepare_url(url.c_str());
|
||||||
|
|
||||||
body.text = (char *)malloc(1);
|
|
||||||
body.size = 0;
|
|
||||||
|
|
||||||
header.text = (char *)malloc(1);
|
|
||||||
header.size = 0;
|
|
||||||
|
|
||||||
curl = create_curl_handle();
|
curl = create_curl_handle();
|
||||||
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&body);
|
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&body);
|
||||||
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
|
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
|
||||||
@ -1236,51 +1186,30 @@ static string upload_part(const char *path, const char *source, int part_number,
|
|||||||
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
|
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
|
||||||
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
||||||
|
|
||||||
result = my_curl_easy_perform(curl, &body, part_file);
|
result = my_curl_easy_perform(curl, &body, &header, part_file);
|
||||||
|
|
||||||
curl_slist_free_all(slist);
|
curl_slist_free_all(slist);
|
||||||
destroy_curl_handle(curl);
|
destroy_curl_handle(curl);
|
||||||
fclose(part_file);
|
fclose(part_file);
|
||||||
|
|
||||||
if(result != 0) {
|
if(result != 0) {
|
||||||
if(header.text)
|
|
||||||
free(header.text);
|
|
||||||
if(body.text)
|
|
||||||
free(body.text);
|
|
||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
|
|
||||||
// calculate local md5sum, if it matches the header
|
// calculate local md5sum, if it matches the header
|
||||||
// ETag value, the upload was successful.
|
// ETag value, the upload was successful.
|
||||||
if((fd = open(source, O_RDONLY)) == -1) {
|
if((fd = open(source, O_RDONLY)) == -1) {
|
||||||
if(header.text)
|
|
||||||
free(header.text);
|
|
||||||
if(body.text)
|
|
||||||
free(body.text);
|
|
||||||
SYSLOGERR("%d###result=%d", __LINE__, -fd);
|
SYSLOGERR("%d###result=%d", __LINE__, -fd);
|
||||||
|
|
||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
|
|
||||||
string md5 = md5sum(fd);
|
string md5 = md5sum(fd);
|
||||||
close(fd);
|
close(fd);
|
||||||
if(!md5.empty() && strstr(header.text, md5.c_str())) {
|
if(!md5.empty() && strstr(header.str(), md5.c_str())) {
|
||||||
ETag.assign(md5);
|
ETag.assign(md5);
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
if(header.text)
|
|
||||||
free(header.text);
|
|
||||||
if(body.text)
|
|
||||||
free(body.text);
|
|
||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
|
|
||||||
// clean up
|
|
||||||
if(header.text)
|
|
||||||
free(header.text);
|
|
||||||
if(body.text)
|
|
||||||
free(body.text);
|
|
||||||
|
|
||||||
return ETag;
|
return ETag;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1294,8 +1223,8 @@ static string copy_part(const char *from, const char *to, int part_number, strin
|
|||||||
string raw_date;
|
string raw_date;
|
||||||
string ETag;
|
string ETag;
|
||||||
string s3_realpath;
|
string s3_realpath;
|
||||||
struct BodyStruct body;
|
BodyData body;
|
||||||
struct BodyStruct header;
|
BodyData header;
|
||||||
|
|
||||||
// Now copy the file as the nth part
|
// Now copy the file as the nth part
|
||||||
FGPRINT("copy_part [from=%s] [to=%s]\n", from, to);
|
FGPRINT("copy_part [from=%s] [to=%s]\n", from, to);
|
||||||
@ -1310,11 +1239,6 @@ static string copy_part(const char *from, const char *to, int part_number, strin
|
|||||||
url = host + resource;
|
url = host + resource;
|
||||||
my_url = prepare_url(url.c_str());
|
my_url = prepare_url(url.c_str());
|
||||||
|
|
||||||
body.text = (char *)malloc(1);
|
|
||||||
body.size = 0;
|
|
||||||
header.text = (char *)malloc(1);
|
|
||||||
header.size = 0;
|
|
||||||
|
|
||||||
auto_curl_slist headers;
|
auto_curl_slist headers;
|
||||||
string date = get_date();
|
string date = get_date();
|
||||||
headers.append("Date: " + date);
|
headers.append("Date: " + date);
|
||||||
@ -1350,30 +1274,18 @@ static string copy_part(const char *from, const char *to, int part_number, strin
|
|||||||
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
|
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
|
||||||
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
||||||
|
|
||||||
result = my_curl_easy_perform(curl, &body);
|
result = my_curl_easy_perform(curl, &body, &header);
|
||||||
destroy_curl_handle(curl);
|
destroy_curl_handle(curl);
|
||||||
|
|
||||||
if(result != 0) {
|
if(result != 0) {
|
||||||
if(body.text)
|
|
||||||
free(body.text);
|
|
||||||
if(header.text)
|
|
||||||
free(header.text);
|
|
||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
|
|
||||||
char *start_etag;
|
char* body_data = (char*)body.str();
|
||||||
char *end_etag;
|
char* start_etag= strstr(body_data, "ETag");
|
||||||
start_etag = strstr(body.text, "ETag");
|
char* end_etag = strstr(body_data, "/ETag>");
|
||||||
end_etag = strstr(body.text, "/ETag>");
|
|
||||||
start_etag += 11;
|
start_etag += 11;
|
||||||
ETag.assign(start_etag, (size_t)(end_etag - start_etag - 7));
|
ETag.assign(start_etag, (size_t)(end_etag - start_etag - 7));
|
||||||
|
|
||||||
// clean up
|
|
||||||
if(body.text)
|
|
||||||
free(body.text);
|
|
||||||
if(header.text)
|
|
||||||
free(header.text);
|
|
||||||
|
|
||||||
return ETag;
|
return ETag;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1384,7 +1296,7 @@ static int list_multipart_uploads(void) {
|
|||||||
CURL *curl = NULL;
|
CURL *curl = NULL;
|
||||||
string resource;
|
string resource;
|
||||||
string url;
|
string url;
|
||||||
struct BodyStruct body;
|
BodyData body;
|
||||||
int result;
|
int result;
|
||||||
string date;
|
string date;
|
||||||
string raw_date;
|
string raw_date;
|
||||||
@ -1392,10 +1304,6 @@ static int list_multipart_uploads(void) {
|
|||||||
string my_url;
|
string my_url;
|
||||||
struct curl_slist *slist=NULL;
|
struct curl_slist *slist=NULL;
|
||||||
|
|
||||||
// Initialization of variables
|
|
||||||
body.text = (char *)malloc(1);
|
|
||||||
body.size = 0;
|
|
||||||
|
|
||||||
printf("List Multipart Uploads\n");
|
printf("List Multipart Uploads\n");
|
||||||
resource = urlEncode(service_path + bucket + "/");
|
resource = urlEncode(service_path + bucket + "/");
|
||||||
resource.append("?uploads");
|
resource.append("?uploads");
|
||||||
@ -1425,16 +1333,11 @@ static int list_multipart_uploads(void) {
|
|||||||
destroy_curl_handle(curl);
|
destroy_curl_handle(curl);
|
||||||
|
|
||||||
if(result != 0) {
|
if(result != 0) {
|
||||||
if(body.text)
|
|
||||||
free(body.text);
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
if(body.size() > 0){
|
||||||
if(body.size > 0)
|
printf("body.text:\n%s\n", body.str());
|
||||||
printf("body.text:\n%s\n", body.text);
|
}
|
||||||
if(body.text)
|
|
||||||
free(body.text);
|
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -2930,16 +2833,13 @@ static int list_bucket(const char *path, struct s3_object **head, const char* de
|
|||||||
CURL *curl;
|
CURL *curl;
|
||||||
int result;
|
int result;
|
||||||
string s3_realpath;
|
string s3_realpath;
|
||||||
struct BodyStruct body;
|
BodyData body;
|
||||||
bool truncated = true;
|
bool truncated = true;
|
||||||
string next_marker = "";
|
string next_marker = "";
|
||||||
|
|
||||||
FGPRINT("list_bucket [path=%s]\n", path);
|
FGPRINT("list_bucket [path=%s]\n", path);
|
||||||
|
|
||||||
body.text = (char *) malloc(1);
|
|
||||||
body.size = 0;
|
|
||||||
s3_realpath = get_realpath(path);
|
s3_realpath = get_realpath(path);
|
||||||
|
|
||||||
string resource = urlEncode(service_path + bucket); // this is what gets signed
|
string resource = urlEncode(service_path + bucket); // this is what gets signed
|
||||||
string query;
|
string query;
|
||||||
if(delimiter && 0 < strlen(delimiter)){
|
if(delimiter && 0 < strlen(delimiter)){
|
||||||
@ -2986,37 +2886,25 @@ static int list_bucket(const char *path, struct s3_object **head, const char* de
|
|||||||
destroy_curl_handle(curl);
|
destroy_curl_handle(curl);
|
||||||
|
|
||||||
if(result != 0) {
|
if(result != 0) {
|
||||||
if(body.text)
|
|
||||||
free(body.text);
|
|
||||||
FGPRINT(" list_bucket my_curl_easy_perform returns with error.\n");
|
FGPRINT(" list_bucket my_curl_easy_perform returns with error.\n");
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
if((append_objects_from_xml(path, body.str(), head)) != 0) {
|
||||||
if((append_objects_from_xml(path, body.text, head)) != 0) {
|
|
||||||
if(body.text)
|
|
||||||
free(body.text);
|
|
||||||
FGPRINT(" list_bucket append_objects_from_xml returns with error.\n");
|
FGPRINT(" list_bucket append_objects_from_xml returns with error.\n");
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
truncated = is_truncated(body.text);
|
truncated = is_truncated(body.str());
|
||||||
if(truncated){
|
if(truncated){
|
||||||
xmlChar* tmpch = get_next_marker(body.text);
|
xmlChar* tmpch = get_next_marker(body.str());
|
||||||
if(tmpch){
|
if(tmpch){
|
||||||
next_marker = (char*)tmpch;
|
next_marker = (char*)tmpch;
|
||||||
xmlFree(tmpch);
|
xmlFree(tmpch);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
body.Clear();
|
||||||
if(body.text)
|
|
||||||
free(body.text);
|
|
||||||
body.size = 0;
|
|
||||||
body.text = (char *) malloc(1);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if(body.text)
|
|
||||||
free(body.text);
|
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -3207,6 +3095,7 @@ static char *get_object_name(xmlDocPtr doc, xmlNodePtr node, const char* path)
|
|||||||
}
|
}
|
||||||
// basepath(path) is as same as fullpath.
|
// basepath(path) is as same as fullpath.
|
||||||
if(0 == strcmp((char*)fullpath, path)){
|
if(0 == strcmp((char*)fullpath, path)){
|
||||||
|
xmlFree(fullpath);
|
||||||
return (char*)c_strErrorObjectName;
|
return (char*)c_strErrorObjectName;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -3534,13 +3423,10 @@ static int s3fs_check_service(void) {
|
|||||||
CURL *curl = NULL;
|
CURL *curl = NULL;
|
||||||
int result = CURLE_OK;
|
int result = CURLE_OK;
|
||||||
CURLcode responseCode;
|
CURLcode responseCode;
|
||||||
|
BodyData body;
|
||||||
|
|
||||||
FGPRINT("s3fs_check_service\n");
|
FGPRINT("s3fs_check_service\n");
|
||||||
|
|
||||||
struct BodyStruct body;
|
|
||||||
body.text = (char *) malloc(1);
|
|
||||||
body.size = 0;
|
|
||||||
|
|
||||||
string resource = urlEncode(service_path + bucket);
|
string resource = urlEncode(service_path + bucket);
|
||||||
string url = host + resource;
|
string url = host + resource;
|
||||||
|
|
||||||
@ -3551,7 +3437,6 @@ static int s3fs_check_service(void) {
|
|||||||
headers.append("Authorization: AWS " + AWSAccessKeyId + ":" +
|
headers.append("Authorization: AWS " + AWSAccessKeyId + ":" +
|
||||||
calc_signature("GET", "", date, headers.get(), resource));
|
calc_signature("GET", "", date, headers.get(), resource));
|
||||||
} else {
|
} else {
|
||||||
if(body.text) free(body.text);
|
|
||||||
return EXIT_SUCCESS;
|
return EXIT_SUCCESS;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -3561,25 +3446,19 @@ static int s3fs_check_service(void) {
|
|||||||
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
|
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
|
||||||
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
|
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
|
||||||
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
|
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
|
||||||
result = my_curl_easy_perform(curl);
|
result = my_curl_easy_perform(curl, &body);
|
||||||
|
|
||||||
// connect either successful or too many timeouts
|
// connect either successful or too many timeouts
|
||||||
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &responseCode);
|
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &responseCode);
|
||||||
|
|
||||||
if(responseCode == 403) {
|
if(responseCode == 403) {
|
||||||
destroy_curl_handle(curl);
|
destroy_curl_handle(curl);
|
||||||
if(body.text)
|
|
||||||
free(body.text);
|
|
||||||
body.text = NULL;
|
|
||||||
fprintf(stderr, "%s: invalid credentials\n", program_name.c_str());
|
fprintf(stderr, "%s: invalid credentials\n", program_name.c_str());
|
||||||
return EXIT_FAILURE;
|
return EXIT_FAILURE;
|
||||||
}
|
}
|
||||||
|
|
||||||
if(responseCode == 404) {
|
if(responseCode == 404) {
|
||||||
destroy_curl_handle(curl);
|
destroy_curl_handle(curl);
|
||||||
if(body.text)
|
|
||||||
free(body.text);
|
|
||||||
body.text = NULL;
|
|
||||||
fprintf(stderr, "%s: bucket not found\n", program_name.c_str());
|
fprintf(stderr, "%s: bucket not found\n", program_name.c_str());
|
||||||
return EXIT_FAILURE;
|
return EXIT_FAILURE;
|
||||||
}
|
}
|
||||||
@ -3587,16 +3466,11 @@ static int s3fs_check_service(void) {
|
|||||||
// unable to connect
|
// unable to connect
|
||||||
if(responseCode == CURLE_OPERATION_TIMEDOUT) {
|
if(responseCode == CURLE_OPERATION_TIMEDOUT) {
|
||||||
destroy_curl_handle(curl);
|
destroy_curl_handle(curl);
|
||||||
if(body.text) free(body.text);
|
|
||||||
body.text = NULL;
|
|
||||||
return EXIT_SUCCESS;
|
return EXIT_SUCCESS;
|
||||||
}
|
}
|
||||||
|
|
||||||
if(responseCode != 200 && responseCode != 301) {
|
if(responseCode != 200 && responseCode != 301) {
|
||||||
SYSLOGDBG("responseCode: %i\n", (int)responseCode);
|
SYSLOGDBG("responseCode: %i\n", (int)responseCode);
|
||||||
if(body.text){
|
|
||||||
free(body.text);
|
|
||||||
}
|
|
||||||
destroy_curl_handle(curl);
|
destroy_curl_handle(curl);
|
||||||
fprintf(stderr, "%s: unable to connect\n", program_name.c_str());
|
fprintf(stderr, "%s: unable to connect\n", program_name.c_str());
|
||||||
return EXIT_FAILURE;
|
return EXIT_FAILURE;
|
||||||
@ -3609,17 +3483,12 @@ static int s3fs_check_service(void) {
|
|||||||
program_name.c_str(), mount_prefix.c_str());
|
program_name.c_str(), mount_prefix.c_str());
|
||||||
|
|
||||||
destroy_curl_handle(curl);
|
destroy_curl_handle(curl);
|
||||||
if(body.text)
|
|
||||||
free(body.text);
|
|
||||||
body.text = NULL;
|
|
||||||
return EXIT_FAILURE;
|
return EXIT_FAILURE;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// success
|
// success
|
||||||
service_validated = true;
|
service_validated = true;
|
||||||
if(body.text) free(body.text);
|
|
||||||
body.text = NULL;
|
|
||||||
destroy_curl_handle(curl);
|
destroy_curl_handle(curl);
|
||||||
|
|
||||||
return EXIT_SUCCESS;
|
return EXIT_SUCCESS;
|
||||||
|
Loading…
Reference in New Issue
Block a user