/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright 2007-2008 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "curl.h" using namespace std; pthread_mutex_t curl_handles_lock; std::map curl_times; std::map curl_progress; std::string curl_ca_bundle; CURL *create_curl_handle(void) { time_t now; CURL *curl_handle; pthread_mutex_lock(&curl_handles_lock); curl_handle = curl_easy_init(); curl_easy_reset(curl_handle); curl_easy_setopt(curl_handle, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(curl_handle, CURLOPT_CONNECTTIMEOUT, connect_timeout); curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 0); curl_easy_setopt(curl_handle, CURLOPT_PROGRESSFUNCTION, my_curl_progress); curl_easy_setopt(curl_handle, CURLOPT_PROGRESSDATA, curl_handle); curl_easy_setopt(curl_handle, CURLOPT_FORBID_REUSE, 1); now = time(0); curl_times[curl_handle] = now; curl_progress[curl_handle] = progress_t(-1, -1); pthread_mutex_unlock(&curl_handles_lock); return curl_handle; } void destroy_curl_handle(CURL *curl_handle) { if(curl_handle != NULL) { pthread_mutex_lock(&curl_handles_lock); curl_times.erase(curl_handle); curl_progress.erase(curl_handle); curl_easy_cleanup(curl_handle); pthread_mutex_unlock(&curl_handles_lock); } return; } /** * @return fuse return code */ int my_curl_easy_perform(CURL* curl, BodyStruct* body, FILE* f) { char url[256]; time_t now; char* ptr_url = url; curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL , &ptr_url); if(debug) syslog(LOG_DEBUG, "connecting to URL %s", ptr_url); // curl_easy_setopt(curl, CURLOPT_VERBOSE, true); if(ssl_verify_hostname.substr(0,1) == "0") curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); if(curl_ca_bundle.size() != 0) curl_easy_setopt(curl, CURLOPT_CAINFO, curl_ca_bundle.c_str()); long responseCode; // 1 attempt + retries... int t = retries + 1; while (t-- > 0) { if (f) { rewind(f); } CURLcode curlCode = curl_easy_perform(curl); switch (curlCode) { case CURLE_OK: // Need to look at the HTTP response code if (curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &responseCode) != 0) { syslog(LOG_ERR, "curl_easy_getinfo failed while trying to retrieve HTTP response code"); return -EIO; } if(debug) syslog(LOG_DEBUG, "HTTP response code %ld", responseCode); if (responseCode < 400) { return 0; } if (responseCode >= 500) { syslog(LOG_ERR, "###HTTP response=%ld", responseCode); sleep(4); break; } // Service response codes which are >= 400 && < 500 switch(responseCode) { case 400: if(debug) syslog(LOG_ERR, "HTTP response code 400 was returned"); if(body) { if(body->size && debug) { syslog(LOG_ERR, "Body Text: %s", body->text); } } if(debug) syslog(LOG_DEBUG, "Now returning EIO"); return -EIO; case 403: if(debug) syslog(LOG_ERR, "HTTP response code 403 was returned"); if(body) { if(body->size && debug) { syslog(LOG_ERR, "Body Text: %s", body->text); } } if(debug) syslog(LOG_DEBUG, "Now returning EIO"); return -EIO; case 404: if(debug) syslog(LOG_DEBUG, "HTTP response code 404 was returned"); if(body) { if(body->size && debug) { syslog(LOG_DEBUG, "Body Text: %s", body->text); } } if(debug) syslog(LOG_DEBUG, "Now returning ENOENT"); return -ENOENT; default: syslog(LOG_ERR, "###response=%ld", responseCode); printf("responseCode %ld\n", responseCode); if(body) { if(body->size) { printf("Body Text %s\n", body->text); } } return -EIO; } break; case CURLE_WRITE_ERROR: syslog(LOG_ERR, "### CURLE_WRITE_ERROR"); sleep(2); break; case CURLE_OPERATION_TIMEDOUT: syslog(LOG_ERR, "### CURLE_OPERATION_TIMEDOUT"); sleep(2); break; case CURLE_COULDNT_RESOLVE_HOST: syslog(LOG_ERR, "### CURLE_COULDNT_RESOLVE_HOST"); sleep(2); break; case CURLE_COULDNT_CONNECT: syslog(LOG_ERR, "### CURLE_COULDNT_CONNECT"); sleep(4); break; case CURLE_GOT_NOTHING: syslog(LOG_ERR, "### CURLE_GOT_NOTHING"); sleep(4); break; case CURLE_ABORTED_BY_CALLBACK: syslog(LOG_ERR, "### CURLE_ABORTED_BY_CALLBACK"); sleep(4); now = time(0); curl_times[curl] = now; break; case CURLE_PARTIAL_FILE: syslog(LOG_ERR, "### CURLE_PARTIAL_FILE"); sleep(4); break; case CURLE_SSL_CACERT: // try to locate cert, if successful, then set the // option and continue if (curl_ca_bundle.size() == 0) { locate_bundle(); if (curl_ca_bundle.size() != 0) { t++; curl_easy_setopt(curl, CURLOPT_CAINFO, curl_ca_bundle.c_str()); continue; } } syslog(LOG_ERR, "curlCode: %i msg: %s", curlCode, curl_easy_strerror(curlCode));; fprintf (stderr, "%s: curlCode: %i -- %s\n", program_name.c_str(), curlCode, curl_easy_strerror(curlCode)); exit(EXIT_FAILURE); break; #ifdef CURLE_PEER_FAILED_VERIFICATION case CURLE_PEER_FAILED_VERIFICATION: first_pos = bucket.find_first_of("."); if (first_pos != string::npos) { fprintf (stderr, "%s: curl returned a CURL_PEER_FAILED_VERIFICATION error\n", program_name.c_str()); fprintf (stderr, "%s: security issue found: buckets with periods in their name are incompatible with https\n", program_name.c_str()); fprintf (stderr, "%s: This check can be over-ridden by using the -o ssl_verify_hostname=0\n", program_name.c_str()); fprintf (stderr, "%s: The certificate will still be checked but the hostname will not be verified.\n", program_name.c_str()); fprintf (stderr, "%s: A more secure method would be to use a bucket name without periods.\n", program_name.c_str()); } else { fprintf (stderr, "%s: my_curl_easy_perform: curlCode: %i -- %s\n", program_name.c_str(), curlCode, curl_easy_strerror(curlCode)); } exit(EXIT_FAILURE); break; #endif // This should be invalid since curl option HTTP FAILONERROR is now off case CURLE_HTTP_RETURNED_ERROR: syslog(LOG_ERR, "### CURLE_HTTP_RETURNED_ERROR"); if (curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &responseCode) != 0) { return -EIO; } syslog(LOG_ERR, "###response=%ld", responseCode); // Let's try to retrieve the if (responseCode == 404) { return -ENOENT; } if (responseCode < 500) { return -EIO; } break; // Unknown CURL return code default: syslog(LOG_ERR, "###curlCode: %i msg: %s", curlCode, curl_easy_strerror(curlCode));; exit(EXIT_FAILURE); break; } syslog(LOG_ERR, "###retrying..."); } syslog(LOG_ERR, "###giving up"); return -EIO; } // libcurl callback size_t WriteMemoryCallback(void *ptr, size_t blockSize, size_t numBlocks, void *data) { size_t realsize = blockSize * numBlocks; struct BodyStruct *mem = (struct BodyStruct *)data; mem->text = (char *)realloc(mem->text, mem->size + realsize + 1); if(mem->text == NULL) { /* out of memory! */ fprintf(stderr, "not enough memory (realloc returned NULL)\n"); exit(EXIT_FAILURE); } memcpy(&(mem->text[mem->size]), ptr, realsize); mem->size += realsize; mem->text[mem->size] = 0; return realsize; } // read_callback // http://curl.haxx.se/libcurl/c/post-callback.html size_t read_callback(void *ptr, size_t size, size_t nmemb, void *userp) { struct WriteThis *pooh = (struct WriteThis *)userp; if(size*nmemb < 1) return 0; if(pooh->sizeleft) { *(char *)ptr = pooh->readptr[0]; /* copy one single byte */ pooh->readptr++; /* advance pointer */ pooh->sizeleft--; /* less data left */ return 1; /* we return 1 byte at a time! */ } return 0; /* no more data left to deliver */ } // homegrown timeout mechanism int my_curl_progress( void *clientp, double dltotal, double dlnow, double ultotal, double ulnow) { CURL* curl = static_cast(clientp); time_t now = time(0); progress_t p(dlnow, ulnow); pthread_mutex_lock(&curl_handles_lock); // any progress? if(p != curl_progress[curl]) { // yes! curl_times[curl] = now; curl_progress[curl] = p; } else { // timeout? if (now - curl_times[curl] > readwrite_timeout) { pthread_mutex_unlock( &curl_handles_lock ); syslog(LOG_ERR, "timeout now: %li curl_times[curl]: %lil readwrite_timeout: %li", (long int)now, curl_times[curl], (long int)readwrite_timeout); return CURLE_ABORTED_BY_CALLBACK; } } pthread_mutex_unlock(&curl_handles_lock); return 0; } void locate_bundle(void) { // See if environment variable CURL_CA_BUNDLE is set // if so, check it, if it is a good path, then set the // curl_ca_bundle variable to it char *CURL_CA_BUNDLE; if(curl_ca_bundle.size() == 0) { CURL_CA_BUNDLE = getenv("CURL_CA_BUNDLE"); if(CURL_CA_BUNDLE != NULL) { // check for existance and readability of the file ifstream BF(CURL_CA_BUNDLE); if(BF.good()) { BF.close(); curl_ca_bundle.assign(CURL_CA_BUNDLE); } else { fprintf(stderr, "%s: file specified by CURL_CA_BUNDLE environment variable is not readable\n", program_name.c_str()); exit(EXIT_FAILURE); } return; } } // not set via environment variable, look in likely locations /////////////////////////////////////////// // from curl's (7.21.2) acinclude.m4 file /////////////////////////////////////////// // dnl CURL_CHECK_CA_BUNDLE // dnl ------------------------------------------------- // dnl Check if a default ca-bundle should be used // dnl // dnl regarding the paths this will scan: // dnl /etc/ssl/certs/ca-certificates.crt Debian systems // dnl /etc/pki/tls/certs/ca-bundle.crt Redhat and Mandriva // dnl /usr/share/ssl/certs/ca-bundle.crt old(er) Redhat // dnl /usr/local/share/certs/ca-root.crt FreeBSD // dnl /etc/ssl/cert.pem OpenBSD // dnl /etc/ssl/certs/ (ca path) SUSE ifstream BF("/etc/pki/tls/certs/ca-bundle.crt"); if(BF.good()) { BF.close(); curl_ca_bundle.assign("/etc/pki/tls/certs/ca-bundle.crt"); return; } return; } // Multi CURL stuff CURLHLL *create_h_element(CURL *handle) { CURLHLL *p; p = (CURLHLL *) malloc(sizeof(CURLHLL)); if(p == NULL) { printf("create_h_element: could not allocation memory\n"); exit(EXIT_FAILURE); } p->handle = handle; p->next = NULL; return p; } CURLMHLL *create_mh_element(CURLM *handle) { CURLMHLL *p; p = (CURLMHLL *) malloc(sizeof(CURLMHLL)); if(p == NULL) { printf("create_mh_element: could not allocation memory\n"); exit(EXIT_FAILURE); } p->handle = handle; p->curlhll_head = NULL; p->next = NULL; return p; } CURLMHLL *add_mh_element(CURLMHLL *head, CURLM *handle) { CURLMHLL *p; CURLMHLL *p_new; p_new = create_mh_element(handle); for (p = head; p->next != NULL; p = p->next); ; p->next = p_new; return p_new; } void add_h_element(CURLHLL *head, CURL *handle) { CURLHLL *p; CURLHLL *p_new; p_new = create_h_element(handle); for (p = head; p->next != NULL; p = p->next); ; p->next = p_new; return; } void add_h_to_mh(CURL *h, CURLMHLL *mh) { CURLHLL *h_head; h_head = mh->curlhll_head; if(h_head == NULL) { h_head = create_h_element(h); mh->curlhll_head = h_head; } else { add_h_element(h_head, h); } return; } void cleanup_multi_stuff(CURLMHLL *mhhead) { // move this to it's own cleanup function CURLMHLL *my_mhhead; CURLMHLL *pnext; CURLHLL *cnext; CURLHLL *chhead; CURLMcode curlm_code; CURL *curl_handle; CURLM *curl_multi_handle; if(mhhead == NULL) return; // Remove all of the easy handles from its multi handle my_mhhead = mhhead; pnext = NULL; cnext = NULL; chhead = NULL; do { chhead = my_mhhead->curlhll_head; while(chhead != NULL) { cnext = chhead->next; curl_multi_handle = my_mhhead->handle; curl_handle = chhead->handle; curlm_code = curl_multi_remove_handle(curl_multi_handle, curl_handle); if(curlm_code != CURLM_OK) { syslog(LOG_ERR, "curl_multi_remove_handle code: %d msg: %s", curlm_code, curl_multi_strerror(curlm_code)); } chhead = cnext; } pnext = my_mhhead->next; my_mhhead = pnext; } while(my_mhhead != NULL); // now clean up the easy handles my_mhhead = mhhead; pnext = NULL; cnext = NULL; chhead = NULL; do { chhead = my_mhhead->curlhll_head; while(chhead != NULL) { cnext = chhead->next; destroy_curl_handle(chhead->handle); chhead = cnext; } pnext = my_mhhead->next; my_mhhead = pnext; } while(my_mhhead != NULL); // now cleanup the multi handles my_mhhead = mhhead; pnext = NULL; cnext = NULL; chhead = NULL; do { pnext = my_mhhead->next; curlm_code = curl_multi_cleanup(my_mhhead->handle); if(curlm_code != CURLM_OK) { syslog(LOG_ERR, "curl_multi_cleanup code: %d msg: %s", curlm_code, curl_multi_strerror(curlm_code)); } my_mhhead = pnext; } while(my_mhhead != NULL); // Now free the memory structures my_mhhead = mhhead; pnext = NULL; cnext = NULL; chhead = NULL; do { chhead = my_mhhead->curlhll_head; while(chhead != NULL) { cnext = chhead->next; free(chhead); chhead = cnext; } pnext = my_mhhead->next; free(my_mhhead); my_mhhead = pnext; } while(my_mhhead != NULL); return; }