2010-11-13 23:59:23 +00:00
|
|
|
/*
|
|
|
|
* s3fs - FUSE-based file system backed by Amazon S3
|
|
|
|
*
|
|
|
|
* Copyright 2007-2008 Randy Rizun <rrizun@gmail.com>
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or
|
|
|
|
* modify it under the terms of the GNU General Public License
|
|
|
|
* as published by the Free Software Foundation; either version 2
|
|
|
|
* of the License, or (at your option) any later version.
|
|
|
|
*
|
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
* along with this program; if not, write to the Free Software
|
|
|
|
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <stdio.h>
|
|
|
|
#include <stdlib.h>
|
|
|
|
#include <string.h>
|
|
|
|
#include <unistd.h>
|
|
|
|
#include <fcntl.h>
|
|
|
|
#include <dirent.h>
|
|
|
|
#include <errno.h>
|
2011-03-10 00:11:55 +00:00
|
|
|
#include <time.h>
|
|
|
|
#include <utime.h>
|
|
|
|
#include <sys/stat.h>
|
2010-11-13 23:59:23 +00:00
|
|
|
#include <libgen.h>
|
2011-07-02 02:11:54 +00:00
|
|
|
#include <libxml/xpath.h>
|
|
|
|
#include <libxml/xpathInternals.h>
|
2010-11-13 23:59:23 +00:00
|
|
|
#include <libxml/parser.h>
|
|
|
|
#include <libxml/tree.h>
|
2011-07-02 02:11:54 +00:00
|
|
|
#include <curl/curl.h>
|
2011-02-15 23:32:27 +00:00
|
|
|
#include <openssl/md5.h>
|
2010-11-13 23:59:23 +00:00
|
|
|
#include <pwd.h>
|
|
|
|
#include <grp.h>
|
|
|
|
#include <syslog.h>
|
|
|
|
#include <getopt.h>
|
|
|
|
|
|
|
|
#include <fstream>
|
|
|
|
#include <iostream>
|
|
|
|
#include <vector>
|
|
|
|
#include <algorithm>
|
2011-02-25 17:35:12 +00:00
|
|
|
#include <string>
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-02-25 17:35:12 +00:00
|
|
|
#include "s3fs.h"
|
2011-03-01 19:35:55 +00:00
|
|
|
#include "curl.h"
|
2011-02-25 17:35:12 +00:00
|
|
|
#include "cache.h"
|
2010-11-13 23:59:23 +00:00
|
|
|
#include "string_util.h"
|
|
|
|
|
|
|
|
using namespace std;
|
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
struct s3_object {
|
|
|
|
char *name;
|
|
|
|
struct s3_object *next;
|
|
|
|
};
|
|
|
|
|
2010-12-19 01:34:27 +00:00
|
|
|
class auto_curl_slist {
|
|
|
|
public:
|
|
|
|
auto_curl_slist() : slist(0) { }
|
|
|
|
~auto_curl_slist() { curl_slist_free_all(slist); }
|
|
|
|
|
|
|
|
struct curl_slist* get() const { return slist; }
|
|
|
|
|
|
|
|
void append(const string& s) {
|
|
|
|
slist = curl_slist_append(slist, s.c_str());
|
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
struct curl_slist* slist;
|
|
|
|
};
|
|
|
|
|
2010-12-21 15:24:46 +00:00
|
|
|
typedef struct mvnode {
|
|
|
|
char *old_path;
|
|
|
|
char *new_path;
|
|
|
|
bool is_dir;
|
|
|
|
struct mvnode *prev;
|
|
|
|
struct mvnode *next;
|
|
|
|
} MVNODE;
|
2010-12-19 01:34:27 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
struct head_data {
|
2010-12-20 05:26:27 +00:00
|
|
|
string path;
|
2011-07-02 02:11:54 +00:00
|
|
|
string *url;
|
|
|
|
struct curl_slist *requestHeaders;
|
|
|
|
headers_t *responseHeaders;
|
2010-12-20 05:26:27 +00:00
|
|
|
};
|
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
typedef map<CURL*, head_data> headMap_t;
|
2010-12-20 05:26:27 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
struct cleanup_head_data {
|
|
|
|
void operator()(pair<CURL*, head_data> qqq) {
|
|
|
|
head_data response = qqq.second;
|
|
|
|
delete response.url;
|
|
|
|
curl_slist_free_all(response.requestHeaders);
|
|
|
|
delete response.responseHeaders;
|
2010-12-20 05:26:27 +00:00
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2011-03-01 19:35:55 +00:00
|
|
|
struct case_insensitive_compare_func {
|
|
|
|
bool operator ()(const string &a, const string &b) {
|
|
|
|
return strcasecmp(a.c_str(), b.c_str()) < 0;
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
typedef map<string, string, case_insensitive_compare_func> mimes_t;
|
|
|
|
static mimes_t mimeTypes;
|
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
class auto_head {
|
2010-12-20 05:26:27 +00:00
|
|
|
public:
|
2011-07-02 02:11:54 +00:00
|
|
|
auto_head() {}
|
|
|
|
~auto_head() {
|
|
|
|
for_each(headMap.begin(), headMap.end(), cleanup_head_data());
|
2010-12-20 05:26:27 +00:00
|
|
|
}
|
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
headMap_t& get() { return headMap; }
|
2010-12-20 05:26:27 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
private:
|
|
|
|
headMap_t headMap;
|
2010-12-20 05:26:27 +00:00
|
|
|
};
|
|
|
|
|
2011-08-31 17:37:55 +00:00
|
|
|
time_t get_mtime(const char *s) {
|
|
|
|
return (time_t) strtoul(s, (char **) NULL, 10);
|
|
|
|
}
|
|
|
|
|
2011-08-31 19:02:30 +00:00
|
|
|
off_t get_size(const char *s) {
|
|
|
|
return (off_t) strtoul(s, (char **) NULL, 10);
|
|
|
|
}
|
|
|
|
|
2011-08-31 17:37:55 +00:00
|
|
|
mode_t get_mode(const char *s) {
|
|
|
|
return (mode_t) strtoul(s, (char **) NULL, 10);
|
|
|
|
}
|
|
|
|
|
|
|
|
uid_t get_uid(const char *s) {
|
|
|
|
return (uid_t) strtoul(s, (char **) NULL, 10);
|
|
|
|
}
|
|
|
|
|
|
|
|
gid_t get_gid(const char *s) {
|
|
|
|
return (gid_t) strtoul(s, (char **) NULL, 10);
|
|
|
|
}
|
|
|
|
|
2011-08-31 19:02:30 +00:00
|
|
|
blkcnt_t get_blocks(off_t size) {
|
|
|
|
return size / 512 + 1;
|
|
|
|
}
|
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
static int insert_object(const char *name, struct s3_object **head) {
|
2011-07-02 02:11:54 +00:00
|
|
|
struct s3_object *new_object;
|
|
|
|
|
|
|
|
new_object = (struct s3_object *) malloc(sizeof(struct s3_object));
|
|
|
|
if(new_object == NULL) {
|
|
|
|
printf("insert_object: could not allocate memory\n");
|
|
|
|
exit(EXIT_FAILURE);
|
|
|
|
}
|
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
if(NULL == (new_object->name = strdup(name))){
|
2013-01-19 16:05:07 +00:00
|
|
|
free(new_object);
|
2011-07-02 02:11:54 +00:00
|
|
|
printf("insert_object: could not allocate memory\n");
|
|
|
|
exit(EXIT_FAILURE);
|
|
|
|
}
|
|
|
|
|
|
|
|
if((*head) == NULL)
|
|
|
|
new_object->next = NULL;
|
|
|
|
else
|
|
|
|
new_object->next = (*head);
|
|
|
|
|
|
|
|
*head = new_object;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static unsigned int count_object_list(struct s3_object *list) {
|
|
|
|
unsigned int count = 0;
|
|
|
|
struct s3_object *head = list;
|
|
|
|
|
|
|
|
while(head != NULL) {
|
|
|
|
count++;
|
|
|
|
head = head->next;
|
|
|
|
}
|
|
|
|
|
|
|
|
return count;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int free_object(struct s3_object *object) {
|
|
|
|
free(object->name);
|
|
|
|
free(object);
|
|
|
|
object = NULL;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int free_object_list(struct s3_object *head) {
|
|
|
|
struct s3_object *tmp = NULL;
|
|
|
|
struct s3_object *current = head;
|
|
|
|
|
|
|
|
current = head;
|
|
|
|
while(current != NULL) {
|
|
|
|
tmp = current;
|
|
|
|
current = current->next;
|
|
|
|
free_object(tmp);
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2010-12-21 15:24:46 +00:00
|
|
|
MVNODE *create_mvnode(char *old_path, char *new_path, bool is_dir) {
|
|
|
|
MVNODE *p;
|
|
|
|
char *p_old_path;
|
|
|
|
char *p_new_path;
|
|
|
|
|
|
|
|
p = (MVNODE *) malloc(sizeof(MVNODE));
|
|
|
|
if (p == NULL) {
|
|
|
|
printf("create_mvnode: could not allocation memory for p\n");
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_FAILURE);
|
2010-12-21 15:24:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
p_old_path = (char *)malloc(strlen(old_path)+1);
|
|
|
|
if (p_old_path == NULL) {
|
2013-01-19 16:05:07 +00:00
|
|
|
free(p);
|
2010-12-21 15:24:46 +00:00
|
|
|
printf("create_mvnode: could not allocation memory for p_old_path\n");
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_FAILURE);
|
2010-12-21 15:24:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
strcpy(p_old_path, old_path);
|
|
|
|
|
|
|
|
p_new_path = (char *)malloc(strlen(new_path)+1);
|
|
|
|
if (p_new_path == NULL) {
|
2013-01-19 16:05:07 +00:00
|
|
|
free(p);
|
|
|
|
free(p_old_path);
|
2010-12-21 15:24:46 +00:00
|
|
|
printf("create_mvnode: could not allocation memory for p_new_path\n");
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_FAILURE);
|
2010-12-21 15:24:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
strcpy(p_new_path, new_path);
|
|
|
|
|
|
|
|
p->old_path = p_old_path;
|
|
|
|
p->new_path = p_new_path;
|
|
|
|
p->is_dir = is_dir;
|
|
|
|
p->prev = NULL;
|
|
|
|
p->next = NULL;
|
|
|
|
return p;
|
|
|
|
}
|
|
|
|
|
|
|
|
MVNODE *add_mvnode(MVNODE *head, char *old_path, char *new_path, bool is_dir) {
|
|
|
|
MVNODE *p;
|
|
|
|
MVNODE *tail;
|
|
|
|
|
|
|
|
tail = create_mvnode(old_path, new_path, is_dir);
|
|
|
|
|
|
|
|
for (p = head; p->next != NULL; p = p->next);
|
|
|
|
;
|
|
|
|
|
|
|
|
p->next = tail;
|
|
|
|
tail->prev = p;
|
|
|
|
return tail;
|
|
|
|
}
|
|
|
|
|
|
|
|
void free_mvnodes(MVNODE *head) {
|
|
|
|
MVNODE *my_head;
|
|
|
|
MVNODE *next;
|
|
|
|
char *p_old_path;
|
|
|
|
char *p_new_path;
|
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
if(head == NULL)
|
|
|
|
return;
|
2010-12-21 15:24:46 +00:00
|
|
|
|
|
|
|
my_head = head;
|
|
|
|
next = NULL;
|
|
|
|
|
|
|
|
do {
|
|
|
|
next = my_head->next;
|
|
|
|
p_old_path = my_head->old_path;
|
|
|
|
p_new_path = my_head->new_path;
|
|
|
|
|
|
|
|
free(p_old_path);
|
|
|
|
free(p_new_path);
|
|
|
|
free(my_head);
|
|
|
|
|
|
|
|
my_head = next;
|
|
|
|
} while(my_head != NULL);
|
|
|
|
|
|
|
|
return;
|
|
|
|
}
|
2011-03-01 19:35:55 +00:00
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
static size_t header_callback(void *data, size_t blockSize, size_t numBlocks, void *userPtr) {
|
|
|
|
headers_t* headers = reinterpret_cast<headers_t*>(userPtr);
|
|
|
|
string header(reinterpret_cast<char*>(data), blockSize * numBlocks);
|
|
|
|
string key;
|
|
|
|
stringstream ss(header);
|
|
|
|
if (getline(ss, key, ':')) {
|
|
|
|
string value;
|
|
|
|
getline(ss, value);
|
|
|
|
(*headers)[key] = trim(value);
|
|
|
|
}
|
|
|
|
return blockSize * numBlocks;
|
|
|
|
}
|
|
|
|
|
|
|
|
// safe variant of dirname
|
2011-08-31 17:00:00 +00:00
|
|
|
// dirname clobbers path so let it operate on a tmp copy
|
2010-11-13 23:59:23 +00:00
|
|
|
static string mydirname(string path) {
|
2011-08-31 17:00:00 +00:00
|
|
|
return string(dirname(&path[0]));
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// safe variant of basename
|
2011-08-31 17:00:00 +00:00
|
|
|
// basename clobbers path so let it operate on a tmp copy
|
2010-11-13 23:59:23 +00:00
|
|
|
static string mybasename(string path) {
|
2011-08-31 17:00:00 +00:00
|
|
|
return string(basename(&path[0]));
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// mkdir --parents
|
|
|
|
static int mkdirp(const string& path, mode_t mode) {
|
|
|
|
string base;
|
|
|
|
string component;
|
|
|
|
stringstream ss(path);
|
|
|
|
while (getline(ss, component, '/')) {
|
|
|
|
base += "/" + component;
|
2011-08-31 17:00:00 +00:00
|
|
|
mkdir(base.c_str(), mode);
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
// get user name from uid
|
|
|
|
static string get_username(uid_t uid)
|
|
|
|
{
|
|
|
|
struct passwd* ppw;
|
|
|
|
if(NULL == (ppw = getpwuid(uid)) || NULL == ppw->pw_name){
|
|
|
|
if(foreground){
|
|
|
|
printf(" could not get username(errno=%d).\n", (int)errno);
|
|
|
|
}
|
|
|
|
if(debug){
|
|
|
|
syslog(LOG_DEBUG, "could not get username(errno=%d).\n", (int)errno);
|
|
|
|
}
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
return string(ppw->pw_name);
|
|
|
|
}
|
|
|
|
|
|
|
|
// check uid in group(gid)
|
|
|
|
static int is_uid_inculde_group(uid_t uid, gid_t gid)
|
|
|
|
{
|
|
|
|
static size_t maxlen = 0; // set onece
|
|
|
|
int result;
|
|
|
|
char* pbuf;
|
|
|
|
struct group ginfo;
|
|
|
|
struct group* pginfo = NULL;
|
|
|
|
|
|
|
|
// make buffer
|
|
|
|
if(0 == maxlen){
|
|
|
|
if(0 > (maxlen = (size_t)sysconf(_SC_GETGR_R_SIZE_MAX))){
|
|
|
|
if(foreground){
|
|
|
|
printf(" could not get max name length.\n");
|
|
|
|
}
|
|
|
|
if(debug){
|
|
|
|
syslog(LOG_DEBUG, "could not get max name length.\n");
|
|
|
|
}
|
|
|
|
maxlen = 0;
|
|
|
|
return -ERANGE;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if(NULL == (pbuf = (char*)malloc(sizeof(char) * maxlen))){
|
|
|
|
if(foreground){
|
|
|
|
printf(" failed to allocate memory.\n");
|
|
|
|
}
|
|
|
|
syslog(LOG_ERR, "failed to allocate memory.\n");
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
// get group infomation
|
|
|
|
if(0 != (result = getgrgid_r(gid, &ginfo, pbuf, maxlen, &pginfo))){
|
|
|
|
if(foreground){
|
|
|
|
printf(" could not get group infomation.\n");
|
|
|
|
}
|
|
|
|
if(debug){
|
|
|
|
syslog(LOG_DEBUG, "could not get group infomation.\n");
|
|
|
|
}
|
|
|
|
free(pbuf);
|
|
|
|
return -result;
|
|
|
|
}
|
|
|
|
|
|
|
|
// check group
|
|
|
|
if(NULL == pginfo){
|
|
|
|
// there is not gid in group.
|
|
|
|
free(pbuf);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
string username = get_username(uid);
|
|
|
|
|
|
|
|
char** ppgr_mem;
|
|
|
|
for(ppgr_mem = pginfo->gr_mem; ppgr_mem && *ppgr_mem; ppgr_mem++){
|
|
|
|
if(username == *ppgr_mem){
|
|
|
|
// Found username in group.
|
|
|
|
free(pbuf);
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
free(pbuf);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
//
|
|
|
|
// Get object attributes with stat cache.
|
|
|
|
// This function is base for s3fs_getattr().
|
|
|
|
//
|
|
|
|
static int get_object_attribute(const char *path, struct stat *stbuf) {
|
|
|
|
int result;
|
|
|
|
headers_t meta;
|
|
|
|
char *s3_realpath;
|
|
|
|
|
|
|
|
//if(foreground)
|
|
|
|
// printf(" get_object_attribute[path=%s]\n", path);
|
|
|
|
|
|
|
|
memset(stbuf, 0, sizeof(struct stat));
|
|
|
|
if(strcmp(path, "/") == 0) {
|
|
|
|
stbuf->st_nlink = 1; // see fuse faq
|
|
|
|
stbuf->st_mode = root_mode | S_IFDIR;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
if(get_stat_cache_entry(path, stbuf) == 0) {
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
s3_realpath = get_realpath(path);
|
|
|
|
if((result = curl_get_headers(s3_realpath, meta)) != 0) {
|
|
|
|
free(s3_realpath);
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
free(s3_realpath);
|
|
|
|
|
|
|
|
stbuf->st_nlink = 1; // see fuse faq
|
|
|
|
stbuf->st_mtime = get_mtime(meta["x-amz-meta-mtime"].c_str());
|
|
|
|
if(stbuf->st_mtime == 0){
|
|
|
|
struct tm tm;
|
|
|
|
strptime(meta["Last-Modified"].c_str(), "%a, %d %b %Y %H:%M:%S %Z", &tm);
|
|
|
|
stbuf->st_mtime = mktime(&tm); // GMT
|
|
|
|
}
|
|
|
|
stbuf->st_mode = get_mode(meta["x-amz-meta-mode"].c_str());
|
|
|
|
if(strstr(meta["Content-Type"].c_str(), "x-directory"))
|
|
|
|
stbuf->st_mode |= S_IFDIR;
|
|
|
|
else
|
|
|
|
stbuf->st_mode |= S_IFREG;
|
|
|
|
|
|
|
|
stbuf->st_size = get_size(meta["Content-Length"].c_str());
|
|
|
|
|
|
|
|
if(S_ISREG(stbuf->st_mode))
|
|
|
|
stbuf->st_blocks = get_blocks(stbuf->st_size);
|
|
|
|
|
|
|
|
stbuf->st_uid = get_uid(meta["x-amz-meta-uid"].c_str());
|
|
|
|
stbuf->st_gid = get_gid(meta["x-amz-meta-gid"].c_str());
|
|
|
|
|
|
|
|
add_stat_cache_entry(path, stbuf);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
//
|
|
|
|
// Check the object uid and gid for write/read/execute.
|
|
|
|
// The param "mask" is as same as access() function.
|
|
|
|
// If there is not a target file, this function returns -ENOENT.
|
|
|
|
// If the target file can be accessed, the result always is 0.
|
|
|
|
//
|
|
|
|
// path: the target object path
|
|
|
|
// mask: bit field(F_OK, R_OK, W_OK, X_OK) like access().
|
|
|
|
// stat: NULL or the pointer of struct stat.
|
|
|
|
//
|
|
|
|
static int check_object_access(const char *path, int mask, struct stat* pstbuf)
|
|
|
|
{
|
|
|
|
int result;
|
|
|
|
struct stat st;
|
|
|
|
struct stat* pst = (pstbuf ? pstbuf : &st);
|
|
|
|
struct fuse_context* pcxt;
|
|
|
|
|
|
|
|
//if(foreground)
|
|
|
|
// printf(" check_object_access[path=%s]\n", path);
|
|
|
|
|
|
|
|
if(NULL == (pcxt = fuse_get_context())){
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
|
|
|
|
memset(pst, 0, sizeof(struct stat));
|
|
|
|
if(0 != (result = get_object_attribute(path, pst))){
|
|
|
|
// If there is not tha target file(object), reusult is -ENOENT.
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
if(0 == pcxt->uid){
|
|
|
|
// root is allowed all accessing.
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
if(F_OK == mask){
|
|
|
|
// if there is a file, always return allowed.
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
// compare file mode and uid/gid + mask.
|
|
|
|
mode_t mode = pst->st_mode;
|
|
|
|
mode_t base_mask = 0;
|
|
|
|
if(pcxt->uid == pst->st_uid){
|
|
|
|
base_mask = S_IRWXU;
|
|
|
|
}else if(pcxt->gid == pst->st_gid){
|
|
|
|
base_mask = S_IRWXG;
|
|
|
|
}else{
|
|
|
|
if(1 == is_uid_inculde_group(pcxt->uid, pst->st_gid)){
|
|
|
|
base_mask = S_IRWXG;
|
|
|
|
}else{
|
|
|
|
base_mask = S_IRWXO;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
mode &= base_mask;
|
|
|
|
|
|
|
|
if(X_OK == (mask & X_OK)){
|
|
|
|
if(0 == (mode & (S_IXUSR | S_IXGRP | S_IXOTH))){
|
|
|
|
return -EPERM;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if(W_OK == (mask & W_OK)){
|
|
|
|
if(0 == (mode & (S_IWUSR | S_IWGRP | S_IWOTH))){
|
|
|
|
return -EACCES;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if(R_OK == (mask & R_OK)){
|
|
|
|
if(0 == (mode & (S_IRUSR | S_IRGRP | S_IROTH))){
|
|
|
|
return -EACCES;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if(0 == mode){
|
|
|
|
return -EACCES;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
//
|
|
|
|
// Check accessing the parent directories of the object by uid and gid.
|
|
|
|
//
|
|
|
|
static int check_parent_object_access(const char *path, int mask)
|
|
|
|
{
|
|
|
|
string parent;
|
|
|
|
int result;
|
|
|
|
|
|
|
|
//if(foreground)
|
|
|
|
// printf(" check_parent_object_access[path=%s]\n", path);
|
|
|
|
|
|
|
|
for(parent = mydirname(path); 0 < parent.size(); parent = mydirname(parent.c_str())){
|
|
|
|
if(parent == "."){
|
|
|
|
parent = "/";
|
|
|
|
}
|
|
|
|
if(0 != (result = check_object_access(parent.c_str(), mask, NULL))){
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
if(parent == "/" || parent == "."){
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
// Get fd in mapping data by path
|
|
|
|
static int get_opened_fd(const char* path)
|
|
|
|
{
|
|
|
|
int fd = -1;
|
|
|
|
|
|
|
|
pthread_mutex_lock( &s3fs_descriptors_lock );
|
|
|
|
if(s3fs_pathtofd.find(string(path)) != s3fs_pathtofd.end()){
|
|
|
|
fd = s3fs_pathtofd[string(path)];
|
|
|
|
if(foreground){
|
|
|
|
cout << " get_opened_fd: found fd [path=" << path << "][fd=" << fd << "]" << endl;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
pthread_mutex_unlock( &s3fs_descriptors_lock );
|
|
|
|
|
|
|
|
return fd;
|
|
|
|
}
|
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
int get_local_fd(const char* path) {
|
2011-02-16 16:52:45 +00:00
|
|
|
int fd = -1;
|
2010-12-19 01:34:27 +00:00
|
|
|
int result;
|
2011-03-10 00:11:55 +00:00
|
|
|
struct stat st;
|
2011-06-26 00:37:52 +00:00
|
|
|
char *s3_realpath;
|
2011-02-16 16:52:45 +00:00
|
|
|
CURL *curl = NULL;
|
2011-06-26 00:37:52 +00:00
|
|
|
string url;
|
|
|
|
string resource;
|
2011-02-16 16:52:45 +00:00
|
|
|
string local_md5;
|
2010-11-13 23:59:23 +00:00
|
|
|
string baseName = mybasename(path);
|
|
|
|
string resolved_path(use_cache + "/" + bucket);
|
|
|
|
string cache_path(resolved_path + path);
|
|
|
|
headers_t responseHeaders;
|
|
|
|
|
2011-02-16 16:52:45 +00:00
|
|
|
if(foreground)
|
|
|
|
cout << " get_local_fd[path=" << path << "]" << endl;
|
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
s3_realpath = get_realpath(path);
|
|
|
|
resource = urlEncode(service_path + bucket + s3_realpath);
|
|
|
|
url = host + resource;
|
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
result = curl_get_headers(s3_realpath, responseHeaders);
|
|
|
|
if(result != 0) {
|
|
|
|
free(s3_realpath);
|
|
|
|
return -result;
|
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
if(use_cache.size() > 0) {
|
2010-11-13 23:59:23 +00:00
|
|
|
fd = open(cache_path.c_str(), O_RDWR); // ### TODO should really somehow obey flags here
|
2011-03-10 00:11:55 +00:00
|
|
|
if(fd != -1) {
|
|
|
|
if((fstat(fd, &st)) == -1) {
|
|
|
|
close(fd);
|
2013-01-19 16:05:07 +00:00
|
|
|
free(s3_realpath);
|
2011-03-10 00:11:55 +00:00
|
|
|
YIKES(-errno);
|
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-03-10 00:11:55 +00:00
|
|
|
// if the local and remote mtime/size
|
|
|
|
// do not match we have an invalid cache entry
|
|
|
|
if(str(st.st_size) != responseHeaders["Content-Length"] ||
|
|
|
|
(str(st.st_mtime) != responseHeaders["x-amz-meta-mtime"])) {
|
2013-01-19 16:05:07 +00:00
|
|
|
if(close(fd) == -1){
|
|
|
|
free(s3_realpath);
|
2010-12-19 22:27:56 +00:00
|
|
|
YIKES(-errno);
|
2013-01-19 16:05:07 +00:00
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
fd = -1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2013-01-19 16:05:07 +00:00
|
|
|
free(s3_realpath);
|
2011-02-10 01:07:46 +00:00
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
// need to download?
|
2011-03-10 00:11:55 +00:00
|
|
|
if(fd == -1) {
|
2011-08-31 17:37:55 +00:00
|
|
|
mode_t mode = get_mode(responseHeaders["x-amz-meta-mode"].c_str());
|
2011-06-26 00:37:52 +00:00
|
|
|
|
2011-03-10 00:11:55 +00:00
|
|
|
if(use_cache.size() > 0) {
|
2010-11-13 23:59:23 +00:00
|
|
|
// only download files, not folders
|
|
|
|
if (S_ISREG(mode)) {
|
2011-08-31 17:00:00 +00:00
|
|
|
mkdirp(resolved_path + mydirname(path), 0777);
|
2010-11-13 23:59:23 +00:00
|
|
|
fd = open(cache_path.c_str(), O_CREAT|O_RDWR|O_TRUNC, mode);
|
|
|
|
} else {
|
2011-08-31 17:00:00 +00:00
|
|
|
// its a folder; do *not* create anything in local cache...
|
|
|
|
// TODO: do this in a better way)
|
2010-11-13 23:59:23 +00:00
|
|
|
fd = fileno(tmpfile());
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
fd = fileno(tmpfile());
|
|
|
|
}
|
|
|
|
|
2011-03-10 00:11:55 +00:00
|
|
|
if(fd == -1)
|
2010-12-19 22:27:56 +00:00
|
|
|
YIKES(-errno);
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-03-10 00:11:55 +00:00
|
|
|
FILE *f = fdopen(fd, "w+");
|
2013-01-19 16:05:07 +00:00
|
|
|
if(f == 0){
|
|
|
|
close(fd);
|
2010-12-19 22:27:56 +00:00
|
|
|
YIKES(-errno);
|
2013-01-19 16:05:07 +00:00
|
|
|
}
|
2011-03-10 00:11:55 +00:00
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
if(foreground)
|
|
|
|
cout << " downloading[path=" << path << "][fd=" << fd << "]" << endl;
|
|
|
|
|
|
|
|
if(debug)
|
|
|
|
syslog(LOG_DEBUG, "LOCAL FD");
|
2010-11-13 23:59:23 +00:00
|
|
|
|
|
|
|
auto_curl_slist headers;
|
|
|
|
string date = get_date();
|
2011-06-26 00:37:52 +00:00
|
|
|
string my_url = prepare_url(url.c_str());
|
2010-11-13 23:59:23 +00:00
|
|
|
headers.append("Date: " + date);
|
|
|
|
headers.append("Content-Type: ");
|
2011-03-10 00:11:55 +00:00
|
|
|
if(public_bucket.substr(0,1) != "1") {
|
2010-11-13 23:59:23 +00:00
|
|
|
headers.append("Authorization: AWS " + AWSAccessKeyId + ":" +
|
|
|
|
calc_signature("GET", "", date, headers.get(), resource));
|
|
|
|
}
|
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
curl = create_curl_handle();
|
|
|
|
curl_easy_setopt(curl, CURLOPT_FILE, f);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
|
2010-11-13 23:59:23 +00:00
|
|
|
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
|
|
|
|
2010-12-19 01:34:27 +00:00
|
|
|
result = my_curl_easy_perform(curl, NULL, f);
|
2011-06-26 00:37:52 +00:00
|
|
|
if(result != 0) {
|
|
|
|
destroy_curl_handle(curl);
|
2013-01-19 16:05:07 +00:00
|
|
|
fclose(f);
|
2011-03-10 00:11:55 +00:00
|
|
|
return -result;
|
2011-06-26 00:37:52 +00:00
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
// only one of these is needed...
|
2010-11-13 23:59:23 +00:00
|
|
|
fflush(f);
|
|
|
|
fsync(fd);
|
|
|
|
|
2011-03-10 00:11:55 +00:00
|
|
|
if(fd == -1)
|
2010-12-19 22:27:56 +00:00
|
|
|
YIKES(-errno);
|
2011-03-10 00:11:55 +00:00
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
if(S_ISREG(mode) && !S_ISLNK(mode)) {
|
2011-03-10 00:11:55 +00:00
|
|
|
// make the file's mtime match that of the file on s3
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
// if fd is tmpfile, but we force tor set mtime.
|
|
|
|
struct timeval tv[2];
|
|
|
|
tv[0].tv_sec = get_mtime(responseHeaders["x-amz-meta-mtime"].c_str());
|
|
|
|
tv[0].tv_usec= 0L;
|
|
|
|
tv[1].tv_sec = tv[0].tv_sec;
|
|
|
|
tv[1].tv_usec= 0L;
|
|
|
|
if(-1 == futimes(fd, tv)){
|
2013-01-19 16:05:07 +00:00
|
|
|
fclose(f);
|
2011-03-10 00:11:55 +00:00
|
|
|
YIKES(-errno);
|
|
|
|
}
|
2010-12-19 22:27:56 +00:00
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
2010-12-19 01:34:27 +00:00
|
|
|
destroy_curl_handle(curl);
|
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
return fd;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* create or update s3 meta
|
|
|
|
* @return fuse return code
|
|
|
|
*/
|
2011-03-10 00:11:55 +00:00
|
|
|
static int put_headers(const char *path, headers_t meta) {
|
2010-12-17 04:40:15 +00:00
|
|
|
int result;
|
2011-06-26 00:37:52 +00:00
|
|
|
char *s3_realpath;
|
|
|
|
string url;
|
|
|
|
string resource;
|
2011-08-29 22:01:32 +00:00
|
|
|
struct stat buf;
|
2011-06-26 00:37:52 +00:00
|
|
|
struct BodyStruct body;
|
|
|
|
CURL *curl = NULL;
|
2010-12-28 04:15:23 +00:00
|
|
|
|
|
|
|
if(foreground)
|
|
|
|
cout << " put_headers[path=" << path << "]" << endl;
|
|
|
|
|
2011-08-29 22:01:32 +00:00
|
|
|
// files larger than 5GB must be modified via the multipart interface
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
get_object_attribute(path, &buf);
|
|
|
|
|
2011-08-30 15:20:49 +00:00
|
|
|
if(buf.st_size >= FIVE_GB)
|
2011-08-29 22:01:32 +00:00
|
|
|
return(put_multipart_headers(path, meta));
|
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
s3_realpath = get_realpath(path);
|
|
|
|
resource = urlEncode(service_path + bucket + s3_realpath);
|
|
|
|
url = host + resource;
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
body.text = (char *)malloc(1);
|
|
|
|
body.size = 0;
|
2010-11-13 23:59:23 +00:00
|
|
|
|
|
|
|
auto_curl_slist headers;
|
|
|
|
string date = get_date();
|
|
|
|
headers.append("Date: " + date);
|
|
|
|
|
|
|
|
meta["x-amz-acl"] = default_acl;
|
2011-06-26 00:37:52 +00:00
|
|
|
string ContentType = meta["Content-Type"];
|
2010-11-13 23:59:23 +00:00
|
|
|
|
|
|
|
for (headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter) {
|
|
|
|
string key = (*iter).first;
|
|
|
|
string value = (*iter).second;
|
|
|
|
if (key == "Content-Type")
|
|
|
|
headers.append(key + ":" + value);
|
|
|
|
if (key.substr(0,9) == "x-amz-acl")
|
|
|
|
headers.append(key + ":" + value);
|
|
|
|
if (key.substr(0,10) == "x-amz-meta")
|
|
|
|
headers.append(key + ":" + value);
|
|
|
|
if (key == "x-amz-copy-source")
|
|
|
|
headers.append(key + ":" + value);
|
|
|
|
}
|
|
|
|
|
2011-03-10 00:11:55 +00:00
|
|
|
if(use_rrs.substr(0,1) == "1")
|
2010-11-13 23:59:23 +00:00
|
|
|
headers.append("x-amz-storage-class:REDUCED_REDUNDANCY");
|
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
if(public_bucket.substr(0,1) != "1")
|
2010-11-13 23:59:23 +00:00
|
|
|
headers.append("Authorization: AWS " + AWSAccessKeyId + ":" +
|
|
|
|
calc_signature("PUT", ContentType, date, headers.get(), resource));
|
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
curl = create_curl_handle();
|
|
|
|
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&body);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_UPLOAD, true); // HTTP PUT
|
|
|
|
curl_easy_setopt(curl, CURLOPT_INFILESIZE, 0); // Content-Length
|
2011-03-10 00:11:55 +00:00
|
|
|
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-03-10 00:11:55 +00:00
|
|
|
if(debug)
|
|
|
|
syslog(LOG_DEBUG, "copy path=%s", path);
|
2010-12-09 20:56:29 +00:00
|
|
|
|
|
|
|
if(foreground)
|
2010-12-28 04:15:23 +00:00
|
|
|
cout << " copying[path=" << path << "]" << endl;
|
2010-11-13 23:59:23 +00:00
|
|
|
|
|
|
|
string my_url = prepare_url(url.c_str());
|
|
|
|
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
|
|
|
|
2010-12-19 01:34:27 +00:00
|
|
|
result = my_curl_easy_perform(curl, &body);
|
2010-12-17 04:40:15 +00:00
|
|
|
|
2010-12-19 01:34:27 +00:00
|
|
|
destroy_curl_handle(curl);
|
2011-07-02 02:11:54 +00:00
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
2011-06-26 00:37:52 +00:00
|
|
|
free(s3_realpath);
|
2010-12-17 04:40:15 +00:00
|
|
|
|
2011-03-10 00:11:55 +00:00
|
|
|
if(result != 0)
|
2011-02-16 16:52:45 +00:00
|
|
|
return result;
|
2011-03-10 00:11:55 +00:00
|
|
|
|
|
|
|
// Update mtime in local file cache.
|
|
|
|
if(meta.count("x-amz-meta-mtime") > 0 && use_cache.size() > 0) {
|
|
|
|
struct stat st;
|
|
|
|
struct utimbuf n_mtime;
|
|
|
|
string cache_path(use_cache + "/" + bucket + path);
|
|
|
|
|
|
|
|
if((stat(cache_path.c_str(), &st)) == 0) {
|
2011-08-31 17:37:55 +00:00
|
|
|
n_mtime.modtime = get_mtime(meta["x-amz-meta-mtime"].c_str());
|
2011-08-02 15:08:28 +00:00
|
|
|
n_mtime.actime = n_mtime.modtime;
|
2011-03-10 00:11:55 +00:00
|
|
|
if((utime(cache_path.c_str(), &n_mtime)) == -1) {
|
|
|
|
YIKES(-errno);
|
|
|
|
}
|
|
|
|
}
|
2010-12-17 04:40:15 +00:00
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2011-08-29 22:01:32 +00:00
|
|
|
static int put_multipart_headers(const char *path, headers_t meta) {
|
|
|
|
int result;
|
|
|
|
char *s3_realpath;
|
|
|
|
string url;
|
|
|
|
string resource;
|
|
|
|
string upload_id;
|
|
|
|
struct stat buf;
|
|
|
|
struct BodyStruct body;
|
|
|
|
vector <file_part> parts;
|
|
|
|
|
|
|
|
if(foreground)
|
|
|
|
cout << " put_multipart_headers[path=" << path << "]" << endl;
|
|
|
|
|
|
|
|
s3_realpath = get_realpath(path);
|
|
|
|
resource = urlEncode(service_path + bucket + s3_realpath);
|
|
|
|
url = host + resource;
|
2013-01-19 16:05:07 +00:00
|
|
|
free(s3_realpath);
|
2011-08-29 22:01:32 +00:00
|
|
|
|
|
|
|
body.text = (char *)malloc(1);
|
|
|
|
body.size = 0;
|
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
// already checked by check_object_access(), so only get attr.
|
|
|
|
get_object_attribute(path, &buf);
|
2011-08-29 22:01:32 +00:00
|
|
|
|
|
|
|
upload_id = initiate_multipart_upload(path, buf.st_size, meta);
|
2013-01-19 16:05:07 +00:00
|
|
|
if(upload_id.size() == 0){
|
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
2011-08-29 22:01:32 +00:00
|
|
|
return(-EIO);
|
2013-01-19 16:05:07 +00:00
|
|
|
}
|
2011-08-29 22:01:32 +00:00
|
|
|
|
|
|
|
off_t chunk = 0;
|
|
|
|
off_t bytes_written = 0;
|
|
|
|
off_t bytes_remaining = buf.st_size;
|
|
|
|
while(bytes_remaining > 0) {
|
|
|
|
file_part part;
|
|
|
|
|
|
|
|
if(bytes_remaining > MAX_COPY_SOURCE_SIZE)
|
|
|
|
chunk = MAX_COPY_SOURCE_SIZE;
|
|
|
|
else
|
|
|
|
chunk = bytes_remaining - 1;
|
|
|
|
|
|
|
|
stringstream ss;
|
|
|
|
ss << "bytes=" << bytes_written << "-" << (bytes_written + chunk);
|
|
|
|
meta["x-amz-copy-source-range"] = ss.str();
|
|
|
|
|
|
|
|
part.etag = copy_part(path, path, parts.size() + 1, upload_id, meta);
|
|
|
|
parts.push_back(part);
|
|
|
|
|
|
|
|
bytes_written += (chunk + 1);
|
|
|
|
bytes_remaining = buf.st_size - bytes_written;
|
|
|
|
}
|
|
|
|
|
|
|
|
result = complete_multipart_upload(path, upload_id, parts);
|
|
|
|
if(result != 0) {
|
2013-01-19 16:05:07 +00:00
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
2011-08-29 22:01:32 +00:00
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Update mtime in local file cache.
|
|
|
|
if(meta.count("x-amz-meta-mtime") > 0 && use_cache.size() > 0) {
|
|
|
|
struct stat st;
|
|
|
|
struct utimbuf n_mtime;
|
|
|
|
string cache_path(use_cache + "/" + bucket + path);
|
|
|
|
|
|
|
|
if((stat(cache_path.c_str(), &st)) == 0) {
|
2011-08-31 17:37:55 +00:00
|
|
|
n_mtime.modtime = get_mtime(meta["x-amz-meta-mtime"].c_str());
|
2011-08-29 22:01:32 +00:00
|
|
|
n_mtime.actime = n_mtime.modtime;
|
|
|
|
if((utime(cache_path.c_str(), &n_mtime)) == -1) {
|
2013-01-19 16:05:07 +00:00
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
2011-08-29 22:01:32 +00:00
|
|
|
YIKES(-errno);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
2011-08-29 22:01:32 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2010-12-28 04:15:23 +00:00
|
|
|
static int put_local_fd_small_file(const char* path, headers_t meta, int fd) {
|
2011-06-26 00:37:52 +00:00
|
|
|
string resource;
|
|
|
|
string url;
|
|
|
|
char *s3_realpath;
|
2010-12-28 04:15:23 +00:00
|
|
|
struct stat st;
|
2011-06-26 00:37:52 +00:00
|
|
|
CURL *curl = NULL;
|
|
|
|
|
|
|
|
if(foreground)
|
|
|
|
printf(" put_local_fd_small_file[path=%s][fd=%d]\n", path, fd);
|
|
|
|
|
|
|
|
if(fstat(fd, &st) == -1)
|
2010-12-28 04:15:23 +00:00
|
|
|
YIKES(-errno);
|
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
s3_realpath = get_realpath(path);
|
|
|
|
resource = urlEncode(service_path + bucket + s3_realpath);
|
|
|
|
url = host + resource;
|
2013-01-19 16:05:07 +00:00
|
|
|
free(s3_realpath);
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2010-12-17 04:40:15 +00:00
|
|
|
int result;
|
2011-06-26 00:37:52 +00:00
|
|
|
struct BodyStruct body;
|
|
|
|
auto_curl_slist headers;
|
|
|
|
string date = get_date();
|
|
|
|
|
|
|
|
body.text = (char *) malloc(1);
|
2010-12-28 04:15:23 +00:00
|
|
|
body.size = 0;
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2010-12-19 01:34:27 +00:00
|
|
|
curl = create_curl_handle();
|
2010-12-17 04:40:15 +00:00
|
|
|
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&body);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
|
2010-11-13 23:59:23 +00:00
|
|
|
curl_easy_setopt(curl, CURLOPT_UPLOAD, true); // HTTP PUT
|
|
|
|
curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, static_cast<curl_off_t>(st.st_size)); // Content-Length
|
|
|
|
|
|
|
|
FILE* f = fdopen(fd, "rb");
|
2013-01-19 16:05:07 +00:00
|
|
|
if(f == 0){
|
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
2010-12-19 22:27:56 +00:00
|
|
|
YIKES(-errno);
|
2013-01-19 16:05:07 +00:00
|
|
|
}
|
2011-06-26 00:37:52 +00:00
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
curl_easy_setopt(curl, CURLOPT_INFILE, f);
|
|
|
|
|
|
|
|
string ContentType = meta["Content-Type"];
|
|
|
|
headers.append("Date: " + date);
|
|
|
|
meta["x-amz-acl"] = default_acl;
|
|
|
|
|
|
|
|
for (headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter) {
|
|
|
|
string key = (*iter).first;
|
|
|
|
string value = (*iter).second;
|
|
|
|
if (key == "Content-Type")
|
|
|
|
headers.append(key + ":" + value);
|
|
|
|
if (key.substr(0,9) == "x-amz-acl")
|
|
|
|
headers.append(key + ":" + value);
|
|
|
|
if (key.substr(0,10) == "x-amz-meta")
|
|
|
|
headers.append(key + ":" + value);
|
|
|
|
}
|
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
if(use_rrs.substr(0,1) == "1")
|
2010-11-13 23:59:23 +00:00
|
|
|
headers.append("x-amz-storage-class:REDUCED_REDUNDANCY");
|
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
if(public_bucket.substr(0,1) != "1")
|
2010-11-13 23:59:23 +00:00
|
|
|
headers.append("Authorization: AWS " + AWSAccessKeyId + ":" +
|
|
|
|
calc_signature("PUT", ContentType, date, headers.get(), resource));
|
2011-06-26 00:37:52 +00:00
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
|
|
|
|
|
2010-12-09 20:56:29 +00:00
|
|
|
if(foreground)
|
2011-08-30 22:09:17 +00:00
|
|
|
printf(" uploading[path=%s][fd=%d][size=%zd]\n", path, fd, st.st_size);
|
2010-11-13 23:59:23 +00:00
|
|
|
|
|
|
|
string my_url = prepare_url(url.c_str());
|
|
|
|
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
|
|
|
|
2010-12-19 01:34:27 +00:00
|
|
|
result = my_curl_easy_perform(curl, &body, f);
|
2010-12-17 04:40:15 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
2010-12-19 01:34:27 +00:00
|
|
|
destroy_curl_handle(curl);
|
|
|
|
|
2011-03-01 19:35:55 +00:00
|
|
|
if(result != 0)
|
2010-12-17 04:40:15 +00:00
|
|
|
return result;
|
2010-11-13 23:59:23 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2010-12-28 04:15:23 +00:00
|
|
|
static int put_local_fd_big_file(const char* path, headers_t meta, int fd) {
|
|
|
|
struct stat st;
|
2011-01-21 00:50:55 +00:00
|
|
|
off_t lSize;
|
2010-12-30 03:13:21 +00:00
|
|
|
int partfd = -1;
|
2010-12-28 04:15:23 +00:00
|
|
|
FILE* pSourceFile;
|
|
|
|
FILE* pPartFile;
|
2011-02-15 23:32:27 +00:00
|
|
|
char *buffer;
|
2010-12-28 04:15:23 +00:00
|
|
|
unsigned long lBufferSize = 0;
|
|
|
|
size_t bytesRead;
|
|
|
|
size_t bytesWritten;
|
|
|
|
string uploadId;
|
2011-02-17 17:31:43 +00:00
|
|
|
vector <file_part> parts;
|
2010-12-28 04:15:23 +00:00
|
|
|
|
|
|
|
if(foreground)
|
2011-06-26 00:37:52 +00:00
|
|
|
printf(" put_local_fd_big_file[path=%s][fd=%d]\n", path, fd);
|
2010-12-28 04:15:23 +00:00
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
if(fstat(fd, &st) == -1)
|
2010-12-28 04:15:23 +00:00
|
|
|
YIKES(-errno);
|
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
uploadId = initiate_multipart_upload(path, st.st_size, meta);
|
2010-12-28 04:15:23 +00:00
|
|
|
if(uploadId.size() == 0) {
|
|
|
|
syslog(LOG_ERR, "Could not determine UploadId");
|
|
|
|
return(-EIO);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Open the source file
|
|
|
|
pSourceFile = fdopen(fd, "rb");
|
2011-02-15 23:32:27 +00:00
|
|
|
if(pSourceFile == NULL) {
|
2010-12-28 04:15:23 +00:00
|
|
|
syslog(LOG_ERR, "%d###result=%d", __LINE__, errno); \
|
|
|
|
return(-errno);
|
|
|
|
}
|
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
// Source sucessfully opened, obtain file size:
|
2011-01-21 00:50:55 +00:00
|
|
|
lSize = st.st_size;
|
2010-12-28 04:15:23 +00:00
|
|
|
|
|
|
|
lBufferSize = 0;
|
2011-02-17 17:31:43 +00:00
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
// cycle through open fd, pulling off 10MB chunks at a time
|
2010-12-28 04:15:23 +00:00
|
|
|
while(lSize > 0) {
|
2011-02-17 17:31:43 +00:00
|
|
|
file_part part;
|
|
|
|
|
|
|
|
if(lSize >= MULTIPART_SIZE)
|
|
|
|
lBufferSize = MULTIPART_SIZE;
|
2011-02-15 23:32:27 +00:00
|
|
|
else
|
2010-12-28 04:15:23 +00:00
|
|
|
lBufferSize = lSize;
|
2011-02-15 23:32:27 +00:00
|
|
|
|
2010-12-28 04:15:23 +00:00
|
|
|
lSize = lSize - lBufferSize;
|
|
|
|
|
2011-02-17 17:31:43 +00:00
|
|
|
if((buffer = (char *) malloc(sizeof(char) * lBufferSize)) == NULL) {
|
2011-02-15 23:32:27 +00:00
|
|
|
syslog(LOG_CRIT, "Could not allocate memory for buffer\n");
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_FAILURE);
|
2010-12-28 04:15:23 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// copy the file portion into the buffer:
|
2011-02-15 23:32:27 +00:00
|
|
|
bytesRead = fread(buffer, 1, lBufferSize, pSourceFile);
|
|
|
|
if(bytesRead != lBufferSize) {
|
2011-01-20 22:40:59 +00:00
|
|
|
syslog(LOG_ERR, "%d ### bytesRead:%zu does not match lBufferSize: %lu\n",
|
2010-12-28 04:15:23 +00:00
|
|
|
__LINE__, bytesRead, lBufferSize);
|
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
if(buffer)
|
|
|
|
free(buffer);
|
|
|
|
|
2010-12-28 04:15:23 +00:00
|
|
|
return(-EIO);
|
|
|
|
}
|
|
|
|
|
2011-02-17 17:31:43 +00:00
|
|
|
// create uniq temporary file
|
|
|
|
strncpy(part.path, "/tmp/s3fs.XXXXXX", sizeof part.path);
|
|
|
|
if((partfd = mkstemp(part.path)) == -1) {
|
2011-02-15 23:32:27 +00:00
|
|
|
if(buffer)
|
|
|
|
free(buffer);
|
|
|
|
|
2011-02-17 17:31:43 +00:00
|
|
|
YIKES(-errno);
|
2010-12-28 04:15:23 +00:00
|
|
|
}
|
|
|
|
|
2011-02-17 17:31:43 +00:00
|
|
|
// open a temporary file for upload
|
|
|
|
if((pPartFile = fdopen(partfd, "wb")) == NULL) {
|
2010-12-28 04:15:23 +00:00
|
|
|
syslog(LOG_ERR, "%d ### Could not open temporary file: errno %i\n",
|
|
|
|
__LINE__, errno);
|
2013-01-19 16:05:07 +00:00
|
|
|
close(partfd);
|
2011-02-15 23:32:27 +00:00
|
|
|
if(buffer)
|
|
|
|
free(buffer);
|
|
|
|
|
2010-12-28 04:15:23 +00:00
|
|
|
return(-errno);
|
|
|
|
}
|
|
|
|
|
2011-02-17 17:31:43 +00:00
|
|
|
// copy buffer to temporary file
|
2011-02-15 23:32:27 +00:00
|
|
|
bytesWritten = fwrite(buffer, 1, (size_t)lBufferSize, pPartFile);
|
|
|
|
if(bytesWritten != lBufferSize) {
|
2011-01-20 22:40:59 +00:00
|
|
|
syslog(LOG_ERR, "%d ### bytesWritten:%zu does not match lBufferSize: %lu\n",
|
2010-12-28 04:15:23 +00:00
|
|
|
__LINE__, bytesWritten, lBufferSize);
|
|
|
|
|
|
|
|
fclose(pPartFile);
|
2011-02-15 23:32:27 +00:00
|
|
|
if(buffer)
|
|
|
|
free(buffer);
|
|
|
|
|
2010-12-28 04:15:23 +00:00
|
|
|
return(-EIO);
|
|
|
|
}
|
|
|
|
|
|
|
|
fclose(pPartFile);
|
2011-02-15 23:32:27 +00:00
|
|
|
if(buffer)
|
|
|
|
free(buffer);
|
2010-12-28 04:15:23 +00:00
|
|
|
|
2011-02-17 17:31:43 +00:00
|
|
|
part.etag = upload_part(path, part.path, parts.size() + 1, uploadId);
|
2010-12-28 04:15:23 +00:00
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
// delete temporary part file
|
2011-02-17 17:31:43 +00:00
|
|
|
if(remove(part.path) != 0)
|
|
|
|
YIKES(-errno);
|
2010-12-28 04:15:23 +00:00
|
|
|
|
2011-02-17 17:31:43 +00:00
|
|
|
parts.push_back(part);
|
2011-02-15 23:32:27 +00:00
|
|
|
} // while(lSize > 0)
|
2010-12-28 04:15:23 +00:00
|
|
|
|
2011-02-17 17:31:43 +00:00
|
|
|
return complete_multipart_upload(path, uploadId, parts);
|
2011-02-15 23:32:27 +00:00
|
|
|
}
|
2010-12-28 04:15:23 +00:00
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
/**
|
|
|
|
* create or update s3 object
|
|
|
|
* @return fuse return code
|
|
|
|
*/
|
|
|
|
static int put_local_fd(const char* path, headers_t meta, int fd) {
|
2011-03-10 00:11:55 +00:00
|
|
|
int result;
|
|
|
|
struct stat st;
|
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
if(foreground)
|
|
|
|
cout << " put_local_fd[path=" << path << "][fd=" << fd << "]" << endl;
|
2010-12-28 04:15:23 +00:00
|
|
|
|
2011-03-10 00:11:55 +00:00
|
|
|
if(fstat(fd, &st) == -1)
|
2010-12-28 04:15:23 +00:00
|
|
|
YIKES(-errno);
|
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
/*
|
|
|
|
* Make decision to do multi upload (or not) based upon file size
|
|
|
|
*
|
|
|
|
* According to the AWS spec:
|
|
|
|
* - 1 to 10,000 parts are allowed
|
|
|
|
* - minimum size of parts is 5MB (expect for the last part)
|
|
|
|
*
|
|
|
|
* For our application, we will define part size to be 10MB (10 * 2^20 Bytes)
|
|
|
|
* maximum file size will be ~64 GB - 2 ** 36
|
|
|
|
*
|
|
|
|
* Initially uploads will be done serially
|
|
|
|
*
|
|
|
|
* If file is > 20MB, then multipart will kick in
|
|
|
|
*/
|
2011-02-15 23:32:27 +00:00
|
|
|
if(st.st_size > 68719476735LL ) { // 64GB - 1
|
|
|
|
// close f ?
|
|
|
|
return -ENOTSUP;
|
|
|
|
}
|
2010-12-28 04:15:23 +00:00
|
|
|
|
2011-06-27 02:21:38 +00:00
|
|
|
if(st.st_size >= 20971520 && !nomultipart) { // 20MB
|
2011-02-15 23:32:27 +00:00
|
|
|
// Additional time is needed for large files
|
|
|
|
if(readwrite_timeout < 120)
|
2011-03-10 00:11:55 +00:00
|
|
|
readwrite_timeout = 120;
|
2011-02-15 23:32:27 +00:00
|
|
|
|
|
|
|
result = put_local_fd_big_file(path, meta, fd);
|
|
|
|
} else {
|
|
|
|
result = put_local_fd_small_file(path, meta, fd);
|
|
|
|
}
|
2011-07-07 22:09:40 +00:00
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* initiate_multipart_upload
|
|
|
|
*
|
|
|
|
* Example :
|
|
|
|
* POST /example-object?uploads HTTP/1.1
|
|
|
|
* Host: example-bucket.s3.amazonaws.com
|
|
|
|
* Date: Mon, 1 Nov 2010 20:34:56 GMT
|
|
|
|
* Authorization: AWS VGhpcyBtZXNzYWdlIHNpZ25lZCBieSBlbHZpbmc=
|
|
|
|
*/
|
|
|
|
string initiate_multipart_upload(const char *path, off_t size, headers_t meta) {
|
|
|
|
CURL *curl = NULL;
|
|
|
|
int result;
|
|
|
|
string auth;
|
|
|
|
string acl;
|
|
|
|
string url;
|
|
|
|
string my_url;
|
|
|
|
string date;
|
|
|
|
string raw_date;
|
|
|
|
string resource;
|
|
|
|
string upload_id = "";
|
|
|
|
string ContentType;
|
2011-06-26 00:37:52 +00:00
|
|
|
char *s3_realpath;
|
2011-02-15 23:32:27 +00:00
|
|
|
struct BodyStruct body;
|
|
|
|
struct curl_slist *slist=NULL;
|
|
|
|
|
|
|
|
if(foreground)
|
|
|
|
cout << " initiate_multipart_upload [path=" << path << "][size=" << size << "]" << endl;
|
|
|
|
|
2010-12-28 04:15:23 +00:00
|
|
|
body.text = (char *)malloc(1);
|
|
|
|
body.size = 0;
|
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
s3_realpath = get_realpath(path);
|
|
|
|
resource = urlEncode(service_path + bucket + s3_realpath);
|
2011-02-15 23:32:27 +00:00
|
|
|
resource.append("?uploads");
|
|
|
|
url = host + resource;
|
|
|
|
my_url = prepare_url(url.c_str());
|
2013-01-19 16:05:07 +00:00
|
|
|
free(s3_realpath);
|
2010-12-28 04:15:23 +00:00
|
|
|
|
|
|
|
curl = create_curl_handle();
|
|
|
|
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&body);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
|
2011-02-15 23:32:27 +00:00
|
|
|
curl_easy_setopt(curl, CURLOPT_POST, true);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, 0);
|
2010-12-28 04:15:23 +00:00
|
|
|
|
|
|
|
date.assign("Date: ");
|
|
|
|
raw_date = get_date();
|
|
|
|
date.append(raw_date);
|
2011-02-15 23:32:27 +00:00
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
slist = curl_slist_append(slist, date.c_str());
|
2010-12-28 04:15:23 +00:00
|
|
|
slist = curl_slist_append(slist, "Accept:");
|
2011-02-15 23:32:27 +00:00
|
|
|
slist = curl_slist_append(slist, "Content-Length:");
|
2010-12-28 04:15:23 +00:00
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
ContentType.assign("Content-Type: ");
|
|
|
|
string ctype_data;
|
|
|
|
ctype_data.assign(lookupMimeType(path));
|
|
|
|
ContentType.append(ctype_data);
|
|
|
|
slist = curl_slist_append(slist, ContentType.c_str());
|
2010-12-28 04:15:23 +00:00
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
// x-amz headers: (a) alphabetical order and (b) no spaces after colon
|
|
|
|
acl.assign("x-amz-acl:");
|
|
|
|
acl.append(default_acl);
|
|
|
|
slist = curl_slist_append(slist, acl.c_str());
|
|
|
|
|
|
|
|
for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter) {
|
|
|
|
string key = (*iter).first;
|
|
|
|
string value = (*iter).second;
|
|
|
|
|
|
|
|
if(key.substr(0,10) == "x-amz-meta") {
|
|
|
|
string entry;
|
|
|
|
entry.assign(key);
|
|
|
|
entry.append(":");
|
|
|
|
entry.append(value);
|
|
|
|
slist = curl_slist_append(slist, entry.c_str());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if(use_rrs.substr(0,1) == "1")
|
|
|
|
slist = curl_slist_append(slist, "x-amz-storage-class:REDUCED_REDUNDANCY");
|
|
|
|
|
|
|
|
if(public_bucket.substr(0,1) != "1") {
|
2010-12-28 04:15:23 +00:00
|
|
|
auth.assign("Authorization: AWS ");
|
|
|
|
auth.append(AWSAccessKeyId);
|
|
|
|
auth.append(":");
|
2011-02-15 23:32:27 +00:00
|
|
|
auth.append(calc_signature("POST", ctype_data, raw_date, slist, resource));
|
2010-12-28 04:15:23 +00:00
|
|
|
slist = curl_slist_append(slist, auth.c_str());
|
|
|
|
}
|
|
|
|
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
result = my_curl_easy_perform(curl, &body);
|
2010-12-28 04:15:23 +00:00
|
|
|
|
|
|
|
curl_slist_free_all(slist);
|
|
|
|
destroy_curl_handle(curl);
|
|
|
|
|
|
|
|
if(result != 0) {
|
2011-07-02 02:11:54 +00:00
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
2011-02-15 23:32:27 +00:00
|
|
|
return upload_id;
|
|
|
|
}
|
2010-12-28 04:15:23 +00:00
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
// XML returns UploadId
|
|
|
|
// Parse XML body for UploadId
|
|
|
|
upload_id.clear();
|
|
|
|
xmlDocPtr doc = xmlReadMemory(body.text, body.size, "", NULL, 0);
|
|
|
|
if(doc != NULL && doc->children != NULL) {
|
|
|
|
for(xmlNodePtr cur_node = doc->children->children;
|
|
|
|
cur_node != NULL;
|
|
|
|
cur_node = cur_node->next) {
|
|
|
|
|
|
|
|
// string cur_node_name(reinterpret_cast<const char *>(cur_node->name));
|
|
|
|
// printf("cur_node_name: %s\n", cur_node_name.c_str());
|
|
|
|
|
|
|
|
if(cur_node->type == XML_ELEMENT_NODE) {
|
|
|
|
string elementName = reinterpret_cast<const char*>(cur_node->name);
|
|
|
|
// printf("elementName: %s\n", elementName.c_str());
|
|
|
|
if(cur_node->children != NULL) {
|
|
|
|
if(cur_node->children->type == XML_TEXT_NODE) {
|
|
|
|
if(elementName == "UploadId") {
|
|
|
|
upload_id = reinterpret_cast<const char *>(cur_node->children->content);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} // for (xmlNodePtr cur_node = doc->children->children;
|
|
|
|
} // if (doc != NULL && doc->children != NULL)
|
|
|
|
xmlFreeDoc(doc);
|
2010-12-28 04:15:23 +00:00
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
// clean up
|
2011-07-02 02:11:54 +00:00
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
2011-02-15 23:32:27 +00:00
|
|
|
body.size = 0;
|
2010-12-28 04:15:23 +00:00
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
return upload_id;
|
|
|
|
}
|
2010-12-28 04:15:23 +00:00
|
|
|
|
2011-02-17 17:31:43 +00:00
|
|
|
static int complete_multipart_upload(const char *path, string upload_id,
|
|
|
|
vector <file_part> parts) {
|
2011-02-15 23:32:27 +00:00
|
|
|
CURL *curl = NULL;
|
|
|
|
char *pData;
|
|
|
|
int result;
|
2011-02-17 17:31:43 +00:00
|
|
|
int i, j;
|
2011-02-15 23:32:27 +00:00
|
|
|
string auth;
|
|
|
|
string date;
|
|
|
|
string raw_date;
|
|
|
|
string url;
|
|
|
|
string my_url;
|
|
|
|
string resource;
|
|
|
|
string postContent;
|
2011-06-26 00:37:52 +00:00
|
|
|
char *s3_realpath;
|
2011-02-15 23:32:27 +00:00
|
|
|
struct BodyStruct body;
|
|
|
|
struct WriteThis pooh;
|
|
|
|
struct curl_slist *slist = NULL;
|
2010-12-28 04:15:23 +00:00
|
|
|
|
|
|
|
if(foreground)
|
2011-02-15 23:32:27 +00:00
|
|
|
cout << " complete_multipart_upload [path=" << path << "]" << endl;
|
2010-12-28 04:15:23 +00:00
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
// initialization of variables
|
|
|
|
curl = NULL;
|
2010-12-28 04:15:23 +00:00
|
|
|
|
|
|
|
postContent.clear();
|
|
|
|
postContent.append("<CompleteMultipartUpload>\n");
|
2011-02-17 17:31:43 +00:00
|
|
|
for(i = 0, j = parts.size(); i < j; i++) {
|
2010-12-28 04:15:23 +00:00
|
|
|
postContent.append(" <Part>\n");
|
|
|
|
postContent.append(" <PartNumber>");
|
|
|
|
postContent.append(IntToStr(i+1));
|
|
|
|
postContent.append("</PartNumber>\n");
|
|
|
|
postContent.append(" <ETag>");
|
2011-02-17 17:31:43 +00:00
|
|
|
postContent.append(parts[i].etag.insert(0, "\"").append("\""));
|
2010-12-28 04:15:23 +00:00
|
|
|
postContent.append("</ETag>\n");
|
|
|
|
postContent.append(" </Part>\n");
|
|
|
|
}
|
|
|
|
postContent.append("</CompleteMultipartUpload>\n");
|
|
|
|
|
2011-02-17 17:31:43 +00:00
|
|
|
if((pData = (char *)malloc(postContent.size() + 1)) == NULL)
|
|
|
|
YIKES(-errno)
|
2010-12-28 04:15:23 +00:00
|
|
|
|
|
|
|
pooh.readptr = pData;
|
|
|
|
pooh.sizeleft = postContent.size();
|
|
|
|
|
|
|
|
strcpy(pData, postContent.c_str());
|
|
|
|
|
|
|
|
postContent.clear();
|
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
s3_realpath = get_realpath(path);
|
|
|
|
resource = urlEncode(service_path + bucket + s3_realpath);
|
2010-12-28 04:15:23 +00:00
|
|
|
resource.append("?uploadId=");
|
2011-02-15 23:32:27 +00:00
|
|
|
resource.append(upload_id);
|
2010-12-28 04:15:23 +00:00
|
|
|
url = host + resource;
|
|
|
|
my_url = prepare_url(url.c_str());
|
|
|
|
|
|
|
|
body.text = (char *)malloc(1);
|
|
|
|
body.size = 0;
|
|
|
|
|
|
|
|
curl = create_curl_handle();
|
|
|
|
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&body);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_POST, true);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, (curl_off_t)pooh.sizeleft);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_callback);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_READDATA, &pooh);
|
|
|
|
|
|
|
|
date.assign("Date: ");
|
|
|
|
raw_date = get_date();
|
|
|
|
date.append(raw_date);
|
|
|
|
slist = NULL;
|
|
|
|
slist = curl_slist_append(slist, date.c_str());
|
|
|
|
|
|
|
|
slist = curl_slist_append(slist, "Accept:");
|
|
|
|
slist = curl_slist_append(slist, "Content-Type:");
|
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
if(public_bucket.substr(0,1) != "1") {
|
2010-12-28 04:15:23 +00:00
|
|
|
auth.assign("Authorization: AWS ");
|
|
|
|
auth.append(AWSAccessKeyId);
|
|
|
|
auth.append(":");
|
|
|
|
auth.append(calc_signature("POST", "", raw_date, slist, resource));
|
|
|
|
slist = curl_slist_append(slist, auth.c_str());
|
|
|
|
}
|
|
|
|
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
|
|
|
|
|
|
|
result = my_curl_easy_perform(curl, &body);
|
|
|
|
|
|
|
|
curl_slist_free_all(slist);
|
|
|
|
destroy_curl_handle(curl);
|
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
2010-12-28 04:15:23 +00:00
|
|
|
free(pData);
|
2011-06-26 00:37:52 +00:00
|
|
|
free(s3_realpath);
|
2011-02-15 23:32:27 +00:00
|
|
|
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
string upload_part(const char *path, const char *source, int part_number, string upload_id) {
|
2011-02-22 21:28:01 +00:00
|
|
|
int fd;
|
2011-02-15 23:32:27 +00:00
|
|
|
CURL *curl = NULL;
|
|
|
|
FILE *part_file;
|
|
|
|
int result;
|
|
|
|
string url;
|
|
|
|
string my_url;
|
|
|
|
string auth;
|
|
|
|
string resource;
|
|
|
|
string date;
|
|
|
|
string raw_date;
|
|
|
|
string ETag;
|
2011-06-26 00:37:52 +00:00
|
|
|
char *s3_realpath;
|
2011-02-15 23:32:27 +00:00
|
|
|
struct stat st;
|
|
|
|
struct BodyStruct body;
|
|
|
|
struct BodyStruct header;
|
|
|
|
struct curl_slist *slist = NULL;
|
|
|
|
|
|
|
|
// Now upload the file as the nth part
|
|
|
|
if(foreground)
|
|
|
|
cout << " multipart upload [path=" << path << "][part=" << part_number << "]" << endl;
|
|
|
|
|
|
|
|
// PUT /ObjectName?partNumber=PartNumber&uploadId=UploadId HTTP/1.1
|
|
|
|
// Host: BucketName.s3.amazonaws.com
|
|
|
|
// Date: date
|
|
|
|
// Content-Length: Size
|
|
|
|
// Authorization: Signature
|
|
|
|
|
|
|
|
// PUT /my-movie.m2ts?partNumber=1&uploadId=VCVsb2FkIElEIGZvciBlbZZpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZR HTTP/1.1
|
|
|
|
// Host: example-bucket.s3.amazonaws.com
|
|
|
|
// Date: Mon, 1 Nov 2010 20:34:56 GMT
|
|
|
|
// Content-Length: 10485760
|
|
|
|
// Content-MD5: pUNXr/BjKK5G2UKvaRRrOA==
|
|
|
|
// Authorization: AWS VGhpcyBtZXNzYWdlIHNpZ25lZGGieSRlbHZpbmc=
|
|
|
|
|
|
|
|
part_file = fopen(source, "rb");
|
|
|
|
if(part_file == NULL) {
|
|
|
|
syslog(LOG_ERR, "%d###result=%d", __LINE__, errno); \
|
|
|
|
return "";
|
2010-12-28 04:15:23 +00:00
|
|
|
}
|
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
if(fstat(fileno(part_file), &st) == -1) {
|
2013-01-19 16:05:07 +00:00
|
|
|
fclose(part_file);
|
2011-02-15 23:32:27 +00:00
|
|
|
syslog(LOG_ERR, "%d###result=%d", __LINE__, errno); \
|
|
|
|
return "";
|
2010-12-28 04:15:23 +00:00
|
|
|
}
|
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
s3_realpath = get_realpath(path);
|
|
|
|
resource = urlEncode(service_path + bucket + s3_realpath);
|
2013-01-19 16:05:07 +00:00
|
|
|
free(s3_realpath);
|
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
resource.append("?partNumber=");
|
|
|
|
resource.append(IntToStr(part_number));
|
|
|
|
resource.append("&uploadId=");
|
|
|
|
resource.append(upload_id);
|
|
|
|
url = host + resource;
|
|
|
|
my_url = prepare_url(url.c_str());
|
2010-12-28 04:15:23 +00:00
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
body.text = (char *)malloc(1);
|
|
|
|
body.size = 0;
|
2010-12-28 04:15:23 +00:00
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
header.text = (char *)malloc(1);
|
|
|
|
header.size = 0;
|
2010-12-28 04:15:23 +00:00
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
curl = create_curl_handle();
|
|
|
|
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&body);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void *)&header);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, WriteMemoryCallback);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_UPLOAD, true); // HTTP PUT
|
|
|
|
curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, static_cast<curl_off_t>(st.st_size)); // Content-Length
|
|
|
|
curl_easy_setopt(curl, CURLOPT_INFILE, part_file);
|
2010-12-28 04:15:23 +00:00
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
date.assign("Date: ");
|
|
|
|
raw_date = get_date();
|
|
|
|
date.append(raw_date);
|
|
|
|
slist = NULL;
|
|
|
|
slist = curl_slist_append(slist, date.c_str());
|
|
|
|
slist = curl_slist_append(slist, "Accept:");
|
2010-12-28 04:15:23 +00:00
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
if(public_bucket.substr(0,1) != "1") {
|
2011-06-26 00:37:52 +00:00
|
|
|
auth.assign("Authorization: AWS ");
|
|
|
|
auth.append(AWSAccessKeyId);
|
|
|
|
auth.append(":");
|
|
|
|
auth.append(calc_signature("PUT", "", raw_date, slist, resource));
|
2011-02-15 23:32:27 +00:00
|
|
|
slist = curl_slist_append(slist, auth.c_str());
|
|
|
|
}
|
|
|
|
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
2010-12-28 04:15:23 +00:00
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
result = my_curl_easy_perform(curl, &body, part_file);
|
2010-12-28 04:15:23 +00:00
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
curl_slist_free_all(slist);
|
|
|
|
destroy_curl_handle(curl);
|
|
|
|
fclose(part_file);
|
2010-12-28 04:15:23 +00:00
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
if(result != 0) {
|
2011-07-02 02:11:54 +00:00
|
|
|
if(header.text)
|
|
|
|
free(header.text);
|
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
2011-02-15 23:32:27 +00:00
|
|
|
return "";
|
2010-12-28 04:15:23 +00:00
|
|
|
}
|
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
// calculate local md5sum, if it matches the header
|
|
|
|
// ETag value, the upload was successful.
|
2011-02-22 21:28:01 +00:00
|
|
|
if((fd = open(source, O_RDONLY)) == -1) {
|
2011-07-02 02:11:54 +00:00
|
|
|
if(header.text)
|
|
|
|
free(header.text);
|
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
2011-02-22 21:28:01 +00:00
|
|
|
syslog(LOG_ERR, "%d###result=%d", __LINE__, -fd);
|
|
|
|
|
|
|
|
return "";
|
|
|
|
}
|
|
|
|
|
|
|
|
string md5 = md5sum(fd);
|
|
|
|
close(fd);
|
2011-02-15 23:32:27 +00:00
|
|
|
if(!md5.empty() && strstr(header.text, md5.c_str())) {
|
|
|
|
ETag.assign(md5);
|
2011-08-29 22:01:32 +00:00
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
} else {
|
2011-07-02 02:11:54 +00:00
|
|
|
if(header.text)
|
|
|
|
free(header.text);
|
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
2011-02-15 23:32:27 +00:00
|
|
|
return "";
|
2010-12-28 04:15:23 +00:00
|
|
|
}
|
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
// clean up
|
2011-07-02 02:11:54 +00:00
|
|
|
if(header.text)
|
|
|
|
free(header.text);
|
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
2010-12-30 03:13:21 +00:00
|
|
|
|
2011-02-15 23:32:27 +00:00
|
|
|
return ETag;
|
|
|
|
}
|
|
|
|
|
2011-08-29 22:01:32 +00:00
|
|
|
string copy_part(const char *from, const char *to, int part_number, string upload_id, headers_t meta) {
|
|
|
|
CURL *curl = NULL;
|
|
|
|
int result;
|
|
|
|
string url;
|
|
|
|
string my_url;
|
|
|
|
string auth;
|
|
|
|
string resource;
|
|
|
|
string raw_date;
|
|
|
|
string ETag;
|
|
|
|
char *s3_realpath;
|
|
|
|
struct BodyStruct body;
|
|
|
|
struct BodyStruct header;
|
|
|
|
|
|
|
|
// Now copy the file as the nth part
|
|
|
|
if(foreground)
|
|
|
|
printf("copy_part [from=%s] [to=%s]\n", from, to);
|
|
|
|
|
|
|
|
s3_realpath = get_realpath(to);
|
|
|
|
resource = urlEncode(service_path + bucket + s3_realpath);
|
2013-01-19 16:05:07 +00:00
|
|
|
free(s3_realpath);
|
|
|
|
|
2011-08-29 22:01:32 +00:00
|
|
|
resource.append("?partNumber=");
|
|
|
|
resource.append(IntToStr(part_number));
|
|
|
|
resource.append("&uploadId=");
|
|
|
|
resource.append(upload_id);
|
|
|
|
url = host + resource;
|
|
|
|
my_url = prepare_url(url.c_str());
|
|
|
|
|
|
|
|
body.text = (char *)malloc(1);
|
|
|
|
body.size = 0;
|
|
|
|
header.text = (char *)malloc(1);
|
|
|
|
header.size = 0;
|
|
|
|
|
|
|
|
auto_curl_slist headers;
|
|
|
|
string date = get_date();
|
|
|
|
headers.append("Date: " + date);
|
|
|
|
|
|
|
|
string ContentType = meta["Content-Type"];
|
|
|
|
meta["x-amz-acl"] = default_acl;
|
|
|
|
|
|
|
|
for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter) {
|
|
|
|
string key = (*iter).first;
|
|
|
|
string value = (*iter).second;
|
|
|
|
if (key == "Content-Type")
|
|
|
|
headers.append(key + ":" + value);
|
|
|
|
if (key == "x-amz-copy-source")
|
|
|
|
headers.append(key + ":" + value);
|
|
|
|
if (key == "x-amz-copy-source-range")
|
|
|
|
headers.append(key + ":" + value);
|
|
|
|
}
|
|
|
|
|
|
|
|
if(use_rrs.substr(0,1) == "1")
|
|
|
|
headers.append("x-amz-storage-class:REDUCED_REDUNDANCY");
|
|
|
|
|
|
|
|
if(public_bucket.substr(0,1) != "1")
|
|
|
|
headers.append("Authorization: AWS " + AWSAccessKeyId + ":" +
|
|
|
|
calc_signature("PUT", ContentType, date, headers.get(), resource));
|
|
|
|
|
|
|
|
curl = create_curl_handle();
|
|
|
|
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&body);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void *)&header);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, WriteMemoryCallback);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_UPLOAD, true); // HTTP PUT
|
|
|
|
curl_easy_setopt(curl, CURLOPT_INFILESIZE, 0); // Content-Length
|
|
|
|
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
|
|
|
|
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
|
|
|
|
|
|
|
result = my_curl_easy_perform(curl, &body);
|
|
|
|
destroy_curl_handle(curl);
|
|
|
|
|
|
|
|
if(result != 0) {
|
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
|
|
|
if(header.text)
|
|
|
|
free(header.text);
|
|
|
|
return "";
|
|
|
|
}
|
|
|
|
|
|
|
|
char *start_etag;
|
|
|
|
char *end_etag;
|
|
|
|
start_etag = strstr(body.text, "ETag");
|
|
|
|
end_etag = strstr(body.text, "/ETag>");
|
|
|
|
start_etag += 11;
|
|
|
|
ETag.assign(start_etag, (size_t)(end_etag - start_etag - 7));
|
|
|
|
|
|
|
|
// clean up
|
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
|
|
|
if(header.text)
|
|
|
|
free(header.text);
|
|
|
|
|
|
|
|
return ETag;
|
|
|
|
}
|
|
|
|
|
2011-02-22 21:28:01 +00:00
|
|
|
string md5sum(int fd) {
|
2011-02-15 23:32:27 +00:00
|
|
|
MD5_CTX c;
|
|
|
|
char buf[512];
|
|
|
|
char hexbuf[3];
|
|
|
|
ssize_t bytes;
|
2011-08-30 15:20:49 +00:00
|
|
|
char md5[2 * MD5_DIGEST_LENGTH + 1];
|
2011-02-15 23:32:27 +00:00
|
|
|
unsigned char *result = (unsigned char *) malloc(MD5_DIGEST_LENGTH);
|
|
|
|
|
|
|
|
memset(buf, 0, 512);
|
|
|
|
MD5_Init(&c);
|
|
|
|
while((bytes = read(fd, buf, 512)) > 0) {
|
|
|
|
MD5_Update(&c, buf, bytes);
|
|
|
|
memset(buf, 0, 512);
|
|
|
|
}
|
|
|
|
|
|
|
|
MD5_Final(result, &c);
|
|
|
|
|
|
|
|
memset(md5, 0, 2 * MD5_DIGEST_LENGTH + 1);
|
|
|
|
for(int i = 0; i < MD5_DIGEST_LENGTH; i++) {
|
|
|
|
snprintf(hexbuf, 3, "%02x", result[i]);
|
|
|
|
strncat(md5, hexbuf, 2);
|
|
|
|
}
|
|
|
|
|
|
|
|
free(result);
|
2011-02-22 21:28:01 +00:00
|
|
|
lseek(fd, 0, 0);
|
2011-02-15 23:32:27 +00:00
|
|
|
|
2011-08-30 15:20:49 +00:00
|
|
|
return string(md5);
|
2010-12-28 04:15:23 +00:00
|
|
|
}
|
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
static int s3fs_getattr(const char *path, struct stat *stbuf)
|
|
|
|
{
|
2011-06-26 00:37:52 +00:00
|
|
|
int result;
|
2010-12-19 01:34:27 +00:00
|
|
|
|
|
|
|
if(foreground)
|
2011-08-31 19:02:30 +00:00
|
|
|
printf("s3fs_getattr[path=%s]\n", path);
|
2010-12-17 04:40:15 +00:00
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
// check parent directory attribute.
|
|
|
|
if(0 != (result = check_parent_object_access(path, X_OK))){
|
2011-06-26 00:37:52 +00:00
|
|
|
return result;
|
2011-09-01 19:24:12 +00:00
|
|
|
}
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
return check_object_access(path, F_OK, stbuf);
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int s3fs_readlink(const char *path, char *buf, size_t size) {
|
2010-12-19 22:27:56 +00:00
|
|
|
int fd = -1;
|
2010-11-13 23:59:23 +00:00
|
|
|
if (size > 0) {
|
|
|
|
--size; // reserve nil terminator
|
|
|
|
|
2010-12-09 20:56:29 +00:00
|
|
|
if(foreground)
|
2013-01-19 16:05:07 +00:00
|
|
|
cout << "s3fs_readlink[path=" << path << "]" << endl;
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2010-12-19 22:27:56 +00:00
|
|
|
fd = get_local_fd(path);
|
|
|
|
if(fd < 0) {
|
2011-06-26 00:37:52 +00:00
|
|
|
syslog(LOG_ERR, "line %d: get_local_fd: %d", __LINE__, -fd);
|
2010-12-19 22:27:56 +00:00
|
|
|
return -EIO;
|
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
|
|
|
|
struct stat st;
|
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
if(fstat(fd, &st) == -1) {
|
|
|
|
syslog(LOG_ERR, "line %d: fstat: %d", __LINE__, -errno);
|
|
|
|
|
|
|
|
if(fd > 0)
|
|
|
|
close(fd);
|
|
|
|
|
2010-12-19 22:27:56 +00:00
|
|
|
return -errno;
|
|
|
|
}
|
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
if(st.st_size < (off_t)size)
|
2010-11-13 23:59:23 +00:00
|
|
|
size = st.st_size;
|
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
if(pread(fd, buf, size, 0) == -1) {
|
|
|
|
syslog(LOG_ERR, "line %d: pread: %d", __LINE__, -errno);
|
|
|
|
|
|
|
|
if(fd > 0)
|
|
|
|
close(fd);
|
|
|
|
|
2010-12-19 22:27:56 +00:00
|
|
|
return -errno;
|
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
|
|
|
|
buf[size] = 0;
|
|
|
|
}
|
|
|
|
|
2011-03-01 19:35:55 +00:00
|
|
|
if(fd > 0)
|
|
|
|
close(fd);
|
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* @param s e.g., "index.html"
|
|
|
|
* @return e.g., "text/html"
|
|
|
|
*/
|
|
|
|
string lookupMimeType(string s) {
|
|
|
|
string result("application/octet-stream");
|
|
|
|
string::size_type last_pos = s.find_last_of('.');
|
|
|
|
string::size_type first_pos = s.find_first_of('.');
|
|
|
|
string prefix, ext, ext2;
|
|
|
|
|
|
|
|
// No dots in name, just return
|
2011-03-01 19:35:55 +00:00
|
|
|
if(last_pos == string::npos)
|
|
|
|
return result;
|
2010-11-13 23:59:23 +00:00
|
|
|
|
|
|
|
// extract the last extension
|
2011-03-01 19:35:55 +00:00
|
|
|
if(last_pos != string::npos)
|
2010-11-13 23:59:23 +00:00
|
|
|
ext = s.substr(1+last_pos, string::npos);
|
|
|
|
|
|
|
|
if (last_pos != string::npos) {
|
|
|
|
// one dot was found, now look for another
|
|
|
|
if (first_pos != string::npos && first_pos < last_pos) {
|
|
|
|
prefix = s.substr(0, last_pos);
|
|
|
|
// Now get the second to last file extension
|
|
|
|
string::size_type next_pos = prefix.find_last_of('.');
|
|
|
|
if (next_pos != string::npos) {
|
|
|
|
ext2 = prefix.substr(1+next_pos, string::npos);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// if we get here, then we have an extension (ext)
|
|
|
|
mimes_t::const_iterator iter = mimeTypes.find(ext);
|
|
|
|
// if the last extension matches a mimeType, then return
|
|
|
|
// that mime type
|
|
|
|
if (iter != mimeTypes.end()) {
|
|
|
|
result = (*iter).second;
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
// return with the default result if there isn't a second extension
|
2011-03-01 19:35:55 +00:00
|
|
|
if(first_pos == last_pos)
|
2010-11-13 23:59:23 +00:00
|
|
|
return result;
|
|
|
|
|
|
|
|
// Didn't find a mime-type for the first extension
|
|
|
|
// Look for second extension in mimeTypes, return if found
|
|
|
|
iter = mimeTypes.find(ext2);
|
|
|
|
if (iter != mimeTypes.end()) {
|
|
|
|
result = (*iter).second;
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
// neither the last extension nor the second-to-last extension
|
|
|
|
// matched a mimeType, return the default mime type
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2010-12-22 17:19:52 +00:00
|
|
|
// common function for creation of a plain object
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
static int create_file_object(const char *path, mode_t mode, uid_t uid, gid_t gid) {
|
2010-12-19 01:34:27 +00:00
|
|
|
int result;
|
2011-06-26 00:37:52 +00:00
|
|
|
char *s3_realpath;
|
|
|
|
CURL *curl = NULL;
|
2010-12-22 17:19:52 +00:00
|
|
|
|
2010-12-09 20:56:29 +00:00
|
|
|
if(foreground)
|
2011-08-31 20:17:53 +00:00
|
|
|
printf(" create_file_object[path=%s][mode=%d]\n", path, mode);
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
s3_realpath = get_realpath(path);
|
|
|
|
string resource = urlEncode(service_path + bucket + s3_realpath);
|
2010-11-13 23:59:23 +00:00
|
|
|
string url = host + resource;
|
|
|
|
|
|
|
|
string date = get_date();
|
2011-06-26 00:37:52 +00:00
|
|
|
string my_url = prepare_url(url.c_str());
|
|
|
|
auto_curl_slist headers;
|
2010-11-13 23:59:23 +00:00
|
|
|
headers.append("Date: " + date);
|
|
|
|
string contentType(lookupMimeType(path));
|
|
|
|
headers.append("Content-Type: " + contentType);
|
|
|
|
// x-amz headers: (a) alphabetical order and (b) no spaces after colon
|
|
|
|
headers.append("x-amz-acl:" + default_acl);
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
headers.append("x-amz-meta-gid:" + str(gid));
|
2010-11-13 23:59:23 +00:00
|
|
|
headers.append("x-amz-meta-mode:" + str(mode));
|
|
|
|
headers.append("x-amz-meta-mtime:" + str(time(NULL)));
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
headers.append("x-amz-meta-uid:" + str(uid));
|
2011-06-26 00:37:52 +00:00
|
|
|
if(public_bucket.substr(0,1) != "1")
|
2010-11-13 23:59:23 +00:00
|
|
|
headers.append("Authorization: AWS " + AWSAccessKeyId + ":" +
|
|
|
|
calc_signature("PUT", contentType, date, headers.get(), resource));
|
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
curl = create_curl_handle();
|
|
|
|
curl_easy_setopt(curl, CURLOPT_UPLOAD, true); // HTTP PUT
|
|
|
|
curl_easy_setopt(curl, CURLOPT_INFILESIZE, 0); // Content-Length: 0
|
|
|
|
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
|
2010-11-13 23:59:23 +00:00
|
|
|
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
|
|
|
|
2010-12-19 01:34:27 +00:00
|
|
|
result = my_curl_easy_perform(curl);
|
|
|
|
destroy_curl_handle(curl);
|
2011-06-26 00:37:52 +00:00
|
|
|
free(s3_realpath);
|
2010-12-19 01:34:27 +00:00
|
|
|
|
2011-03-01 19:35:55 +00:00
|
|
|
if(result != 0)
|
|
|
|
return result;
|
2010-11-13 23:59:23 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
static int s3fs_mknod(const char *path, mode_t mode, dev_t rdev)
|
|
|
|
{
|
2010-12-22 17:19:52 +00:00
|
|
|
if(foreground)
|
2011-08-31 20:17:53 +00:00
|
|
|
printf("s3fs_mknod[path=%s][mode=%d]\n", path, mode);
|
2010-12-21 15:24:46 +00:00
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
// Could not make block or character special files on S3,
|
|
|
|
// always return a error.
|
|
|
|
return -EPERM;
|
2010-12-22 17:19:52 +00:00
|
|
|
}
|
2010-12-21 15:24:46 +00:00
|
|
|
|
2010-12-22 17:19:52 +00:00
|
|
|
static int s3fs_create(const char *path, mode_t mode, struct fuse_file_info *fi) {
|
|
|
|
int result;
|
|
|
|
headers_t meta;
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
struct fuse_context* pcxt;
|
2010-12-21 15:24:46 +00:00
|
|
|
|
2010-12-22 17:19:52 +00:00
|
|
|
if(foreground)
|
2010-12-28 04:15:23 +00:00
|
|
|
cout << "s3fs_create[path=" << path << "][mode=" << mode << "]" << "[flags=" << fi->flags << "]" << endl;
|
2010-12-21 15:24:46 +00:00
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
if(NULL == (pcxt = fuse_get_context())){
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
|
|
|
|
// check parent directory attribute.
|
|
|
|
if(0 != (result = check_parent_object_access(path, W_OK | X_OK))){
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
result = check_object_access(path, W_OK, NULL);
|
|
|
|
if(0 != result && -ENOENT != result){
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
result = create_file_object(path, mode, pcxt->uid, pcxt->gid);
|
2010-12-21 15:24:46 +00:00
|
|
|
|
2011-06-27 02:21:38 +00:00
|
|
|
if(result != 0)
|
2011-07-07 22:09:40 +00:00
|
|
|
return result;
|
2010-12-22 17:19:52 +00:00
|
|
|
|
2011-08-31 20:17:53 +00:00
|
|
|
// object created, open it
|
|
|
|
if((fi->fh = get_local_fd(path)) <= 0)
|
|
|
|
return -EIO;
|
2010-12-21 15:24:46 +00:00
|
|
|
|
|
|
|
// remember flags and headers...
|
|
|
|
pthread_mutex_lock( &s3fs_descriptors_lock );
|
|
|
|
s3fs_descriptors[fi->fh] = fi->flags;
|
2013-01-19 16:05:07 +00:00
|
|
|
s3fs_pathtofd[string(path)] = fi->fh;
|
2010-12-21 15:24:46 +00:00
|
|
|
pthread_mutex_unlock( &s3fs_descriptors_lock );
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
static int create_directory_object(const char *path, mode_t mode, time_t time, uid_t uid, gid_t gid)
|
|
|
|
{
|
2010-12-19 01:34:27 +00:00
|
|
|
CURL *curl = NULL;
|
|
|
|
int result;
|
2011-06-26 00:37:52 +00:00
|
|
|
char *s3_realpath;
|
|
|
|
string url;
|
|
|
|
string resource;
|
2011-02-23 17:16:12 +00:00
|
|
|
string date = get_date();
|
|
|
|
auto_curl_slist headers;
|
2010-12-19 01:34:27 +00:00
|
|
|
|
2010-12-09 20:56:29 +00:00
|
|
|
if(foreground)
|
2013-01-19 16:05:07 +00:00
|
|
|
cout << " create_directory_object[path=" << path << "][mode=" << mode <<
|
|
|
|
"][time=" << time << "][uid=" << uid << "][gid=" << gid << "]" << endl;
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
s3_realpath = get_realpath(path);
|
|
|
|
resource = urlEncode(service_path + bucket + s3_realpath);
|
2013-01-19 16:05:07 +00:00
|
|
|
free(s3_realpath);
|
2011-06-26 00:37:52 +00:00
|
|
|
url = host + resource;
|
|
|
|
|
2010-12-19 01:34:27 +00:00
|
|
|
curl = create_curl_handle();
|
2010-11-13 23:59:23 +00:00
|
|
|
curl_easy_setopt(curl, CURLOPT_UPLOAD, true); // HTTP PUT
|
|
|
|
curl_easy_setopt(curl, CURLOPT_INFILESIZE, 0); // Content-Length: 0
|
|
|
|
|
|
|
|
headers.append("Date: " + date);
|
|
|
|
headers.append("Content-Type: application/x-directory");
|
|
|
|
// x-amz headers: (a) alphabetical order and (b) no spaces after colon
|
|
|
|
headers.append("x-amz-acl:" + default_acl);
|
2013-01-19 16:05:07 +00:00
|
|
|
headers.append("x-amz-meta-gid:" + str(gid));
|
2010-11-13 23:59:23 +00:00
|
|
|
headers.append("x-amz-meta-mode:" + str(mode));
|
2013-01-19 16:05:07 +00:00
|
|
|
headers.append("x-amz-meta-mtime:" + str(time));
|
|
|
|
headers.append("x-amz-meta-uid:" + str(uid));
|
2010-12-28 04:15:23 +00:00
|
|
|
if (use_rrs.substr(0,1) == "1") {
|
|
|
|
headers.append("x-amz-storage-class:REDUCED_REDUNDANCY");
|
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
if (public_bucket.substr(0,1) != "1") {
|
|
|
|
headers.append("Authorization: AWS " + AWSAccessKeyId + ":" +
|
|
|
|
calc_signature("PUT", "application/x-directory", date, headers.get(), resource));
|
|
|
|
}
|
|
|
|
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
|
|
|
|
|
|
|
|
string my_url = prepare_url(url.c_str());
|
|
|
|
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
|
|
|
|
2010-12-19 01:34:27 +00:00
|
|
|
result = my_curl_easy_perform(curl);
|
|
|
|
|
|
|
|
destroy_curl_handle(curl);
|
|
|
|
|
2011-02-23 17:16:12 +00:00
|
|
|
if(result != 0)
|
|
|
|
return result;
|
2010-11-13 23:59:23 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
static int s3fs_mkdir(const char *path, mode_t mode)
|
|
|
|
{
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
int result;
|
|
|
|
struct fuse_context* pcxt;
|
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
if(foreground)
|
|
|
|
cout << "s3fs_mkdir[path=" << path << "][mode=" << mode << "]" << endl;
|
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
if(NULL == (pcxt = fuse_get_context())){
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
|
|
|
|
// check parent directory attribute.
|
|
|
|
if(0 != (result = check_parent_object_access(path, W_OK | X_OK))){
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
if(-ENOENT != (result = check_object_access(path, F_OK, NULL))){
|
|
|
|
if(0 == result){
|
|
|
|
result = -EEXIST;
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
return create_directory_object(path, mode, time(NULL), pcxt->uid, pcxt->gid);
|
2013-01-19 16:05:07 +00:00
|
|
|
}
|
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
static int s3fs_unlink(const char *path) {
|
2010-12-19 01:34:27 +00:00
|
|
|
int result;
|
2011-06-26 00:37:52 +00:00
|
|
|
char *s3_realpath;
|
2011-02-23 17:16:12 +00:00
|
|
|
|
|
|
|
if(foreground)
|
2013-01-19 16:05:07 +00:00
|
|
|
printf("s3fs_unlink[path=%s]\n", path);
|
2010-11-13 23:59:23 +00:00
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
if(0 != (result = check_parent_object_access(path, W_OK | X_OK))){
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
s3_realpath = get_realpath(path);
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-08-31 22:20:20 +00:00
|
|
|
result = curl_delete(s3_realpath);
|
2011-06-26 00:37:52 +00:00
|
|
|
free(s3_realpath);
|
2011-02-10 01:07:46 +00:00
|
|
|
delete_stat_cache_entry(path);
|
|
|
|
|
2011-08-31 22:20:20 +00:00
|
|
|
return result;
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
|
2011-08-30 19:08:01 +00:00
|
|
|
static int directory_empty(const char *path) {
|
2010-12-17 04:40:15 +00:00
|
|
|
int result;
|
2011-06-26 00:37:52 +00:00
|
|
|
char *s3_realpath;
|
2011-08-30 19:08:01 +00:00
|
|
|
string url;
|
|
|
|
string my_url;
|
|
|
|
string date;
|
|
|
|
string resource = urlEncode(service_path + bucket);
|
|
|
|
string query = "delimiter=/&prefix=";
|
|
|
|
CURL *curl = NULL;
|
2011-06-26 00:37:52 +00:00
|
|
|
struct BodyStruct body;
|
2011-08-30 19:08:01 +00:00
|
|
|
auto_curl_slist headers;
|
2010-12-17 04:40:15 +00:00
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
s3_realpath = get_realpath(path);
|
|
|
|
body.text = (char *)malloc(1);
|
|
|
|
body.size = 0;
|
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
if(strcmp(path, "/") != 0){
|
2011-08-30 19:08:01 +00:00
|
|
|
query += urlEncode(string(s3_realpath).substr(1) + "/");
|
2013-01-19 16:05:07 +00:00
|
|
|
}else{
|
2011-08-30 19:08:01 +00:00
|
|
|
query += urlEncode(string(s3_realpath).substr(1));
|
2013-01-19 16:05:07 +00:00
|
|
|
}
|
|
|
|
free(s3_realpath);
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-08-30 19:08:01 +00:00
|
|
|
query += "&max-keys=1";
|
|
|
|
url = host + resource + "?"+ query;
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-08-30 19:08:01 +00:00
|
|
|
curl = create_curl_handle();
|
|
|
|
my_url = prepare_url(url.c_str());
|
|
|
|
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
|
|
|
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&body);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
|
2011-06-26 00:37:52 +00:00
|
|
|
|
2011-08-30 19:08:01 +00:00
|
|
|
date = get_date();
|
|
|
|
headers.append("Date: " + date);
|
|
|
|
headers.append("ContentType: ");
|
|
|
|
if(public_bucket.substr(0,1) != "1") {
|
|
|
|
headers.append("Authorization: AWS " + AWSAccessKeyId + ":" +
|
|
|
|
calc_signature("GET", "", date, headers.get(), resource + "/"));
|
|
|
|
}
|
|
|
|
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-08-30 19:08:01 +00:00
|
|
|
result = my_curl_easy_perform(curl, &body);
|
2013-01-19 16:05:07 +00:00
|
|
|
destroy_curl_handle(curl);
|
2011-08-30 19:08:01 +00:00
|
|
|
if(result != 0) {
|
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
|
|
|
return result;
|
|
|
|
}
|
2010-12-09 20:56:29 +00:00
|
|
|
|
2011-08-30 19:08:01 +00:00
|
|
|
// is the directory empty?
|
|
|
|
if(strstr(body.text, "<CommonPrefixes>") != NULL ||
|
|
|
|
strstr(body.text, "<ETag>") != NULL ) {
|
2013-01-19 16:05:07 +00:00
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
2011-08-30 19:08:01 +00:00
|
|
|
return -ENOTEMPTY;
|
|
|
|
}
|
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
|
|
|
|
2011-08-30 19:08:01 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int s3fs_rmdir(const char *path) {
|
|
|
|
int result;
|
|
|
|
char *s3_realpath;
|
|
|
|
|
|
|
|
if(foreground)
|
|
|
|
printf("s3fs_rmdir [path=%s]\n", path);
|
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
if(0 != (result = check_parent_object_access(path, W_OK | X_OK))){
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2011-08-30 19:08:01 +00:00
|
|
|
s3_realpath = get_realpath(path);
|
|
|
|
|
|
|
|
// directory must be empty
|
|
|
|
if(directory_empty(path) != 0)
|
|
|
|
return -ENOTEMPTY;
|
2011-02-08 18:23:38 +00:00
|
|
|
|
2011-08-31 22:20:20 +00:00
|
|
|
result = curl_delete(s3_realpath);
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
delete_stat_cache_entry(path);
|
|
|
|
free(s3_realpath);
|
2010-12-17 04:40:15 +00:00
|
|
|
|
2011-08-31 22:20:20 +00:00
|
|
|
return result;
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int s3fs_symlink(const char *from, const char *to) {
|
2010-12-19 22:27:56 +00:00
|
|
|
int result;
|
|
|
|
int fd = -1;
|
|
|
|
|
2010-12-09 20:56:29 +00:00
|
|
|
if(foreground)
|
2011-06-26 00:37:52 +00:00
|
|
|
cout << "s3fs_symlink[from=" << from << "][to=" << to << "]" << endl;
|
2010-11-13 23:59:23 +00:00
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
if(0 != (result = check_parent_object_access(to, W_OK | X_OK))){
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
if(-ENOENT != (result = check_object_access(to, F_OK, NULL))){
|
|
|
|
if(0 == result){
|
|
|
|
result = -EEXIST;
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
headers_t headers;
|
2013-01-19 16:05:07 +00:00
|
|
|
headers["x-amz-meta-mode"] = str(S_IFLNK | S_IRWXU | S_IRWXG | S_IRWXO);
|
2010-11-13 23:59:23 +00:00
|
|
|
headers["x-amz-meta-mtime"] = str(time(NULL));
|
|
|
|
|
2010-12-19 22:27:56 +00:00
|
|
|
fd = fileno(tmpfile());
|
|
|
|
if(fd == -1) {
|
2011-06-26 00:37:52 +00:00
|
|
|
syslog(LOG_ERR, "line %d: error: fileno(tmpfile()): %d", __LINE__, -errno);
|
2010-12-19 22:27:56 +00:00
|
|
|
return -errno;
|
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-03-01 19:35:55 +00:00
|
|
|
if(pwrite(fd, from, strlen(from), 0) == -1) {
|
2011-06-26 00:37:52 +00:00
|
|
|
syslog(LOG_ERR, "line %d: error: pwrite: %d", __LINE__, -errno);
|
2011-03-01 19:35:55 +00:00
|
|
|
if(fd > 0)
|
|
|
|
close(fd);
|
|
|
|
|
2010-12-19 22:27:56 +00:00
|
|
|
return -errno;
|
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2010-12-19 22:27:56 +00:00
|
|
|
result = put_local_fd(to, headers, fd);
|
2011-03-01 19:35:55 +00:00
|
|
|
if(result != 0) {
|
|
|
|
if(fd > 0)
|
|
|
|
close(fd);
|
|
|
|
|
2010-12-19 22:27:56 +00:00
|
|
|
return result;
|
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-03-01 19:35:55 +00:00
|
|
|
if(fd > 0)
|
|
|
|
close(fd);
|
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
static int rename_object(const char *from, const char *to) {
|
2010-11-23 22:41:58 +00:00
|
|
|
int result;
|
2011-06-26 00:37:52 +00:00
|
|
|
char *s3_realpath;
|
2010-12-20 05:26:27 +00:00
|
|
|
headers_t meta;
|
2010-12-17 04:40:15 +00:00
|
|
|
|
2011-03-01 19:35:55 +00:00
|
|
|
if(foreground)
|
2011-08-29 22:01:32 +00:00
|
|
|
printf("rename_object [from=%s] [to=%s]\n", from , to);
|
2010-11-23 22:41:58 +00:00
|
|
|
|
2011-03-01 19:35:55 +00:00
|
|
|
if(debug)
|
|
|
|
syslog(LOG_DEBUG, "rename_object [from=%s] [to=%s]", from, to);
|
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
if(0 != (result = check_parent_object_access(to, W_OK | X_OK))){
|
|
|
|
// not permmit writing "to" object parent dir.
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
if(0 != (result = check_parent_object_access(from, W_OK | X_OK))){
|
|
|
|
// not permmit removing "from" object parent dir.
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2011-09-01 19:24:12 +00:00
|
|
|
s3_realpath = get_realpath(from);
|
|
|
|
result = curl_get_headers(s3_realpath, meta);
|
2011-03-01 19:35:55 +00:00
|
|
|
|
2011-09-01 19:24:12 +00:00
|
|
|
if(result != 0) {
|
|
|
|
free(s3_realpath);
|
2011-03-01 19:35:55 +00:00
|
|
|
return result;
|
2011-09-01 19:24:12 +00:00
|
|
|
}
|
2010-12-19 22:27:56 +00:00
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
meta["x-amz-copy-source"] = urlEncode("/" + bucket + s3_realpath);
|
2010-11-13 23:59:23 +00:00
|
|
|
meta["Content-Type"] = lookupMimeType(to);
|
|
|
|
meta["x-amz-metadata-directive"] = "REPLACE";
|
2011-09-01 19:24:12 +00:00
|
|
|
free(s3_realpath);
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2010-11-23 22:41:58 +00:00
|
|
|
result = put_headers(to, meta);
|
2011-09-01 19:24:12 +00:00
|
|
|
if(result != 0)
|
2010-11-13 23:59:23 +00:00
|
|
|
return result;
|
|
|
|
|
2010-12-20 05:26:27 +00:00
|
|
|
result = s3fs_unlink(from);
|
|
|
|
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
static int rename_object_nocopy(const char *from, const char *to) {
|
|
|
|
int result;
|
|
|
|
char* s3_realpath;
|
|
|
|
headers_t meta;
|
|
|
|
int fd;
|
|
|
|
int isclose = 1;
|
|
|
|
|
|
|
|
if(foreground)
|
|
|
|
printf("rename_object_nocopy [from=%s] [to=%s]\n", from , to);
|
|
|
|
|
|
|
|
if(debug)
|
|
|
|
syslog(LOG_DEBUG, "rename_object_nocopy [from=%s] [to=%s]", from, to);
|
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
if(0 != (result = check_parent_object_access(to, W_OK | X_OK))){
|
|
|
|
// not permmit writing "to" object parent dir.
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
if(0 != (result = check_parent_object_access(from, W_OK | X_OK))){
|
|
|
|
// not permmit removing "from" object parent dir.
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
// Downloading
|
|
|
|
if(0 > (fd = get_opened_fd(from))){
|
|
|
|
if(0 > (fd = get_local_fd(from))){
|
|
|
|
if(foreground) {
|
|
|
|
printf(" rename_object_nocopy line %d: get_local_fd result: %d", __LINE__, fd);
|
|
|
|
}
|
|
|
|
syslog(LOG_ERR, "rename_object_nocopy line %d: get_local_fd result: %d", __LINE__, fd);
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
}else{
|
|
|
|
isclose = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get attributes
|
|
|
|
s3_realpath = get_realpath(from);
|
|
|
|
result = curl_get_headers(s3_realpath, meta);
|
|
|
|
free(s3_realpath);
|
|
|
|
if(result != 0){
|
|
|
|
if(isclose){
|
|
|
|
close(fd);
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Set header
|
|
|
|
meta["Content-Type"] = lookupMimeType(to);
|
|
|
|
|
|
|
|
// Re-uploading
|
|
|
|
result = put_local_fd(to, meta, fd);
|
|
|
|
if(isclose){
|
|
|
|
close(fd);
|
|
|
|
}
|
|
|
|
if(0 != result){
|
|
|
|
if(foreground) {
|
|
|
|
printf(" rename_object_nocopy line %d: put_local_fd result: %d\n", __LINE__, result);
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Remove file
|
|
|
|
result = s3fs_unlink(from);
|
|
|
|
|
|
|
|
// Stats
|
|
|
|
delete_stat_cache_entry(to);
|
|
|
|
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2011-08-29 22:01:32 +00:00
|
|
|
static int rename_large_object(const char *from, const char *to) {
|
|
|
|
int result;
|
|
|
|
char *s3_realpath;
|
|
|
|
struct stat buf;
|
|
|
|
headers_t meta;
|
|
|
|
string upload_id;
|
|
|
|
vector <file_part> parts;
|
|
|
|
|
|
|
|
if(foreground)
|
|
|
|
printf("rename_large_object [from=%s] [to=%s]\n", from , to);
|
|
|
|
|
|
|
|
if(debug)
|
|
|
|
syslog(LOG_DEBUG, "rename_large_object [from=%s] [to=%s]", from, to);
|
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
if(0 != (result = check_parent_object_access(to, W_OK | X_OK))){
|
|
|
|
// not permmit writing "to" object parent dir.
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
if(0 != (result = check_parent_object_access(from, W_OK | X_OK))){
|
|
|
|
// not permmit removing "from" object parent dir.
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
get_object_attribute(from, &buf);
|
2011-08-29 22:01:32 +00:00
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
s3_realpath = get_realpath(from);
|
2011-09-01 19:24:12 +00:00
|
|
|
if((curl_get_headers(s3_realpath, meta) != 0)) {
|
|
|
|
free(s3_realpath);
|
2011-08-29 22:01:32 +00:00
|
|
|
return -1;
|
2011-09-01 19:24:12 +00:00
|
|
|
}
|
2011-08-29 22:01:32 +00:00
|
|
|
|
|
|
|
meta["Content-Type"] = lookupMimeType(to);
|
|
|
|
meta["x-amz-copy-source"] = urlEncode("/" + bucket + s3_realpath);
|
2011-09-01 19:24:12 +00:00
|
|
|
free(s3_realpath);
|
2011-08-29 22:01:32 +00:00
|
|
|
|
|
|
|
upload_id = initiate_multipart_upload(to, buf.st_size, meta);
|
|
|
|
if(upload_id.size() == 0)
|
|
|
|
return(-EIO);
|
|
|
|
|
|
|
|
off_t chunk = 0;
|
|
|
|
off_t bytes_written = 0;
|
|
|
|
off_t bytes_remaining = buf.st_size;
|
|
|
|
while(bytes_remaining > 0) {
|
|
|
|
file_part part;
|
|
|
|
|
|
|
|
if(bytes_remaining > MAX_COPY_SOURCE_SIZE)
|
|
|
|
chunk = MAX_COPY_SOURCE_SIZE;
|
|
|
|
else
|
|
|
|
chunk = bytes_remaining - 1;
|
|
|
|
|
|
|
|
stringstream ss;
|
|
|
|
ss << "bytes=" << bytes_written << "-" << (bytes_written + chunk);
|
|
|
|
meta["x-amz-copy-source-range"] = ss.str();
|
|
|
|
|
|
|
|
part.etag = copy_part(from, to, parts.size() + 1, upload_id, meta);
|
|
|
|
parts.push_back(part);
|
|
|
|
|
|
|
|
bytes_written += (chunk + 1);
|
|
|
|
bytes_remaining = buf.st_size - bytes_written;
|
|
|
|
}
|
|
|
|
|
|
|
|
result = complete_multipart_upload(to, upload_id, parts);
|
|
|
|
if(result != 0)
|
|
|
|
return -EIO;
|
|
|
|
|
|
|
|
return s3fs_unlink(from);
|
|
|
|
}
|
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
static int clone_directory_object(const char *from, const char *to)
|
|
|
|
{
|
2010-12-21 15:24:46 +00:00
|
|
|
int result;
|
|
|
|
mode_t mode;
|
2013-01-19 16:05:07 +00:00
|
|
|
time_t time;
|
|
|
|
uid_t uid;
|
|
|
|
gid_t gid;
|
2010-12-21 15:24:46 +00:00
|
|
|
headers_t meta;
|
2011-09-01 19:24:12 +00:00
|
|
|
char *s3_realpath;
|
2010-12-21 15:24:46 +00:00
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
if(foreground)
|
|
|
|
printf("clone_directory_object [from=%s] [to=%s]\n", from, to);
|
|
|
|
|
|
|
|
if(debug)
|
|
|
|
syslog(LOG_DEBUG, "clone_directory_object [from=%s] [to=%s]", from, to);
|
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
// get target's attributes
|
2011-09-01 19:24:12 +00:00
|
|
|
s3_realpath = get_realpath(from);
|
|
|
|
result = curl_get_headers(s3_realpath, meta);
|
2013-01-19 16:05:07 +00:00
|
|
|
free(s3_realpath);
|
2011-09-01 19:24:12 +00:00
|
|
|
if(result != 0) {
|
2011-03-01 19:35:55 +00:00
|
|
|
return result;
|
2011-09-01 19:24:12 +00:00
|
|
|
}
|
2010-12-21 15:24:46 +00:00
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
mode = get_mode(meta["x-amz-meta-mode"].c_str());
|
|
|
|
time = get_mtime(meta["x-amz-meta-mtime"].c_str());
|
|
|
|
uid = get_uid(meta["x-amz-meta-uid"].c_str());
|
|
|
|
gid = get_gid(meta["x-amz-meta-gid"].c_str());
|
|
|
|
result = create_directory_object(to, mode, time, uid, gid);
|
2010-12-21 15:24:46 +00:00
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
return result;
|
2010-12-21 15:24:46 +00:00
|
|
|
}
|
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
static int rename_directory(const char *from, const char *to) {
|
2010-12-20 05:26:27 +00:00
|
|
|
int result;
|
2010-12-30 03:13:21 +00:00
|
|
|
// mode_t mode;
|
2010-12-20 05:26:27 +00:00
|
|
|
headers_t meta;
|
|
|
|
int num_keys = 0;
|
2010-12-21 15:24:46 +00:00
|
|
|
int max_keys = 50;
|
|
|
|
string path;
|
|
|
|
string new_path;
|
|
|
|
string to_path;
|
|
|
|
string from_path;
|
|
|
|
MVNODE *head = NULL;
|
|
|
|
MVNODE *tail = NULL;
|
2010-12-20 05:26:27 +00:00
|
|
|
|
|
|
|
if(foreground)
|
|
|
|
cout << "rename_directory[from=" << from << "][to=" << to << "]" << endl;
|
|
|
|
|
2011-03-01 19:35:55 +00:00
|
|
|
if(debug)
|
|
|
|
syslog(LOG_DEBUG, "rename_directory [from=%s] [to=%s]", from, to);
|
2010-12-20 05:26:27 +00:00
|
|
|
|
|
|
|
CURL *curl;
|
|
|
|
struct BodyStruct body;
|
|
|
|
string NextMarker;
|
|
|
|
string IsTruncated("true");
|
2010-12-21 15:24:46 +00:00
|
|
|
string object_type;
|
|
|
|
string Key;
|
|
|
|
bool is_dir;
|
2010-12-20 05:26:27 +00:00
|
|
|
|
|
|
|
body.text = (char *)malloc(1);
|
|
|
|
body.size = 0;
|
|
|
|
|
2010-12-21 15:24:46 +00:00
|
|
|
// create the head/tail of the linked list
|
|
|
|
from_path.assign(from);
|
|
|
|
to_path.assign(to);
|
|
|
|
is_dir = 1;
|
|
|
|
|
|
|
|
// printf("calling create_mvnode\n");
|
|
|
|
// head = create_mvnode((char *)from, (char *)to, is_dir);
|
|
|
|
head = create_mvnode((char *)from_path.c_str(), (char *)to_path.c_str(), is_dir);
|
|
|
|
tail = head;
|
|
|
|
// printf("back from create_mvnode\n");
|
|
|
|
|
2010-12-20 05:26:27 +00:00
|
|
|
while (IsTruncated == "true") {
|
2011-06-26 00:37:52 +00:00
|
|
|
string query;
|
2010-12-20 05:26:27 +00:00
|
|
|
string resource = urlEncode(service_path + bucket);
|
2011-06-26 00:37:52 +00:00
|
|
|
|
|
|
|
if(mount_prefix.size() > 0)
|
|
|
|
query = "prefix=" + mount_prefix .substr(1, mount_prefix.size() - 1) + "/";
|
|
|
|
else
|
|
|
|
query = "prefix=";
|
2010-12-20 05:26:27 +00:00
|
|
|
|
|
|
|
if (strcmp(from, "/") != 0)
|
|
|
|
query += urlEncode(string(from).substr(1) + "/");
|
|
|
|
|
|
|
|
if (NextMarker.size() > 0)
|
|
|
|
query += "&marker=" + urlEncode(NextMarker);
|
|
|
|
|
2010-12-21 15:24:46 +00:00
|
|
|
query += "&max-keys=";
|
|
|
|
query.append(IntToStr(max_keys));
|
2010-12-20 05:26:27 +00:00
|
|
|
|
|
|
|
string url = host + resource + "?" + query;
|
|
|
|
|
|
|
|
{
|
|
|
|
curl = create_curl_handle();
|
|
|
|
|
|
|
|
string my_url = prepare_url(url.c_str());
|
|
|
|
|
|
|
|
if(body.text) {
|
|
|
|
free(body.text);
|
|
|
|
body.size = 0;
|
|
|
|
body.text = (char *)malloc(1);
|
|
|
|
}
|
|
|
|
|
|
|
|
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
|
|
|
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&body);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
|
|
|
|
|
|
|
|
auto_curl_slist headers;
|
|
|
|
string date = get_date();
|
|
|
|
headers.append("Date: " + date);
|
|
|
|
headers.append("ContentType: ");
|
|
|
|
if (public_bucket.substr(0,1) != "1") {
|
|
|
|
headers.append("Authorization: AWS " + AWSAccessKeyId + ":" +
|
|
|
|
calc_signature("GET", "", date, headers.get(), resource + "/"));
|
|
|
|
}
|
|
|
|
|
|
|
|
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
|
|
|
|
|
|
|
|
result = my_curl_easy_perform(curl, &body);
|
|
|
|
|
|
|
|
destroy_curl_handle(curl);
|
|
|
|
|
|
|
|
if(result != 0) {
|
2011-07-02 02:11:54 +00:00
|
|
|
if(body.text)
|
2010-12-20 05:26:27 +00:00
|
|
|
free(body.text);
|
2010-12-21 15:24:46 +00:00
|
|
|
free_mvnodes(head);
|
2010-12-20 05:26:27 +00:00
|
|
|
return result;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
xmlDocPtr doc = xmlReadMemory(body.text, body.size, "", NULL, 0);
|
|
|
|
if (doc != NULL && doc->children != NULL) {
|
|
|
|
for (xmlNodePtr cur_node = doc->children->children;
|
|
|
|
cur_node != NULL;
|
|
|
|
cur_node = cur_node->next) {
|
2010-12-20 05:26:27 +00:00
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
string cur_node_name(reinterpret_cast<const char *>(cur_node->name));
|
|
|
|
if(cur_node_name == "IsTruncated")
|
|
|
|
IsTruncated = reinterpret_cast<const char *>(cur_node->children->content);
|
|
|
|
|
|
|
|
if (cur_node_name == "Contents") {
|
|
|
|
if (cur_node->children != NULL) {
|
|
|
|
string LastModified;
|
|
|
|
string Size;
|
|
|
|
for (xmlNodePtr sub_node = cur_node->children;
|
|
|
|
sub_node != NULL;
|
|
|
|
sub_node = sub_node->next) {
|
|
|
|
|
|
|
|
if (sub_node->type == XML_ELEMENT_NODE) {
|
|
|
|
string elementName = reinterpret_cast<const char*>(sub_node->name);
|
|
|
|
if (sub_node->children != NULL) {
|
|
|
|
if (sub_node->children->type == XML_TEXT_NODE) {
|
|
|
|
if (elementName == "Key") {
|
|
|
|
Key = reinterpret_cast<const char *>(sub_node->children->content);
|
|
|
|
}
|
|
|
|
if (elementName == "LastModified") {
|
|
|
|
LastModified = reinterpret_cast<const char *>(sub_node->children->content);
|
|
|
|
}
|
|
|
|
if (elementName == "Size") {
|
|
|
|
Size = reinterpret_cast<const char *>(sub_node->children->content);
|
2010-12-20 05:26:27 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2011-06-26 00:37:52 +00:00
|
|
|
}
|
2010-12-20 05:26:27 +00:00
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
if (Key.size() > 0) {
|
|
|
|
num_keys++;
|
|
|
|
path = "/" + Key;
|
|
|
|
new_path = path;
|
|
|
|
if(mount_prefix.size() > 0)
|
|
|
|
new_path.replace(0, mount_prefix.substr(0, mount_prefix.size()).size() + from_path.size(), to_path);
|
|
|
|
else
|
2010-12-21 15:24:46 +00:00
|
|
|
new_path.replace(0, from_path.size(), to_path);
|
2011-06-26 00:37:52 +00:00
|
|
|
|
|
|
|
if(mount_prefix.size() > 0)
|
2011-09-01 19:24:12 +00:00
|
|
|
result = curl_get_headers(path.replace(0, mount_prefix.size(), "").c_str(), meta);
|
2011-06-26 00:37:52 +00:00
|
|
|
else
|
2011-09-01 19:24:12 +00:00
|
|
|
result = curl_get_headers(path.c_str(), meta);
|
2011-06-26 00:37:52 +00:00
|
|
|
|
|
|
|
if(result != 0) {
|
2013-01-19 16:05:07 +00:00
|
|
|
xmlFreeDoc(doc);
|
2011-06-26 00:37:52 +00:00
|
|
|
free_mvnodes(head);
|
2011-07-02 02:11:54 +00:00
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
2011-06-26 00:37:52 +00:00
|
|
|
body.text = NULL;
|
2010-12-20 05:26:27 +00:00
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
// process the Key appropriately
|
|
|
|
// if it is a directory move the directory object
|
|
|
|
object_type = meta["Content-Type"];
|
|
|
|
if(object_type.compare("application/x-directory") == 0)
|
|
|
|
is_dir = 1;
|
|
|
|
else
|
|
|
|
is_dir = 0;
|
|
|
|
|
|
|
|
// push this one onto the stack
|
|
|
|
tail = add_mvnode(head, (char *)path.c_str(), (char *)new_path.c_str(), is_dir);
|
|
|
|
}
|
|
|
|
|
|
|
|
} // if (cur_node->children != NULL) {
|
|
|
|
} // if (cur_node_name == "Contents") {
|
|
|
|
} // for (xmlNodePtr cur_node = doc->children->children;
|
|
|
|
} // if (doc != NULL && doc->children != NULL) {
|
|
|
|
xmlFreeDoc(doc);
|
2010-12-21 15:24:46 +00:00
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
if(IsTruncated == "true")
|
2010-12-21 15:24:46 +00:00
|
|
|
NextMarker = Key;
|
2010-12-20 05:26:27 +00:00
|
|
|
|
|
|
|
} // while (IsTruncated == "true") {
|
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
2011-06-26 00:37:52 +00:00
|
|
|
body.text = NULL;
|
2010-12-20 05:26:27 +00:00
|
|
|
|
2010-12-21 15:24:46 +00:00
|
|
|
// iterate over the list - clone directories first - top down
|
2011-03-01 19:35:55 +00:00
|
|
|
if(head == NULL)
|
|
|
|
return 0;
|
2010-12-20 05:26:27 +00:00
|
|
|
|
2010-12-21 15:24:46 +00:00
|
|
|
MVNODE *my_head;
|
|
|
|
MVNODE *my_tail;
|
|
|
|
MVNODE *next;
|
|
|
|
MVNODE *prev;
|
|
|
|
my_head = head;
|
|
|
|
my_tail = tail;
|
|
|
|
next = NULL;
|
|
|
|
prev = NULL;
|
|
|
|
|
|
|
|
do {
|
|
|
|
if(my_head->is_dir) {
|
|
|
|
result = clone_directory_object( my_head->old_path, my_head->new_path);
|
|
|
|
if(result != 0) {
|
|
|
|
free_mvnodes(head);
|
|
|
|
syslog(LOG_ERR, "clone_directory_object returned an error");
|
|
|
|
return -EIO;
|
|
|
|
}
|
2010-12-20 05:26:27 +00:00
|
|
|
}
|
2010-12-21 15:24:46 +00:00
|
|
|
next = my_head->next;
|
|
|
|
my_head = next;
|
|
|
|
} while(my_head != NULL);
|
2010-12-20 05:26:27 +00:00
|
|
|
|
2010-12-21 15:24:46 +00:00
|
|
|
// iterate over the list - copy the files with rename_object
|
|
|
|
// does a safe copy - copies first and then deletes old
|
|
|
|
my_head = head;
|
|
|
|
next = NULL;
|
|
|
|
|
|
|
|
do {
|
|
|
|
if(my_head->is_dir != 1) {
|
2013-01-19 16:05:07 +00:00
|
|
|
if(!nocopyapi){
|
|
|
|
result = rename_object( my_head->old_path, my_head->new_path);
|
|
|
|
}else{
|
|
|
|
result = rename_object_nocopy( my_head->old_path, my_head->new_path);
|
|
|
|
}
|
2010-12-21 15:24:46 +00:00
|
|
|
if(result != 0) {
|
|
|
|
free_mvnodes(head);
|
|
|
|
syslog(LOG_ERR, "rename_dir: rename_object returned an error");
|
|
|
|
return -EIO;
|
|
|
|
}
|
2010-12-20 05:26:27 +00:00
|
|
|
}
|
2010-12-21 15:24:46 +00:00
|
|
|
next = my_head->next;
|
|
|
|
my_head = next;
|
|
|
|
} while(my_head != NULL);
|
2010-12-20 05:26:27 +00:00
|
|
|
|
2010-12-21 15:24:46 +00:00
|
|
|
// Iterate over old the directories, bottoms up and remove
|
|
|
|
do {
|
|
|
|
if(my_tail->is_dir) {
|
|
|
|
result = s3fs_unlink( my_tail->old_path);
|
|
|
|
if(result != 0) {
|
|
|
|
free_mvnodes(head);
|
|
|
|
syslog(LOG_ERR, "rename_dir: s3fs_unlink returned an error");
|
|
|
|
return -EIO;
|
|
|
|
}
|
2010-12-20 05:26:27 +00:00
|
|
|
}
|
2010-12-21 15:24:46 +00:00
|
|
|
prev = my_tail->prev;
|
|
|
|
my_tail = prev;
|
|
|
|
} while(my_tail != NULL);
|
2010-12-20 05:26:27 +00:00
|
|
|
|
2010-12-21 15:24:46 +00:00
|
|
|
free_mvnodes(head);
|
2010-12-20 05:26:27 +00:00
|
|
|
|
2010-12-21 15:24:46 +00:00
|
|
|
return 0;
|
2010-12-20 05:26:27 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int s3fs_rename(const char *from, const char *to) {
|
2011-03-01 19:35:55 +00:00
|
|
|
struct stat buf;
|
|
|
|
int result;
|
|
|
|
|
2010-12-20 05:26:27 +00:00
|
|
|
if(foreground)
|
2011-08-29 22:01:32 +00:00
|
|
|
printf("s3fs_rename [from=%s] [to=%s]\n", from, to);
|
2010-12-20 05:26:27 +00:00
|
|
|
|
2011-03-01 19:35:55 +00:00
|
|
|
if(debug)
|
2011-08-29 22:01:32 +00:00
|
|
|
syslog(LOG_DEBUG, "s3fs_rename [from=%s] [to=%s]", from, to);
|
2010-12-20 05:26:27 +00:00
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
if(0 != (result = check_parent_object_access(to, W_OK | X_OK))){
|
|
|
|
// not permmit writing "to" object parent dir.
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
if(0 != (result = check_parent_object_access(from, W_OK | X_OK))){
|
|
|
|
// not permmit removing "from" object parent dir.
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
get_object_attribute(from, &buf);
|
2011-08-29 22:01:32 +00:00
|
|
|
|
|
|
|
// files larger than 5GB must be modified via the multipart interface
|
2013-01-19 16:05:07 +00:00
|
|
|
if(S_ISDIR(buf.st_mode)){
|
2010-12-21 15:24:46 +00:00
|
|
|
result = rename_directory(from, to);
|
2013-01-19 16:05:07 +00:00
|
|
|
}else if(!nomultipart && buf.st_size >= FIVE_GB){
|
2011-08-29 22:01:32 +00:00
|
|
|
result = rename_large_object(from, to);
|
2013-01-19 16:05:07 +00:00
|
|
|
}else{
|
|
|
|
if(!nocopyapi){
|
|
|
|
result = rename_object(from, to);
|
|
|
|
}else{
|
|
|
|
result = rename_object_nocopy(from, to);
|
|
|
|
}
|
|
|
|
}
|
2011-02-08 18:23:38 +00:00
|
|
|
|
2010-12-20 05:26:27 +00:00
|
|
|
return result;
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int s3fs_link(const char *from, const char *to) {
|
2010-12-09 20:56:29 +00:00
|
|
|
if(foreground)
|
2013-01-19 16:05:07 +00:00
|
|
|
cout << "s3fs_link[from=" << from << "][to=" << to << "]" << endl;
|
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
return -EPERM;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int s3fs_chmod(const char *path, mode_t mode) {
|
2010-12-19 22:27:56 +00:00
|
|
|
int result;
|
2011-06-26 00:37:52 +00:00
|
|
|
char *s3_realpath;
|
2011-03-01 19:35:55 +00:00
|
|
|
headers_t meta;
|
|
|
|
|
2010-12-09 20:56:29 +00:00
|
|
|
if(foreground)
|
2011-06-27 02:21:38 +00:00
|
|
|
printf("s3fs_chmod [path=%s] [mode=%d]\n", path, mode);
|
2010-12-19 22:27:56 +00:00
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
if(0 != (result = check_parent_object_access(path, W_OK | X_OK))){
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
if(0 != (result = check_object_access(path, W_OK, NULL))){
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2011-09-01 19:24:12 +00:00
|
|
|
s3_realpath = get_realpath(path);
|
|
|
|
result = curl_get_headers(s3_realpath, meta);
|
|
|
|
if(result != 0) {
|
|
|
|
free(s3_realpath);
|
2011-03-01 19:35:55 +00:00
|
|
|
return result;
|
2011-09-01 19:24:12 +00:00
|
|
|
}
|
2010-12-19 22:27:56 +00:00
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
meta["x-amz-meta-mode"] = str(mode);
|
2011-06-26 00:37:52 +00:00
|
|
|
meta["x-amz-copy-source"] = urlEncode("/" + bucket + s3_realpath);
|
2010-11-13 23:59:23 +00:00
|
|
|
meta["x-amz-metadata-directive"] = "REPLACE";
|
2011-06-26 00:37:52 +00:00
|
|
|
free(s3_realpath);
|
2011-02-11 03:30:02 +00:00
|
|
|
|
2011-08-29 22:01:32 +00:00
|
|
|
if(put_headers(path, meta) != 0)
|
|
|
|
return -EIO;
|
|
|
|
|
2011-02-11 03:30:02 +00:00
|
|
|
delete_stat_cache_entry(path);
|
|
|
|
|
2011-08-29 22:01:32 +00:00
|
|
|
return 0;
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
static int s3fs_chmod_nocopy(const char *path, mode_t mode) {
|
|
|
|
int result;
|
|
|
|
char *s3_realpath;
|
|
|
|
headers_t meta;
|
|
|
|
int fd;
|
|
|
|
int isclose = 1;
|
|
|
|
|
|
|
|
if(foreground)
|
|
|
|
printf("s3fs_chmod_nocopy [path=%s] [mode=%d]\n", path, mode);
|
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
if(0 != (result = check_parent_object_access(path, W_OK | X_OK))){
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
if(0 != (result = check_object_access(path, W_OK, NULL))){
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
// Downloading
|
|
|
|
if(0 > (fd = get_opened_fd(path))){
|
|
|
|
if(0 > (fd = get_local_fd(path))){
|
|
|
|
if(foreground) {
|
|
|
|
printf(" s3fs_chmod_nocopy line %d: get_local_fd result: %d\n", __LINE__, fd);
|
|
|
|
}
|
|
|
|
syslog(LOG_ERR, "s3fs_chmod_nocopy line %d: get_local_fd result: %d", __LINE__, fd);
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
}else{
|
|
|
|
isclose = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get attributes
|
|
|
|
s3_realpath = get_realpath(path);
|
|
|
|
result = curl_get_headers(s3_realpath, meta);
|
|
|
|
free(s3_realpath);
|
|
|
|
if(result != 0){
|
|
|
|
if(isclose){
|
|
|
|
close(fd);
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Change file mode
|
|
|
|
meta["x-amz-meta-mode"] = str(mode);
|
|
|
|
// Change local file mode
|
|
|
|
if(-1 == fchmod(fd, mode)){
|
|
|
|
if(isclose){
|
|
|
|
close(fd);
|
|
|
|
}
|
|
|
|
if(foreground) {
|
|
|
|
printf(" s3fs_chmod_nocopy line %d: fchmod(fd=%d) error(%d)\n", __LINE__, fd, errno);
|
|
|
|
}
|
|
|
|
syslog(LOG_ERR, "s3fs_chmod_nocopy line %d: fchmod(fd=%d) error(%d)", __LINE__, fd, errno);
|
|
|
|
return -errno;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Re-uploading
|
|
|
|
if(0 != (result = put_local_fd(path, meta, fd))){
|
|
|
|
if(foreground) {
|
|
|
|
printf(" s3fs_chmod_nocopy line %d: put_local_fd result: %d\n", __LINE__, result);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if(isclose){
|
|
|
|
close(fd);
|
|
|
|
}
|
|
|
|
delete_stat_cache_entry(path);
|
|
|
|
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
static int s3fs_chown(const char *path, uid_t uid, gid_t gid) {
|
2010-12-19 22:27:56 +00:00
|
|
|
int result;
|
2011-06-26 00:37:52 +00:00
|
|
|
char *s3_realpath;
|
2011-06-27 02:21:38 +00:00
|
|
|
|
2010-12-09 20:56:29 +00:00
|
|
|
if(foreground)
|
2011-06-27 02:21:38 +00:00
|
|
|
printf("s3fs_chown [path=%s] [uid=%d] [gid=%d]\n", path, uid, gid);
|
2010-11-13 23:59:23 +00:00
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
if(0 != (result = check_parent_object_access(path, W_OK | X_OK))){
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
if(0 != (result = check_object_access(path, W_OK, NULL))){
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2011-09-01 19:24:12 +00:00
|
|
|
s3_realpath = get_realpath(path);
|
2010-11-13 23:59:23 +00:00
|
|
|
headers_t meta;
|
2011-09-01 19:24:12 +00:00
|
|
|
result = curl_get_headers(s3_realpath, meta);
|
|
|
|
if(result != 0) {
|
|
|
|
free(s3_realpath);
|
|
|
|
return result;
|
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-06-27 02:21:38 +00:00
|
|
|
struct passwd *aaa = getpwuid(uid);
|
|
|
|
if(aaa != 0)
|
2010-11-13 23:59:23 +00:00
|
|
|
meta["x-amz-meta-uid"] = str((*aaa).pw_uid);
|
|
|
|
|
2011-06-27 02:21:38 +00:00
|
|
|
struct group *bbb = getgrgid(gid);
|
|
|
|
if(bbb != 0)
|
2010-11-13 23:59:23 +00:00
|
|
|
meta["x-amz-meta-gid"] = str((*bbb).gr_gid);
|
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
meta["x-amz-copy-source"] = urlEncode("/" + bucket + s3_realpath);
|
2010-11-13 23:59:23 +00:00
|
|
|
meta["x-amz-metadata-directive"] = "REPLACE";
|
2011-06-26 00:37:52 +00:00
|
|
|
free(s3_realpath);
|
2011-02-11 03:30:02 +00:00
|
|
|
|
2011-08-29 22:01:32 +00:00
|
|
|
if(put_headers(path, meta) != 0)
|
|
|
|
return -EIO;
|
|
|
|
|
2011-02-11 03:30:02 +00:00
|
|
|
delete_stat_cache_entry(path);
|
|
|
|
|
2011-08-29 22:01:32 +00:00
|
|
|
return 0;
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
static int s3fs_chown_nocopy(const char *path, uid_t uid, gid_t gid) {
|
2010-12-19 22:27:56 +00:00
|
|
|
int result;
|
2011-09-01 19:24:12 +00:00
|
|
|
char *s3_realpath;
|
2013-01-19 16:05:07 +00:00
|
|
|
headers_t meta;
|
|
|
|
int fd;
|
|
|
|
int isclose = 1;
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2010-12-09 20:56:29 +00:00
|
|
|
if(foreground)
|
2013-01-19 16:05:07 +00:00
|
|
|
printf("s3fs_chown_nocopy [path=%s] [uid=%d] [gid=%d]\n", path, uid, gid);
|
2010-11-13 23:59:23 +00:00
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
if(0 != (result = check_parent_object_access(path, W_OK | X_OK))){
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
if(0 != (result = check_object_access(path, W_OK, NULL))){
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
// Downloading
|
|
|
|
if(0 > (fd = get_opened_fd(path))){
|
|
|
|
if(0 > (fd = get_local_fd(path))){
|
|
|
|
if(foreground) {
|
|
|
|
printf(" s3fs_chown_nocopy line %d: get_local_fd result: %d\n", __LINE__, fd);
|
|
|
|
}
|
|
|
|
syslog(LOG_ERR, "s3fs_chown_nocopy line %d: get_local_fd result: %d", __LINE__, fd);
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
}else{
|
|
|
|
isclose = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get attributes
|
2011-09-01 19:24:12 +00:00
|
|
|
s3_realpath = get_realpath(path);
|
|
|
|
result = curl_get_headers(s3_realpath, meta);
|
|
|
|
free(s3_realpath);
|
2013-01-19 16:05:07 +00:00
|
|
|
if(result != 0){
|
|
|
|
if(isclose){
|
|
|
|
close(fd);
|
|
|
|
}
|
2011-09-01 19:24:12 +00:00
|
|
|
return result;
|
2013-01-19 16:05:07 +00:00
|
|
|
}
|
2010-12-19 22:27:56 +00:00
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
// Change owner
|
|
|
|
struct passwd *aaa = getpwuid(uid);
|
|
|
|
if(aaa != 0){
|
|
|
|
meta["x-amz-meta-uid"] = str((*aaa).pw_uid);
|
|
|
|
}
|
|
|
|
struct group *bbb = getgrgid(gid);
|
|
|
|
if(bbb != 0){
|
|
|
|
meta["x-amz-meta-gid"] = str((*bbb).gr_gid);
|
|
|
|
}
|
|
|
|
// Change local file owner
|
|
|
|
if(-1 == fchown(fd, uid, gid)){
|
|
|
|
if(isclose){
|
|
|
|
close(fd);
|
|
|
|
}
|
|
|
|
if(foreground) {
|
|
|
|
printf(" s3fs_chown_nocopy line %d: fchown(fd=%d, uid=%d, gid=%d) is error(%d)\n", __LINE__, fd, (int)uid, (int)gid, errno);
|
|
|
|
}
|
|
|
|
syslog(LOG_ERR, "s3fs_chown_nocopy line %d: fchown(fd=%d, uid=%d, gid=%d) is error(%d)", __LINE__, fd, (int)uid, (int)gid, errno);
|
2010-12-19 22:27:56 +00:00
|
|
|
return -errno;
|
|
|
|
}
|
2011-08-31 20:17:53 +00:00
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
// Re-uploading
|
|
|
|
if(0 != (result = put_local_fd(path, meta, fd))){
|
|
|
|
if(foreground) {
|
|
|
|
printf(" s3fs_chown_nocopy line %d: put_local_fd result: %d\n", __LINE__, result);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if(isclose){
|
|
|
|
close(fd);
|
|
|
|
}
|
|
|
|
delete_stat_cache_entry(path);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int s3fs_truncate(const char *path, off_t size) {
|
|
|
|
int fd = -1;
|
|
|
|
int result;
|
|
|
|
headers_t meta;
|
|
|
|
char *s3_realpath;
|
|
|
|
int isclose = 1;
|
|
|
|
|
|
|
|
if(foreground)
|
|
|
|
printf("s3fs_truncate[path=%s][size=%zd]\n", path, size);
|
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
if(0 != (result = check_parent_object_access(path, W_OK | X_OK))){
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
if(0 != (result = check_object_access(path, W_OK, NULL))){
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
// Get file information
|
|
|
|
s3_realpath = get_realpath(path);
|
|
|
|
result = curl_get_headers(s3_realpath, meta);
|
|
|
|
free(s3_realpath);
|
|
|
|
if(0 == result){
|
|
|
|
// Exists -> Get file
|
|
|
|
if(0 > (fd = get_opened_fd(path))){
|
|
|
|
if(0 > (fd = get_local_fd(path))){
|
|
|
|
if(foreground) {
|
|
|
|
printf(" s3fs_truncate line %d: get_local_fd result: %d\n", __LINE__, fd);
|
|
|
|
}
|
|
|
|
syslog(LOG_ERR, "s3fs_truncate line %d: get_local_fd result: %d", __LINE__, fd);
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
}else{
|
|
|
|
isclose = 0;
|
|
|
|
}
|
|
|
|
}else{
|
|
|
|
// Not found -> Make tmpfile
|
|
|
|
if(-1 == (fd = fileno(tmpfile()))){
|
|
|
|
syslog(LOG_ERR, "error: line %d: %d", __LINE__, -errno);
|
|
|
|
return -errno;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Truncate
|
|
|
|
if(0 != ftruncate(fd, size) || 0 != fsync(fd)){
|
|
|
|
if(foreground) {
|
|
|
|
printf(" s3fs_truncate line %d: ftruncate or fsync returned err(%d)\n", __LINE__, errno);
|
|
|
|
}
|
|
|
|
syslog(LOG_ERR, "s3fs_truncate line %d: ftruncate or fsync returned err(%d)", __LINE__, errno);
|
|
|
|
if(isclose){
|
2011-08-31 20:17:53 +00:00
|
|
|
close(fd);
|
2013-01-19 16:05:07 +00:00
|
|
|
}
|
|
|
|
return -errno;
|
|
|
|
}
|
2011-03-10 00:11:55 +00:00
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
// Re-uploading
|
|
|
|
if(0 != (result = put_local_fd(path, meta, fd))){
|
|
|
|
if(foreground) {
|
|
|
|
printf(" s3fs_truncate line %d: put_local_fd result: %d\n", __LINE__, result);
|
|
|
|
}
|
2010-12-19 22:27:56 +00:00
|
|
|
}
|
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
if(isclose){
|
2011-03-10 00:11:55 +00:00
|
|
|
close(fd);
|
2013-01-19 16:05:07 +00:00
|
|
|
}
|
|
|
|
delete_stat_cache_entry(path);
|
2010-11-13 23:59:23 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int s3fs_open(const char *path, struct fuse_file_info *fi) {
|
2011-01-19 05:26:01 +00:00
|
|
|
int result;
|
2011-06-26 00:37:52 +00:00
|
|
|
headers_t meta;
|
|
|
|
|
|
|
|
if(foreground)
|
|
|
|
cout << "s3fs_open[path=" << path << "][flags=" << fi->flags << "]" << endl;
|
2011-01-19 05:26:01 +00:00
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
int mask = (O_RDONLY != (fi->flags & O_ACCMODE) ? W_OK : R_OK);
|
|
|
|
if(0 != (result = check_parent_object_access(path, mask | X_OK))){
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
if(0 != (result = check_object_access(path, mask, NULL))){
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2011-01-19 05:26:01 +00:00
|
|
|
// Go do the truncation if called for
|
2011-06-26 00:37:52 +00:00
|
|
|
if((unsigned int)fi->flags & O_TRUNC) {
|
2011-01-19 05:26:01 +00:00
|
|
|
result = s3fs_truncate(path, 0);
|
2011-06-26 00:37:52 +00:00
|
|
|
if(result != 0)
|
2011-01-19 05:26:01 +00:00
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2011-08-31 20:17:53 +00:00
|
|
|
if((fi->fh = get_local_fd(path)) <= 0)
|
|
|
|
return -EIO;
|
2010-11-13 23:59:23 +00:00
|
|
|
|
|
|
|
// remember flags and headers...
|
2010-12-19 22:27:56 +00:00
|
|
|
pthread_mutex_lock( &s3fs_descriptors_lock );
|
2010-11-13 23:59:23 +00:00
|
|
|
s3fs_descriptors[fi->fh] = fi->flags;
|
2013-01-19 16:05:07 +00:00
|
|
|
s3fs_pathtofd[string(path)] = fi->fh;
|
2010-12-19 22:27:56 +00:00
|
|
|
pthread_mutex_unlock( &s3fs_descriptors_lock );
|
2010-11-13 23:59:23 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int s3fs_read(
|
|
|
|
const char *path, char *buf, size_t size, off_t offset, struct fuse_file_info *fi) {
|
2011-06-26 00:37:52 +00:00
|
|
|
int res;
|
2011-03-10 00:11:55 +00:00
|
|
|
|
2010-12-11 04:42:52 +00:00
|
|
|
if(foreground)
|
2010-12-28 04:15:23 +00:00
|
|
|
cout << "s3fs_read[path=" << path << "]" << endl;
|
2011-03-10 00:11:55 +00:00
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
res = pread(fi->fh, buf, size, offset);
|
2011-03-10 00:11:55 +00:00
|
|
|
if(res == -1)
|
2010-12-19 22:27:56 +00:00
|
|
|
YIKES(-errno);
|
2011-03-10 00:11:55 +00:00
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int s3fs_write(
|
|
|
|
const char *path, const char *buf, size_t size, off_t offset, struct fuse_file_info *fi) {
|
|
|
|
int res = pwrite(fi->fh, buf, size, offset);
|
2011-03-10 00:11:55 +00:00
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
// Commented - This message is output too much
|
|
|
|
//if(foreground)
|
|
|
|
// cout << "s3fs_write[path=" << path << "]" << endl;
|
2011-03-10 00:11:55 +00:00
|
|
|
|
|
|
|
if(res == -1)
|
2010-12-19 22:27:56 +00:00
|
|
|
YIKES(-errno);
|
2011-03-10 00:11:55 +00:00
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int s3fs_statfs(const char *path, struct statvfs *stbuf) {
|
|
|
|
// 256T
|
2011-06-27 02:21:38 +00:00
|
|
|
stbuf->f_bsize = 0X1000000;
|
2010-11-13 23:59:23 +00:00
|
|
|
stbuf->f_blocks = 0X1000000;
|
2011-06-27 02:21:38 +00:00
|
|
|
stbuf->f_bfree = 0x1000000;
|
2010-11-13 23:59:23 +00:00
|
|
|
stbuf->f_bavail = 0x1000000;
|
2011-07-19 19:52:38 +00:00
|
|
|
stbuf->f_namemax = NAME_MAX;
|
2010-11-13 23:59:23 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int get_flags(int fd) {
|
2010-12-19 22:27:56 +00:00
|
|
|
int flags;
|
|
|
|
pthread_mutex_lock( &s3fs_descriptors_lock );
|
|
|
|
flags = s3fs_descriptors[fd];
|
|
|
|
pthread_mutex_unlock( &s3fs_descriptors_lock );
|
|
|
|
return flags;
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int s3fs_flush(const char *path, struct fuse_file_info *fi) {
|
2011-02-23 17:16:12 +00:00
|
|
|
int flags;
|
2010-12-19 22:27:56 +00:00
|
|
|
int result;
|
2010-11-13 23:59:23 +00:00
|
|
|
int fd = fi->fh;
|
2011-09-01 19:24:12 +00:00
|
|
|
char *s3_realpath;
|
2010-12-09 20:56:29 +00:00
|
|
|
|
|
|
|
if(foreground)
|
2010-12-28 04:15:23 +00:00
|
|
|
cout << "s3fs_flush[path=" << path << "][fd=" << fd << "]" << endl;
|
2010-12-09 20:56:29 +00:00
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
int mask = (O_RDONLY != (fi->flags & O_ACCMODE) ? W_OK : R_OK);
|
|
|
|
if(0 != (result = check_parent_object_access(path, mask | X_OK))){
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
if(0 != (result = check_object_access(path, mask, NULL))){
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
// NOTE- fi->flags is not available here
|
2011-02-23 17:16:12 +00:00
|
|
|
flags = get_flags(fd);
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
if(O_RDONLY != (flags & O_ACCMODE)) {
|
2010-11-13 23:59:23 +00:00
|
|
|
headers_t meta;
|
2013-01-19 16:05:07 +00:00
|
|
|
s3_realpath = get_realpath(path);
|
2011-09-01 19:24:12 +00:00
|
|
|
result = curl_get_headers(s3_realpath, meta);
|
|
|
|
free(s3_realpath);
|
2011-02-22 21:28:01 +00:00
|
|
|
|
2011-03-10 00:11:55 +00:00
|
|
|
if(result != 0)
|
2011-02-22 22:28:57 +00:00
|
|
|
return result;
|
2011-02-22 21:28:01 +00:00
|
|
|
|
2011-03-10 00:11:55 +00:00
|
|
|
// if the cached file matches the remote file skip uploading
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
struct stat st;
|
|
|
|
if((fstat(fd, &st)) == -1)
|
|
|
|
YIKES(-errno);
|
2011-03-10 00:11:55 +00:00
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
if(str(st.st_size) == meta["Content-Length"] &&
|
|
|
|
(str(st.st_mtime) == meta["x-amz-meta-mtime"])) {
|
|
|
|
return result;
|
2011-03-10 00:11:55 +00:00
|
|
|
}
|
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
// If both mtime are not same, force to change mtime based on fd.
|
|
|
|
if(str(st.st_mtime) != meta["x-amz-meta-mtime"]){
|
|
|
|
meta["x-amz-meta-mtime"] = str(st.st_mtime);
|
2011-03-10 00:11:55 +00:00
|
|
|
}
|
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
return put_local_fd(path, meta, fd);
|
|
|
|
}
|
2011-02-22 21:28:01 +00:00
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int s3fs_release(const char *path, struct fuse_file_info *fi) {
|
2010-12-09 20:56:29 +00:00
|
|
|
if(foreground)
|
2011-07-07 22:09:40 +00:00
|
|
|
cout << "s3fs_release[path=" << path << "][fd=" << fi->fh << "]" << endl;
|
2010-12-09 20:56:29 +00:00
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
// clear file discriptor mapping.
|
|
|
|
s3fs_pathtofd_t::iterator it;
|
|
|
|
pthread_mutex_lock( &s3fs_descriptors_lock );
|
|
|
|
if(s3fs_pathtofd.end() != (it = s3fs_pathtofd.find(string(path)))){
|
|
|
|
if(fi->fh == (uint)s3fs_pathtofd[string(path)]){
|
|
|
|
s3fs_pathtofd.erase(it);
|
|
|
|
}else{
|
|
|
|
if(foreground) {
|
|
|
|
printf("s3fs_release line %d: file discriptor is not same(%d : %d)\n", __LINE__, (int)fi->fh, s3fs_pathtofd[string(path)]);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
pthread_mutex_unlock( &s3fs_descriptors_lock );
|
|
|
|
|
2011-07-07 22:09:40 +00:00
|
|
|
if(close(fi->fh) == -1)
|
2010-12-19 22:27:56 +00:00
|
|
|
YIKES(-errno);
|
2011-02-10 01:07:46 +00:00
|
|
|
|
2011-02-22 23:01:42 +00:00
|
|
|
if((fi->flags & O_RDWR) || (fi->flags & O_WRONLY))
|
|
|
|
delete_stat_cache_entry(path);
|
2011-02-10 01:07:46 +00:00
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
static CURL *create_head_handle(head_data *request_data) {
|
|
|
|
CURL *curl_handle = create_curl_handle();
|
|
|
|
string resource = urlEncode(service_path + bucket + request_data->path);
|
|
|
|
string url = host + resource;
|
|
|
|
|
|
|
|
// libcurl 7.17 does deep copy of url, deep copy "stable" url
|
|
|
|
string my_url = prepare_url(url.c_str());
|
|
|
|
request_data->url = new string(my_url.c_str());
|
|
|
|
request_data->requestHeaders = 0;
|
|
|
|
request_data->responseHeaders = new headers_t;
|
|
|
|
|
|
|
|
curl_easy_setopt(curl_handle, CURLOPT_URL, request_data->url->c_str());
|
|
|
|
curl_easy_setopt(curl_handle, CURLOPT_NOBODY, true); // HEAD
|
|
|
|
curl_easy_setopt(curl_handle, CURLOPT_FILETIME, true); // Last-Modified
|
|
|
|
|
|
|
|
// requestHeaders
|
|
|
|
string date = get_date();
|
|
|
|
request_data->requestHeaders = curl_slist_append(
|
|
|
|
request_data->requestHeaders, string("Date: " + date).c_str());
|
|
|
|
request_data->requestHeaders = curl_slist_append(
|
|
|
|
request_data->requestHeaders, string("Content-Type: ").c_str());
|
|
|
|
if(public_bucket.substr(0,1) != "1") {
|
|
|
|
request_data->requestHeaders = curl_slist_append(
|
|
|
|
request_data->requestHeaders, string("Authorization: AWS " + AWSAccessKeyId + ":" +
|
|
|
|
calc_signature("HEAD", "", date, request_data->requestHeaders, resource)).c_str());
|
|
|
|
}
|
|
|
|
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, request_data->requestHeaders);
|
|
|
|
|
|
|
|
// responseHeaders
|
|
|
|
curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, request_data->responseHeaders);
|
|
|
|
curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, header_callback);
|
|
|
|
|
|
|
|
return curl_handle;
|
|
|
|
}
|
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
static int s3fs_opendir(const char *path, struct fuse_file_info *fi)
|
|
|
|
{
|
|
|
|
int result;
|
|
|
|
int mask = (O_RDONLY != (fi->flags & O_ACCMODE) ? W_OK : R_OK) | X_OK;
|
|
|
|
|
|
|
|
if(foreground)
|
|
|
|
cout << "s3fs_opendir [path=" << path << "][flags=" << fi->flags << "]" << endl;
|
|
|
|
|
|
|
|
if(0 == (result = check_object_access(path, mask, NULL))){
|
|
|
|
result = check_parent_object_access(path, mask);
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
static int s3fs_readdir(
|
|
|
|
const char *path, void *buf, fuse_fill_dir_t filler, off_t offset, struct fuse_file_info *fi) {
|
2011-07-02 02:11:54 +00:00
|
|
|
CURLM *mh;
|
|
|
|
CURLMsg *msg;
|
2010-12-19 01:34:27 +00:00
|
|
|
CURLMcode curlm_code;
|
2011-07-02 02:11:54 +00:00
|
|
|
int n_reqs;
|
|
|
|
int n_objects;
|
|
|
|
int remaining_messages;
|
|
|
|
struct s3_object *head = NULL;
|
|
|
|
struct s3_object *headref = NULL;
|
|
|
|
auto_head curl_map;
|
2010-12-19 01:34:27 +00:00
|
|
|
|
2010-12-09 20:56:29 +00:00
|
|
|
if(foreground)
|
2013-01-19 16:05:07 +00:00
|
|
|
cout << "s3fs_readdir[path=" << path << "]" << endl;
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-08-30 19:44:26 +00:00
|
|
|
int result;
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
if(0 != (result = check_object_access(path, X_OK, NULL))){
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
// get a list of all the objects
|
2013-01-19 16:05:07 +00:00
|
|
|
if((result = list_bucket(path, &head)) != 0){
|
|
|
|
if(foreground)
|
|
|
|
printf(" s3fs_readdir list_bucket returns error.\n");
|
2011-08-30 19:44:26 +00:00
|
|
|
return result;
|
2013-01-19 16:05:07 +00:00
|
|
|
}
|
2011-07-28 15:43:52 +00:00
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
// force to add "." and ".." name.
|
|
|
|
if(0 != insert_object("..", &head) || 0 != insert_object(".", &head) || !head){
|
2013-01-19 16:05:07 +00:00
|
|
|
if(foreground)
|
|
|
|
printf(" s3fs_readdir list_bucket returns empty head.\n");
|
2011-07-02 02:11:54 +00:00
|
|
|
return 0;
|
2013-01-19 16:05:07 +00:00
|
|
|
}
|
2010-12-17 04:40:15 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
n_objects = count_object_list(head);
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
// populate fuse buffer
|
|
|
|
headref = head;
|
|
|
|
while(headref != NULL) {
|
|
|
|
filler(buf, headref->name, 0, 0);
|
|
|
|
headref = headref->next;
|
|
|
|
}
|
|
|
|
headref = head;
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
// populate the multi interface with an initial set of requests
|
|
|
|
n_reqs = 0;
|
|
|
|
mh = curl_multi_init();
|
|
|
|
while(n_reqs < MAX_REQUESTS && head != NULL) {
|
|
|
|
string fullpath = path;
|
2011-06-26 00:37:52 +00:00
|
|
|
if(strcmp(path, "/") != 0)
|
2011-07-02 02:11:54 +00:00
|
|
|
fullpath += "/" + string(head->name);
|
2011-06-26 00:37:52 +00:00
|
|
|
else
|
2011-07-02 02:11:54 +00:00
|
|
|
fullpath += string(head->name);
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
if(get_stat_cache_entry(fullpath.c_str(), NULL) == 0) {
|
|
|
|
head = head->next;
|
|
|
|
continue;
|
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
// file not cached, prepare a call to get_headers
|
|
|
|
head_data request_data;
|
|
|
|
request_data.path = fullpath;
|
|
|
|
CURL *curl_handle = create_head_handle(&request_data);
|
|
|
|
curl_map.get()[curl_handle] = request_data;
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
// add this handle to the multi handle
|
|
|
|
n_reqs++;
|
|
|
|
curlm_code = curl_multi_add_handle(mh, curl_handle);
|
|
|
|
if(curlm_code != CURLM_OK) {
|
|
|
|
syslog(LOG_ERR, "readdir: curl_multi_add_handle code: %d msg: %s",
|
|
|
|
curlm_code, curl_multi_strerror(curlm_code));
|
|
|
|
return -EIO;
|
|
|
|
}
|
2010-12-19 01:34:27 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
// go to the next object.
|
|
|
|
head = head->next;
|
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
// Start making requests.
|
|
|
|
int still_running = 0;
|
2011-07-29 20:03:38 +00:00
|
|
|
do {
|
|
|
|
curlm_code = curl_multi_perform(mh, &still_running);
|
|
|
|
} while(curlm_code == CURLM_CALL_MULTI_PERFORM);
|
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
if(curlm_code != CURLM_OK) {
|
|
|
|
syslog(LOG_ERR, "readdir: curl_multi_perform code: %d msg: %s",
|
|
|
|
curlm_code, curl_multi_strerror(curlm_code));
|
|
|
|
}
|
2010-12-17 04:40:15 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
while(still_running) {
|
2011-07-29 20:03:38 +00:00
|
|
|
do {
|
|
|
|
curlm_code = curl_multi_perform(mh, &still_running);
|
|
|
|
} while(curlm_code == CURLM_CALL_MULTI_PERFORM);
|
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
if(curlm_code != CURLM_OK) {
|
|
|
|
syslog(LOG_ERR, "s3fs_readdir: curl_multi_perform code: %d msg: %s",
|
|
|
|
curlm_code, curl_multi_strerror(curlm_code));
|
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
if(still_running) {
|
|
|
|
fd_set r_fd;
|
|
|
|
fd_set w_fd;
|
|
|
|
fd_set e_fd;
|
|
|
|
FD_ZERO(&r_fd);
|
|
|
|
FD_ZERO(&w_fd);
|
|
|
|
FD_ZERO(&e_fd);
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
long milliseconds;
|
|
|
|
curlm_code = curl_multi_timeout(mh, &milliseconds);
|
|
|
|
if(curlm_code != CURLM_OK) {
|
|
|
|
syslog(LOG_ERR, "readdir: curl_multi_perform code: %d msg: %s",
|
|
|
|
curlm_code, curl_multi_strerror(curlm_code));
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
if(milliseconds < 0)
|
|
|
|
milliseconds = 50;
|
|
|
|
if(milliseconds > 0) {
|
|
|
|
struct timeval timeout;
|
|
|
|
timeout.tv_sec = 1000 * milliseconds / 1000000;
|
|
|
|
timeout.tv_usec = 1000 * milliseconds % 1000000;
|
2010-12-19 01:34:27 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
int max_fd;
|
|
|
|
curlm_code = curl_multi_fdset(mh, &r_fd, &w_fd, &e_fd, &max_fd);
|
|
|
|
if(curlm_code != CURLM_OK) {
|
|
|
|
syslog(LOG_ERR, "readdir: curl_multi_fdset code: %d msg: %s",
|
|
|
|
curlm_code, curl_multi_strerror(curlm_code));
|
|
|
|
return -EIO;
|
|
|
|
}
|
2011-02-23 17:16:12 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
if(select(max_fd + 1, &r_fd, &w_fd, &e_fd, &timeout) == -1)
|
|
|
|
YIKES(-errno);
|
2010-12-17 04:40:15 +00:00
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
while((msg = curl_multi_info_read(mh, &remaining_messages))) {
|
|
|
|
if(msg->msg == CURLMSG_DONE) {
|
|
|
|
CURLcode code = msg->data.result;
|
|
|
|
if(code != 0) {
|
|
|
|
syslog(LOG_DEBUG, "s3fs_readdir: remaining_msgs: %i code: %d msg: %s",
|
|
|
|
remaining_messages, code, curl_easy_strerror(code));
|
|
|
|
return -EIO;
|
|
|
|
}
|
2010-12-19 01:34:27 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
CURL *curl_handle = msg->easy_handle;
|
|
|
|
head_data response = curl_map.get()[curl_handle];
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
struct stat st;
|
|
|
|
memset(&st, 0, sizeof(st));
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
st.st_nlink = 1; // see fuse FAQ
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
// mode
|
2011-08-31 17:37:55 +00:00
|
|
|
st.st_mode = get_mode((*response.responseHeaders)["x-amz-meta-mode"].c_str());
|
2011-02-11 03:52:31 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
// content-type
|
|
|
|
char *ContentType = 0;
|
|
|
|
if(curl_easy_getinfo(curl_handle, CURLINFO_CONTENT_TYPE, &ContentType) == 0)
|
|
|
|
if(ContentType)
|
|
|
|
st.st_mode |= strcmp(ContentType, "application/x-directory") == 0 ? S_IFDIR : S_IFREG;
|
2011-02-11 03:52:31 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
// mtime
|
2011-08-31 17:37:55 +00:00
|
|
|
st.st_mtime = get_mtime((*response.responseHeaders)["x-amz-meta-mtime"].c_str());
|
2011-07-02 02:11:54 +00:00
|
|
|
if(st.st_mtime == 0) {
|
|
|
|
long LastModified;
|
|
|
|
if(curl_easy_getinfo(curl_handle, CURLINFO_FILETIME, &LastModified) == 0)
|
|
|
|
st.st_mtime = LastModified;
|
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
// size
|
|
|
|
double ContentLength;
|
|
|
|
if(curl_easy_getinfo(curl_handle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &ContentLength) == 0)
|
|
|
|
st.st_size = static_cast<off_t>(ContentLength);
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
if(S_ISREG(st.st_mode))
|
2011-08-31 19:02:30 +00:00
|
|
|
st.st_blocks = get_blocks(st.st_size);
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-08-31 17:37:55 +00:00
|
|
|
st.st_uid = get_uid((*response.responseHeaders)["x-amz-meta-uid"].c_str());
|
|
|
|
st.st_gid = get_gid((*response.responseHeaders)["x-amz-meta-gid"].c_str());
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
add_stat_cache_entry(response.path.c_str(), &st);
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
// cleanup
|
|
|
|
curl_multi_remove_handle(mh, curl_handle);
|
|
|
|
destroy_curl_handle(curl_handle);
|
|
|
|
n_reqs--;
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
// add additional requests
|
|
|
|
while(n_reqs < MAX_REQUESTS && head != NULL) {
|
|
|
|
string fullpath = path;
|
|
|
|
if(strcmp(path, "/") != 0)
|
|
|
|
fullpath += "/" + string(head->name);
|
|
|
|
else
|
|
|
|
fullpath += string(head->name);
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
if(get_stat_cache_entry(fullpath.c_str(), NULL) == 0) {
|
|
|
|
head = head->next;
|
|
|
|
continue;
|
|
|
|
}
|
2010-12-19 01:34:27 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
// file not cached, prepare a call to get_headers
|
|
|
|
head_data request_data;
|
|
|
|
request_data.path = fullpath;
|
|
|
|
CURL *curl_handle = create_head_handle(&request_data);
|
|
|
|
curl_map.get()[curl_handle] = request_data;
|
|
|
|
|
|
|
|
// add this handle to the multi handle
|
|
|
|
n_reqs++;
|
|
|
|
curlm_code = curl_multi_add_handle(mh, curl_handle);
|
|
|
|
if(curlm_code != CURLM_OK) {
|
|
|
|
syslog(LOG_ERR, "readdir: curl_multi_add_handle code: %d msg: %s",
|
|
|
|
curlm_code, curl_multi_strerror(curlm_code));
|
|
|
|
return -EIO;
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
2011-07-02 02:11:54 +00:00
|
|
|
|
|
|
|
// prevent this from sitting at 0, we may have requests to finish
|
2011-07-07 16:25:04 +00:00
|
|
|
still_running++;
|
2011-07-02 02:11:54 +00:00
|
|
|
|
|
|
|
// go to the next object.
|
|
|
|
head = head->next;
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
2011-07-02 02:11:54 +00:00
|
|
|
} else {
|
|
|
|
syslog(LOG_ERR, "readdir: curl_multi_add_handle code: %d msg: %s",
|
|
|
|
curlm_code, curl_multi_strerror(curlm_code));
|
|
|
|
|
|
|
|
curl_multi_cleanup(mh);
|
|
|
|
free_object_list(headref);
|
|
|
|
return -EIO;
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
}
|
2011-07-02 02:11:54 +00:00
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
curl_multi_cleanup(mh);
|
|
|
|
free_object_list(headref);
|
2010-12-19 01:34:27 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
return 0;
|
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
static int list_bucket(const char *path, struct s3_object **head) {
|
|
|
|
CURL *curl;
|
|
|
|
int result;
|
|
|
|
char *s3_realpath;
|
|
|
|
struct BodyStruct body;
|
|
|
|
bool truncated = true;
|
|
|
|
string next_marker = "";
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
if(foreground)
|
|
|
|
printf("list_bucket [path=%s]\n", path);
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
body.text = (char *) malloc(1);
|
|
|
|
body.size = 0;
|
|
|
|
s3_realpath = get_realpath(path);
|
2010-12-19 01:34:27 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
string resource = urlEncode(service_path + bucket); // this is what gets signed
|
|
|
|
string query = "delimiter=/&prefix=";
|
2010-12-19 01:34:27 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
if(strcmp(path, "/") != 0)
|
|
|
|
query += urlEncode(string(s3_realpath).substr(1) + "/");
|
|
|
|
else
|
|
|
|
query += urlEncode(string(s3_realpath).substr(1));
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
free(s3_realpath);
|
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
query += "&max-keys=1000";
|
2010-12-19 01:34:27 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
while(truncated) {
|
|
|
|
string url = host + resource + "?" + query;
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
if(next_marker != ""){
|
2011-07-02 02:11:54 +00:00
|
|
|
url += "&marker=" + urlEncode(next_marker);
|
2013-01-19 16:05:07 +00:00
|
|
|
next_marker = "";
|
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
string my_url = prepare_url(url.c_str());
|
|
|
|
|
|
|
|
auto_curl_slist headers;
|
|
|
|
string date = get_date();
|
|
|
|
headers.append("Date: " + date);
|
|
|
|
headers.append("ContentType: ");
|
|
|
|
if(public_bucket.substr(0,1) != "1") {
|
|
|
|
headers.append("Authorization: AWS " + AWSAccessKeyId + ":" +
|
|
|
|
calc_signature("GET", "", date, headers.get(), resource + "/"));
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
curl = create_curl_handle();
|
|
|
|
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
|
|
|
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &body);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
|
2011-06-26 00:37:52 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
result = my_curl_easy_perform(curl, &body);
|
|
|
|
destroy_curl_handle(curl);
|
2011-06-26 00:37:52 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
if(result != 0) {
|
2013-01-19 16:05:07 +00:00
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
|
|
|
if(foreground)
|
|
|
|
printf(" list_bucket my_curl_easy_perform returns with error.\n");
|
2011-07-02 02:11:54 +00:00
|
|
|
return result;
|
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
if((append_objects_from_xml(path, body.text, head)) != 0) {
|
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
|
|
|
if(foreground)
|
|
|
|
printf(" list_bucket append_objects_from_xml returns with error.\n");
|
2011-07-02 02:11:54 +00:00
|
|
|
return -1;
|
2011-08-31 17:00:00 +00:00
|
|
|
}
|
2011-02-14 18:54:30 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
truncated = is_truncated(body.text);
|
2013-01-19 16:05:07 +00:00
|
|
|
if(truncated){
|
|
|
|
xmlChar* tmpch = (xmlChar*)get_next_marker(body.text);
|
|
|
|
if(tmpch){
|
|
|
|
next_marker = (char*)tmpch;
|
|
|
|
xmlFree(tmpch);
|
|
|
|
}
|
|
|
|
}
|
2011-07-02 02:11:54 +00:00
|
|
|
|
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
|
|
|
body.size = 0;
|
|
|
|
body.text = (char *) malloc(1);
|
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
2011-06-26 00:37:52 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
const char* c_strErrorObjectName = "FILE or SUBDIR in DIR";
|
|
|
|
|
|
|
|
static int append_objects_from_xml(const char* path, const char *xml, struct s3_object **head) {
|
2011-07-02 02:11:54 +00:00
|
|
|
xmlDocPtr doc;
|
|
|
|
xmlXPathContextPtr ctx;
|
|
|
|
xmlXPathObjectPtr contents_xp;
|
|
|
|
xmlNodeSetPtr content_nodes;
|
2013-01-19 16:05:07 +00:00
|
|
|
string exp_contents = noxmlns ? "//Contents" : "//s3:Contents";
|
|
|
|
string exp_Key = noxmlns ? "Key" : "s3:Key";
|
2011-07-02 02:11:54 +00:00
|
|
|
|
|
|
|
doc = xmlReadMemory(xml, strlen(xml), "", NULL, 0);
|
2013-01-19 16:05:07 +00:00
|
|
|
if(doc == NULL){
|
|
|
|
if(foreground)
|
|
|
|
printf(" append_objects_from_xml xmlReadMemory returns with error.\n");
|
2011-07-28 15:43:52 +00:00
|
|
|
return -1;
|
2013-01-19 16:05:07 +00:00
|
|
|
}
|
2011-07-28 15:43:52 +00:00
|
|
|
|
2011-07-02 02:11:54 +00:00
|
|
|
ctx = xmlXPathNewContext(doc);
|
2013-01-19 16:05:07 +00:00
|
|
|
if(!noxmlns){
|
|
|
|
xmlXPathRegisterNs(ctx, (xmlChar *) "s3",
|
|
|
|
(xmlChar *) "http://s3.amazonaws.com/doc/2006-03-01/");
|
|
|
|
}
|
|
|
|
contents_xp = xmlXPathEvalExpression((xmlChar *)exp_contents.c_str(), ctx);
|
2011-07-02 02:11:54 +00:00
|
|
|
content_nodes = contents_xp->nodesetval;
|
|
|
|
|
|
|
|
int i;
|
|
|
|
for(i = 0; i < content_nodes->nodeNr; i++) {
|
|
|
|
ctx->node = content_nodes->nodeTab[i];
|
|
|
|
|
|
|
|
// object name
|
2013-01-19 16:05:07 +00:00
|
|
|
xmlXPathObjectPtr key = xmlXPathEvalExpression((xmlChar *)exp_Key.c_str(), ctx);
|
2011-07-02 02:11:54 +00:00
|
|
|
xmlNodeSetPtr key_nodes = key->nodesetval;
|
2013-01-19 16:05:07 +00:00
|
|
|
char *name = get_object_name(doc, key_nodes->nodeTab[0]->xmlChildrenNode, path);
|
|
|
|
|
|
|
|
if(!name){
|
|
|
|
if(foreground)
|
|
|
|
printf(" append_objects_from_xml name is something wrong. but continue.\n");
|
|
|
|
|
|
|
|
}else if((const char*)name != c_strErrorObjectName){
|
|
|
|
if((insert_object(name, head)) != 0){
|
|
|
|
if(foreground)
|
|
|
|
printf(" append_objects_from_xml insert_object returns with error.\n");
|
|
|
|
|
|
|
|
xmlXPathFreeObject(key);
|
|
|
|
xmlXPathFreeObject(contents_xp);
|
|
|
|
xmlXPathFreeContext(ctx);
|
|
|
|
xmlFreeDoc(doc);
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
free(name);
|
2013-01-19 16:05:07 +00:00
|
|
|
return -1;
|
|
|
|
}
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
free(name);
|
2013-01-19 16:05:07 +00:00
|
|
|
}else{
|
|
|
|
//if(foreground)
|
|
|
|
// printf("append_objects_from_xml name is file or subdir in dir. but continue.\n");
|
2011-08-31 17:00:00 +00:00
|
|
|
}
|
2011-07-02 02:11:54 +00:00
|
|
|
xmlXPathFreeObject(key);
|
|
|
|
}
|
|
|
|
|
|
|
|
xmlXPathFreeObject(contents_xp);
|
|
|
|
xmlXPathFreeContext(ctx);
|
|
|
|
xmlFreeDoc(doc);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static const char *get_next_marker(const char *xml) {
|
|
|
|
xmlDocPtr doc;
|
|
|
|
xmlXPathContextPtr ctx;
|
|
|
|
xmlXPathObjectPtr marker_xp;
|
|
|
|
xmlNodeSetPtr nodes;
|
|
|
|
char *next_marker;
|
2013-01-19 16:05:07 +00:00
|
|
|
string exp_nextmarker = noxmlns ? "//NextMarker" : "//s3:NextMarker";
|
2011-07-02 02:11:54 +00:00
|
|
|
|
|
|
|
doc = xmlReadMemory(xml, strlen(xml), "", NULL, 0);
|
|
|
|
ctx = xmlXPathNewContext(doc);
|
2013-01-19 16:05:07 +00:00
|
|
|
if(!noxmlns){
|
|
|
|
xmlXPathRegisterNs(ctx, (xmlChar *) "s3",
|
|
|
|
(xmlChar *) "http://s3.amazonaws.com/doc/2006-03-01/");
|
|
|
|
}
|
|
|
|
marker_xp = xmlXPathEvalExpression((xmlChar *)exp_nextmarker.c_str(), ctx);
|
2011-07-02 02:11:54 +00:00
|
|
|
nodes = marker_xp->nodesetval;
|
|
|
|
|
|
|
|
if(nodes->nodeNr < 1)
|
|
|
|
return "";
|
|
|
|
|
|
|
|
next_marker = (char *) xmlNodeListGetString(doc, nodes->nodeTab[0]->xmlChildrenNode, 1);
|
|
|
|
|
|
|
|
xmlXPathFreeObject(marker_xp);
|
|
|
|
xmlXPathFreeContext(ctx);
|
|
|
|
xmlFreeDoc(doc);
|
|
|
|
|
|
|
|
return next_marker;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool is_truncated(const char *xml) {
|
|
|
|
if(strstr(xml, "<IsTruncated>true</IsTruncated>"))
|
|
|
|
return true;
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
// return: the pointer to object name on allocated memory.
|
|
|
|
// the pointer to "c_strErrorObjectName".(not allocated)
|
|
|
|
// NULL(a case of something error occured)
|
2013-01-19 16:05:07 +00:00
|
|
|
static char *get_object_name(xmlDocPtr doc, xmlNodePtr node, const char* path)
|
|
|
|
{
|
|
|
|
// Get full path
|
|
|
|
xmlChar* fullpath= xmlNodeListGetString(doc, node, 1);
|
|
|
|
if(!fullpath){
|
|
|
|
if(foreground)
|
|
|
|
printf(" get_object_name could not get object full path name..\n");
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
// Make dir path and filename
|
|
|
|
string strfullpath= (char*)fullpath;
|
|
|
|
string strdirpath = mydirname((char*)fullpath);
|
|
|
|
string strmybpath = mybasename((char*)fullpath);
|
|
|
|
const char* dirpath = strdirpath.c_str();
|
|
|
|
const char* mybname = strmybpath.c_str();
|
|
|
|
const char* basepath= (!path || '\0' == path[0] || '/' != path[0] ? path : &path[1]);
|
|
|
|
xmlFree(fullpath);
|
|
|
|
|
|
|
|
if(!mybname || '\0' == mybname[0]){
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
// check subdir & file in subdir
|
|
|
|
if(dirpath && 0 < strlen(dirpath)){
|
|
|
|
// case of "/"
|
|
|
|
if(0 == strcmp(mybname, "/") && 0 == strcmp(dirpath, "/")){
|
|
|
|
return (char*)c_strErrorObjectName;
|
|
|
|
}
|
|
|
|
// case of "."
|
|
|
|
if(0 == strcmp(mybname, ".") && 0 == strcmp(dirpath, ".")){
|
|
|
|
return (char*)c_strErrorObjectName;
|
|
|
|
}
|
|
|
|
// case of ".."
|
|
|
|
if(0 == strcmp(mybname, "..") && 0 == strcmp(dirpath, ".")){
|
|
|
|
return (char*)c_strErrorObjectName;
|
|
|
|
}
|
|
|
|
// case of "name"
|
|
|
|
if(0 == strcmp(dirpath, ".")){
|
|
|
|
// OK
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
return strdup(mybname);
|
2013-01-19 16:05:07 +00:00
|
|
|
}else{
|
|
|
|
if(0 == strcmp(dirpath, basepath)){
|
|
|
|
// OK
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
return strdup(mybname);
|
2013-01-19 16:05:07 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// case of something wrong
|
|
|
|
return (char*)c_strErrorObjectName;
|
2011-07-02 02:11:54 +00:00
|
|
|
}
|
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
static int remote_mountpath_exists(const char *path) {
|
2011-08-31 19:27:50 +00:00
|
|
|
struct stat stbuf;
|
2011-06-26 00:37:52 +00:00
|
|
|
|
|
|
|
if(foreground)
|
|
|
|
printf("remote_mountpath_exists [path=%s]\n", path);
|
|
|
|
|
2011-08-31 19:27:50 +00:00
|
|
|
// getattr will prefix the path with the remote mountpoint
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
get_object_attribute("", &stbuf);
|
2011-06-26 00:37:52 +00:00
|
|
|
if(!S_ISDIR(stbuf.st_mode))
|
|
|
|
return -1;
|
2010-12-19 01:34:27 +00:00
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* OpenSSL locking function.
|
|
|
|
*
|
|
|
|
* @param mode lock mode
|
|
|
|
* @param n lock number
|
|
|
|
* @param file source file name
|
|
|
|
* @param line source file line number
|
|
|
|
* @return none
|
|
|
|
*/
|
|
|
|
static void locking_function(int mode, int n, const char *file, int line) {
|
2011-09-01 19:24:12 +00:00
|
|
|
if(mode & CRYPTO_LOCK)
|
2010-11-13 23:59:23 +00:00
|
|
|
pthread_mutex_lock(&mutex_buf[n]);
|
2011-09-01 19:24:12 +00:00
|
|
|
else
|
2010-11-13 23:59:23 +00:00
|
|
|
pthread_mutex_unlock(&mutex_buf[n]);
|
|
|
|
}
|
|
|
|
|
2011-08-31 19:27:50 +00:00
|
|
|
// OpenSSL uniq thread id function.
|
2011-02-23 17:16:12 +00:00
|
|
|
static unsigned long id_function(void) {
|
|
|
|
return((unsigned long) pthread_self());
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void* s3fs_init(struct fuse_conn_info *conn) {
|
|
|
|
syslog(LOG_INFO, "init $Rev$");
|
2011-09-01 19:24:12 +00:00
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
if(foreground)
|
|
|
|
cout << "s3fs_init" << endl;
|
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
// openssl
|
|
|
|
mutex_buf = static_cast<pthread_mutex_t*>(malloc(CRYPTO_num_locks() * sizeof(pthread_mutex_t)));
|
|
|
|
for (int i = 0; i < CRYPTO_num_locks(); i++)
|
|
|
|
pthread_mutex_init(&mutex_buf[i], NULL);
|
|
|
|
CRYPTO_set_locking_callback(locking_function);
|
|
|
|
CRYPTO_set_id_callback(id_function);
|
|
|
|
curl_global_init(CURL_GLOBAL_ALL);
|
|
|
|
pthread_mutex_init(&curl_handles_lock, NULL);
|
|
|
|
pthread_mutex_init(&s3fs_descriptors_lock, NULL);
|
|
|
|
pthread_mutex_init(&stat_cache_lock, NULL);
|
2011-08-30 22:09:17 +00:00
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
string line;
|
2010-11-22 18:28:07 +00:00
|
|
|
ifstream MT("/etc/mime.types");
|
|
|
|
if (MT.good()) {
|
|
|
|
while (getline(MT, line)) {
|
2011-09-01 19:24:12 +00:00
|
|
|
if(line[0]=='#')
|
2010-11-13 23:59:23 +00:00
|
|
|
continue;
|
2011-09-01 19:24:12 +00:00
|
|
|
if(line.size() == 0)
|
2010-11-22 18:28:07 +00:00
|
|
|
continue;
|
|
|
|
stringstream tmp(line);
|
|
|
|
string mimeType;
|
|
|
|
tmp >> mimeType;
|
|
|
|
while (tmp) {
|
|
|
|
string ext;
|
|
|
|
tmp >> ext;
|
|
|
|
if (ext.size() == 0)
|
|
|
|
continue;
|
|
|
|
mimeTypes[ext] = mimeType;
|
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
}
|
2011-01-19 05:26:01 +00:00
|
|
|
|
|
|
|
// Investigate system capabilities
|
2011-08-30 22:09:17 +00:00
|
|
|
if((unsigned int)conn->capable & FUSE_CAP_ATOMIC_O_TRUNC)
|
2011-01-19 05:26:01 +00:00
|
|
|
conn->want |= FUSE_CAP_ATOMIC_O_TRUNC;
|
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void s3fs_destroy(void*) {
|
2011-06-26 00:37:52 +00:00
|
|
|
if(debug)
|
|
|
|
syslog(LOG_DEBUG, "destroy");
|
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
if(foreground)
|
|
|
|
cout << "s3fs_destroy" << endl;
|
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
// openssl
|
|
|
|
CRYPTO_set_id_callback(NULL);
|
|
|
|
CRYPTO_set_locking_callback(NULL);
|
2011-08-30 22:09:17 +00:00
|
|
|
for(int i = 0; i < CRYPTO_num_locks(); i++)
|
2010-11-13 23:59:23 +00:00
|
|
|
pthread_mutex_destroy(&mutex_buf[i]);
|
|
|
|
free(mutex_buf);
|
|
|
|
mutex_buf = NULL;
|
|
|
|
curl_global_cleanup();
|
|
|
|
pthread_mutex_destroy(&curl_handles_lock);
|
|
|
|
pthread_mutex_destroy(&s3fs_descriptors_lock);
|
|
|
|
pthread_mutex_destroy(&stat_cache_lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int s3fs_access(const char *path, int mask) {
|
2010-12-11 04:42:52 +00:00
|
|
|
if(foreground)
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
printf("s3fs_access[path=%s][mask=%s%s%s%s]\n", path,
|
|
|
|
((mask & R_OK) == R_OK) ? "R_OK " : "",
|
|
|
|
((mask & W_OK) == W_OK) ? "W_OK " : "",
|
|
|
|
((mask & X_OK) == X_OK) ? "X_OK " : "",
|
|
|
|
(mask == F_OK) ? "F_OK" : "");
|
2011-08-30 22:09:17 +00:00
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
return check_object_access(path, mask, NULL);
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int s3fs_utimens(const char *path, const struct timespec ts[2]) {
|
2010-12-19 22:27:56 +00:00
|
|
|
int result;
|
2011-06-26 00:37:52 +00:00
|
|
|
char *s3_realpath;
|
2010-12-28 04:15:23 +00:00
|
|
|
headers_t meta;
|
|
|
|
|
2010-12-09 20:56:29 +00:00
|
|
|
if(foreground)
|
2011-08-31 19:27:50 +00:00
|
|
|
printf("s3fs_utimens[path=%s][mtime=%zd]\n", path, ts[1].tv_sec);
|
2010-12-09 20:56:29 +00:00
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
if(0 != (result = check_parent_object_access(path, W_OK | X_OK))){
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
if(0 != (result = check_object_access(path, W_OK, NULL))){
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2011-09-01 19:24:12 +00:00
|
|
|
s3_realpath = get_realpath(path);
|
2013-01-19 16:05:07 +00:00
|
|
|
if((result = curl_get_headers(s3_realpath, meta) != 0)) {
|
2011-09-01 19:24:12 +00:00
|
|
|
free(s3_realpath);
|
2011-03-10 00:11:55 +00:00
|
|
|
return result;
|
2011-09-01 19:24:12 +00:00
|
|
|
}
|
2010-12-28 04:15:23 +00:00
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
meta["x-amz-meta-mtime"] = str(ts[1].tv_sec);
|
2011-06-26 00:37:52 +00:00
|
|
|
meta["x-amz-copy-source"] = urlEncode("/" + bucket + s3_realpath);
|
2010-11-13 23:59:23 +00:00
|
|
|
meta["x-amz-metadata-directive"] = "REPLACE";
|
2011-06-26 00:37:52 +00:00
|
|
|
free(s3_realpath);
|
2010-12-28 04:15:23 +00:00
|
|
|
|
2011-08-31 20:17:53 +00:00
|
|
|
return put_headers(path, meta);
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
static int s3fs_utimens_nocopy(const char *path, const struct timespec ts[2]) {
|
|
|
|
int result;
|
|
|
|
char *s3_realpath;
|
|
|
|
headers_t meta;
|
|
|
|
int fd;
|
|
|
|
int isclose = 1;
|
|
|
|
struct timeval tv[2];
|
|
|
|
|
|
|
|
if(foreground)
|
|
|
|
cout << "s3fs_utimens_nocopy [path=" << path << "][mtime=" << str(ts[1].tv_sec) << "]" << endl;
|
|
|
|
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
if(0 != (result = check_parent_object_access(path, W_OK | X_OK))){
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
if(0 != (result = check_object_access(path, W_OK, NULL))){
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
// Downloading
|
|
|
|
if(0 > (fd = get_opened_fd(path))){
|
|
|
|
if(0 > (fd = get_local_fd(path))){
|
|
|
|
if(foreground) {
|
|
|
|
printf(" s3fs_utimens_nocopy line %d: get_local_fd result: %d\n", __LINE__, fd);
|
|
|
|
}
|
|
|
|
syslog(LOG_ERR, "s3fs_utimens_nocopy line %d: get_local_fd result: %d", __LINE__, fd);
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
}else{
|
|
|
|
isclose = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get attributes
|
|
|
|
s3_realpath = get_realpath(path);
|
|
|
|
result = curl_get_headers(s3_realpath, meta);
|
|
|
|
free(s3_realpath);
|
|
|
|
if(result != 0){
|
|
|
|
if(isclose){
|
|
|
|
close(fd);
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Change date
|
|
|
|
meta["x-amz-meta-mtime"] = str(ts[1].tv_sec);
|
|
|
|
// Change local file date
|
|
|
|
TIMESPEC_TO_TIMEVAL(&tv[0], &ts[0]);
|
|
|
|
TIMESPEC_TO_TIMEVAL(&tv[1], &ts[1]);
|
|
|
|
if(-1 == futimes(fd, tv)){
|
|
|
|
if(isclose){
|
|
|
|
close(fd);
|
|
|
|
}
|
|
|
|
if(foreground) {
|
|
|
|
printf(" s3fs_utimens_nocopy line %d: futimes(fd=%d, ...) is error(%d)\n", __LINE__, fd, errno);
|
|
|
|
}
|
|
|
|
syslog(LOG_ERR, "s3fs_utimens_nocopy line %d: futimes(fd=%d, ...) is error(%d)", __LINE__, fd, errno);
|
|
|
|
return -errno;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Re-uploading
|
|
|
|
if(0 != (result = put_local_fd(path, meta, fd))){
|
|
|
|
if(foreground) {
|
|
|
|
printf(" s3fs_utimens_nocopy line %d: put_local_fd result: %d\n", __LINE__, result);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if(isclose){
|
|
|
|
close(fd);
|
|
|
|
}
|
|
|
|
delete_stat_cache_entry(path);
|
|
|
|
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2010-12-30 03:13:21 +00:00
|
|
|
///////////////////////////////////////////////////////////
|
|
|
|
// List Multipart Uploads for bucket
|
|
|
|
///////////////////////////////////////////////////////////
|
|
|
|
static int list_multipart_uploads(void) {
|
|
|
|
CURL *curl = NULL;
|
|
|
|
string resource;
|
|
|
|
string url;
|
|
|
|
struct BodyStruct body;
|
|
|
|
int result;
|
|
|
|
string date;
|
|
|
|
string raw_date;
|
|
|
|
string auth;
|
|
|
|
string my_url;
|
|
|
|
struct curl_slist *slist=NULL;
|
|
|
|
|
|
|
|
// Initialization of variables
|
|
|
|
body.text = (char *)malloc(1);
|
|
|
|
body.size = 0;
|
|
|
|
|
2011-02-17 17:31:43 +00:00
|
|
|
printf("List Multipart Uploads\n");
|
2010-12-30 03:13:21 +00:00
|
|
|
resource = urlEncode(service_path + bucket + "/");
|
|
|
|
resource.append("?uploads");
|
|
|
|
url = host + resource;
|
|
|
|
my_url = prepare_url(url.c_str());
|
|
|
|
curl = create_curl_handle();
|
|
|
|
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&body);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
|
|
|
|
|
|
|
|
date.assign("Date: ");
|
|
|
|
raw_date = get_date();
|
|
|
|
date.append(raw_date);
|
|
|
|
slist = curl_slist_append(slist, date.c_str());
|
|
|
|
slist = curl_slist_append(slist, "Accept:");
|
|
|
|
|
|
|
|
if (public_bucket.substr(0,1) != "1") {
|
|
|
|
auth.assign("Authorization: AWS ");
|
|
|
|
auth.append(AWSAccessKeyId);
|
|
|
|
auth.append(":");
|
|
|
|
auth.append(calc_signature("GET", "", raw_date, slist, resource));
|
|
|
|
slist = curl_slist_append(slist, auth.c_str());
|
|
|
|
}
|
|
|
|
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
|
|
|
|
result = my_curl_easy_perform(curl, &body);
|
|
|
|
curl_slist_free_all(slist);
|
|
|
|
destroy_curl_handle(curl);
|
|
|
|
|
|
|
|
if(result != 0) {
|
2011-07-02 02:11:54 +00:00
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
2010-12-30 03:13:21 +00:00
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2011-08-30 22:09:17 +00:00
|
|
|
if(body.size > 0)
|
2010-12-30 03:13:21 +00:00
|
|
|
printf("body.text:\n%s\n", body.text);
|
2013-01-19 16:05:07 +00:00
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
2010-12-30 03:13:21 +00:00
|
|
|
|
2011-08-30 22:09:17 +00:00
|
|
|
return 0;
|
2010-12-30 03:13:21 +00:00
|
|
|
}
|
|
|
|
|
2010-11-20 17:55:15 +00:00
|
|
|
static void s3fs_check_service(void) {
|
2010-12-19 01:34:27 +00:00
|
|
|
CURL *curl = NULL;
|
2011-08-30 22:09:17 +00:00
|
|
|
int result = CURLE_OK;
|
|
|
|
CURLcode responseCode;
|
2010-12-19 01:34:27 +00:00
|
|
|
|
2010-12-09 20:56:29 +00:00
|
|
|
if(foreground)
|
2011-08-30 22:09:17 +00:00
|
|
|
printf("s3fs_check_service\n");
|
2010-11-20 17:55:15 +00:00
|
|
|
|
2010-12-17 04:40:15 +00:00
|
|
|
struct BodyStruct body;
|
2011-09-01 19:24:12 +00:00
|
|
|
body.text = (char *) malloc(1);
|
2011-06-26 00:37:52 +00:00
|
|
|
body.size = 0;
|
2010-12-17 04:40:15 +00:00
|
|
|
|
2011-08-30 22:09:17 +00:00
|
|
|
string resource = urlEncode(service_path + bucket);
|
2010-11-20 17:55:15 +00:00
|
|
|
string url = host + resource;
|
|
|
|
|
|
|
|
auto_curl_slist headers;
|
|
|
|
string date = get_date();
|
|
|
|
headers.append("Date: " + date);
|
|
|
|
if (public_bucket.substr(0,1) != "1") {
|
|
|
|
headers.append("Authorization: AWS " + AWSAccessKeyId + ":" +
|
|
|
|
calc_signature("GET", "", date, headers.get(), resource));
|
|
|
|
} else {
|
2011-09-01 19:24:12 +00:00
|
|
|
if(body.text) free(body.text);
|
2010-11-20 17:55:15 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
curl = create_curl_handle();
|
|
|
|
curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&body);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
|
|
|
|
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
|
|
|
|
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
|
2011-08-30 22:09:17 +00:00
|
|
|
result = my_curl_easy_perform(curl);
|
2010-11-20 17:55:15 +00:00
|
|
|
|
2011-08-30 22:09:17 +00:00
|
|
|
// connect either successful or too many timeouts
|
2010-12-19 01:34:27 +00:00
|
|
|
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &responseCode);
|
2011-06-26 00:37:52 +00:00
|
|
|
|
2011-08-30 22:09:17 +00:00
|
|
|
if(responseCode == 403) {
|
2013-01-19 16:05:07 +00:00
|
|
|
destroy_curl_handle(curl);
|
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
|
|
|
body.text = NULL;
|
2011-08-30 22:09:17 +00:00
|
|
|
fprintf(stderr, "%s: invalid credentials\n", program_name.c_str());
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_FAILURE);
|
2010-11-20 17:55:15 +00:00
|
|
|
}
|
2010-11-26 22:11:48 +00:00
|
|
|
|
2011-08-30 22:09:17 +00:00
|
|
|
if(responseCode == 404) {
|
2013-01-19 16:05:07 +00:00
|
|
|
destroy_curl_handle(curl);
|
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
|
|
|
body.text = NULL;
|
2011-08-30 22:09:17 +00:00
|
|
|
fprintf(stderr, "%s: bucket not found\n", program_name.c_str());
|
|
|
|
exit(EXIT_FAILURE);
|
2010-11-26 22:11:48 +00:00
|
|
|
}
|
|
|
|
|
2011-08-30 22:09:17 +00:00
|
|
|
// unable to connect
|
|
|
|
if(responseCode == CURLE_OPERATION_TIMEDOUT) {
|
2013-01-19 16:05:07 +00:00
|
|
|
destroy_curl_handle(curl);
|
2011-08-30 22:09:17 +00:00
|
|
|
if(body.text) free(body.text);
|
2011-06-26 00:37:52 +00:00
|
|
|
body.text = NULL;
|
|
|
|
|
2010-12-17 04:40:15 +00:00
|
|
|
return;
|
2010-11-26 22:11:48 +00:00
|
|
|
}
|
|
|
|
|
2013-01-19 16:05:07 +00:00
|
|
|
if(responseCode != 200 && responseCode != 301) {
|
2011-08-30 22:09:17 +00:00
|
|
|
if(debug) syslog(LOG_DEBUG, "responseCode: %i\n", (int)responseCode);
|
|
|
|
if(body.text) free(body.text);
|
|
|
|
destroy_curl_handle(curl);
|
|
|
|
fprintf(stderr, "%s: unable to connect\n", program_name.c_str());
|
|
|
|
|
|
|
|
exit(EXIT_FAILURE);
|
2010-11-26 22:11:48 +00:00
|
|
|
}
|
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
// make sure remote mountpath exists and is a directory
|
|
|
|
if(mount_prefix.size() > 0) {
|
|
|
|
if(remote_mountpath_exists(mount_prefix.c_str()) != 0) {
|
|
|
|
fprintf(stderr, "%s: remote mountpath %s not found.\n",
|
|
|
|
program_name.c_str(), mount_prefix.c_str());
|
|
|
|
|
2011-08-25 20:32:56 +00:00
|
|
|
destroy_curl_handle(curl);
|
2013-01-19 16:05:07 +00:00
|
|
|
if(body.text)
|
|
|
|
free(body.text);
|
|
|
|
body.text = NULL;
|
2011-06-26 00:37:52 +00:00
|
|
|
exit(EXIT_FAILURE);
|
2010-12-17 04:40:15 +00:00
|
|
|
}
|
2010-11-26 22:11:48 +00:00
|
|
|
}
|
2010-11-20 17:55:15 +00:00
|
|
|
|
2011-08-30 22:09:17 +00:00
|
|
|
// success
|
2011-06-26 00:37:52 +00:00
|
|
|
service_validated = true;
|
2011-08-30 22:09:17 +00:00
|
|
|
if(body.text) free(body.text);
|
2011-06-26 00:37:52 +00:00
|
|
|
body.text = NULL;
|
2010-12-19 01:34:27 +00:00
|
|
|
destroy_curl_handle(curl);
|
2011-06-26 00:37:52 +00:00
|
|
|
|
2010-11-20 17:55:15 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2011-04-06 15:06:28 +00:00
|
|
|
static bool check_for_aws_format (void) {
|
|
|
|
size_t first_pos = string::npos;
|
|
|
|
string line;
|
|
|
|
bool got_access_key_id_line = 0;
|
|
|
|
bool got_secret_key_line = 0;
|
|
|
|
string str1 ("AWSAccessKeyId=");
|
|
|
|
string str2 ("AWSSecretKey=");
|
|
|
|
size_t found;
|
|
|
|
|
|
|
|
|
|
|
|
ifstream PF(passwd_file.c_str());
|
|
|
|
if (PF.good()) {
|
|
|
|
while (getline(PF, line)) {
|
2011-08-30 22:09:17 +00:00
|
|
|
if(line[0]=='#')
|
2011-04-06 15:06:28 +00:00
|
|
|
continue;
|
2011-08-30 22:09:17 +00:00
|
|
|
if(line.size() == 0)
|
2011-04-06 15:06:28 +00:00
|
|
|
continue;
|
|
|
|
|
|
|
|
first_pos = line.find_first_of(" \t");
|
|
|
|
if (first_pos != string::npos) {
|
|
|
|
printf ("%s: invalid line in passwd file, found whitespace character\n",
|
|
|
|
program_name.c_str());
|
|
|
|
exit(EXIT_FAILURE);
|
|
|
|
}
|
|
|
|
|
|
|
|
first_pos = line.find_first_of("[");
|
|
|
|
if (first_pos != string::npos && first_pos == 0) {
|
|
|
|
printf ("%s: invalid line in passwd file, found a bracket \"[\" character\n",
|
|
|
|
program_name.c_str());
|
|
|
|
exit(EXIT_FAILURE);
|
|
|
|
}
|
|
|
|
|
|
|
|
found = line.find(str1);
|
|
|
|
if (found != string::npos) {
|
|
|
|
first_pos = line.find_first_of("=");
|
|
|
|
AWSAccessKeyId = line.substr(first_pos + 1, string::npos);
|
|
|
|
got_access_key_id_line = 1;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
found = line.find(str2);
|
|
|
|
if (found != string::npos) {
|
|
|
|
first_pos = line.find_first_of("=");
|
|
|
|
AWSSecretAccessKey = line.substr(first_pos + 1, string::npos);
|
|
|
|
got_secret_key_line = 1;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-08-30 22:09:17 +00:00
|
|
|
if(got_access_key_id_line && got_secret_key_line)
|
2011-04-06 15:06:28 +00:00
|
|
|
return 1;
|
2011-08-30 22:09:17 +00:00
|
|
|
else
|
2011-04-06 15:06:28 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
//////////////////////////////////////////////////////////////////
|
|
|
|
// check_passwd_file_perms
|
|
|
|
//
|
|
|
|
// expect that global passwd_file variable contains
|
|
|
|
// a non-empty value and is readable by the current user
|
|
|
|
//
|
|
|
|
// Check for too permissive access to the file
|
|
|
|
// help save users from themselves via a security hole
|
|
|
|
//
|
|
|
|
// only two options: return or error out
|
|
|
|
//////////////////////////////////////////////////////////////////
|
|
|
|
static void check_passwd_file_perms (void) {
|
|
|
|
struct stat info;
|
|
|
|
|
|
|
|
// let's get the file info
|
|
|
|
if (stat(passwd_file.c_str(), &info) != 0) {
|
|
|
|
fprintf (stderr, "%s: unexpected error from stat(%s, ) \n",
|
|
|
|
program_name.c_str(), passwd_file.c_str());
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_FAILURE);
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// return error if any file has others permissions
|
|
|
|
if ((info.st_mode & S_IROTH) ||
|
|
|
|
(info.st_mode & S_IWOTH) ||
|
|
|
|
(info.st_mode & S_IXOTH)) {
|
|
|
|
fprintf (stderr, "%s: credentials file %s should not have others permissions\n",
|
|
|
|
program_name.c_str(), passwd_file.c_str());
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_FAILURE);
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Any local file should not have any group permissions
|
2011-02-23 16:03:08 +00:00
|
|
|
// /etc/passwd-s3fs can have group permissions
|
2010-11-13 23:59:23 +00:00
|
|
|
if (passwd_file != "/etc/passwd-s3fs") {
|
|
|
|
if ((info.st_mode & S_IRGRP) ||
|
|
|
|
(info.st_mode & S_IWGRP) ||
|
|
|
|
(info.st_mode & S_IXGRP)) {
|
|
|
|
fprintf (stderr, "%s: credentials file %s should not have group permissions\n",
|
|
|
|
program_name.c_str(), passwd_file.c_str());
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_FAILURE);
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
//////////////////////////////////////////////////////////////////
|
|
|
|
// read_passwd_file
|
|
|
|
//
|
|
|
|
// Support for per bucket credentials
|
|
|
|
//
|
|
|
|
// Format for the credentials file:
|
|
|
|
// [bucket:]AccessKeyId:SecretAccessKey
|
|
|
|
//
|
|
|
|
// Lines beginning with # are considered comments
|
|
|
|
// and ignored, as are empty lines
|
|
|
|
//
|
|
|
|
// Uncommented lines without the ":" character are flagged as
|
|
|
|
// an error, so are lines with spaces or tabs
|
|
|
|
//
|
|
|
|
// only one default key pair is allowed, but not required
|
|
|
|
//////////////////////////////////////////////////////////////////
|
|
|
|
static void read_passwd_file (void) {
|
|
|
|
string line;
|
|
|
|
string field1, field2, field3;
|
|
|
|
size_t first_pos = string::npos;
|
|
|
|
size_t last_pos = string::npos;
|
|
|
|
bool default_found = 0;
|
2011-04-06 15:06:28 +00:00
|
|
|
bool aws_format;
|
2010-11-13 23:59:23 +00:00
|
|
|
|
|
|
|
// if you got here, the password file
|
|
|
|
// exists and is readable by the
|
|
|
|
// current user, check for permissions
|
|
|
|
check_passwd_file_perms();
|
|
|
|
|
2011-04-06 15:06:28 +00:00
|
|
|
aws_format = check_for_aws_format();
|
|
|
|
|
|
|
|
if (aws_format)
|
|
|
|
return;
|
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
ifstream PF(passwd_file.c_str());
|
|
|
|
if (PF.good()) {
|
|
|
|
while (getline(PF, line)) {
|
|
|
|
if (line[0]=='#') {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (line.size() == 0) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
first_pos = line.find_first_of(" \t");
|
|
|
|
if (first_pos != string::npos) {
|
|
|
|
printf ("%s: invalid line in passwd file, found whitespace character\n",
|
|
|
|
program_name.c_str());
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_FAILURE);
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
|
2010-11-24 02:44:15 +00:00
|
|
|
first_pos = line.find_first_of("[");
|
|
|
|
if (first_pos != string::npos && first_pos == 0) {
|
|
|
|
printf ("%s: invalid line in passwd file, found a bracket \"[\" character\n",
|
|
|
|
program_name.c_str());
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_FAILURE);
|
2010-11-24 02:44:15 +00:00
|
|
|
}
|
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
first_pos = line.find_first_of(":");
|
|
|
|
if (first_pos == string::npos) {
|
|
|
|
printf ("%s: invalid line in passwd file, no \":\" separator found\n",
|
|
|
|
program_name.c_str());
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_FAILURE);
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
last_pos = line.find_last_of(":");
|
|
|
|
|
|
|
|
if (first_pos != last_pos) {
|
|
|
|
// bucket specified
|
|
|
|
field1 = line.substr(0,first_pos);
|
|
|
|
field2 = line.substr(first_pos + 1, last_pos - first_pos - 1);
|
|
|
|
field3 = line.substr(last_pos + 1, string::npos);
|
|
|
|
} else {
|
|
|
|
// no bucket specified - original style - found default key
|
|
|
|
if (default_found == 1) {
|
|
|
|
printf ("%s: more than one default key pair found in passwd file\n",
|
|
|
|
program_name.c_str());
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_FAILURE);
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
default_found = 1;
|
|
|
|
field1.assign("");
|
|
|
|
field2 = line.substr(0,first_pos);
|
|
|
|
field3 = line.substr(first_pos + 1, string::npos);
|
|
|
|
AWSAccessKeyId = field2;
|
|
|
|
AWSSecretAccessKey = field3;
|
|
|
|
}
|
|
|
|
|
|
|
|
// does the bucket we are mounting match this passwd file entry?
|
|
|
|
// if so, use that key pair, otherwise use the default key, if found,
|
|
|
|
// will be used
|
|
|
|
if (field1.size() != 0 && field1 == bucket) {
|
|
|
|
AWSAccessKeyId = field2;
|
|
|
|
AWSSecretAccessKey = field3;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/////////////////////////////////////////////////////////////
|
|
|
|
// get_access_keys
|
|
|
|
//
|
|
|
|
// called only when were are not mounting a
|
|
|
|
// public bucket
|
|
|
|
//
|
|
|
|
// Here is the order precedence for getting the
|
|
|
|
// keys:
|
|
|
|
//
|
|
|
|
// 1 - from the command line (security risk)
|
|
|
|
// 2 - from a password file specified on the command line
|
|
|
|
// 3 - from environment variables
|
|
|
|
// 4 - from the users ~/.passwd-s3fs
|
|
|
|
// 5 - from /etc/passwd-s3fs
|
|
|
|
/////////////////////////////////////////////////////////////
|
|
|
|
static void get_access_keys (void) {
|
|
|
|
|
|
|
|
// should be redundant
|
|
|
|
if (public_bucket.substr(0,1) == "1") {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
// 1 - keys specified on the command line
|
|
|
|
if (AWSAccessKeyId.size() > 0 && AWSSecretAccessKey.size() > 0) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
// 2 - was specified on the command line
|
|
|
|
if (passwd_file.size() > 0) {
|
|
|
|
ifstream PF(passwd_file.c_str());
|
|
|
|
if (PF.good()) {
|
|
|
|
PF.close();
|
|
|
|
read_passwd_file();
|
|
|
|
return;
|
|
|
|
} else {
|
|
|
|
fprintf(stderr, "%s: specified passwd_file is not readable\n",
|
|
|
|
program_name.c_str());
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_FAILURE);
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// 3 - environment variables
|
|
|
|
char * AWSACCESSKEYID;
|
|
|
|
char * AWSSECRETACCESSKEY;
|
|
|
|
|
|
|
|
AWSACCESSKEYID = getenv("AWSACCESSKEYID");
|
|
|
|
AWSSECRETACCESSKEY = getenv("AWSSECRETACCESSKEY");
|
|
|
|
if (AWSACCESSKEYID != NULL || AWSSECRETACCESSKEY != NULL) {
|
|
|
|
if ((AWSACCESSKEYID == NULL && AWSSECRETACCESSKEY != NULL) ||
|
|
|
|
(AWSACCESSKEYID != NULL && AWSSECRETACCESSKEY == NULL) ){
|
|
|
|
|
|
|
|
fprintf(stderr, "%s: if environment variable AWSACCESSKEYID is set then AWSSECRETACCESSKEY must be set too\n",
|
|
|
|
program_name.c_str());
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_FAILURE);
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
AWSAccessKeyId.assign(AWSACCESSKEYID);
|
|
|
|
AWSSecretAccessKey.assign(AWSSECRETACCESSKEY);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2011-04-06 15:06:28 +00:00
|
|
|
// 3a - from the AWS_CREDENTIAL_FILE environment variable
|
|
|
|
char * AWS_CREDENTIAL_FILE;
|
|
|
|
AWS_CREDENTIAL_FILE = getenv("AWS_CREDENTIAL_FILE");
|
|
|
|
if (AWS_CREDENTIAL_FILE != NULL) {
|
|
|
|
passwd_file.assign(AWS_CREDENTIAL_FILE);
|
|
|
|
if (passwd_file.size() > 0) {
|
|
|
|
ifstream PF(passwd_file.c_str());
|
|
|
|
if (PF.good()) {
|
|
|
|
PF.close();
|
|
|
|
read_passwd_file();
|
|
|
|
return;
|
|
|
|
} else {
|
|
|
|
fprintf(stderr, "%s: AWS_CREDENTIAL_FILE: \"%s\" is not readable\n",
|
|
|
|
program_name.c_str(), passwd_file.c_str());
|
|
|
|
exit(EXIT_FAILURE);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
// 4 - from the default location in the users home directory
|
|
|
|
char * HOME;
|
|
|
|
HOME = getenv ("HOME");
|
|
|
|
if (HOME != NULL) {
|
|
|
|
passwd_file.assign(HOME);
|
|
|
|
passwd_file.append("/.passwd-s3fs");
|
|
|
|
ifstream PF(passwd_file.c_str());
|
|
|
|
if (PF.good()) {
|
|
|
|
PF.close();
|
|
|
|
read_passwd_file();
|
|
|
|
// It is possible that the user's file was there but
|
|
|
|
// contained no key pairs i.e. commented out
|
|
|
|
// in that case, go look in the final location
|
|
|
|
if (AWSAccessKeyId.size() > 0 && AWSSecretAccessKey.size() > 0) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// 5 - from the system default location
|
|
|
|
passwd_file.assign("/etc/passwd-s3fs");
|
|
|
|
ifstream PF(passwd_file.c_str());
|
|
|
|
if (PF.good()) {
|
|
|
|
PF.close();
|
|
|
|
read_passwd_file();
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
fprintf(stderr, "%s: could not determine how to establish security credentials\n",
|
|
|
|
program_name.c_str());
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_FAILURE);
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void show_usage (void) {
|
2011-06-26 00:37:52 +00:00
|
|
|
printf("Usage: %s BUCKET:[PATH] MOUNTPOINT [OPTION]...\n",
|
2010-11-13 23:59:23 +00:00
|
|
|
program_name.c_str());
|
|
|
|
}
|
|
|
|
|
|
|
|
static void show_help (void) {
|
|
|
|
show_usage();
|
|
|
|
printf(
|
|
|
|
"\n"
|
|
|
|
"Mount an Amazon S3 bucket as a file system.\n"
|
|
|
|
"\n"
|
|
|
|
" General forms for s3fs and FUSE/mount options:\n"
|
|
|
|
" -o opt[,opt...]\n"
|
|
|
|
" -o opt [-o opt] ...\n"
|
|
|
|
"\n"
|
|
|
|
"s3fs Options:\n"
|
|
|
|
"\n"
|
2011-06-27 02:21:38 +00:00
|
|
|
" Most s3fs options are given in the form where \"opt\" is:\n"
|
2010-11-13 23:59:23 +00:00
|
|
|
"\n"
|
|
|
|
" <option_name>=<option_value>\n"
|
|
|
|
"\n"
|
|
|
|
" default_acl (default=\"private\")\n"
|
|
|
|
" - the default canned acl to apply to all written s3 objects\n"
|
|
|
|
" see http://aws.amazon.com/documentation/s3/ for the \n"
|
|
|
|
" full list of canned acls\n"
|
|
|
|
"\n"
|
|
|
|
" retries (default=\"2\")\n"
|
|
|
|
" - number of times to retry a failed s3 transaction\n"
|
|
|
|
"\n"
|
|
|
|
" use_cache (default=\"\" which means disabled)\n"
|
|
|
|
" - local folder to use for local file cache\n"
|
|
|
|
"\n"
|
|
|
|
" use_rrs (default=\"\" which means diabled)\n"
|
|
|
|
" - use Amazon's Reduced Redundancy Storage when set to 1\n"
|
|
|
|
"\n"
|
|
|
|
" public_bucket (default=\"\" which means disabled)\n"
|
|
|
|
" - anonymously mount a public bucket when set to 1\n"
|
|
|
|
"\n"
|
|
|
|
" passwd_file (default=\"\")\n"
|
|
|
|
" - specify which s3fs password file to use\n"
|
|
|
|
"\n"
|
2010-12-30 03:13:21 +00:00
|
|
|
" connect_timeout (default=\"10\" seconds)\n"
|
2010-11-13 23:59:23 +00:00
|
|
|
" - time to wait for connection before giving up\n"
|
|
|
|
"\n"
|
2010-12-30 03:13:21 +00:00
|
|
|
" readwrite_timeout (default=\"30\" seconds)\n"
|
2010-11-13 23:59:23 +00:00
|
|
|
" - time to wait between read/write activity before giving up\n"
|
|
|
|
"\n"
|
2011-02-12 16:48:23 +00:00
|
|
|
" max_stat_cache_size (default=\"10000\" entries (about 4MB))\n"
|
|
|
|
" - maximum number of entries in the stat cache\n"
|
|
|
|
"\n"
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
" stat_cache_expire (default is no expire)\n"
|
|
|
|
" - specify expire time(seconds) for entries in the stat cache.\n"
|
|
|
|
"\n"
|
2010-11-13 23:59:23 +00:00
|
|
|
" url (default=\"http://s3.amazonaws.com\")\n"
|
|
|
|
" - sets the url to use to access amazon s3\n"
|
|
|
|
"\n"
|
2011-06-27 02:21:38 +00:00
|
|
|
" nomultipart - disable multipart uploads\n"
|
|
|
|
"\n"
|
2013-01-19 16:05:07 +00:00
|
|
|
" noxmlns - disable registing xml name space.\n"
|
|
|
|
" disable registing xml name space for response of \n"
|
|
|
|
" ListBucketResult and ListVersionsResult etc. Default name \n"
|
|
|
|
" space is looked up from \"http://s3.amazonaws.com/doc/2006-03-01\".\n"
|
|
|
|
"\n"
|
|
|
|
" nocopyapi - for other incomplete compatibility object storage.\n"
|
|
|
|
" For a distributed object storage which is compatibility S3\n"
|
|
|
|
" API without PUT(copy api).\n"
|
|
|
|
" If you set this option, s3fs do not use PUT with \n"
|
|
|
|
" \"x-amz-copy-source\"(copy api). Because traffic is increased\n"
|
|
|
|
" 2-3 times by this option, we do not recommend this.\n"
|
|
|
|
"\n"
|
2010-11-13 23:59:23 +00:00
|
|
|
"FUSE/mount Options:\n"
|
|
|
|
"\n"
|
|
|
|
" Most of the generic mount options described in 'man mount' are\n"
|
|
|
|
" supported (ro, rw, suid, nosuid, dev, nodev, exec, noexec, atime,\n"
|
|
|
|
" noatime, sync async, dirsync). Filesystems are mounted with\n"
|
|
|
|
" '-onodev,nosuid' by default, which can only be overridden by a\n"
|
|
|
|
" privileged user.\n"
|
|
|
|
" \n"
|
|
|
|
" There are many FUSE specific mount options that can be specified.\n"
|
|
|
|
" e.g. allow_other See the FUSE's README for the full set.\n"
|
|
|
|
"\n"
|
|
|
|
"Miscellaneous Options:\n"
|
|
|
|
"\n"
|
|
|
|
" -h, --help Output this help.\n"
|
|
|
|
" --version Output version info.\n"
|
|
|
|
" -d --debug Turn on DEBUG messages to syslog. Specifying -d\n"
|
|
|
|
" twice turns on FUSE debug messages to STDOUT.\n"
|
2010-11-19 22:23:38 +00:00
|
|
|
" -f FUSE foreground option - do not run as daemon.\n"
|
2010-12-04 20:07:08 +00:00
|
|
|
" -s FUSE singlethread option\n"
|
|
|
|
" disable multi-threaded operation\n"
|
2010-11-13 23:59:23 +00:00
|
|
|
"\n"
|
|
|
|
"\n"
|
|
|
|
"Report bugs to <s3fs-devel@googlegroups.com>\n"
|
|
|
|
"s3fs home page: <http://code.google.com/p/s3fs/>\n"
|
|
|
|
);
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_SUCCESS);
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void show_version(void) {
|
|
|
|
printf(
|
|
|
|
"Amazon Simple Storage Service File System %s\n"
|
|
|
|
"Copyright (C) 2010 Randy Rizun <rrizun@gmail.com>\n"
|
|
|
|
"License GPL2: GNU GPL version 2 <http://gnu.org/licenses/gpl.html>\n"
|
|
|
|
"This is free software: you are free to change and redistribute it.\n"
|
2011-06-26 00:37:52 +00:00
|
|
|
"There is NO WARRANTY, to the extent permitted by law.\n", VERSION );
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_SUCCESS);
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
char *get_realpath(const char *path) {
|
|
|
|
size_t size;
|
|
|
|
char *realpath;
|
|
|
|
|
|
|
|
size = (strlen(path) + 1) + (mount_prefix.size() + 1);
|
|
|
|
realpath = (char *) malloc(size);
|
|
|
|
snprintf(realpath, size, "%s%s", mount_prefix.c_str(), path);
|
|
|
|
|
|
|
|
return(realpath);
|
|
|
|
}
|
|
|
|
|
|
|
|
// This is repeatedly called by the fuse option parser
|
|
|
|
// if the key is equal to FUSE_OPT_KEY_OPT, it's an option passed in prefixed by
|
|
|
|
// '-' or '--' e.g.: -f -d -ousecache=/tmp
|
|
|
|
//
|
|
|
|
// if the key is equal to FUSE_OPT_KEY_NONOPT, it's either the bucket name
|
|
|
|
// or the mountpoint. The bucket name will always come before the mountpoint
|
2010-11-13 23:59:23 +00:00
|
|
|
static int my_fuse_opt_proc(void *data, const char *arg, int key, struct fuse_args *outargs) {
|
2011-06-26 00:37:52 +00:00
|
|
|
if(key == FUSE_OPT_KEY_NONOPT) {
|
|
|
|
// the first NONOPT option is the bucket name
|
|
|
|
if(bucket.size() == 0) {
|
|
|
|
// extract remote mount path
|
|
|
|
char *bucket_name = (char *) arg;
|
|
|
|
if(strstr(arg, ":")) {
|
|
|
|
bucket = strtok(bucket_name, ":");
|
|
|
|
mount_prefix = strtok(NULL, ":");
|
|
|
|
// remove trailing slash
|
|
|
|
if(mount_prefix.at(mount_prefix.size() - 1) == '/')
|
|
|
|
mount_prefix = mount_prefix.substr(0, mount_prefix.size() - 1);
|
2010-11-13 23:59:23 +00:00
|
|
|
} else {
|
2011-06-26 00:37:52 +00:00
|
|
|
bucket = arg;
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
2011-06-26 00:37:52 +00:00
|
|
|
|
|
|
|
return 0;
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
|
2011-06-26 00:37:52 +00:00
|
|
|
// save the mountpoint and do some basic error checking
|
|
|
|
mountpoint = arg;
|
|
|
|
struct stat stbuf;
|
|
|
|
|
|
|
|
if(stat(arg, &stbuf) == -1) {
|
|
|
|
fprintf(stderr, "%s: unable to access MOUNTPOINT %s: %s\n",
|
|
|
|
program_name.c_str(), mountpoint.c_str(), strerror(errno));
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_FAILURE);
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
2011-06-26 00:37:52 +00:00
|
|
|
|
|
|
|
root_mode = stbuf.st_mode; // save mode for later usage
|
|
|
|
|
|
|
|
if(!(S_ISDIR(stbuf.st_mode ))) {
|
|
|
|
fprintf(stderr, "%s: MOUNTPOINT: %s is not a directory\n",
|
|
|
|
program_name.c_str(), mountpoint.c_str());
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_FAILURE);
|
2011-06-26 00:37:52 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
struct dirent *ent;
|
|
|
|
DIR *dp = opendir(mountpoint.c_str());
|
|
|
|
if(dp == NULL) {
|
|
|
|
fprintf(stderr, "%s: failed to open MOUNTPOINT: %s: %s\n",
|
|
|
|
program_name.c_str(), mountpoint.c_str(), strerror(errno));
|
|
|
|
exit(EXIT_FAILURE);
|
|
|
|
}
|
|
|
|
|
|
|
|
while((ent = readdir(dp)) != NULL) {
|
|
|
|
if(strcmp(ent->d_name, ".") != 0 && strcmp(ent->d_name, "..") != 0) {
|
|
|
|
closedir(dp);
|
|
|
|
fprintf(stderr, "%s: MOUNTPOINT directory %s is not empty\n",
|
|
|
|
program_name.c_str(), mountpoint.c_str());
|
|
|
|
exit(EXIT_FAILURE);
|
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
2011-08-25 17:11:25 +00:00
|
|
|
|
|
|
|
closedir(dp);
|
2011-06-26 00:37:52 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (key == FUSE_OPT_KEY_OPT) {
|
2010-11-13 23:59:23 +00:00
|
|
|
if (strstr(arg, "default_acl=") != 0) {
|
|
|
|
default_acl = strchr(arg, '=') + 1;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
if (strstr(arg, "retries=") != 0) {
|
|
|
|
retries = atoi(strchr(arg, '=') + 1);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
if (strstr(arg, "use_cache=") != 0) {
|
|
|
|
use_cache = strchr(arg, '=') + 1;
|
|
|
|
return 0;
|
|
|
|
}
|
2011-06-27 02:21:38 +00:00
|
|
|
|
|
|
|
if(strstr(arg, "nomultipart") != 0) {
|
|
|
|
nomultipart = true;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
if (strstr(arg, "use_rrs=") != 0) {
|
|
|
|
use_rrs = strchr(arg, '=') + 1;
|
|
|
|
if (strcmp(use_rrs.c_str(), "1") == 0 ||
|
|
|
|
strcmp(use_rrs.c_str(), "") == 0 ) {
|
|
|
|
return 0;
|
|
|
|
} else {
|
|
|
|
fprintf(stderr, "%s: poorly formed argument to option: use_rrs\n",
|
|
|
|
program_name.c_str());
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_FAILURE);
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
}
|
2010-11-26 22:11:48 +00:00
|
|
|
if (strstr(arg, "ssl_verify_hostname=") != 0) {
|
|
|
|
ssl_verify_hostname = strchr(arg, '=') + 1;
|
|
|
|
if (strcmp(ssl_verify_hostname.c_str(), "1") == 0 ||
|
|
|
|
strcmp(ssl_verify_hostname.c_str(), "0") == 0 ) {
|
|
|
|
return 0;
|
|
|
|
} else {
|
|
|
|
fprintf(stderr, "%s: poorly formed argument to option: ssl_verify_hostname\n",
|
|
|
|
program_name.c_str());
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_FAILURE);
|
2010-11-26 22:11:48 +00:00
|
|
|
}
|
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
if (strstr(arg, "passwd_file=") != 0) {
|
|
|
|
passwd_file = strchr(arg, '=') + 1;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
if (strstr(arg, "public_bucket=") != 0) {
|
|
|
|
public_bucket = strchr(arg, '=') + 1;
|
|
|
|
if (strcmp(public_bucket.c_str(), "1") == 0 ||
|
|
|
|
strcmp(public_bucket.c_str(), "") == 0 ) {
|
|
|
|
return 0;
|
|
|
|
} else {
|
|
|
|
fprintf(stderr, "%s: poorly formed argument to option: public_bucket\n",
|
|
|
|
program_name.c_str());
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_FAILURE);
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
if (strstr(arg, "host=") != 0) {
|
|
|
|
host = strchr(arg, '=') + 1;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
if (strstr(arg, "servicepath=") != 0) {
|
|
|
|
service_path = strchr(arg, '=') + 1;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
if (strstr(arg, "connect_timeout=") != 0) {
|
|
|
|
connect_timeout = strtol(strchr(arg, '=') + 1, 0, 10);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
if (strstr(arg, "readwrite_timeout=") != 0) {
|
|
|
|
readwrite_timeout = strtoul(strchr(arg, '=') + 1, 0, 10);
|
|
|
|
return 0;
|
|
|
|
}
|
2011-02-12 16:48:23 +00:00
|
|
|
if (strstr(arg, "max_stat_cache_size=") != 0) {
|
|
|
|
max_stat_cache_size = strtoul(strchr(arg, '=') + 1, 0, 10);
|
|
|
|
return 0;
|
|
|
|
}
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
if (strstr(arg, "stat_cache_expire=") != 0) {
|
|
|
|
is_stat_cache_expire_time = 1;
|
|
|
|
stat_cache_expire_time = strtoul(strchr(arg, '=') + 1, 0, 10);
|
|
|
|
return 0;
|
|
|
|
}
|
2013-01-19 16:05:07 +00:00
|
|
|
if(strstr(arg, "noxmlns") != 0) {
|
|
|
|
noxmlns = true;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
if(strstr(arg, "nocopyapi") != 0) {
|
|
|
|
nocopyapi = true;
|
|
|
|
return 0;
|
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
if (strstr(arg, "url=") != 0) {
|
|
|
|
host = strchr(arg, '=') + 1;
|
2011-03-09 20:55:31 +00:00
|
|
|
// strip the trailing '/', if any, off the end of the host
|
|
|
|
// string
|
|
|
|
size_t found, length;
|
|
|
|
found = host.find_last_of('/');
|
|
|
|
length = host.length();
|
|
|
|
while ( found == (length - 1) && length > 0 ) {
|
|
|
|
host.erase(found);
|
|
|
|
found = host.find_last_of('/');
|
|
|
|
length = host.length();
|
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
return 0;
|
|
|
|
}
|
2011-06-26 00:37:52 +00:00
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
// debug option
|
|
|
|
//
|
|
|
|
// The first -d (or --debug) enables s3fs debug
|
|
|
|
// the second -d option is passed to fuse to turn on its
|
|
|
|
// debug output
|
|
|
|
if ( (strcmp(arg, "-d") == 0) || (strcmp(arg, "--debug") == 0) ) {
|
|
|
|
if (!debug) {
|
|
|
|
debug = 1;
|
|
|
|
return 0;
|
|
|
|
} else {
|
|
|
|
// fuse doesn't understand "--debug", but it
|
|
|
|
// understands -d, but we can't pass -d back
|
|
|
|
// to fuse, in this case just ignore the
|
|
|
|
// second --debug if is was provided. If we
|
|
|
|
// do not ignore this, fuse emits an error
|
|
|
|
if(strcmp(arg, "--debug") == 0) {
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2011-06-26 00:37:52 +00:00
|
|
|
|
|
|
|
if (strstr(arg, "accessKeyId=") != 0) {
|
|
|
|
fprintf(stderr, "%s: option accessKeyId is no longer supported\n",
|
|
|
|
program_name.c_str());
|
|
|
|
exit(EXIT_FAILURE);
|
|
|
|
}
|
|
|
|
if (strstr(arg, "secretAccessKey=") != 0) {
|
|
|
|
fprintf(stderr, "%s: option secretAccessKey is no longer supported\n",
|
|
|
|
program_name.c_str());
|
|
|
|
exit(EXIT_FAILURE);
|
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
2011-02-12 15:02:44 +00:00
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
int main(int argc, char *argv[]) {
|
|
|
|
int ch;
|
2011-08-25 20:32:56 +00:00
|
|
|
int fuse_res;
|
2010-11-13 23:59:23 +00:00
|
|
|
int option_index = 0;
|
|
|
|
|
|
|
|
static const struct option long_opts[] = {
|
|
|
|
{"help", no_argument, NULL, 'h'},
|
2011-06-26 00:37:52 +00:00
|
|
|
{"version", no_argument, 0, 0},
|
2010-11-13 23:59:23 +00:00
|
|
|
{"debug", no_argument, NULL, 'd'},
|
|
|
|
{0, 0, 0, 0}};
|
|
|
|
|
|
|
|
// get progam name - emulate basename
|
2010-11-26 22:11:48 +00:00
|
|
|
size_t found = string::npos;
|
2010-11-13 23:59:23 +00:00
|
|
|
program_name.assign(argv[0]);
|
|
|
|
found = program_name.find_last_of("/");
|
|
|
|
if(found != string::npos) {
|
|
|
|
program_name.replace(0, found+1, "");
|
|
|
|
}
|
|
|
|
|
2010-12-30 03:13:21 +00:00
|
|
|
while ((ch = getopt_long(argc, argv, "dho:fsu", long_opts, &option_index)) != -1) {
|
2010-11-13 23:59:23 +00:00
|
|
|
switch (ch) {
|
|
|
|
case 0:
|
2011-06-26 00:37:52 +00:00
|
|
|
if(strcmp(long_opts[option_index].name, "version") == 0)
|
2010-11-13 23:59:23 +00:00
|
|
|
show_version();
|
|
|
|
break;
|
|
|
|
case 'h':
|
|
|
|
show_help();
|
|
|
|
break;
|
|
|
|
case 'o':
|
|
|
|
break;
|
|
|
|
case 'd':
|
|
|
|
break;
|
2010-11-19 22:23:38 +00:00
|
|
|
case 'f':
|
2010-12-09 20:56:29 +00:00
|
|
|
foreground = 1;
|
2010-11-19 22:23:38 +00:00
|
|
|
break;
|
2010-12-04 20:07:08 +00:00
|
|
|
case 's':
|
|
|
|
break;
|
2010-12-30 03:13:21 +00:00
|
|
|
case 'u':
|
|
|
|
utility_mode = 1;
|
|
|
|
break;
|
2010-11-13 23:59:23 +00:00
|
|
|
default:
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_FAILURE);
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// clear this structure
|
|
|
|
memset(&s3fs_oper, 0, sizeof(s3fs_oper));
|
|
|
|
|
|
|
|
// This is the fuse-style parser for the arguments
|
|
|
|
// after which the bucket name and mountpoint names
|
|
|
|
// should have been set
|
|
|
|
struct fuse_args custom_args = FUSE_ARGS_INIT(argc, argv);
|
|
|
|
fuse_opt_parse(&custom_args, NULL, NULL, my_fuse_opt_proc);
|
|
|
|
|
|
|
|
// The first plain argument is the bucket
|
|
|
|
if (bucket.size() == 0) {
|
|
|
|
fprintf(stderr, "%s: missing BUCKET argument\n", program_name.c_str());
|
|
|
|
show_usage();
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_FAILURE);
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// bucket names cannot contain upper case characters
|
|
|
|
if (lower(bucket) != bucket) {
|
|
|
|
fprintf(stderr, "%s: BUCKET %s, upper case characters are not supported\n",
|
|
|
|
program_name.c_str(), bucket.c_str());
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_FAILURE);
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
|
2011-02-26 14:48:02 +00:00
|
|
|
// check bucket name for illegal characters
|
|
|
|
found = bucket.find_first_of("/:\\;!@#$%^&*?|+=");
|
|
|
|
if(found != string::npos) {
|
|
|
|
fprintf(stderr, "%s: BUCKET %s -- bucket name contains an illegal character\n",
|
|
|
|
program_name.c_str(), bucket.c_str());
|
|
|
|
exit(EXIT_FAILURE);
|
|
|
|
}
|
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
// The second plain argument is the mountpoint
|
|
|
|
// if the option was given, we all ready checked for a
|
|
|
|
// readable, non-empty directory, this checks determines
|
|
|
|
// if the mountpoint option was ever supplied
|
2010-12-30 03:13:21 +00:00
|
|
|
if (utility_mode == 0) {
|
|
|
|
if (mountpoint.size() == 0) {
|
|
|
|
fprintf(stderr, "%s: missing MOUNTPOINT argument\n", program_name.c_str());
|
|
|
|
show_usage();
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_FAILURE);
|
2010-12-30 03:13:21 +00:00
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// error checking of command line arguments for compatability
|
|
|
|
if ((AWSSecretAccessKey.size() > 0 && AWSAccessKeyId.size() == 0) ||
|
|
|
|
(AWSSecretAccessKey.size() == 0 && AWSAccessKeyId.size() > 0)) {
|
|
|
|
fprintf(stderr, "%s: if one access key is specified, both keys need to be specified\n",
|
|
|
|
program_name.c_str());
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_FAILURE);
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (public_bucket.substr(0,1) == "1" &&
|
|
|
|
(AWSSecretAccessKey.size() > 0 || AWSAccessKeyId.size() > 0)) {
|
|
|
|
fprintf(stderr, "%s: specifying both public_bucket and the access keys options is invalid\n",
|
|
|
|
program_name.c_str());
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_FAILURE);
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (passwd_file.size() > 0 &&
|
|
|
|
(AWSSecretAccessKey.size() > 0 || AWSAccessKeyId.size() > 0)) {
|
|
|
|
fprintf(stderr, "%s: specifying both passwd_file and the access keys options is invalid\n",
|
|
|
|
program_name.c_str());
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_FAILURE);
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (public_bucket.substr(0,1) != "1") {
|
|
|
|
get_access_keys();
|
|
|
|
if(AWSSecretAccessKey.size() == 0 || AWSAccessKeyId.size() == 0) {
|
|
|
|
fprintf(stderr, "%s: could not establish security credentials, check documentation\n",
|
|
|
|
program_name.c_str());
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_FAILURE);
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
|
|
|
// More error checking on the access key pair can be done
|
|
|
|
// like checking for appropriate lengths and characters
|
|
|
|
}
|
|
|
|
|
|
|
|
// There's room for more command line error checking
|
|
|
|
|
2010-11-26 22:11:48 +00:00
|
|
|
// Check to see if the bucket name contains periods and https (SSL) is
|
|
|
|
// being used. This is a known limitation:
|
|
|
|
// http://docs.amazonwebservices.com/AmazonS3/latest/dev/
|
|
|
|
// The Developers Guide suggests that either use HTTP of for us to write
|
|
|
|
// our own certificate verification logic.
|
|
|
|
// For now, this will be unsupported unless we get a request for it to
|
|
|
|
// be supported. In that case, we have a couple of options:
|
|
|
|
// - implement a command line option that bypasses the verify host
|
|
|
|
// but doesn't bypass verifying the certificate
|
|
|
|
// - write our own host verification (this might be complex)
|
2011-06-26 00:37:52 +00:00
|
|
|
// See issue #128strncasecmp
|
2010-11-26 22:11:48 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
if (ssl_verify_hostname.substr(0,1) == "1") {
|
|
|
|
found = bucket.find_first_of(".");
|
|
|
|
if(found != string::npos) {
|
|
|
|
found = host.find("https:");
|
|
|
|
if(found != string::npos) {
|
|
|
|
fprintf(stderr, "%s: Using https and a bucket name with periods is unsupported.\n",
|
|
|
|
program_name.c_str());
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
*/
|
2010-11-20 17:55:15 +00:00
|
|
|
|
|
|
|
// Does the bucket exist?
|
2011-06-26 00:37:52 +00:00
|
|
|
// if the network is up, check for valid credentials and if the bucket
|
|
|
|
// exists. skip check if mounting a public bucket
|
|
|
|
if(public_bucket.substr(0,1) != "1")
|
2010-11-20 17:55:15 +00:00
|
|
|
s3fs_check_service();
|
|
|
|
|
2010-12-30 03:13:21 +00:00
|
|
|
if (utility_mode) {
|
|
|
|
printf("Utility Mode\n");
|
|
|
|
int result;
|
|
|
|
result = list_multipart_uploads();
|
2011-02-23 16:03:08 +00:00
|
|
|
exit(EXIT_SUCCESS);
|
2010-12-30 03:13:21 +00:00
|
|
|
}
|
|
|
|
|
2010-11-13 23:59:23 +00:00
|
|
|
s3fs_oper.getattr = s3fs_getattr;
|
|
|
|
s3fs_oper.readlink = s3fs_readlink;
|
|
|
|
s3fs_oper.mknod = s3fs_mknod;
|
|
|
|
s3fs_oper.mkdir = s3fs_mkdir;
|
|
|
|
s3fs_oper.unlink = s3fs_unlink;
|
|
|
|
s3fs_oper.rmdir = s3fs_rmdir;
|
|
|
|
s3fs_oper.symlink = s3fs_symlink;
|
|
|
|
s3fs_oper.rename = s3fs_rename;
|
|
|
|
s3fs_oper.link = s3fs_link;
|
2013-01-19 16:05:07 +00:00
|
|
|
if(!nocopyapi){
|
|
|
|
s3fs_oper.chmod = s3fs_chmod;
|
|
|
|
s3fs_oper.chown = s3fs_chown;
|
|
|
|
s3fs_oper.utimens = s3fs_utimens;
|
|
|
|
}else{
|
|
|
|
s3fs_oper.chmod = s3fs_chmod_nocopy;
|
|
|
|
s3fs_oper.chown = s3fs_chown_nocopy;
|
|
|
|
s3fs_oper.utimens = s3fs_utimens_nocopy;
|
|
|
|
}
|
2010-11-13 23:59:23 +00:00
|
|
|
s3fs_oper.truncate = s3fs_truncate;
|
|
|
|
s3fs_oper.open = s3fs_open;
|
|
|
|
s3fs_oper.read = s3fs_read;
|
|
|
|
s3fs_oper.write = s3fs_write;
|
|
|
|
s3fs_oper.statfs = s3fs_statfs;
|
|
|
|
s3fs_oper.flush = s3fs_flush;
|
|
|
|
s3fs_oper.release = s3fs_release;
|
Summary of Changes(1.62 -> 1.63)
1) Lifetime for the stats cache
Added the new option "stat_cache_expire".
This option which is specified by seconds means the lifetime for each stats cache entry.
If this option is not specified, the stats cache is kept in s3fs process until the stats cache grown to maximum size. (default)
If this option is specified, the stats cache entry is out from the memory when the entry expires time.
2) Enable file permission
s3fs before 1.62 did not consider the file access permission.
s3fs after this version can consider it.
For access permission, the s3fs_getattr() function was divided into sub function which can check the file access permission.
It is like access() function.
And the function calling the s3fs_getattr() calls this new sub function instead of s3fs_getattr().
Last the s3fs_opendir() function which is called by FUSE was added for checking directory access permission when listing the files in directory.
3) UID/GUID
When a file or a directory was created, the s3fs could not set the UID/GID as the user who executed a command.
(Almost the UID/GID are root, because the s3fs run by root.)
After this version, the s3fs set correct UID/GID as the user who executes the commond.
4) About the mtime
If the object does not have "x-amz-meta-mtime" meta, the s3fs uses the "Last-Modified" header instead of it.
But the s3fs had a bug in this code, and this version fixed this bug.
When user modified the file, the s3fs did not update the mtime of the file.
This version fixed this bug.
In the get_local_fd() function, the local file's mtime was changed only when s3fs run with "use_cache" option.
This version always updates the mtime whether the local cache file is used or not.
And s3fs_flush ( ) function set the mtime of local cache file from S3 object mtime, but it was wrong .
This version is that the s3fs_flush ( ) changes the mtime of S3 object from the local cache file or the tmpfile .
The s3fs cuts some requests, because the s3fs can always check mtime whether the s3fs uses or does not use the local cache file.
5) A case of no "x-amz-meta-mode"
If the object did not have "x-amz-meta-mtime" mete, the s3fs recognized the file as not regular file.
After this version, the s3fs recognizes the file as regular file.
6) "." and ".." directory
The s3fs_readdir() did not return "X" and "XX" directory name.
After this version, the s3fs is changed that it returns "X" and "XX".
Example, the result of "ls" lists "X" and "XX" directory.
7) Fixed a bug
The insert_object() had a bug, and it is fixed.
git-svn-id: http://s3fs.googlecode.com/svn/trunk@390 df820570-a93a-0410-bd06-b72b767a4274
2013-02-24 08:58:54 +00:00
|
|
|
s3fs_oper.opendir = s3fs_opendir;
|
2010-11-13 23:59:23 +00:00
|
|
|
s3fs_oper.readdir = s3fs_readdir;
|
|
|
|
s3fs_oper.init = s3fs_init;
|
|
|
|
s3fs_oper.destroy = s3fs_destroy;
|
|
|
|
s3fs_oper.access = s3fs_access;
|
2010-12-21 15:24:46 +00:00
|
|
|
s3fs_oper.create = s3fs_create;
|
2010-11-13 23:59:23 +00:00
|
|
|
|
|
|
|
// now passing things off to fuse, fuse will finish evaluating the command line args
|
2011-08-25 20:32:56 +00:00
|
|
|
fuse_res = fuse_main(custom_args.argc, custom_args.argv, &s3fs_oper, NULL);
|
|
|
|
fuse_opt_free_args(&custom_args);
|
|
|
|
|
|
|
|
return fuse_res;
|
2010-11-13 23:59:23 +00:00
|
|
|
}
|
2013-01-19 16:05:07 +00:00
|
|
|
|