octoleo/s3fs-fuse
1
0
Fork 0
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git synced 2024-06-06 11:00:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixed Issue 31 and Cleanup codes

Browse Source 
1) s3sync'ed files not supported(Issue 31)
    Supports HTTP headers which made by s3sync.
    Supported new HTTP headers are x-amz-meta-owner, 
    x-amz-meta-permissions and x-amz-meta-group.
    s3fs read and understand these headers, but s3fs 
    gives priority to s3fs's headers over these headers.

2) Cleanups codes
    Cleanups some codes about issue 31.




git-svn-id: http://s3fs.googlecode.com/svn/trunk@422 df820570-a93a-0410-bd06-b72b767a4274
This commit is contained in:
ggtakec@gmail.com 2013-05-09 08:35:17 +00:00
parent c862ee40ea
commit 715b837a2b
4 changed files with 144 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
src/cache.cpp
View File

@ -367,8 +367,6 @@ bool StatCache::DelStat(const char* key)
//-------------------------------------------------------------------
bool convert_header_to_stat(const char* path, headers_t& meta, struct stat* pst, bool forcedir)
{
headers_t::const_iterator iter;
if(!path || !pst){
return false;
}
@ -377,32 +375,7 @@ bool convert_header_to_stat(const char* path, headers_t& meta, struct stat* pst,
pst->st_nlink = 1; // see fuse FAQ
// mode
iter = meta.find("x-amz-meta-mode");
if(iter != meta.end()){
pst->st_mode = get_mode((*iter).second.c_str());
}
// content-type
string strConType;
iter = meta.find("Content-Type");
if(iter != meta.end()){
strConType = (*iter).second;
}
if(forcedir){
pst->st_mode |= S_IFDIR;
}else{
if(strConType == "application/x-directory"){
pst->st_mode |= S_IFDIR;
}else if(0 < strlen(path) && '/' == path[strlen(path) - 1]){
if(strConType == "binary/octet-stream" || strConType == "application/octet-stream"){
pst->st_mode |= S_IFDIR;
}else{
pst->st_mode |= S_IFREG;
}
}else{
pst->st_mode |= S_IFREG;
}
}
pst->st_mode = get_mode(meta, path, true, forcedir);
// blocks
if(S_ISREG(pst->st_mode)){
@ -410,35 +383,14 @@ bool convert_header_to_stat(const char* path, headers_t& meta, struct stat* pst,
}
// mtime
iter = meta.find("x-amz-meta-mtime");
if(iter != meta.end()){
pst->st_mtime = get_mtime((*iter).second.c_str());
}
if(pst->st_mtime == 0) {
iter = meta.find("Last-Modified");
if(iter != meta.end()){
pst->st_mtime = get_lastmodified((*iter).second.c_str());
}
}
if(-1 == pst->st_mtime){
pst->st_mtime = 0;
}
pst->st_mtime = get_mtime(meta);
// size
iter = meta.find("Content-Length");
if(iter != meta.end()){
pst->st_size = get_size((*iter).second.c_str());
}
pst->st_size = get_size(meta);
// uid/gid
iter = meta.find("x-amz-meta-uid");
if(iter != meta.end()){
pst->st_uid = get_uid((*iter).second.c_str());
}
iter = meta.find("x-amz-meta-gid");
if(iter != meta.end()){
pst->st_gid = get_gid((*iter).second.c_str());
}
pst->st_uid = get_uid(meta);
pst->st_gid = get_gid(meta);
return true;
}

82
src/s3fs.cpp
View File

@ -573,6 +573,7 @@ static int get_local_fd(const char* path) {
int fd = -1;
int result;
struct stat st;
struct stat stobj;
CURL *curl = NULL;
string url;
string resource;
@ -580,7 +581,6 @@ static int get_local_fd(const char* path) {
string baseName = mybasename(path);
string resolved_path(use_cache + "/" + bucket);
string cache_path(resolved_path + path);
headers_t responseHeaders;
FGPRINT(" get_local_fd[path=%s]\n", path);
@ -588,7 +588,7 @@ static int get_local_fd(const char* path) {
resource = urlEncode(service_path + bucket + s3_realpath);
url = host + resource;
if(0 != (result = get_object_attribute(path, NULL, &responseHeaders))){
if(0 != (result = get_object_attribute(path, &stobj))){
return result;
}
@ -602,8 +602,7 @@ static int get_local_fd(const char* path) {
// if the local and remote mtime/size
// do not match we have an invalid cache entry
if(str(st.st_size) != responseHeaders["Content-Length"] ||
(str(st.st_mtime) != responseHeaders["x-amz-meta-mtime"])) {
if(st.st_size != stobj.st_size || st.st_mtime != stobj.st_mtime){
if(close(fd) == -1){
YIKES(-errno);
}
@ -614,13 +613,11 @@ static int get_local_fd(const char* path) {
// need to download?
if(fd == -1) {
mode_t mode = get_mode(responseHeaders["x-amz-meta-mode"].c_str());
if(use_cache.size() > 0) {
// only download files, not folders
if (S_ISREG(mode)) {
if (S_ISREG(stobj.st_mode)) {
mkdirp(resolved_path + mydirname(path), 0777);
fd = open(cache_path.c_str(), O_CREAT|O_RDWR|O_TRUNC, mode);
fd = open(cache_path.c_str(), O_CREAT|O_RDWR|O_TRUNC, stobj.st_mode);
} else {
// its a folder; do *not* create anything in local cache...
// TODO: do this in a better way)
@ -675,11 +672,11 @@ static int get_local_fd(const char* path) {
fflush(f);
fsync(fd);
if(S_ISREG(mode) && !S_ISLNK(mode)) {
if(S_ISREG(stobj.st_mode) && !S_ISLNK(stobj.st_mode)) {
// make the file's mtime match that of the file on s3
// if fd is tmpfile, but we force tor set mtime.
struct timeval tv[2];
tv[0].tv_sec = get_mtime(responseHeaders["x-amz-meta-mtime"].c_str());
tv[0].tv_sec = stobj.st_mtime;
tv[0].tv_usec= 0L;
tv[1].tv_sec = tv[0].tv_sec;
tv[1].tv_usec= 0L;
@ -775,29 +772,28 @@ static int put_headers(const char *path, headers_t meta) {
return result;
// Update mtime in local file cache.
if(meta.count("x-amz-meta-mtime") > 0){
int fd;
if(0 <= (fd = get_opened_fd(path))){
// The file already is opened, so update fd before close(flush);
struct timeval tv[2];
memset(tv, 0, sizeof(struct timeval) * 2);
tv[0].tv_sec = get_mtime(meta["x-amz-meta-mtime"].c_str());
tv[1].tv_sec = tv[0].tv_sec;
if(-1 == futimes(fd, tv)){
YIKES(-errno);
}
}else if(use_cache.size() > 0){
// Use local cache file.
struct stat st;
struct utimbuf n_mtime;
string cache_path(use_cache + "/" + bucket + path);
int fd;
time_t mtime = get_mtime(meta);
if(0 <= (fd = get_opened_fd(path))){
// The file already is opened, so update fd before close(flush);
struct timeval tv[2];
memset(tv, 0, sizeof(struct timeval) * 2);
tv[0].tv_sec = mtime;
tv[1].tv_sec = tv[0].tv_sec;
if(-1 == futimes(fd, tv)){
YIKES(-errno);
}
}else if(use_cache.size() > 0){
// Use local cache file.
struct stat st;
struct utimbuf n_mtime;
string cache_path(use_cache + "/" + bucket + path);
if((stat(cache_path.c_str(), &st)) == 0) {
n_mtime.modtime = get_mtime(meta["x-amz-meta-mtime"].c_str());
n_mtime.actime = n_mtime.modtime;
if((utime(cache_path.c_str(), &n_mtime)) == -1) {
YIKES(-errno);
}
if((stat(cache_path.c_str(), &st)) == 0) {
n_mtime.modtime = mtime;
n_mtime.actime = n_mtime.modtime;
if((utime(cache_path.c_str(), &n_mtime)) == -1) {
YIKES(-errno);
}
}
}
@ -858,20 +854,19 @@ static int put_multipart_headers(const char *path, headers_t meta) {
}
// Update mtime in local file cache.
if(meta.count("x-amz-meta-mtime") > 0 && use_cache.size() > 0) {
if(use_cache.size() > 0) {
struct stat st;
struct utimbuf n_mtime;
string cache_path(use_cache + "/" + bucket + path);
if((stat(cache_path.c_str(), &st)) == 0) {
n_mtime.modtime = get_mtime(meta["x-amz-meta-mtime"].c_str());
n_mtime.actime = n_mtime.modtime;
n_mtime.modtime = get_mtime(meta);
n_mtime.actime = n_mtime.modtime;
if((utime(cache_path.c_str(), &n_mtime)) == -1) {
YIKES(-errno);
}
}
}
return 0;
}
@ -2116,25 +2111,16 @@ static int rename_large_object(const char *from, const char *to) {
static int clone_directory_object(const char *from, const char *to)
{
int result = -1;
mode_t mode;
time_t time;
uid_t uid;
gid_t gid;
headers_t meta;
struct stat stbuf;
FGPRINT("clone_directory_object [from=%s] [to=%s]\n", from, to);
SYSLOGDBG("clone_directory_object [from=%s] [to=%s]", from, to);
// get target's attributes
if(0 != (result = get_object_attribute(from, NULL, &meta))){
if(0 != (result = get_object_attribute(from, &stbuf))){
return result;
}
mode = get_mode(meta["x-amz-meta-mode"].c_str());
time = get_mtime(meta["x-amz-meta-mtime"].c_str());
uid = get_uid(meta["x-amz-meta-uid"].c_str());
gid = get_gid(meta["x-amz-meta-gid"].c_str());
result = create_directory_object(to, mode, time, uid, gid);
result = create_directory_object(to, stbuf.st_mode, stbuf.st_mtime, stbuf.st_uid, stbuf.st_gid);
StatCache::getStatCacheData()->DelStat(to);
return result;

99
src/s3fs_util.cpp
View File

@ -501,26 +501,116 @@ time_t get_mtime(const char *s)
return (time_t) strtoul(s, (char **) NULL, 10);
}
time_t get_mtime(headers_t& meta, bool overcheck)
{
headers_t::const_iterator iter;
if(meta.end() == (iter = meta.find("x-amz-meta-mtime"))){
if(overcheck){
return get_lastmodified(meta);
}
return 0;
}
return get_mtime((*iter).second.c_str());
}
off_t get_size(const char *s)
{
return (off_t) strtoul(s, (char **) NULL, 10);
}
off_t get_size(headers_t& meta)
{
headers_t::const_iterator iter;
if(meta.end() == (iter = meta.find("Content-Length"))){
return 0;
}
return get_size((*iter).second.c_str());
}
mode_t get_mode(const char *s)
{
return (mode_t) strtoul(s, (char **) NULL, 10);
}
mode_t get_mode(headers_t& meta, const char* path, bool checkdir, bool forcedir)
{
mode_t mode = 0;
bool isS3sync = false;
headers_t::const_iterator iter;
if(meta.end() != (iter = meta.find("x-amz-meta-mode"))){
mode = get_mode((*iter).second.c_str());
}else{
if(meta.end() != (iter = meta.find("x-amz-meta-permissions"))){ // for s3sync
mode = get_mode((*iter).second.c_str());
isS3sync = true;
}
}
if(!isS3sync){
if(checkdir){
if(forcedir){
mode |= S_IFDIR;
}else{
if(meta.end() != (iter = meta.find("Content-Type"))){
string strConType = (*iter).second;
if(strConType == "application/x-directory"){
mode |= S_IFDIR;
}else if(path && 0 < strlen(path) && '/' == path[strlen(path) - 1]){
if(strConType == "binary/octet-stream" || strConType == "application/octet-stream"){
mode |= S_IFDIR;
}else{
mode |= S_IFREG;
}
}else{
mode |= S_IFREG;
}
}else{
mode |= S_IFREG;
}
}
}
}else{
if(!checkdir){
// cut dir/reg flag.
mode &= ~S_IFDIR;
mode &= ~S_IFREG;
}
}
return mode;
}
uid_t get_uid(const char *s)
{
return (uid_t) strtoul(s, (char **) NULL, 10);
}
uid_t get_uid(headers_t& meta)
{
headers_t::const_iterator iter;
if(meta.end() == (iter = meta.find("x-amz-meta-uid"))){
if(meta.end() == (iter = meta.find("x-amz-meta-owner"))){ // for s3sync
return 0;
}
}
return get_uid((*iter).second.c_str());
}
gid_t get_gid(const char *s)
{
return (gid_t) strtoul(s, (char **) NULL, 10);
}
gid_t get_gid(headers_t& meta)
{
headers_t::const_iterator iter;
if(meta.end() == (iter = meta.find("x-amz-meta-gid"))){
if(meta.end() == (iter = meta.find("x-amz-meta-group"))){ // for s3sync
return 0;
}
}
return get_gid((*iter).second.c_str());
}
blkcnt_t get_blocks(off_t size)
{
return size / 512 + 1;
@ -536,6 +626,15 @@ time_t get_lastmodified(const char* s)
return mktime(&tm); // GMT
}
time_t get_lastmodified(headers_t& meta)
{
headers_t::const_iterator iter;
if(meta.end() == (iter = meta.find("Last-Modified"))){
return 0;
}
return get_lastmodified((*iter).second.c_str());
}
//-------------------------------------------------------------------
// Help
//-------------------------------------------------------------------

6
src/s3fs_util.h
View File

@ -81,12 +81,18 @@ std::string mybasename(std::string path);
int mkdirp(const std::string& path, mode_t mode);
time_t get_mtime(const char *s);
time_t get_mtime(headers_t& meta, bool overcheck = true);
off_t get_size(const char *s);
off_t get_size(headers_t& meta);
mode_t get_mode(const char *s);
mode_t get_mode(headers_t& meta, const char* path = NULL, bool checkdir = false, bool forcedir = false);
uid_t get_uid(const char *s);
uid_t get_uid(headers_t& meta);
gid_t get_gid(const char *s);
gid_t get_gid(headers_t& meta);
blkcnt_t get_blocks(off_t size);
time_t get_lastmodified(const char* s);
time_t get_lastmodified(headers_t& meta);
void show_usage(void);
void show_help(void);