mirror of
https://github.com/s3fs-fuse/s3fs-fuse.git
synced 2024-06-13 14:22:23 +00:00
Checkpoint for large file (> 2GB) upload capability
needs more testing. git-svn-id: http://s3fs.googlecode.com/svn/trunk@302 df820570-a93a-0410-bd06-b72b767a4274
This commit is contained in:
parent
0f18298886
commit
8e4c89fdec
|
@ -1,7 +1,7 @@
|
||||||
dnl Process this file with autoconf to produce a configure script.
|
dnl Process this file with autoconf to produce a configure script.
|
||||||
|
|
||||||
AC_PREREQ(2.59)
|
AC_PREREQ(2.59)
|
||||||
AC_INIT(s3fs, 1.33)
|
AC_INIT(s3fs, 1.34)
|
||||||
|
|
||||||
|
|
||||||
AC_CANONICAL_SYSTEM
|
AC_CANONICAL_SYSTEM
|
||||||
|
@ -9,6 +9,8 @@ AM_INIT_AUTOMAKE()
|
||||||
|
|
||||||
AC_PROG_CXX
|
AC_PROG_CXX
|
||||||
|
|
||||||
|
CXXFLAGS="$CXXFLAGS -Wall -D_FILE_OFFSET_BITS=64"
|
||||||
|
|
||||||
PKG_CHECK_MODULES([DEPS], [fuse >= 2.8.4 libcurl >= 7.0 libxml-2.0 >= 2.6 libcrypto >= 0.9])
|
PKG_CHECK_MODULES([DEPS], [fuse >= 2.8.4 libcurl >= 7.0 libxml-2.0 >= 2.6 libcrypto >= 0.9])
|
||||||
|
|
||||||
AC_CONFIG_FILES(Makefile src/Makefile test/Makefile)
|
AC_CONFIG_FILES(Makefile src/Makefile test/Makefile)
|
||||||
|
|
13
src/s3fs.cpp
13
src/s3fs.cpp
|
@ -1367,7 +1367,7 @@ static int put_local_fd_small_file(const char* path, headers_t meta, int fd) {
|
||||||
|
|
||||||
//###rewind(f);
|
//###rewind(f);
|
||||||
|
|
||||||
if(debug) syslog(LOG_DEBUG, "upload path=%s size=%llu", path, st.st_size);
|
// if(debug) syslog(LOG_DEBUG, "upload path=%s size=%zu", path, st.st_size);
|
||||||
|
|
||||||
if(foreground)
|
if(foreground)
|
||||||
cout << " uploading[path=" << path << "][fd=" << fd << "][size="<<st.st_size <<"]" << endl;
|
cout << " uploading[path=" << path << "][fd=" << fd << "][size="<<st.st_size <<"]" << endl;
|
||||||
|
@ -1403,6 +1403,7 @@ static int put_local_fd_big_file(const char* path, headers_t meta, int fd) {
|
||||||
string ETag;
|
string ETag;
|
||||||
int result;
|
int result;
|
||||||
unsigned long lSize;
|
unsigned long lSize;
|
||||||
|
// long lSize;
|
||||||
int partfd = -1;
|
int partfd = -1;
|
||||||
FILE* pSourceFile;
|
FILE* pSourceFile;
|
||||||
FILE* pPartFile;
|
FILE* pPartFile;
|
||||||
|
@ -1639,7 +1640,7 @@ static int put_local_fd_big_file(const char* path, headers_t meta, int fd) {
|
||||||
// copy the file portion into the buffer:
|
// copy the file portion into the buffer:
|
||||||
bytesRead = fread (buffer, 1, lBufferSize, pSourceFile);
|
bytesRead = fread (buffer, 1, lBufferSize, pSourceFile);
|
||||||
if (bytesRead != lBufferSize) {
|
if (bytesRead != lBufferSize) {
|
||||||
syslog(LOG_ERR, "%d ### bytesRead:%i does not match lBufferSize: %lu\n",
|
syslog(LOG_ERR, "%d ### bytesRead:%zu does not match lBufferSize: %lu\n",
|
||||||
__LINE__, bytesRead, lBufferSize);
|
__LINE__, bytesRead, lBufferSize);
|
||||||
|
|
||||||
if(buffer) free(buffer);
|
if(buffer) free(buffer);
|
||||||
|
@ -1689,7 +1690,7 @@ static int put_local_fd_big_file(const char* path, headers_t meta, int fd) {
|
||||||
|
|
||||||
|
|
||||||
if (bytesWritten != lBufferSize) {
|
if (bytesWritten != lBufferSize) {
|
||||||
syslog(LOG_ERR, "%d ### bytesWritten:%i does not match lBufferSize: %lu\n",
|
syslog(LOG_ERR, "%d ### bytesWritten:%zu does not match lBufferSize: %lu\n",
|
||||||
__LINE__, bytesWritten, lBufferSize);
|
__LINE__, bytesWritten, lBufferSize);
|
||||||
|
|
||||||
fclose(pPartFile);
|
fclose(pPartFile);
|
||||||
|
@ -1974,14 +1975,14 @@ static int put_local_fd(const char* path, headers_t meta, int fd) {
|
||||||
// - minimum size of parts is 5MB (expect for the last part)
|
// - minimum size of parts is 5MB (expect for the last part)
|
||||||
//
|
//
|
||||||
// For our application, we will define part size to be 10MB (10 * 2^20 Bytes)
|
// For our application, we will define part size to be 10MB (10 * 2^20 Bytes)
|
||||||
// maximum file size will be ~2 GB - 2 ** 31
|
// maximum file size will be ~64 GB - 2 ** 36
|
||||||
//
|
//
|
||||||
// Initially uploads will be done serially
|
// Initially uploads will be done serially
|
||||||
//
|
//
|
||||||
// If file is > 20MB, then multipart will kick in
|
// If file is > 20MB, then multipart will kick in
|
||||||
/////////////////////////////////////////////////////////////
|
/////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
if(st.st_size > 2147483647) { // 2GB - 1
|
if(st.st_size > 68719476735LL ) { // 64GB - 1
|
||||||
// close f ?
|
// close f ?
|
||||||
return -ENOTSUP;
|
return -ENOTSUP;
|
||||||
}
|
}
|
||||||
|
@ -2125,7 +2126,7 @@ static int s3fs_readlink(const char *path, char *buf, size_t size) {
|
||||||
return -errno;
|
return -errno;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (st.st_size < size) {
|
if (st.st_size < (off_t)size) {
|
||||||
size = st.st_size;
|
size = st.st_size;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue
Block a user