octoleo/s3fs-fuse
1
0
Fork 0
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git synced 2025-02-02 10:38:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Checkpoint for large file (> 2GB) upload capability

Browse Source 
needs more testing.


git-svn-id: http://s3fs.googlecode.com/svn/trunk@302 df820570-a93a-0410-bd06-b72b767a4274
This commit is contained in:
mooredan@suncup.net 2011-01-20 22:40:59 +00:00
parent 0f18298886
commit 8e4c89fdec
2 changed files with 10 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
configure.ac
View File

@ -1,7 +1,7 @@
dnl Process this file with autoconf to produce a configure script.
AC_PREREQ(2.59)
AC_INIT(s3fs, 1.33)
AC_INIT(s3fs, 1.34)
AC_CANONICAL_SYSTEM
@ -9,6 +9,8 @@ AM_INIT_AUTOMAKE()
AC_PROG_CXX
CXXFLAGS="$CXXFLAGS -Wall -D_FILE_OFFSET_BITS=64"
PKG_CHECK_MODULES([DEPS], [fuse >= 2.8.4 libcurl >= 7.0 libxml-2.0 >= 2.6 libcrypto >= 0.9])
AC_CONFIG_FILES(Makefile src/Makefile test/Makefile)

13
src/s3fs.cpp
View File

@ -1367,7 +1367,7 @@ static int put_local_fd_small_file(const char* path, headers_t meta, int fd) {
//###rewind(f);
if(debug) syslog(LOG_DEBUG, "upload path=%s size=%llu", path, st.st_size);
// if(debug) syslog(LOG_DEBUG, "upload path=%s size=%zu", path, st.st_size);
if(foreground)
cout << " uploading[path=" << path << "][fd=" << fd << "][size="<<st.st_size <<"]" << endl;
@ -1403,6 +1403,7 @@ static int put_local_fd_big_file(const char* path, headers_t meta, int fd) {
string ETag;
int result;
unsigned long lSize;
// long lSize;
int partfd = -1;
FILE* pSourceFile;
FILE* pPartFile;
@ -1639,7 +1640,7 @@ static int put_local_fd_big_file(const char* path, headers_t meta, int fd) {
// copy the file portion into the buffer:
bytesRead = fread (buffer, 1, lBufferSize, pSourceFile);
if (bytesRead != lBufferSize) {
syslog(LOG_ERR, "%d ### bytesRead:%i does not match lBufferSize: %lu\n",
syslog(LOG_ERR, "%d ### bytesRead:%zu does not match lBufferSize: %lu\n",
__LINE__, bytesRead, lBufferSize);
if(buffer) free(buffer);
@ -1689,7 +1690,7 @@ static int put_local_fd_big_file(const char* path, headers_t meta, int fd) {
if (bytesWritten != lBufferSize) {
syslog(LOG_ERR, "%d ### bytesWritten:%i does not match lBufferSize: %lu\n",
syslog(LOG_ERR, "%d ### bytesWritten:%zu does not match lBufferSize: %lu\n",
__LINE__, bytesWritten, lBufferSize);
fclose(pPartFile);
@ -1974,14 +1975,14 @@ static int put_local_fd(const char* path, headers_t meta, int fd) {
// - minimum size of parts is 5MB (expect for the last part)
//
// For our application, we will define part size to be 10MB (10 * 2^20 Bytes)
// maximum file size will be ~2 GB - 2 ** 31
// maximum file size will be ~64 GB - 2 ** 36
//
// Initially uploads will be done serially
//
// If file is > 20MB, then multipart will kick in
/////////////////////////////////////////////////////////////
if(st.st_size > 2147483647) { // 2GB - 1
if(st.st_size > 68719476735LL ) { // 64GB - 1
// close f ?
return -ENOTSUP;
}
@ -2125,7 +2126,7 @@ static int s3fs_readlink(const char *path, char *buf, size_t size) {
return -errno;
}
if (st.st_size < size) {
if (st.st_size < (off_t)size) {
size = st.st_size;
}