mirror of
https://github.com/s3fs-fuse/s3fs-fuse.git
synced 2024-06-01 00:20:49 +00:00
Add a test that is multi-block writing by one flush
This commit is contained in:
parent
ea64886469
commit
34ea2acd75
|
@ -30,6 +30,9 @@ EXTRA_DIST = \
|
|||
|
||||
testdir = test
|
||||
|
||||
noinst_PROGRAMS = write_multiblock
|
||||
write_multiblock_SOURCES = write_multiblock.cc
|
||||
|
||||
#
|
||||
# Local variables:
|
||||
# tab-width: 4
|
||||
|
|
|
@ -1485,6 +1485,143 @@ function test_ut_ossfs {
|
|||
../../ut_test.py
|
||||
}
|
||||
|
||||
#
|
||||
# This test opens a file and writes multiple sets of data.
|
||||
# The file is opened only once and multiple blocks of data are written
|
||||
# to the file descriptor with a gap.
|
||||
#
|
||||
# That is, the data sets are written discontinuously.
|
||||
# The data to be written uses multiple data that is less than or larger
|
||||
# than the part size of the multi-part upload.
|
||||
# The gap should be at least the part size of the multi-part upload.
|
||||
# Write as shown below:
|
||||
# <SOF>....<write data>....<write data>....<write data><EOF>
|
||||
#
|
||||
# There are two types of tests: new files and existing files.
|
||||
# For existing files, the file size must be larger than where this test
|
||||
# writes last position.
|
||||
# <SOF>....<write data>....<write data>....<write data>...<EOF>
|
||||
#
|
||||
function test_write_data_with_skip() {
|
||||
describe "Testing write data block with skipping block..."
|
||||
|
||||
#
|
||||
# The first argument of the script is "testrun-<random>" the directory name.
|
||||
#
|
||||
CACHE_TESTRUN_DIR=$1
|
||||
|
||||
_SKIPWRITE_FILE="test_skipwrite"
|
||||
_TMP_SKIPWRITE_FILE="/tmp/${_SKIPWRITE_FILE}"
|
||||
|
||||
#------------------------------------------------------
|
||||
# (1) test new file
|
||||
#------------------------------------------------------
|
||||
#
|
||||
# Clean files
|
||||
#
|
||||
rm_test_file ${_SKIPWRITE_FILE}
|
||||
rm_test_file ${_TMP_SKIPWRITE_FILE}
|
||||
|
||||
#
|
||||
# Create new file in bucket and temporary directory(/tmp)
|
||||
#
|
||||
# Writing to the file is as follows:
|
||||
# |<-- skip(12MB) --><-- write(1MB) --><-- skip(22MB) --><-- write(20MB) --><-- skip(23MB) --><-- write(1MB) -->| (79MB)
|
||||
#
|
||||
# As a result, areas that are not written to the file are mixed.
|
||||
# The part that is not written has a HOLE that is truncate and filled
|
||||
# with 0x00.
|
||||
# Assuming that multipart upload is performed on a part-by-part basis,
|
||||
# it will be as follows:
|
||||
# part 1) 0x0.. 0x9FFFFF : <not write area(0x00)>
|
||||
# part 2) 0xA00000..0x13FFFFF : 0xA00000..0xBFFFFF <not write area(0x00)>
|
||||
# 0xC00000..0xCFFFFF <write area>
|
||||
# 0xD00000..0x13FFFFF <not write area(0x00)>
|
||||
# part 3) 0x1400000..0x1DFFFFF : <not write area(0x00)>
|
||||
# part 4) 0x1E00000..0x27FFFFF : 0x1E00000..0x22FFFFF <not write area(0x00)>
|
||||
# 0x2300000..0x27FFFFF <write area>
|
||||
# part 5) 0x2800000..0x31FFFFF : <write area>
|
||||
# part 6) 0x3200000..0x3BFFFFF : 0x3200000..0x36FFFFF <write area>
|
||||
# 0x3700000..0x3BFFFFF <not write area(0x00)>
|
||||
# part 7) 0x3C00000..0x45FFFFF : <not write area(0x00)>
|
||||
# part 8) 0x4600000..0x4BFFFFF : 0x4600000..0x4AFFFFF <not write area(0x00)>
|
||||
# 0x4B00000..0x4BFFFFF <write area>
|
||||
#
|
||||
./write_multiblock -f "${_SKIPWRITE_FILE}" -f "${_TMP_SKIPWRITE_FILE}" -p 12582912:65536 -p 36700160:20971520 -p 78643200:65536
|
||||
|
||||
#
|
||||
# delete cache file if using cache
|
||||
#
|
||||
if ps u $S3FS_PID | grep -q use_cache; then
|
||||
rm -f ${CACHE_DIR}/${TEST_BUCKET_1}/${CACHE_TESTRUN_DIR}/${_SKIPWRITE_FILE}
|
||||
rm -f ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${_SKIPWRITE_FILE}
|
||||
fi
|
||||
|
||||
#
|
||||
# Compare
|
||||
#
|
||||
cmp ${_SKIPWRITE_FILE} ${_TMP_SKIPWRITE_FILE}
|
||||
|
||||
#------------------------------------------------------
|
||||
# (2) test existed file
|
||||
#------------------------------------------------------
|
||||
# [NOTE]
|
||||
# This test uses the file used in the previous test as an existing file.
|
||||
#
|
||||
if ps u $S3FS_PID | grep -q use_cache; then
|
||||
rm -f ${CACHE_DIR}/${TEST_BUCKET_1}/${CACHE_TESTRUN_DIR}/${_SKIPWRITE_FILE}
|
||||
rm -f ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${_SKIPWRITE_FILE}
|
||||
fi
|
||||
|
||||
#
|
||||
# Over write data to existed file in bucket and temporary directory(/tmp)
|
||||
#
|
||||
# Writing to the file is as follows:
|
||||
# |<----------------------------------------------- existed file ----------------------------------------------------------->| (79MB)
|
||||
# |<-- skip(12MB) --><-- write(1MB) --><-- skip(22MB) --><-- write(20MB) --><-- skip(22MB) --><-- write(1MB) --><-- 1MB -->| (79MB)
|
||||
#
|
||||
# As a result, areas that are not written to the file are mixed.
|
||||
# The part that is not written has a HOLE that is truncate and filled
|
||||
# with 0x00.
|
||||
# Assuming that multipart upload is performed on a part-by-part basis,
|
||||
# it will be as follows:
|
||||
# part 1) 0x0.. 0x9FFFFF : <not write area(0x00)>
|
||||
# part 2) 0xA00000..0x13FFFFF : 0xA00000..0xBFFFFF <not write area(0x00)>
|
||||
# 0xC00000..0xCFFFFF <write area>
|
||||
# 0xD00000..0x13FFFFF <not write area(0x00)>
|
||||
# part 3) 0x1400000..0x1DFFFFF : <not write area(0x00)>
|
||||
# part 4) 0x1E00000..0x27FFFFF : 0x1E00000..0x22FFFFF <not write area(0x00)>
|
||||
# 0x2300000..0x27FFFFF <write area>
|
||||
# part 5) 0x2800000..0x31FFFFF : <write area>
|
||||
# part 6) 0x3200000..0x3BFFFFF : 0x3200000..0x36FFFFF <write area>
|
||||
# 0x3700000..0x3BFFFFF <not write area(0x00)>
|
||||
# part 7) 0x3C00000..0x45FFFFF : <not write area(0x00)>
|
||||
# part 8) 0x4600000..0x4BFFFFF : 0x4600000..0x49FFFFF <not write area(0x00)>
|
||||
# part 8) 0x4600000..0x4BFFFFF : 0x4A00000..0x4AFFFFF <write area>
|
||||
# 0x4B00000..0x4BFFFFF <not write area(0x00)>
|
||||
#
|
||||
./write_multiblock -f "${_SKIPWRITE_FILE}" -f "${_TMP_SKIPWRITE_FILE}" -p 12582912:65536 -p 36700160:20971520 -p 77594624:65536
|
||||
|
||||
#
|
||||
# delete cache file if using cache
|
||||
#
|
||||
if ps u $S3FS_PID | grep -q use_cache; then
|
||||
rm -f ${CACHE_DIR}/${TEST_BUCKET_1}/${CACHE_TESTRUN_DIR}/${_SKIPWRITE_FILE}
|
||||
rm -f ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${_SKIPWRITE_FILE}
|
||||
fi
|
||||
|
||||
#
|
||||
# Compare
|
||||
#
|
||||
cmp ${_SKIPWRITE_FILE} ${_TMP_SKIPWRITE_FILE}
|
||||
|
||||
#
|
||||
# Clean files
|
||||
#
|
||||
rm_test_file ${_SKIPWRITE_FILE}
|
||||
rm_test_file ${_TMP_SKIPWRITE_FILE}
|
||||
}
|
||||
|
||||
function add_all_tests {
|
||||
if ps u $S3FS_PID | grep -q use_cache; then
|
||||
add_tests test_cache_file_stat
|
||||
|
@ -1546,6 +1683,7 @@ function add_all_tests {
|
|||
if ! ps u $S3FS_PID | grep -q ensure_diskfree && ! uname | grep -q Darwin; then
|
||||
add_tests test_ensurespace_move_file
|
||||
fi
|
||||
test_write_data_with_skip
|
||||
}
|
||||
|
||||
init_suite
|
||||
|
|
268
test/write_multiblock.cc
Normal file
268
test/write_multiblock.cc
Normal file
|
@ -0,0 +1,268 @@
|
|||
/*
|
||||
* s3fs - FUSE-based file system backed by Amazon S3
|
||||
*
|
||||
* Copyright(C) 2007 Randy Rizun <rrizun@gmail.com>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <iostream>
|
||||
#include <climits>
|
||||
#include <string>
|
||||
#include <list>
|
||||
|
||||
#include <unistd.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <fcntl.h>
|
||||
#include <string.h>
|
||||
#include <errno.h>
|
||||
|
||||
//---------------------------------------------------------
|
||||
// Structures and Typedefs
|
||||
//---------------------------------------------------------
|
||||
struct write_block_part
|
||||
{
|
||||
off_t start;
|
||||
off_t size;
|
||||
};
|
||||
|
||||
typedef std::list<write_block_part> wbpart_list_t;
|
||||
typedef std::list<std::string> strlist_t;
|
||||
|
||||
//---------------------------------------------------------
|
||||
// Const
|
||||
//---------------------------------------------------------
|
||||
const char usage_string[] = "Usage : \"write_multiblock -f <file path> -p <start offset:size>\" (allows -f and -p multiple times.)";
|
||||
|
||||
//---------------------------------------------------------
|
||||
// Utility functions
|
||||
//---------------------------------------------------------
|
||||
static unsigned char* create_random_data(off_t size)
|
||||
{
|
||||
int fd;
|
||||
if(-1 == (fd = open("/dev/urandom", O_RDONLY))){
|
||||
std::cerr << "[ERROR] Could not open /dev/urandom" << std::endl;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
unsigned char* pbuff;
|
||||
if(NULL == (pbuff = reinterpret_cast<unsigned char*>(malloc(size)))){
|
||||
std::cerr << "[ERROR] Could not allocate memory." << std::endl;
|
||||
close(fd);
|
||||
return NULL;
|
||||
}
|
||||
for(ssize_t readpos = 0, readcnt = 0; readpos < size; readpos += readcnt){
|
||||
if(-1 == (readcnt = read(fd, &(pbuff[readpos]), static_cast<size_t>(size - readpos)))){
|
||||
if(EAGAIN != errno && EWOULDBLOCK != errno && EINTR != errno){
|
||||
std::cerr << "[ERROR] Failed reading from /dev/urandom with errno: " << errno << std::endl;
|
||||
free(pbuff);
|
||||
close(fd);
|
||||
return NULL;
|
||||
}
|
||||
readcnt = 0;
|
||||
}
|
||||
}
|
||||
return pbuff;
|
||||
}
|
||||
|
||||
static off_t cvt_string_to_number(const char* pstr)
|
||||
{
|
||||
if(!pstr){
|
||||
return -1;
|
||||
}
|
||||
|
||||
errno = 0;
|
||||
char* ptemp = NULL;
|
||||
long long result = strtoll(pstr, &ptemp, 10);
|
||||
|
||||
if(!ptemp || ptemp == pstr || *ptemp != '\0'){
|
||||
return -1;
|
||||
}
|
||||
if((result == LLONG_MIN || result == LLONG_MAX) && errno == ERANGE){
|
||||
return -1;
|
||||
}
|
||||
|
||||
return static_cast<off_t>(result);
|
||||
}
|
||||
|
||||
static bool parse_string(const char* pstr, char delim, strlist_t& strlist)
|
||||
{
|
||||
if(!pstr){
|
||||
return false;
|
||||
}
|
||||
char* ptmp = strdup(pstr);
|
||||
char* pbup = ptmp;
|
||||
for(char* pfound = strchr(ptmp, delim); pfound; pfound = strchr(ptmp, delim)){
|
||||
*pfound = '\0';
|
||||
if(0 < strlen(ptmp)){
|
||||
strlist.push_back(std::string(ptmp));
|
||||
}
|
||||
ptmp = ++pfound;
|
||||
}
|
||||
if(0 < strlen(ptmp)){
|
||||
strlist.push_back(std::string(ptmp));
|
||||
}
|
||||
free(pbup);
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool parse_write_blocks(const char* pstr, wbpart_list_t& wbparts, off_t& max_size)
|
||||
{
|
||||
if(!pstr){
|
||||
return false;
|
||||
}
|
||||
|
||||
strlist_t partlist;
|
||||
if(!parse_string(pstr, ',', partlist)){
|
||||
return false;
|
||||
}
|
||||
|
||||
for(strlist_t::const_iterator iter = partlist.begin(); iter != partlist.end(); ++iter){
|
||||
strlist_t partpair;
|
||||
if(parse_string(iter->c_str(), ':', partpair) && 2 == partpair.size()){
|
||||
write_block_part tmp_part;
|
||||
|
||||
tmp_part.start = cvt_string_to_number(partpair.front().c_str());
|
||||
partpair.pop_front();
|
||||
tmp_part.size = cvt_string_to_number(partpair.front().c_str());
|
||||
|
||||
if(tmp_part.start < 0 || tmp_part.size <= 0){
|
||||
std::cerr << "[ERROR] -p option parameter(" << pstr << ") is something wrong." << std::endl;
|
||||
return false;
|
||||
}
|
||||
if(max_size < tmp_part.size){
|
||||
max_size = tmp_part.size;
|
||||
}
|
||||
wbparts.push_back(tmp_part);
|
||||
}else{
|
||||
std::cerr << "[ERROR] -p option parameter(" << pstr << ") is something wrong." << std::endl;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool parse_arguments(int argc, char** argv, strlist_t& files, wbpart_list_t& wbparts, off_t& max_size)
|
||||
{
|
||||
if(argc < 2 || !argv){
|
||||
std::cerr << "[ERROR] The -f option and -p option are required as arguments." << std::endl;
|
||||
std::cerr << usage_string << std::endl;
|
||||
return false;
|
||||
}
|
||||
files.clear();
|
||||
wbparts.clear();
|
||||
max_size = 0;
|
||||
|
||||
int opt;
|
||||
while(-1 != (opt = getopt(argc, argv, "f:p:"))){
|
||||
switch(opt){
|
||||
case 'f':
|
||||
files.push_back(std::string(optarg));
|
||||
break;
|
||||
case 'p':
|
||||
if(!parse_write_blocks(optarg, wbparts, max_size)){
|
||||
return false;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
std::cerr << usage_string << std::endl;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if(files.empty() || wbparts.empty()){
|
||||
std::cerr << "[ERROR] The -f option and -p option are required as arguments." << std::endl;
|
||||
std::cerr << usage_string << std::endl;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
//---------------------------------------------------------
|
||||
// Main
|
||||
//---------------------------------------------------------
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
// parse arguments
|
||||
strlist_t files;
|
||||
wbpart_list_t wbparts;
|
||||
off_t max_size = 0;
|
||||
if(!parse_arguments(argc, argv, files, wbparts, max_size)){
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
// make data and buffer
|
||||
unsigned char* pData;
|
||||
if(NULL == (pData = create_random_data(max_size))){
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
for(strlist_t::const_iterator fiter = files.begin(); fiter != files.end(); ++fiter){
|
||||
// open/create file
|
||||
int fd;
|
||||
struct stat st;
|
||||
if(0 == stat(fiter->c_str(), &st)){
|
||||
if(!S_ISREG(st.st_mode)){
|
||||
std::cerr << "[ERROR] File " << fiter->c_str() << " is existed, but it is not regular file." << std::endl;
|
||||
free(pData);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if(-1 == (fd = open(fiter->c_str(), O_WRONLY))){
|
||||
std::cerr << "[ERROR] Could not open " << fiter->c_str() << std::endl;
|
||||
free(pData);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}else{
|
||||
if(-1 == (fd = open(fiter->c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644))){
|
||||
std::cerr << "[ERROR] Could not create " << fiter->c_str() << std::endl;
|
||||
free(pData);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
// write blocks
|
||||
for(wbpart_list_t::const_iterator piter = wbparts.begin(); piter != wbparts.end(); ++piter){
|
||||
// write one block
|
||||
for(ssize_t writepos = 0, writecnt = 0; writepos < piter->size; writepos += writecnt){
|
||||
if(-1 == (writecnt = pwrite(fd, &(pData[writepos]), static_cast<size_t>(piter->size - writepos), (piter->start + writepos)))){
|
||||
if(EAGAIN != errno && EWOULDBLOCK != errno && EINTR != errno){
|
||||
std::cerr << "[ERROR] Failed writing to " << fiter->c_str() << " by errno : " << errno << std::endl;
|
||||
close(fd);
|
||||
free(pData);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
writecnt = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
// close file
|
||||
close(fd);
|
||||
}
|
||||
free(pData);
|
||||
|
||||
exit(EXIT_SUCCESS);
|
||||
}
|
||||
|
||||
/*
|
||||
* Local variables:
|
||||
* tab-width: 4
|
||||
* c-basic-offset: 4
|
||||
* End:
|
||||
* vim600: expandtab sw=4 ts=4 fdm=marker
|
||||
* vim<600: expandtab sw=4 ts=4
|
||||
*/
|
Loading…
Reference in New Issue
Block a user