octoleo
/
s3fs-fuse
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Added mime option for strict checking of mime types file

This commit is contained in:
Takeshi Nakatani 2020-03-19 15:13:21 +00:00
parent fe2b269b6e
commit 7fbda230f5
8 changed files with 137 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
COMPILATION.md
View File

@ -15,7 +15,11 @@ Keep in mind using the pre-built packages when available.
* libcurl
* libxml2
* openssl
* /etc/mime.types (the package providing depends on the OS)
* mime.types (the package providing depends on the OS)
* s3fs tries to detect `/etc/mime.types` as default regardless of the OS
* Else s3fs tries to detect `/etc/apache2/mime.types` if OS is macOS
* s3fs exits with an error if these files are not exist
* Alternatively, you can set mime.types file path with `mime` option without detecting these default files
* pkg-config (or your OS equivalent)
2. Then compile from master via the following commands:

5
doc/man/s3fs.1
View File

@ -326,6 +326,11 @@ Otherwise an error is returned.
\fB\-o\fR requester_pays (default is disable)
This option instructs s3fs to enable requests involving Requester Pays buckets (It includes the 'x-amz-request-payer=requester' entry in the request header).
.TP
\fB\-o\fR mime (default is "/etc/mime.types")
Specify the path of the mime.types file.
If this option is not specified, the existence of "/etc/mime.types" is checked, and that file is loaded as mime information.
If this file does not exist on macOS, then "/etc/apache2/mime.types" is checked as well.
.TP
\fB\-o\fR dbglevel (default="crit")
Set the debug message level. set value as crit (critical), err (error), warn (warning), info (information) to debug level. default debug level is critical.
If s3fs run with "-d" option, the debug level is set information.

55
src/curl.cpp
View File

@ -343,6 +343,20 @@ static const long S3FSCURL_RESPONSECODE_NOTSET = -1;
static const long S3FSCURL_RESPONSECODE_FATAL_ERROR = -2;
static const int S3FSCURL_PERFORM_RESULT_NOTSET = 1;
// [NOTE] about default mime.types file
// If no mime.types file is specified in the mime option, s3fs
// will look for /etc/mime.types on all operating systems and
// load mime information.
// However, in the case of macOS, when this file does not exist,
// it tries to detect the /etc/apache2/mime.types file.
// The reason for this is that apache2 is preinstalled on macOS,
// and the mime.types file is expected to exist in this path.
// If the mime.types file is not found, s3fs will exit with an
// error.
//
static const char* DEFAULT_MIME_FILE = "/etc/mime.types";
static const char* SPECIAL_DARWIN_MIME_FILE = "/etc/apache2/mime.types";
// [NOTICE]
// This symbol is for libcurl under 7.23.0
#ifndef CURLSHE_NOT_BUILT_IN
@ -397,7 +411,7 @@ bool S3fsCurl::requester_pays = false; // default
//-------------------------------------------------------------------
// Class methods for S3fsCurl
//-------------------------------------------------------------------
bool S3fsCurl::InitS3fsCurl(const char* MimeFile)
bool S3fsCurl::InitS3fsCurl()
{
pthread_mutexattr_t attr;
pthread_mutexattr_init(&attr);
@ -413,9 +427,6 @@ bool S3fsCurl::InitS3fsCurl(const char* MimeFile)
if(0 != pthread_mutex_init(&S3fsCurl::curl_share_lock[SHARE_MUTEX_SSL_SESSION], &attr)){
return false;
}
if(!S3fsCurl::InitMimeType(MimeFile)){
return false;
}
if(!S3fsCurl::InitGlobalCurl()){
return false;
}
@ -618,15 +629,39 @@ int S3fsCurl::CurlProgress(void *clientp, double dltotal, double dlnow, double u
return 0;
}
bool S3fsCurl::InitMimeType(const char* MimeFile)
bool S3fsCurl::InitMimeType(const std::string& strFile)
{
if(!MimeFile){
MimeFile = "/etc/mime.types"; // default
string MimeFile;
if(!strFile.empty()){
MimeFile = strFile;
}else{
// search default mime.types
string errPaths = DEFAULT_MIME_FILE;
struct stat st;
if(0 == stat(DEFAULT_MIME_FILE, &st)){
MimeFile = DEFAULT_MIME_FILE;
}else if(compare_sysname("Darwin")){
// for macos, search another default file.
if(0 == stat(SPECIAL_DARWIN_MIME_FILE, &st)){
MimeFile = SPECIAL_DARWIN_MIME_FILE;
}else{
errPaths += " and ";
errPaths += SPECIAL_DARWIN_MIME_FILE;
}
}
if(MimeFile.empty()){
S3FS_PRN_ERR("Could not find miime.types files, you have to create file(%s) or specify mime option for existing mime.types file.", errPaths.c_str());
return false;
}
}
S3FS_PRN_DBG("Try to load mime types from %s file.", MimeFile.c_str());
string line;
ifstream MT(MimeFile);
ifstream MT(MimeFile.c_str());
if(MT.good()){
S3FS_PRN_DBG("The old mime types are cleared to load new mime types.");
S3fsCurl::mimeTypes.clear();
while(getline(MT, line)){
if(line[0]=='#'){
continue;
@ -647,6 +682,10 @@ bool S3fsCurl::InitMimeType(const char* MimeFile)
S3fsCurl::mimeTypes[ext] = mimeType;
}
}
S3FS_PRN_INIT_INFO("Loaded mime information from %s", MimeFile.c_str());
}else{
S3FS_PRN_ERR("Could not load mime types from %s, please check the existence and permissions of this file.", MimeFile.c_str());
return false;
}
return true;
}

4
src/curl.h
View File

@ -355,7 +355,6 @@ class S3fsCurl
static bool DestroyCryptMutex(void);
static int CurlProgress(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow);
static bool InitMimeType(const char* MimeFile = NULL);
static bool LocateBundle(void);
static size_t HeaderCallback(void *data, size_t blockSize, size_t numBlocks, void *userPtr);
static size_t WriteMemoryCallback(void *ptr, size_t blockSize, size_t numBlocks, void *data);
@ -408,7 +407,8 @@ class S3fsCurl
public:
// class methods
static bool InitS3fsCurl(const char* MimeFile = NULL);
static bool InitS3fsCurl(void);
static bool InitMimeType(const std::string& strFile);
static bool DestroyS3fsCurl(void);
static int ParallelMultipartUploadRequest(const char* tpath, headers_t& meta, int fd);
static int ParallelMixMultipartUploadRequest(const char* tpath, headers_t& meta, int fd, const PageList& pagelist);

32
src/s3fs.cpp
View File

@ -124,6 +124,7 @@ static mode_t mp_umask = 0; // umask for mount point
static bool is_mp_umask = false;// default does not set.
static std::string mountpoint;
static std::string passwd_file;
static std::string mimetype_file;
static utility_incomp_type utility_mode = NO_UTILITY_MODE;
static bool noxmlns = false;
static bool nocopyapi = false;
@ -5021,6 +5022,10 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
instance_name = "[" + instance_name + "]";
return 0;
}
if(0 == STR2NCMP(arg, "mime=")){
mimetype_file = strchr(arg, '=') + sizeof(char);
return 0;
}
//
// debug option for s3fs
//
@ -5199,8 +5204,23 @@ int main(int argc, char* argv[])
exit(EXIT_FAILURE);
}
// init curl
if(!S3fsCurl::InitS3fsCurl("/etc/mime.types")){
// init curl (without mime types)
//
// [NOTE]
// The curl initialization here does not load mime types.
// The mime types file parameter are dynamic values according
// to the user's environment, and are analyzed by the my_fuse_opt_proc
// function.
// The my_fuse_opt_proc function is executed after this curl
// initialization. Because the curl method is used in the
// my_fuse_opt_proc function, then it must be called here to
// initialize. Fortunately, the processing using mime types
// is only PUT/POST processing, and it is not used until the
// call of my_fuse_opt_proc function is completed. Therefore,
// the mime type is loaded just after calling the my_fuse_opt_proc
// function.
//
if(!S3fsCurl::InitS3fsCurl()){
S3FS_PRN_EXIT("Could not initiate curl library.");
s3fs_destroy_global_ssl();
exit(EXIT_FAILURE);
@ -5219,6 +5239,14 @@ int main(int argc, char* argv[])
exit(EXIT_FAILURE);
}
// init mime types for curl
if(!S3fsCurl::InitMimeType(mimetype_file)){
S3FS_PRN_EXIT("Could not load mime types for curl library.");
S3fsCurl::DestroyS3fsCurl();
s3fs_destroy_global_ssl();
exit(EXIT_FAILURE);
}
// [NOTE]
// exclusive option check here.
//

42
src/s3fs_util.cpp
View File

@ -30,6 +30,7 @@
#include <syslog.h>
#include <pthread.h>
#include <sys/types.h>
#include <sys/utsname.h>
#include <dirent.h>
#include <libxml/xpath.h>
#include <libxml/xpathInternals.h>
@ -744,6 +745,36 @@ bool delete_files_in_dir(const char* dir, bool is_remove_own)
return true;
}
//-------------------------------------------------------------------
// Utility for system information
//-------------------------------------------------------------------
bool compare_sysname(const char* target)
{
// [NOTE]
// The buffer size of sysname member in struct utsname is
// OS dependent, but 512 bytes is sufficient for now.
//
static char* psysname = NULL;
static char sysname[512];
if(!psysname){
struct utsname sysinfo;
if(0 != uname(&sysinfo)){
S3FS_PRN_ERR("could not initialize system name to internal buffer(errno:%d), thus use \"Linux\".", errno);
strcpy(sysname, "Linux");
}else{
S3FS_PRN_INFO("system name is %s", sysinfo.sysname);
sysname[sizeof(sysname) - 1] = '\0';
strncpy(sysname, sysinfo.sysname, sizeof(sysname) - 1);
}
psysname = &sysname[0];
}
if(!target || 0 != strcmp(psysname, target)){
return false;
}
return true;
}
//-------------------------------------------------------------------
// Utility functions for convert
//-------------------------------------------------------------------
@ -1421,13 +1452,20 @@ void show_help ()
" use_session_token - indicate that session token should be provided.\n"
" If credentials are provided by environment variables this switch\n"
" forces presence check of AWSSESSIONTOKEN variable.\n"
" Otherwise an error is returned."
" Otherwise an error is returned.\n"
"\n"
" requester_pays (default is disable)\n"
" This option instructs s3fs to enable requests involving\n"
" Requester Pays buckets.\n"
" It includes the 'x-amz-request-payer=requester' entry in the\n"
" request header."
" request header.\n"
"\n"
" mime (default is \"/etc/mime.types\")\n"
" Specify the path of the mime.types file.\n"
" If this option is not specified, the existence of \"/etc/mime.types\"\n"
" is checked, and that file is loaded as mime information.\n"
" If this file does not exist on macOS, then \"/etc/apache2/mime.types\"\n"
" is checked as well.\n"
"\n"
" dbglevel (default=\"crit\")\n"
" Set the debug message level. set value as crit (critical), err\n"

2
src/s3fs_util.h
View File

@ -123,6 +123,8 @@ std::string get_exist_directory_path(const std::string& path);
bool check_exist_dir_permission(const char* dirpath);
bool delete_files_in_dir(const char* dir, bool is_remove_own);
bool compare_sysname(const char* target);
time_t get_mtime(const char *s);
time_t get_mtime(headers_t& meta, bool overcheck = true);
time_t get_ctime(headers_t& meta, bool overcheck = true);

26
test/integration-test-main.sh
View File

@ -707,30 +707,16 @@ function test_content_type() {
touch "test.txt"
CONTENT_TYPE=$(aws_cli s3api head-object --bucket "${TEST_BUCKET_1}" --key "${DIR_NAME}/test.txt" | grep "ContentType")
if [ `uname` = "Darwin" ]; then
if ! echo $CONTENT_TYPE | grep -q "application/octet-stream"; then
echo "Unexpected Content-Type(MacOS): $CONTENT_TYPE"
return 1;
fi
else
if ! echo $CONTENT_TYPE | grep -q "text/plain"; then
echo "Unexpected Content-Type: $CONTENT_TYPE"
return 1;
fi
if ! echo $CONTENT_TYPE | grep -q "text/plain"; then
echo "Unexpected Content-Type: $CONTENT_TYPE"
return 1;
fi
touch "test.jpg"
CONTENT_TYPE=$(aws_cli s3api head-object --bucket "${TEST_BUCKET_1}" --key "${DIR_NAME}/test.jpg" | grep "ContentType")
if [ `uname` = "Darwin" ]; then
if ! echo $CONTENT_TYPE | grep -q "application/octet-stream"; then
echo "Unexpected Content-Type(MacOS): $CONTENT_TYPE"
return 1;
fi
else
if ! echo $CONTENT_TYPE | grep -q "image/jpeg"; then
echo "Unexpected Content-Type: $CONTENT_TYPE"
return 1;
fi
if ! echo $CONTENT_TYPE | grep -q "image/jpeg"; then
echo "Unexpected Content-Type: $CONTENT_TYPE"
return 1;
fi
touch "test.bin"