From 7fbda230f5de31b722a979e8780593c2504f0a89 Mon Sep 17 00:00:00 2001 From: Takeshi Nakatani Date: Thu, 19 Mar 2020 15:13:21 +0000 Subject: [PATCH] Added mime option for strict checking of mime types file --- COMPILATION.md | 6 +++- doc/man/s3fs.1 | 5 ++++ src/curl.cpp | 55 ++++++++++++++++++++++++++++++----- src/curl.h | 4 +-- src/s3fs.cpp | 32 ++++++++++++++++++-- src/s3fs_util.cpp | 42 ++++++++++++++++++++++++-- src/s3fs_util.h | 2 ++ test/integration-test-main.sh | 26 ++++------------- 8 files changed, 137 insertions(+), 35 deletions(-) diff --git a/COMPILATION.md b/COMPILATION.md index 52c2509..0ba6fbe 100644 --- a/COMPILATION.md +++ b/COMPILATION.md @@ -15,7 +15,11 @@ Keep in mind using the pre-built packages when available. * libcurl * libxml2 * openssl -* /etc/mime.types (the package providing depends on the OS) +* mime.types (the package providing depends on the OS) + * s3fs tries to detect `/etc/mime.types` as default regardless of the OS + * Else s3fs tries to detect `/etc/apache2/mime.types` if OS is macOS + * s3fs exits with an error if these files are not exist + * Alternatively, you can set mime.types file path with `mime` option without detecting these default files * pkg-config (or your OS equivalent) 2. Then compile from master via the following commands: diff --git a/doc/man/s3fs.1 b/doc/man/s3fs.1 index a690a4b..fc40a00 100644 --- a/doc/man/s3fs.1 +++ b/doc/man/s3fs.1 @@ -326,6 +326,11 @@ Otherwise an error is returned. \fB\-o\fR requester_pays (default is disable) This option instructs s3fs to enable requests involving Requester Pays buckets (It includes the 'x-amz-request-payer=requester' entry in the request header). .TP +\fB\-o\fR mime (default is "/etc/mime.types") +Specify the path of the mime.types file. +If this option is not specified, the existence of "/etc/mime.types" is checked, and that file is loaded as mime information. +If this file does not exist on macOS, then "/etc/apache2/mime.types" is checked as well. +.TP \fB\-o\fR dbglevel (default="crit") Set the debug message level. set value as crit (critical), err (error), warn (warning), info (information) to debug level. default debug level is critical. If s3fs run with "-d" option, the debug level is set information. diff --git a/src/curl.cpp b/src/curl.cpp index c80c9f2..9533db0 100644 --- a/src/curl.cpp +++ b/src/curl.cpp @@ -343,6 +343,20 @@ static const long S3FSCURL_RESPONSECODE_NOTSET = -1; static const long S3FSCURL_RESPONSECODE_FATAL_ERROR = -2; static const int S3FSCURL_PERFORM_RESULT_NOTSET = 1; +// [NOTE] about default mime.types file +// If no mime.types file is specified in the mime option, s3fs +// will look for /etc/mime.types on all operating systems and +// load mime information. +// However, in the case of macOS, when this file does not exist, +// it tries to detect the /etc/apache2/mime.types file. +// The reason for this is that apache2 is preinstalled on macOS, +// and the mime.types file is expected to exist in this path. +// If the mime.types file is not found, s3fs will exit with an +// error. +// +static const char* DEFAULT_MIME_FILE = "/etc/mime.types"; +static const char* SPECIAL_DARWIN_MIME_FILE = "/etc/apache2/mime.types"; + // [NOTICE] // This symbol is for libcurl under 7.23.0 #ifndef CURLSHE_NOT_BUILT_IN @@ -397,7 +411,7 @@ bool S3fsCurl::requester_pays = false; // default //------------------------------------------------------------------- // Class methods for S3fsCurl //------------------------------------------------------------------- -bool S3fsCurl::InitS3fsCurl(const char* MimeFile) +bool S3fsCurl::InitS3fsCurl() { pthread_mutexattr_t attr; pthread_mutexattr_init(&attr); @@ -413,9 +427,6 @@ bool S3fsCurl::InitS3fsCurl(const char* MimeFile) if(0 != pthread_mutex_init(&S3fsCurl::curl_share_lock[SHARE_MUTEX_SSL_SESSION], &attr)){ return false; } - if(!S3fsCurl::InitMimeType(MimeFile)){ - return false; - } if(!S3fsCurl::InitGlobalCurl()){ return false; } @@ -618,15 +629,39 @@ int S3fsCurl::CurlProgress(void *clientp, double dltotal, double dlnow, double u return 0; } -bool S3fsCurl::InitMimeType(const char* MimeFile) +bool S3fsCurl::InitMimeType(const std::string& strFile) { - if(!MimeFile){ - MimeFile = "/etc/mime.types"; // default + string MimeFile; + if(!strFile.empty()){ + MimeFile = strFile; + }else{ + // search default mime.types + string errPaths = DEFAULT_MIME_FILE; + struct stat st; + if(0 == stat(DEFAULT_MIME_FILE, &st)){ + MimeFile = DEFAULT_MIME_FILE; + }else if(compare_sysname("Darwin")){ + // for macos, search another default file. + if(0 == stat(SPECIAL_DARWIN_MIME_FILE, &st)){ + MimeFile = SPECIAL_DARWIN_MIME_FILE; + }else{ + errPaths += " and "; + errPaths += SPECIAL_DARWIN_MIME_FILE; + } + } + if(MimeFile.empty()){ + S3FS_PRN_ERR("Could not find miime.types files, you have to create file(%s) or specify mime option for existing mime.types file.", errPaths.c_str()); + return false; + } } + S3FS_PRN_DBG("Try to load mime types from %s file.", MimeFile.c_str()); string line; - ifstream MT(MimeFile); + ifstream MT(MimeFile.c_str()); if(MT.good()){ + S3FS_PRN_DBG("The old mime types are cleared to load new mime types."); + S3fsCurl::mimeTypes.clear(); + while(getline(MT, line)){ if(line[0]=='#'){ continue; @@ -647,6 +682,10 @@ bool S3fsCurl::InitMimeType(const char* MimeFile) S3fsCurl::mimeTypes[ext] = mimeType; } } + S3FS_PRN_INIT_INFO("Loaded mime information from %s", MimeFile.c_str()); + }else{ + S3FS_PRN_ERR("Could not load mime types from %s, please check the existence and permissions of this file.", MimeFile.c_str()); + return false; } return true; } diff --git a/src/curl.h b/src/curl.h index 02c1019..44dc0c3 100644 --- a/src/curl.h +++ b/src/curl.h @@ -355,7 +355,6 @@ class S3fsCurl static bool DestroyCryptMutex(void); static int CurlProgress(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow); - static bool InitMimeType(const char* MimeFile = NULL); static bool LocateBundle(void); static size_t HeaderCallback(void *data, size_t blockSize, size_t numBlocks, void *userPtr); static size_t WriteMemoryCallback(void *ptr, size_t blockSize, size_t numBlocks, void *data); @@ -408,7 +407,8 @@ class S3fsCurl public: // class methods - static bool InitS3fsCurl(const char* MimeFile = NULL); + static bool InitS3fsCurl(void); + static bool InitMimeType(const std::string& strFile); static bool DestroyS3fsCurl(void); static int ParallelMultipartUploadRequest(const char* tpath, headers_t& meta, int fd); static int ParallelMixMultipartUploadRequest(const char* tpath, headers_t& meta, int fd, const PageList& pagelist); diff --git a/src/s3fs.cpp b/src/s3fs.cpp index d043c02..1349717 100644 --- a/src/s3fs.cpp +++ b/src/s3fs.cpp @@ -124,6 +124,7 @@ static mode_t mp_umask = 0; // umask for mount point static bool is_mp_umask = false;// default does not set. static std::string mountpoint; static std::string passwd_file; +static std::string mimetype_file; static utility_incomp_type utility_mode = NO_UTILITY_MODE; static bool noxmlns = false; static bool nocopyapi = false; @@ -5021,6 +5022,10 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar instance_name = "[" + instance_name + "]"; return 0; } + if(0 == STR2NCMP(arg, "mime=")){ + mimetype_file = strchr(arg, '=') + sizeof(char); + return 0; + } // // debug option for s3fs // @@ -5199,8 +5204,23 @@ int main(int argc, char* argv[]) exit(EXIT_FAILURE); } - // init curl - if(!S3fsCurl::InitS3fsCurl("/etc/mime.types")){ + // init curl (without mime types) + // + // [NOTE] + // The curl initialization here does not load mime types. + // The mime types file parameter are dynamic values according + // to the user's environment, and are analyzed by the my_fuse_opt_proc + // function. + // The my_fuse_opt_proc function is executed after this curl + // initialization. Because the curl method is used in the + // my_fuse_opt_proc function, then it must be called here to + // initialize. Fortunately, the processing using mime types + // is only PUT/POST processing, and it is not used until the + // call of my_fuse_opt_proc function is completed. Therefore, + // the mime type is loaded just after calling the my_fuse_opt_proc + // function. + // + if(!S3fsCurl::InitS3fsCurl()){ S3FS_PRN_EXIT("Could not initiate curl library."); s3fs_destroy_global_ssl(); exit(EXIT_FAILURE); @@ -5219,6 +5239,14 @@ int main(int argc, char* argv[]) exit(EXIT_FAILURE); } + // init mime types for curl + if(!S3fsCurl::InitMimeType(mimetype_file)){ + S3FS_PRN_EXIT("Could not load mime types for curl library."); + S3fsCurl::DestroyS3fsCurl(); + s3fs_destroy_global_ssl(); + exit(EXIT_FAILURE); + } + // [NOTE] // exclusive option check here. // diff --git a/src/s3fs_util.cpp b/src/s3fs_util.cpp index 1eeaae0..dc7fbf8 100644 --- a/src/s3fs_util.cpp +++ b/src/s3fs_util.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -744,6 +745,36 @@ bool delete_files_in_dir(const char* dir, bool is_remove_own) return true; } +//------------------------------------------------------------------- +// Utility for system information +//------------------------------------------------------------------- +bool compare_sysname(const char* target) +{ + // [NOTE] + // The buffer size of sysname member in struct utsname is + // OS dependent, but 512 bytes is sufficient for now. + // + static char* psysname = NULL; + static char sysname[512]; + if(!psysname){ + struct utsname sysinfo; + if(0 != uname(&sysinfo)){ + S3FS_PRN_ERR("could not initialize system name to internal buffer(errno:%d), thus use \"Linux\".", errno); + strcpy(sysname, "Linux"); + }else{ + S3FS_PRN_INFO("system name is %s", sysinfo.sysname); + sysname[sizeof(sysname) - 1] = '\0'; + strncpy(sysname, sysinfo.sysname, sizeof(sysname) - 1); + } + psysname = &sysname[0]; + } + + if(!target || 0 != strcmp(psysname, target)){ + return false; + } + return true; +} + //------------------------------------------------------------------- // Utility functions for convert //------------------------------------------------------------------- @@ -1421,13 +1452,20 @@ void show_help () " use_session_token - indicate that session token should be provided.\n" " If credentials are provided by environment variables this switch\n" " forces presence check of AWSSESSIONTOKEN variable.\n" - " Otherwise an error is returned." + " Otherwise an error is returned.\n" "\n" " requester_pays (default is disable)\n" " This option instructs s3fs to enable requests involving\n" " Requester Pays buckets.\n" " It includes the 'x-amz-request-payer=requester' entry in the\n" - " request header." + " request header.\n" + "\n" + " mime (default is \"/etc/mime.types\")\n" + " Specify the path of the mime.types file.\n" + " If this option is not specified, the existence of \"/etc/mime.types\"\n" + " is checked, and that file is loaded as mime information.\n" + " If this file does not exist on macOS, then \"/etc/apache2/mime.types\"\n" + " is checked as well.\n" "\n" " dbglevel (default=\"crit\")\n" " Set the debug message level. set value as crit (critical), err\n" diff --git a/src/s3fs_util.h b/src/s3fs_util.h index ed70f59..e6842c8 100644 --- a/src/s3fs_util.h +++ b/src/s3fs_util.h @@ -123,6 +123,8 @@ std::string get_exist_directory_path(const std::string& path); bool check_exist_dir_permission(const char* dirpath); bool delete_files_in_dir(const char* dir, bool is_remove_own); +bool compare_sysname(const char* target); + time_t get_mtime(const char *s); time_t get_mtime(headers_t& meta, bool overcheck = true); time_t get_ctime(headers_t& meta, bool overcheck = true); diff --git a/test/integration-test-main.sh b/test/integration-test-main.sh index 41c3ca3..58e5f11 100755 --- a/test/integration-test-main.sh +++ b/test/integration-test-main.sh @@ -707,30 +707,16 @@ function test_content_type() { touch "test.txt" CONTENT_TYPE=$(aws_cli s3api head-object --bucket "${TEST_BUCKET_1}" --key "${DIR_NAME}/test.txt" | grep "ContentType") - if [ `uname` = "Darwin" ]; then - if ! echo $CONTENT_TYPE | grep -q "application/octet-stream"; then - echo "Unexpected Content-Type(MacOS): $CONTENT_TYPE" - return 1; - fi - else - if ! echo $CONTENT_TYPE | grep -q "text/plain"; then - echo "Unexpected Content-Type: $CONTENT_TYPE" - return 1; - fi + if ! echo $CONTENT_TYPE | grep -q "text/plain"; then + echo "Unexpected Content-Type: $CONTENT_TYPE" + return 1; fi touch "test.jpg" CONTENT_TYPE=$(aws_cli s3api head-object --bucket "${TEST_BUCKET_1}" --key "${DIR_NAME}/test.jpg" | grep "ContentType") - if [ `uname` = "Darwin" ]; then - if ! echo $CONTENT_TYPE | grep -q "application/octet-stream"; then - echo "Unexpected Content-Type(MacOS): $CONTENT_TYPE" - return 1; - fi - else - if ! echo $CONTENT_TYPE | grep -q "image/jpeg"; then - echo "Unexpected Content-Type: $CONTENT_TYPE" - return 1; - fi + if ! echo $CONTENT_TYPE | grep -q "image/jpeg"; then + echo "Unexpected Content-Type: $CONTENT_TYPE" + return 1; fi touch "test.bin"