Merge pull request #1254 from ggtakec/modify_mimetypes

Added mime option for strict checking of mime types file

COMPILATION.md

@@ -15,7 +15,11 @@ Keep in mind using the pre-built packages when available.
 * libcurl
 * libxml2
 * openssl
-* /etc/mime.types (the package providing depends on the OS)
+* mime.types (the package providing depends on the OS)
+	* s3fs tries to detect `/etc/mime.types` as default regardless of the OS
+	* Else s3fs tries to detect `/etc/apache2/mime.types` if OS is macOS
+	* s3fs exits with an error if these files are not exist
+	* Alternatively, you can set mime.types file path with `mime` option without detecting these default files
 * pkg-config (or your OS equivalent)
 
 2. Then compile from master via the following commands:

doc/man/s3fs.1

@@ -326,6 +326,11 @@ Otherwise an error is returned.
 \fB\-o\fR requester_pays (default is disable)
 This option instructs s3fs to enable requests involving Requester Pays buckets (It includes the 'x-amz-request-payer=requester' entry in the request header).
 .TP
+\fB\-o\fR mime (default is "/etc/mime.types")
+Specify the path of the mime.types file.
+If this option is not specified, the existence of "/etc/mime.types" is checked, and that file is loaded as mime information.
+If this file does not exist on macOS, then "/etc/apache2/mime.types" is checked as well.
+.TP
 \fB\-o\fR dbglevel (default="crit")
 Set the debug message level. set value as crit (critical), err (error), warn (warning), info (information) to debug level. default debug level is critical.
 If s3fs run with "-d" option, the debug level is set information.

src/curl.cpp

@@ -343,6 +343,20 @@ static const long S3FSCURL_RESPONSECODE_NOTSET      = -1;
 static const long S3FSCURL_RESPONSECODE_FATAL_ERROR = -2;
 static const int  S3FSCURL_PERFORM_RESULT_NOTSET    = 1;
 
+// [NOTE] about default mime.types file
+// If no mime.types file is specified in the mime option, s3fs
+// will look for /etc/mime.types on all operating systems and
+// load mime information.
+// However, in the case of macOS, when this file does not exist,
+// it tries to detect the /etc/apache2/mime.types file.
+// The reason for this is that apache2 is preinstalled on macOS,
+// and the mime.types file is expected to exist in this path.
+// If the mime.types file is not found, s3fs will exit with an
+// error.
+//
+static const char* DEFAULT_MIME_FILE        = "/etc/mime.types";
+static const char* SPECIAL_DARWIN_MIME_FILE = "/etc/apache2/mime.types";
+
 // [NOTICE]
 // This symbol is for libcurl under 7.23.0
 #ifndef CURLSHE_NOT_BUILT_IN
@@ -397,7 +411,7 @@ bool             S3fsCurl::requester_pays      = false;          // default
 //-------------------------------------------------------------------
 // Class methods for S3fsCurl
 //-------------------------------------------------------------------
-bool S3fsCurl::InitS3fsCurl(const char* MimeFile)
+bool S3fsCurl::InitS3fsCurl()
 {
   pthread_mutexattr_t attr;
   pthread_mutexattr_init(&attr);
@@ -413,9 +427,6 @@ bool S3fsCurl::InitS3fsCurl(const char* MimeFile)
   if(0 != pthread_mutex_init(&S3fsCurl::curl_share_lock[SHARE_MUTEX_SSL_SESSION], &attr)){
     return false;
   }
-  if(!S3fsCurl::InitMimeType(MimeFile)){
-    return false;
-  }
   if(!S3fsCurl::InitGlobalCurl()){
     return false;
   }
@@ -618,15 +629,39 @@ int S3fsCurl::CurlProgress(void *clientp, double dltotal, double dlnow, double u
   return 0;
 }
 
-bool S3fsCurl::InitMimeType(const char* MimeFile)
+bool S3fsCurl::InitMimeType(const std::string& strFile)
 {
-  if(!MimeFile){
+  string MimeFile;
-    MimeFile = "/etc/mime.types";  // default
+  if(!strFile.empty()){
+    MimeFile = strFile;
+  }else{
+    // search default mime.types
+    string errPaths = DEFAULT_MIME_FILE;
+    struct stat st;
+    if(0 == stat(DEFAULT_MIME_FILE, &st)){
+      MimeFile = DEFAULT_MIME_FILE;
+    }else if(compare_sysname("Darwin")){
+      // for macos, search another default file.
+      if(0 == stat(SPECIAL_DARWIN_MIME_FILE, &st)){
+        MimeFile = SPECIAL_DARWIN_MIME_FILE;
+      }else{
+        errPaths += " and ";
+        errPaths += SPECIAL_DARWIN_MIME_FILE;
       }
+    }
+    if(MimeFile.empty()){
+      S3FS_PRN_ERR("Could not find miime.types files, you have to create file(%s) or specify mime option for existing mime.types file.", errPaths.c_str());
+      return false;
+    }
+  }
+  S3FS_PRN_DBG("Try to load mime types from %s file.", MimeFile.c_str());
 
   string line;
-  ifstream MT(MimeFile);
+  ifstream MT(MimeFile.c_str());
   if(MT.good()){
+    S3FS_PRN_DBG("The old mime types are cleared to load new mime types.");
+    S3fsCurl::mimeTypes.clear();
+
     while(getline(MT, line)){
       if(line[0]=='#'){
         continue;
@@ -647,6 +682,10 @@ bool S3fsCurl::InitMimeType(const char* MimeFile)
         S3fsCurl::mimeTypes[ext] = mimeType;
       }
     }
+    S3FS_PRN_INIT_INFO("Loaded mime information from %s", MimeFile.c_str());
+  }else{
+    S3FS_PRN_ERR("Could not load mime types from %s, please check the existence and permissions of this file.", MimeFile.c_str());
+    return false;
   }
   return true;
 }

src/curl.h

@@ -355,7 +355,6 @@ class S3fsCurl
     static bool DestroyCryptMutex(void);
     static int CurlProgress(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow);
 
-    static bool InitMimeType(const char* MimeFile = NULL);
     static bool LocateBundle(void);
     static size_t HeaderCallback(void *data, size_t blockSize, size_t numBlocks, void *userPtr);
     static size_t WriteMemoryCallback(void *ptr, size_t blockSize, size_t numBlocks, void *data);
@@ -408,7 +407,8 @@ class S3fsCurl
 
   public:
     // class methods
-    static bool InitS3fsCurl(const char* MimeFile = NULL);
+    static bool InitS3fsCurl(void);
+    static bool InitMimeType(const std::string& strFile);
     static bool DestroyS3fsCurl(void);
     static int ParallelMultipartUploadRequest(const char* tpath, headers_t& meta, int fd);
     static int ParallelMixMultipartUploadRequest(const char* tpath, headers_t& meta, int fd, const PageList& pagelist);

src/s3fs.cpp

@@ -124,6 +124,7 @@ static mode_t mp_umask            = 0;    // umask for mount point
 static bool is_mp_umask           = false;// default does not set.
 static std::string mountpoint;
 static std::string passwd_file;
+static std::string mimetype_file;
 static utility_incomp_type utility_mode = NO_UTILITY_MODE;
 static bool noxmlns               = false;
 static bool nocopyapi             = false;
@@ -5021,6 +5022,10 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
       instance_name = "[" + instance_name + "]";
       return 0;
     }
+    if(0 == STR2NCMP(arg, "mime=")){
+      mimetype_file = strchr(arg, '=') + sizeof(char);
+      return 0;
+    }
     //
     // debug option for s3fs
     //
@@ -5199,8 +5204,23 @@ int main(int argc, char* argv[])
     exit(EXIT_FAILURE);
   }
 
-  // init curl
+  // init curl (without mime types)
-  if(!S3fsCurl::InitS3fsCurl("/etc/mime.types")){
+  //
+  // [NOTE]
+  // The curl initialization here does not load mime types.
+  // The mime types file parameter are dynamic values according
+  // to the user's environment, and are analyzed by the my_fuse_opt_proc
+  // function.
+  // The my_fuse_opt_proc function is executed after this curl
+  // initialization. Because the curl method is used in the
+  // my_fuse_opt_proc function, then it must be called here to
+  // initialize. Fortunately, the processing using mime types
+  // is only PUT/POST processing, and it is not used until the
+  // call of my_fuse_opt_proc function is completed. Therefore,
+  // the mime type is loaded just after calling the my_fuse_opt_proc
+  // function.
+  // 
+  if(!S3fsCurl::InitS3fsCurl()){
     S3FS_PRN_EXIT("Could not initiate curl library.");
     s3fs_destroy_global_ssl();
     exit(EXIT_FAILURE);
@@ -5219,6 +5239,14 @@ int main(int argc, char* argv[])
     exit(EXIT_FAILURE);
   }
 
+  // init mime types for curl
+  if(!S3fsCurl::InitMimeType(mimetype_file)){
+    S3FS_PRN_EXIT("Could not load mime types for curl library.");
+    S3fsCurl::DestroyS3fsCurl();
+    s3fs_destroy_global_ssl();
+    exit(EXIT_FAILURE);
+  }
+
   // [NOTE]
   // exclusive option check here.
   //

src/s3fs_util.cpp

@@ -30,6 +30,7 @@
 #include <syslog.h>
 #include <pthread.h>
 #include <sys/types.h>
+#include <sys/utsname.h>
 #include <dirent.h>
 #include <libxml/xpath.h>
 #include <libxml/xpathInternals.h>
@@ -744,6 +745,36 @@ bool delete_files_in_dir(const char* dir, bool is_remove_own)
   return true;
 }
 
+//-------------------------------------------------------------------
+// Utility for system information
+//-------------------------------------------------------------------
+bool compare_sysname(const char* target)
+{
+  // [NOTE]
+  // The buffer size of sysname member in struct utsname is
+  // OS dependent, but 512 bytes is sufficient for now.
+  //
+  static char* psysname = NULL;
+  static char  sysname[512];
+  if(!psysname){
+    struct utsname sysinfo;
+    if(0 != uname(&sysinfo)){
+      S3FS_PRN_ERR("could not initialize system name to internal buffer(errno:%d), thus use \"Linux\".", errno);
+      strcpy(sysname, "Linux");
+    }else{
+      S3FS_PRN_INFO("system name is %s", sysinfo.sysname);
+      sysname[sizeof(sysname) - 1] = '\0';
+      strncpy(sysname, sysinfo.sysname, sizeof(sysname) - 1);
+    }
+    psysname = &sysname[0];
+  }
+
+  if(!target || 0 != strcmp(psysname, target)){
+    return false;
+  }
+  return true;
+}
+
 //-------------------------------------------------------------------
 // Utility functions for convert
 //-------------------------------------------------------------------
@@ -1421,13 +1452,20 @@ void show_help ()
     "   use_session_token - indicate that session token should be provided.\n"
     "        If credentials are provided by environment variables this switch\n"
     "        forces presence check of AWSSESSIONTOKEN variable.\n"
-    "        Otherwise an error is returned."
+    "        Otherwise an error is returned.\n"
     "\n"
     "   requester_pays (default is disable)\n"
     "        This option instructs s3fs to enable requests involving\n"
     "        Requester Pays buckets.\n"
     "        It includes the 'x-amz-request-payer=requester' entry in the\n"
-    "        request header."
+    "        request header.\n"
+    "\n"
+    "   mime (default is \"/etc/mime.types\")\n"
+    "        Specify the path of the mime.types file.\n"
+    "        If this option is not specified, the existence of \"/etc/mime.types\"\n"
+    "        is checked, and that file is loaded as mime information.\n"
+    "        If this file does not exist on macOS, then \"/etc/apache2/mime.types\"\n"
+    "        is checked as well.\n"
     "\n"
     "   dbglevel (default=\"crit\")\n"
     "        Set the debug message level. set value as crit (critical), err\n"

src/s3fs_util.h

@@ -123,6 +123,8 @@ std::string get_exist_directory_path(const std::string& path);
 bool check_exist_dir_permission(const char* dirpath);
 bool delete_files_in_dir(const char* dir, bool is_remove_own);
 
+bool compare_sysname(const char* target);
+
 time_t get_mtime(const char *s);
 time_t get_mtime(headers_t& meta, bool overcheck = true);
 time_t get_ctime(headers_t& meta, bool overcheck = true);

test/integration-test-main.sh

@@ -707,31 +707,17 @@ function test_content_type() {
 
     touch "test.txt"
     CONTENT_TYPE=$(aws_cli s3api head-object --bucket "${TEST_BUCKET_1}" --key "${DIR_NAME}/test.txt" | grep "ContentType")
-    if [ `uname` = "Darwin" ]; then
-        if ! echo $CONTENT_TYPE | grep -q "application/octet-stream"; then
-            echo "Unexpected Content-Type(MacOS): $CONTENT_TYPE"
-            return 1;
-        fi
-    else
     if ! echo $CONTENT_TYPE | grep -q "text/plain"; then
         echo "Unexpected Content-Type: $CONTENT_TYPE"
         return 1;
     fi
-    fi
 
     touch "test.jpg"
     CONTENT_TYPE=$(aws_cli s3api head-object --bucket "${TEST_BUCKET_1}" --key "${DIR_NAME}/test.jpg" | grep "ContentType")
-    if [ `uname` = "Darwin" ]; then
-        if ! echo $CONTENT_TYPE | grep -q "application/octet-stream"; then
-            echo "Unexpected Content-Type(MacOS): $CONTENT_TYPE"
-            return 1;
-        fi
-    else
     if ! echo $CONTENT_TYPE | grep -q "image/jpeg"; then
         echo "Unexpected Content-Type: $CONTENT_TYPE"
         return 1;
     fi
-    fi
 
     touch "test.bin"
     CONTENT_TYPE=$(aws_cli s3api head-object --bucket "${TEST_BUCKET_1}" --key "${DIR_NAME}/test.bin" | grep "ContentType")