Merge pull request #631 from s3fs-fuse/macosx

Merged macosx branch into master branch #601

.travis.yml

@@ -1,17 +1,45 @@
 language: cpp
-sudo: required
-dist: trusty
-cache: apt
-before_install:
+
+matrix:
+  include:
+    - os: linux
+      sudo: required
+      dist: trusty
+      cache: apt
+      before_install:
         - sudo apt-get update -qq
         - sudo apt-get install -qq cppcheck libfuse-dev openjdk-7-jdk
         - sudo update-alternatives --set java /usr/lib/jvm/java-7-openjdk-amd64/jre/bin/java
-script:
+      script:
         - ./autogen.sh
- - ./configure
+        - ./configure CPPFLAGS='-I/usr/local/opt/openssl/include'
         - make
         - make cppcheck
         - make check -C src
         - modprobe fuse
         - make check -C test
         - cat test/test-suite.log
+
+    - os: osx
+      osx_image: xcode8.3
+      before_install:
+        - curl -sSL https://rvm.io/mpapis.asc | gpg2 --import -
+        - rvm get head
+        - brew update
+        - brew install truncate
+        - brew tap caskroom/cask
+        - brew cask install osxfuse
+        - if [ -f /Library/Filesystems/osxfusefs.fs/Support/load_osxfusefs ]; then sudo chmod +s /Library/Filesystems/osxfusefs.fs/Support/load_osxfusefs ; elif [ -f /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse ]; then sudo chmod +s /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse ; fi
+        - brew install gnu-sed
+        - sudo ln -s /usr/local/opt/gnu-sed/bin/gsed /usr/local/bin/sed
+        - sudo ln -s /usr/local/opt/coreutils/bin/gstdbuf /usr/local/bin/stdbuf
+        - brew install cppcheck
+      script:
+        - ./autogen.sh
+        - PKG_CONFIG_PATH=/usr/local/opt/curl/lib/pkgconfig:/usr/local/opt/openssl/lib/pkgconfig ./configure
+        - make
+        - make cppcheck
+        - make check -C src
+        - if [ -f /Library/Filesystems/osxfusefs.fs/Support/load_osxfusefs ]; then /Library/Filesystems/osxfusefs.fs/Support/load_osxfusefs ; elif [ -f /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse ]; then /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse ; fi
+        - make check -C test
+        - cat test/test-suite.log

Makefile.am

@@ -34,6 +34,7 @@ cppcheck:
             --std=c++03 \
             -U CURLE_PEER_FAILED_VERIFICATION \
             -U P_tmpdir \
+            -U ENOATTR \
             --enable=all \
             --suppress=missingIncludeSystem \
             --suppress=unusedFunction \

src/test_util.h

@@ -34,8 +34,8 @@ void assert_strequals(const char *x, const char *y, const char *file, int line)
   if(x == NULL && y == NULL){
     return;
   // cppcheck-suppress nullPointerRedundantCheck
-  } else if((x == NULL || y == NULL) || strcmp(x, y) != 0){
-    std::cerr << x << " != " << y << " at " << file << ":" << line << std::endl;
+  } else if(x == NULL || y == NULL || strcmp(x, y) != 0){
+    std::cerr << (x ? x : "null") << " != " << (y ? y : "null") << " at " << file << ":" << line << std::endl;
     std::exit(1);
   }
 }

test/integration-test-common.sh

@@ -122,11 +122,12 @@ function start_s3proxy {
             sleep 1
         done
 
-        S3PROXY_PID=$(netstat -lpnt | grep :8080 | awk '{ print $7 }' | sed -u 's|/java||')
+        S3PROXY_PID=$(lsof -nP -iTCP -sTCP:LISTEN 2>/dev/null | grep :8080 | awk '{ print $2 }')
     fi
 }
 
 function stop_s3proxy {
+    S3PROXY_PID=$(lsof -nP -iTCP -sTCP:LISTEN 2>/dev/null | grep :8080 | awk '{ print $2 }') 
     if [ -n "${S3PROXY_PID}" ]
     then
         kill $S3PROXY_PID
@@ -184,12 +185,29 @@ function start_s3fs {
             -o createbucket \
             ${AUTH_OPT} \
             -o dbglevel=${DBGLEVEL:=info} \
+            -o retries=3 \
             -f \
-            ${@} \
-        |& stdbuf -oL -eL sed -u "s/^/s3fs: /" &
+            ${@} | stdbuf -oL -eL sed -u "s/^/s3fs: /" &
     )
 
+    if [ `uname` = "Darwin" ]; then
+         set +o errexit
+         TRYCOUNT=0
+         while [ $TRYCOUNT -le 20 ]; do
+             df | grep -q $TEST_BUCKET_MOUNT_POINT_1
+             if [ $? -eq 0 ]; then
+                 break;
+             fi
+             sleep 1
+             TRYCOUNT=`expr ${TRYCOUNT} + 1`
+         done
+         if [ $? -ne 0 ]; then
+             exit 1
+         fi
+         set -o errexit
+    else
         retry 5 grep -q $TEST_BUCKET_MOUNT_POINT_1 /proc/mounts || exit 1
+    fi
 
     # Quick way to start system up for manual testing with options under test
     if [[ -n ${INTERACT} ]]; then
@@ -202,14 +220,21 @@ function start_s3fs {
 
 function stop_s3fs {
     # Retry in case file system is in use
+    if [ `uname` = "Darwin" ]; then
+        df | grep -q $TEST_BUCKET_MOUNT_POINT_1
+        if [ $? -eq 0 ]; then
+            retry 10 df | grep -q $TEST_BUCKET_MOUNT_POINT_1 && umount $TEST_BUCKET_MOUNT_POINT_1
+        fi
+    else
         if grep -q $TEST_BUCKET_MOUNT_POINT_1 /proc/mounts; then 
             retry 10 grep -q $TEST_BUCKET_MOUNT_POINT_1 /proc/mounts && fusermount -u $TEST_BUCKET_MOUNT_POINT_1
         fi
+    fi
 }
 
 # trap handlers do not stack.  If a test sets its own, the new handler should call common_exit_handler
 function common_exit_handler {
-    stop_s3proxy
     stop_s3fs
+    stop_s3proxy
 }
 trap common_exit_handler EXIT

test/integration-test-main.sh

@@ -6,7 +6,11 @@ source test-utils.sh
 
 function test_append_file {
     describe "Testing append to file ..."
+
     # Write a small test file
+    if [ `uname` = "Darwin" ]; then
+       cat /dev/null > ${TEST_TEXT_FILE}
+    fi
     for x in `seq 1 $TEST_TEXT_FILE_LENGTH`
     do
        echo "echo ${TEST_TEXT} to ${TEST_TEXT_FILE}"
@@ -51,7 +55,11 @@ function test_truncate_empty_file {
     truncate ${TEST_TEXT_FILE} -s $t_size
 
     # Verify file is zero length
+    if [ `uname` = "Darwin" ]; then
+        size=$(stat -f "%z" ${TEST_TEXT_FILE})
+    else
         size=$(stat -c %s ${TEST_TEXT_FILE})
+    fi
     if [ $t_size -ne $size ]
     then
         echo "error: expected ${TEST_TEXT_FILE} to be $t_size length, got $size"
@@ -77,6 +85,9 @@ function test_mv_file {
     # create the test file again
     mk_test_file
 
+    # save file length
+    ALT_TEXT_LENGTH=`wc -c $TEST_TEXT_FILE | awk '{print $1}'`
+
     #rename the test file
     mv $TEST_TEXT_FILE $ALT_TEST_TEXT_FILE
     if [ ! -e $ALT_TEST_TEXT_FILE ]
@@ -86,7 +97,6 @@ function test_mv_file {
     fi
 
     # Check the contents of the alt file
-    ALT_TEXT_LENGTH=`echo $TEST_TEXT | wc -c | awk '{print $1}'`
     ALT_FILE_LENGTH=`wc -c $ALT_TEST_TEXT_FILE | awk '{print $1}'`
     if [ "$ALT_FILE_LENGTH" -ne "$ALT_TEXT_LENGTH" ]
     then
@@ -179,12 +189,21 @@ function test_chmod {
     # create the test file again
     mk_test_file
 
+    if [ `uname` = "Darwin" ]; then
+        ORIGINAL_PERMISSIONS=$(stat -f "%p" $TEST_TEXT_FILE)
+    else
         ORIGINAL_PERMISSIONS=$(stat --format=%a $TEST_TEXT_FILE)
+    fi
 
     chmod 777 $TEST_TEXT_FILE;
 
     # if they're the same, we have a problem.
-    if [ $(stat --format=%a $TEST_TEXT_FILE) == $ORIGINAL_PERMISSIONS ]
+    if [ `uname` = "Darwin" ]; then
+        CHANGED_PERMISSIONS=$(stat -f "%p" $TEST_TEXT_FILE)
+    else
+        CHANGED_PERMISSIONS=$(stat --format=%a $TEST_TEXT_FILE)
+    fi
+    if [ $CHANGED_PERMISSIONS == $ORIGINAL_PERMISSIONS ]
     then
       echo "Could not modify $TEST_TEXT_FILE permissions"
       return 1
@@ -200,12 +219,21 @@ function test_chown {
     # create the test file again
     mk_test_file
 
+    if [ `uname` = "Darwin" ]; then
+        ORIGINAL_PERMISSIONS=$(stat -f "%u:%g" $TEST_TEXT_FILE)
+    else
         ORIGINAL_PERMISSIONS=$(stat --format=%u:%g $TEST_TEXT_FILE)
+    fi
 
     chown 1000:1000 $TEST_TEXT_FILE;
 
     # if they're the same, we have a problem.
-    if [ $(stat --format=%u:%g $TEST_TEXT_FILE) == $ORIGINAL_PERMISSIONS ]
+    if [ `uname` = "Darwin" ]; then
+        CHANGED_PERMISSIONS=$(stat -f "%u:%g" $TEST_TEXT_FILE)
+    else
+        CHANGED_PERMISSIONS=$(stat --format=%u:%g $TEST_TEXT_FILE)
+    fi
+    if [ $CHANGED_PERMISSIONS == $ORIGINAL_PERMISSIONS ]
     then
       if [ $ORIGINAL_PERMISSIONS == "1000:1000" ]
       then
@@ -262,6 +290,10 @@ function test_rename_before_close {
 
 function test_multipart_upload {
     describe "Testing multi-part upload ..."
+
+    if [ `uname` = "Darwin" ]; then
+       cat /dev/null > $BIG_FILE
+    fi
     dd if=/dev/urandom of="/tmp/${BIG_FILE}" bs=$BIG_FILE_LENGTH count=1
     dd if="/tmp/${BIG_FILE}" of="${BIG_FILE}" bs=$BIG_FILE_LENGTH count=1
 
@@ -278,6 +310,10 @@ function test_multipart_upload {
 
 function test_multipart_copy {
     describe "Testing multi-part copy ..."
+
+    if [ `uname` = "Darwin" ]; then
+       cat /dev/null > $BIG_FILE
+    fi
     dd if=/dev/urandom of="/tmp/${BIG_FILE}" bs=$BIG_FILE_LENGTH count=1
     dd if="/tmp/${BIG_FILE}" of="${BIG_FILE}" bs=$BIG_FILE_LENGTH count=1
     mv "${BIG_FILE}" "${BIG_FILE}-copy"
@@ -364,8 +400,13 @@ function test_mtime_file {
 
     #copy the test file with preserve mode
     cp -p $TEST_TEXT_FILE $ALT_TEST_TEXT_FILE
+    if [ `uname` = "Darwin" ]; then
+        testmtime=`stat -f "%m" $TEST_TEXT_FILE`
+        altmtime=`stat -f "%m" $ALT_TEST_TEXT_FILE`
+    else
         testmtime=`stat -c %Y $TEST_TEXT_FILE`
         altmtime=`stat -c %Y $ALT_TEST_TEXT_FILE`
+    fi
     if [ "$testmtime" -ne "$altmtime" ]
     then
        echo "File times do not match:  $testmtime != $altmtime"

test/test-utils.sh

@@ -24,6 +24,21 @@ function mk_test_file {
         echo "Could not create file ${TEST_TEXT_FILE}, it does not exist"
         exit 1
     fi
+
+    # wait & check
+    BASE_TEXT_LENGTH=`echo $TEXT | wc -c | awk '{print $1}'`
+    TRY_COUNT=10
+    while true; do
+        MK_TEXT_LENGTH=`wc -c $TEST_TEXT_FILE | awk '{print $1}'`
+        if [ $BASE_TEXT_LENGTH -eq $MK_TEXT_LENGTH ]; then
+            break
+        fi
+        TRY_COUNT=`expr $TRY_COUNT - 1`
+        if [ $TRY_COUNT -le 0 ]; then
+            echo "Could not create file ${TEST_TEXT_FILE}, that file size is something wrong"
+        fi
+        sleep 1
+    done
 }
 
 function rm_test_file {
@@ -65,7 +80,7 @@ function cd_run_dir {
         echo "TEST_BUCKET_MOUNT_POINT variable not set"
         exit 1
     fi
-    RUN_DIR=$(mktemp --directory ${TEST_BUCKET_MOUNT_POINT_1}/testrun-XXXXXX)
+    RUN_DIR=$(mktemp -d ${TEST_BUCKET_MOUNT_POINT_1}/testrun-XXXXXX)
     cd ${RUN_DIR}
 }