Add a partial page_list unit test (#1735)

.gitignore

@@ -74,6 +74,7 @@ missing
 default_commit_hash
 src/s3fs
 src/test_curl_util
+src/test_page_list
 src/test_string_util
 test/chaos-http-proxy-*
 test/s3proxy-*

src/Makefile.am

@@ -68,6 +68,7 @@ s3fs_LDADD = $(DEPS_LIBS)
 
 noinst_PROGRAMS = \
     test_curl_util \
+    test_page_list \
     test_string_util
 
 test_curl_util_SOURCES = common_auth.cpp curl_util.cpp string_util.cpp test_curl_util.cpp s3fs_global.cpp s3fs_logger.cpp
@@ -83,10 +84,18 @@ endif
 
 test_curl_util_LDADD = $(DEPS_LIBS)
 
+test_page_list_SOURCES = \
+    fdcache_page.cpp \
+    s3fs_global.cpp \
+    s3fs_logger.cpp \
+    string_util.cpp \
+    test_page_list.cpp
+
 test_string_util_SOURCES = string_util.cpp test_string_util.cpp s3fs_logger.cpp
 
 TESTS = \
     test_curl_util \
+    test_page_list \
     test_string_util
 
 clang-tidy:

src/common.h

@@ -21,6 +21,8 @@
 #ifndef S3FS_COMMON_H_
 #define S3FS_COMMON_H_
 
+#include <fcntl.h>
+
 #include "../config.h"
 #include "types.h"
 #include "s3fs_logger.h"

src/curl.h

@@ -21,8 +21,14 @@
 #ifndef S3FS_CURL_H_
 #define S3FS_CURL_H_
 
+#include <cassert>
 #include <curl/curl.h>
+#include <list>
+#include <map>
+#include <strings.h>
+#include <vector>
 
+#include "common.h"
 #include "curl_handlerpool.h"
 #include "bodydata.h"
 #include "psemaphore.h"

src/fdcache_page.h

@@ -21,6 +21,9 @@
 #ifndef S3FS_FDCACHE_PAGE_H_
 #define S3FS_FDCACHE_PAGE_H_
 
+#include <list>
+#include <sys/types.h>
+
 #include "fdcache_stat.h"
 
 //------------------------------------------------
@@ -89,7 +92,6 @@ class PageList
         static bool CheckAreaInSparseFile(const struct fdpage& checkpage, const fdpage_list_t& sparse_list, int fd, fdpage_list_t& err_area_list, fdpage_list_t& warn_area_list);
 
         void Clear();
-        bool Compress();
         bool Parse(off_t new_pos);
 
     public:
@@ -115,6 +117,7 @@ class PageList
         bool IsModified() const;
         bool ClearAllModified();
 
+        bool Compress();
         bool Serialize(CacheFileStat& file, bool is_output, ino_t inode);
         void Dump() const;
         bool CompareSparseFile(int fd, size_t file_size, fdpage_list_t& err_area_list, fdpage_list_t& warn_area_list);

src/fdcache_untreated.h

@@ -21,6 +21,9 @@
 #ifndef S3FS_FDCACHE_UNTREATED_H_
 #define S3FS_FDCACHE_UNTREATED_H_
 
+#include "common.h"
+#include "types.h"
+
 //------------------------------------------------
 // Class UntreatedParts
 //------------------------------------------------

src/test_page_list.cpp

@@ -0,0 +1,82 @@
+/*
+ * s3fs - FUSE-based file system backed by Amazon S3
+ *
+ * Copyright(C) 2021 Andrew Gaul <andrew@gaul.org>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#include <limits>
+#include <stdint.h>
+#include <string>
+
+#include "fdcache.h"
+#include "test_util.h"
+
+bool CacheFileStat::Open() { return false; }
+
+void test_compress()
+{
+  PageList list;
+  ASSERT_EQUALS(off_t(0), list.Size());
+
+  list.Init(42, /*is_loaded=*/ false, /*is_modified=*/ false);
+  ASSERT_EQUALS(off_t(42), list.Size());
+  ASSERT_FALSE(list.IsPageLoaded(0, 1));
+
+  list.SetPageLoadedStatus(0, 1, /*pstatus=*/ PageList::PAGE_LOADED);
+  ASSERT_TRUE(list.IsPageLoaded(0, 1));
+  ASSERT_FALSE(list.IsPageLoaded(0, 2));
+
+  off_t start = 0;
+  off_t size = 0;
+  ASSERT_TRUE(list.FindUnloadedPage(0, start, size));
+  ASSERT_EQUALS(off_t(1), start);
+  ASSERT_EQUALS(off_t(41), size);
+
+  // test adding subsequent page then compressing
+  list.SetPageLoadedStatus(1, 3, /*pstatus=*/ PageList::PAGE_LOADED);
+  list.Compress();
+  ASSERT_TRUE(list.IsPageLoaded(0, 3));
+
+  ASSERT_TRUE(list.FindUnloadedPage(0, start, size));
+  ASSERT_EQUALS(off_t(4), start);
+  ASSERT_EQUALS(off_t(38), size);
+
+  // test adding non-contiguous page then compressing
+  list.SetPageLoadedStatus(5, 1, /*pstatus=*/ PageList::PAGE_LOADED);
+  list.Compress();
+
+  ASSERT_TRUE(list.FindUnloadedPage(0, start, size));
+  ASSERT_EQUALS(off_t(4), start);
+  ASSERT_EQUALS(off_t(1), size);
+  list.Dump();
+  printf("\n");
+
+  // test adding page between two pages then compressing
+  list.SetPageLoadedStatus(4, 1, /*pstatus=*/ PageList::PAGE_LOADED);
+  list.Compress();
+
+  list.Dump();
+  ASSERT_TRUE(list.FindUnloadedPage(0, start, size));
+  ASSERT_EQUALS(off_t(6), start);
+  ASSERT_EQUALS(off_t(36), size);
+}
+
+int main(int argc, char *argv[])
+{
+  test_compress();
+  return 0;
+}

src/types.h

@@ -21,6 +21,7 @@
 #ifndef S3FS_TYPES_H_
 #define S3FS_TYPES_H_
 
+#include <cstdlib>
 #include <cstring>
 #include <string>
 #include <map>