octoleo/s3fs-fuse
1
0
Fork 0
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git synced 2025-02-02 18:48:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #354 from ggtakec/master

Browse Source 
Supported regex type for additional header format - #343
This commit is contained in:
Takeshi Nakatani 2016-02-07 15:16:51 +09:00
commit bf056b213a
9 changed files with 399 additions and 244 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
doc/man/s3fs.1
View File

@ -105,16 +105,17 @@ specify the path to the password file, which which takes precedence over the pas
This option specifies the configuration file path which file is the additional HTTP header by file(object) extension. This option specifies the configuration file path which file is the additional HTTP header by file(object) extension.
The configuration file format is below: The configuration file format is below:
----------- -----------
line = [file suffix] HTTP-header [HTTP-values] line = [file suffix or regex] HTTP-header [HTTP-values]
file suffix = file(object) suffix, if this field is empty, it means "*"(all object). file suffix = file(object) suffix, if this field is empty, it means "reg:(.*)".(=all object).
regex = regular expression to match the file(object) path. this type starts with "reg:" prefix.
HTTP-header = additional HTTP header name HTTP-header = additional HTTP header name
HTTP-values = additional HTTP header value HTTP-values = additional HTTP header value
----------- -----------
Sample: Sample:
----------- -----------
.gz Content-Encoding gzip .gz Content-Encoding gzip
.Z Content-Encoding compress .Z Content-Encoding compress
X-S3FS-MYHTTPHEAD myvalue reg:^/MYDIR/(.*)[.]t2$ Content-Encoding text2
----------- -----------
A sample configuration file is uploaded in "test" directory. A sample configuration file is uploaded in "test" directory.
If you specify this option for set "Content-Encoding" HTTP header, please take care for RFC 2616. If you specify this option for set "Content-Encoding" HTTP header, please take care for RFC 2616.

2
src/Makefile.am
View File

@ -24,7 +24,7 @@ if USE_GNUTLS_NETTLE
AM_CPPFLAGS += -DUSE_GNUTLS_NETTLE AM_CPPFLAGS += -DUSE_GNUTLS_NETTLE
endif endif
s3fs_SOURCES = s3fs.cpp s3fs.h curl.cpp curl.h cache.cpp cache.h string_util.cpp string_util.h s3fs_util.cpp s3fs_util.h fdcache.cpp fdcache.h common_auth.cpp s3fs_auth.h common.h s3fs_SOURCES = s3fs.cpp s3fs.h curl.cpp curl.h cache.cpp cache.h string_util.cpp string_util.h s3fs_util.cpp s3fs_util.h fdcache.cpp fdcache.h common_auth.cpp s3fs_auth.h addhead.cpp addhead.h common.h
if USE_SSL_OPENSSL if USE_SSL_OPENSSL
s3fs_SOURCES += openssl_auth.cpp s3fs_SOURCES += openssl_auth.cpp
endif endif

284
src/addhead.cpp Normal file
View File

@ -0,0 +1,284 @@
/*
* s3fs - FUSE-based file system backed by Amazon S3
*
* Copyright 2007-2008 Randy Rizun <rrizun@gmail.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <syslog.h>
#include <assert.h>
#include <curl/curl.h>
#include <sstream>
#include <fstream>
#include <string>
#include <map>
#include <list>
#include <vector>
#include "common.h"
#include "addhead.h"
#include "curl.h"
#include "s3fs.h"
using namespace std;
//-------------------------------------------------------------------
// Symbols
//-------------------------------------------------------------------
#define ADD_HEAD_REGEX "reg:"
//-------------------------------------------------------------------
// Class AdditionalHeader
//-------------------------------------------------------------------
AdditionalHeader AdditionalHeader::singleton;
//-------------------------------------------------------------------
// Class AdditionalHeader method
//-------------------------------------------------------------------
AdditionalHeader::AdditionalHeader()
{
if(this == AdditionalHeader::get()){
is_enable = false;
}else{
assert(false);
}
}
AdditionalHeader::~AdditionalHeader()
{
if(this == AdditionalHeader::get()){
Unload();
}else{
assert(false);
}
}
bool AdditionalHeader::Load(const char* file)
{
if(!file){
S3FS_PRN_WARN("file is NULL.");
return false;
}
Unload();
ifstream AH(file);
if(!AH.good()){
S3FS_PRN_WARN("Could not open file(%s).", file);
return false;
}
// read file
string line;
PADDHEAD paddhead;
while(getline(AH, line)){
if('#' == line[0]){
continue;
}
if(0 == line.size()){
continue;
}
// load a line
stringstream ss(line);
string key(""); // suffix(key)
string head; // additional HTTP header
string value; // header value
if(0 == isblank(line[0])){
ss >> key;
}
if(ss){
ss >> head;
if(ss && static_cast<size_t>(ss.tellg()) < line.size()){
value = line.substr(static_cast<int>(ss.tellg()) + 1);
}
}
// check it
if(0 == head.size()){
if(0 == key.size()){
continue;
}
S3FS_PRN_ERR("file format error: %s key(suffix) is no HTTP header value.", key.c_str());
Unload();
return false;
}
paddhead = new ADDHEAD;
if(0 == strncasecmp(key.c_str(), ADD_HEAD_REGEX, strlen(ADD_HEAD_REGEX))){
// regex
if(key.size() <= strlen(ADD_HEAD_REGEX)){
S3FS_PRN_ERR("file format error: %s key(suffix) does not have key string.", key.c_str());
continue;
}
key = key.substr(strlen(ADD_HEAD_REGEX));
// compile
regex_t* preg = new regex_t;
int result;
char errbuf[256];
if(0 != (result = regcomp(preg, key.c_str(), REG_EXTENDED | REG_NOSUB))){ // we do not need matching info
regerror(result, preg, errbuf, sizeof(errbuf));
S3FS_PRN_ERR("failed to compile regex from %s key by %s.", key.c_str(), errbuf);
delete preg;
delete paddhead;
continue;
}
// set
paddhead->pregex = preg;
paddhead->basestring = key;
paddhead->headkey = head;
paddhead->headvalue = value;
}else{
// not regex, directly comparing
paddhead->pregex = NULL;
paddhead->basestring = key;
paddhead->headkey = head;
paddhead->headvalue = value;
}
// add list
addheadlist.push_back(paddhead);
// set flag
if(!is_enable){
is_enable = true;
}
}
return true;
}
void AdditionalHeader::Unload(void)
{
is_enable = false;
for(addheadlist_t::iterator iter = addheadlist.begin(); iter != addheadlist.end(); iter = addheadlist.erase(iter)){
PADDHEAD paddhead = *iter;
if(paddhead){
if(paddhead->pregex){
regfree(paddhead->pregex);
delete paddhead->pregex;
}
delete paddhead;
}
}
}
bool AdditionalHeader::AddHeader(headers_t& meta, const char* path) const
{
if(!is_enable){
return true;
}
if(!path){
S3FS_PRN_WARN("path is NULL.");
return false;
}
size_t pathlength = strlen(path);
// loop
for(addheadlist_t::const_iterator iter = addheadlist.begin(); iter != addheadlist.end(); ++iter){
const PADDHEAD paddhead = *iter;
if(!paddhead){
continue;
}
if(paddhead->pregex){
// regex
regmatch_t match; // not use
if(0 == regexec(paddhead->pregex, path, 1, &match, 0)){
// match -> adding header
meta[paddhead->headkey] = paddhead->headvalue;
break;
}
}else{
// directly comparing
if(paddhead->basestring.length() < pathlength){
if(0 == paddhead->basestring.length() || 0 == strcmp(&path[pathlength - paddhead->basestring.length()], paddhead->basestring.c_str())){
// match -> adding header
meta[paddhead->headkey] = paddhead->headvalue;
break;
}
}
}
}
return true;
}
struct curl_slist* AdditionalHeader::AddHeader(struct curl_slist* list, const char* path) const
{
headers_t meta;
if(!AddHeader(meta, path)){
return list;
}
for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){
// Adding header
list = curl_slist_sort_insert(list, iter->first.c_str(), iter->second.c_str());
}
meta.clear();
S3FS_MALLOCTRIM(0);
return list;
}
bool AdditionalHeader::Dump(void) const
{
if(!IS_S3FS_LOG_DBG()){
return true;
}
stringstream ssdbg;
int cnt = 1;
ssdbg << "Additional Header list[" << addheadlist.size() << "] = {" << endl;
for(addheadlist_t::const_iterator iter = addheadlist.begin(); iter != addheadlist.end(); ++iter, ++cnt){
const PADDHEAD paddhead = *iter;
ssdbg << " [" << cnt << "] = {" << endl;
if(paddhead){
if(paddhead->pregex){
ssdbg << " type\t\t--->\tregex" << endl;
}else{
ssdbg << " type\t\t--->\tsuffix matching" << endl;
}
ssdbg << " base string\t--->\t" << paddhead->basestring << endl;
ssdbg << " add header\t--->\t" << paddhead->headkey << ": " << paddhead->headvalue << endl;
}
ssdbg << " }" << endl;
}
ssdbg << "}" << endl;
// print all
S3FS_PRN_DBG("%s", ssdbg.str().c_str());
return true;
}
/*
* Local variables:
* tab-width: 4
* c-basic-offset: 4
* End:
* vim600: noet sw=4 ts=4 fdm=marker
* vim<600: noet sw=4 ts=4
*/

70
src/addhead.h Normal file
View File

@ -0,0 +1,70 @@
/*
* s3fs - FUSE-based file system backed by Amazon S3
*
* Copyright 2007-2008 Randy Rizun <rrizun@gmail.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#ifndef S3FS_ADDHEAD_H_
#define S3FS_ADDHEAD_H_
#include <regex.h>
//----------------------------------------------
// class AdditionalHeader
//----------------------------------------------
typedef struct add_header{
regex_t* pregex; // not NULL means using regex, NULL means comparing suffix directly.
std::string basestring;
std::string headkey;
std::string headvalue;
}ADDHEAD, *PADDHEAD;
typedef std::vector<PADDHEAD> addheadlist_t;
class AdditionalHeader
{
private:
static AdditionalHeader singleton;
bool is_enable;
addheadlist_t addheadlist;
protected:
AdditionalHeader();
~AdditionalHeader();
public:
// Reference singleton
static AdditionalHeader* get(void) { return &singleton; }
bool Load(const char* file);
void Unload(void);
bool AddHeader(headers_t& meta, const char* path) const;
struct curl_slist* AddHeader(struct curl_slist* list, const char* path) const;
bool Dump(void) const;
};
#endif // S3FS_ADDHEAD_H_
/*
* Local variables:
* tab-width: 4
* c-basic-offset: 4
* End:
* vim600: noet sw=4 ts=4 fdm=marker
* vim<600: noet sw=4 ts=4
*/

187
src/curl.cpp
View File

@ -48,6 +48,7 @@
#include "s3fs.h" #include "s3fs.h"
#include "s3fs_util.h" #include "s3fs_util.h"
#include "s3fs_auth.h" #include "s3fs_auth.h"
#include "addhead.h"
using namespace std; using namespace std;
@ -3780,192 +3781,6 @@ int S3fsMultiCurl::Request(void)
return 0; return 0;
} }
//-------------------------------------------------------------------
// Class AdditionalHeader
//-------------------------------------------------------------------
AdditionalHeader AdditionalHeader::singleton;
//-------------------------------------------------------------------
// Class AdditionalHeader method
//-------------------------------------------------------------------
AdditionalHeader::AdditionalHeader()
{
if(this == AdditionalHeader::get()){
is_enable = false;
}else{
assert(false);
}
}
AdditionalHeader::~AdditionalHeader()
{
if(this == AdditionalHeader::get()){
Unload();
}else{
assert(false);
}
}
bool AdditionalHeader::Load(const char* file)
{
if(!file){
S3FS_PRN_WARN("file is NULL.");
return false;
}
Unload();
ifstream AH(file);
if(!AH.good()){
S3FS_PRN_WARN("Could not open file(%s).", file);
return false;
}
// read file
string line;
while(getline(AH, line)){
if('#' == line[0]){
continue;
}
if(0 == line.size()){
continue;
}
// load a line
stringstream ss(line);
string key(""); // suffix(key)
string head; // additional HTTP header
string value; // header value
if(0 == isblank(line[0])){
ss >> key;
}
if(ss){
ss >> head;
if(ss && static_cast<size_t>(ss.tellg()) < line.size()){
value = line.substr(static_cast<int>(ss.tellg()) + 1);
}
}
// check it
if(0 == head.size()){
if(0 == key.size()){
continue;
}
S3FS_PRN_ERR("file format error: %s key(suffix) is no HTTP header value.", key.c_str());
Unload();
return false;
}
// set charcntlist
int keylen = key.size();
charcnt_list_t::iterator iter;
for(iter = charcntlist.begin(); iter != charcntlist.end(); ++iter){
if(keylen == (*iter)){
break;
}
}
if(iter == charcntlist.end()){
charcntlist.push_back(keylen);
}
// set addheader
addheader_t::iterator aiter;
if(addheader.end() == (aiter = addheader.find(key))){
headerpair_t hpair;
hpair[head] = value;
addheader[key] = hpair;
}else{
aiter->second[head] = value;
}
// set flag
if(!is_enable){
is_enable = true;
}
}
return true;
}
void AdditionalHeader::Unload(void)
{
is_enable = false;
charcntlist.clear();
addheader.clear();
}
bool AdditionalHeader::AddHeader(headers_t& meta, const char* path) const
{
if(!is_enable){
return true;
}
if(!path){
S3FS_PRN_WARN("path is NULL.");
return false;
}
int nPathLen = strlen(path);
for(charcnt_list_t::const_iterator iter = charcntlist.begin(); iter != charcntlist.end(); ++iter){
// get target character count
if(nPathLen < (*iter)){
continue;
}
// make target suffix(same character count) & find
string suffix(&path[nPathLen - (*iter)]);
addheader_t::const_iterator aiter;
if(addheader.end() == (aiter = addheader.find(suffix))){
continue;
}
for(headerpair_t::const_iterator piter = aiter->second.begin(); piter != aiter->second.end(); ++piter){
// Adding header
meta[(*piter).first] = (*piter).second;
}
}
return true;
}
struct curl_slist* AdditionalHeader::AddHeader(struct curl_slist* list, const char* path) const
{
headers_t meta;
if(!AddHeader(meta, path)){
return list;
}
for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){
// Adding header
list = curl_slist_sort_insert(list, iter->first.c_str(), iter->second.c_str());
}
meta.clear();
S3FS_MALLOCTRIM(0);
return list;
}
bool AdditionalHeader::Dump(void) const
{
if(!IS_S3FS_LOG_DBG()){
return true;
}
// character count list
stringstream ssdbg;
ssdbg << "Character count list[" << charcntlist.size() << "] = {";
for(charcnt_list_t::const_iterator citer = charcntlist.begin(); citer != charcntlist.end(); ++citer){
ssdbg << " " << (*citer);
}
ssdbg << " }\n";
// additional header
ssdbg << "Additional Header list[" << addheader.size() << "] = {\n";
for(addheader_t::const_iterator aiter = addheader.begin(); aiter != addheader.end(); ++aiter){
string key = (*aiter).first;
if(0 == key.size()){
key = "*";
}
for(headerpair_t::const_iterator piter = (*aiter).second.begin(); piter != (*aiter).second.end(); ++piter){
ssdbg << " " << key << "\t--->\t" << (*piter).first << ": " << (*piter).second << "\n";
}
}
ssdbg << "}";
// print all
S3FS_PRN_DBG("%s", ssdbg.str().c_str());
return true;
}
//------------------------------------------------------------------- //-------------------------------------------------------------------
// Utility functions // Utility functions
//------------------------------------------------------------------- //-------------------------------------------------------------------

31
src/curl.h
View File

@ -17,6 +17,7 @@
* along with this program; if not, write to the Free Software * along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/ */
#ifndef S3FS_CURL_H_ #ifndef S3FS_CURL_H_
#define S3FS_CURL_H_ #define S3FS_CURL_H_
@ -425,36 +426,6 @@ class S3fsMultiCurl
int Request(void); int Request(void);
}; };
//----------------------------------------------
// class AdditionalHeader
//----------------------------------------------
typedef std::list<int> charcnt_list_t;
typedef std::map<std::string, std::string> headerpair_t;
typedef std::map<std::string, headerpair_t> addheader_t;
class AdditionalHeader
{
private:
static AdditionalHeader singleton;
bool is_enable;
charcnt_list_t charcntlist;
addheader_t addheader;
public:
// Reference singleton
static AdditionalHeader* get(void) { return &singleton; }
AdditionalHeader();
~AdditionalHeader();
bool Load(const char* file);
void Unload(void);
bool AddHeader(headers_t& meta, const char* path) const;
struct curl_slist* AddHeader(struct curl_slist* list, const char* path) const;
bool Dump(void) const;
};
//---------------------------------------------- //----------------------------------------------
// Utility Functions // Utility Functions
//---------------------------------------------- //----------------------------------------------

1
src/s3fs.cpp
View File

@ -52,6 +52,7 @@
#include "s3fs_util.h" #include "s3fs_util.h"
#include "fdcache.h" #include "fdcache.h"
#include "s3fs_auth.h" #include "s3fs_auth.h"
#include "addhead.h"
using namespace std; using namespace std;

12
src/s3fs_util.cpp
View File

@ -954,17 +954,19 @@ void show_help (void)
" file is the additional HTTP header by file(object) extension.\n" " file is the additional HTTP header by file(object) extension.\n"
" The configuration file format is below:\n" " The configuration file format is below:\n"
" -----------\n" " -----------\n"
" line = [file suffix] HTTP-header [HTTP-values]\n" " line = [file suffix or regex] HTTP-header [HTTP-values]\n"
" file suffix = file(object) suffix, if this field is empty,\n" " file suffix = file(object) suffix, if this field is empty,\n"
" it means \"*\"(all object).\n" " it means \"reg:(.*)\".(=all object).\n"
" regex = regular expression to match the file(object) path.\n"
" this type starts with \"reg:\" prefix.\n"
" HTTP-header = additional HTTP header name\n" " HTTP-header = additional HTTP header name\n"
" HTTP-values = additional HTTP header value\n" " HTTP-values = additional HTTP header value\n"
" -----------\n" " -----------\n"
" Sample:\n" " Sample:\n"
" -----------\n" " -----------\n"
" .gz Content-Encoding gzip\n" " .gz Content-Encoding gzip\n"
" .Z Content-Encoding compress\n" " .Z Content-Encoding compress\n"
" X-S3FS-MYHTTPHEAD myvalue\n" " reg:^/MYDIR/(.*)[.]t2$ Content-Encoding text2\n"
" -----------\n" " -----------\n"
" A sample configuration file is uploaded in \"test\" directory.\n" " A sample configuration file is uploaded in \"test\" directory.\n"
" If you specify this option for set \"Content-Encoding\" HTTP \n" " If you specify this option for set \"Content-Encoding\" HTTP \n"

45
test/sample_ahbe.conf
View File

@ -4,21 +4,27 @@
# s3fs loads this file at starting. # s3fs loads this file at starting.
# #
# Format: # Format:
# line = [file suffix] HTTP-header [HTTP-header-values] # line = [file suffix or regex] HTTP-header [HTTP-header-values]
# file suffix = file(object) suffix, if this field is empty, # file suffix = file(object) suffix, if this field is empty,
# it means "*"(all object). # it means "reg:(.*)".(=all object).
# regex = regular expression to match the file(object) path.
# this type starts with "reg:" prefix.
# HTTP-header = additional HTTP header name # HTTP-header = additional HTTP header name
# HTTP-header-values = additional HTTP header value # HTTP-header-values = additional HTTP header value
# #
# <suffix(extension)> <HTTP header> <HTTP header values> # <suffix(extension)> <HTTP header> <HTTP header values>
# #
# Verification is done in the order in which they are described in the file.
# That order is very important.
#
# Example: # Example:
# " Content-Encoding gzip" --> all object # " Content-Encoding gzip" --> all object
# ".gz Content-Encoding gzip" --> only ".gz" extension file # ".gz Content-Encoding gzip" --> only ".gz" extension file
# "reg:^/DIR/(.*).t2$ Content-Encoding text2" --> "/DIR/*.t2" extension file
# #
# Notice: # Notice:
# If you need to set all object, you can specify without "suffix". # If you need to set all object, you can specify without "suffix" or regex
# Then all of object(file) is added additional header. # type "reg:(.*)". Then all of object(file) is added additional header.
# If you have this configuration file for Content-Encoding, you should # If you have this configuration file for Content-Encoding, you should
# know about RFC 2616. # know about RFC 2616.
# #
@ -27,15 +33,20 @@
# Encoding header, and SHOULD NOT be used in the Content-Encoding # Encoding header, and SHOULD NOT be used in the Content-Encoding
# header." # header."
# #
.gz Content-Encoding gzip
.Z Content-Encoding compress # file suffix type
.bz2 Content-Encoding bzip2 .gz Content-Encoding gzip
.svgz Content-Encoding gzip .Z Content-Encoding compress
.svg.gz Content-Encoding gzip .bz2 Content-Encoding bzip2
.tgz Content-Encoding gzip .svgz Content-Encoding gzip
.tar.gz Content-Encoding gzip .svg.gz Content-Encoding gzip
.taz Content-Encoding gzip .tgz Content-Encoding gzip
.tz Content-Encoding gzip .tar.gz Content-Encoding gzip
.tbz2 Content-Encoding gzip .taz Content-Encoding gzip
gz.js Content-Encoding gzip .tz Content-Encoding gzip
.tbz2 Content-Encoding gzip
gz.js Content-Encoding gzip
# regex type(test)
reg:^/MYDIR/(.*)[.]t2$ Content-Encoding text2