octoleo/s3fs-fuse
1
0
Fork 0
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git synced 2024-12-23 01:08:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Supported regex type for additional header format.

Browse Source
This commit is contained in:
Takeshi Nakatani 2016-02-07 05:08:52 +00:00
parent 938554e569
commit 6472eedddc
5 changed files with 158 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
doc/man/s3fs.1
View File

@ -105,8 +105,9 @@ specify the path to the password file, which which takes precedence over the pas
This option specifies the configuration file path which file is the additional HTTP header by file(object) extension. This option specifies the configuration file path which file is the additional HTTP header by file(object) extension.
The configuration file format is below: The configuration file format is below:
----------- -----------
line = [file suffix] HTTP-header [HTTP-values] line = [file suffix or regex] HTTP-header [HTTP-values]
file suffix = file(object) suffix, if this field is empty, it means "*"(all object). file suffix = file(object) suffix, if this field is empty, it means "reg:(.*)".(=all object).
regex = regular expression to match the file(object) path. this type starts with "reg:" prefix.
HTTP-header = additional HTTP header name HTTP-header = additional HTTP header name
HTTP-values = additional HTTP header value HTTP-values = additional HTTP header value
----------- -----------
@ -114,7 +115,7 @@ This option specifies the configuration file path which file is the additional H
----------- -----------
.gz Content-Encoding gzip .gz Content-Encoding gzip
.Z Content-Encoding compress .Z Content-Encoding compress
X-S3FS-MYHTTPHEAD myvalue reg:^/MYDIR/(.*)[.]t2$ Content-Encoding text2
----------- -----------
A sample configuration file is uploaded in "test" directory. A sample configuration file is uploaded in "test" directory.
If you specify this option for set "Content-Encoding" HTTP header, please take care for RFC 2616. If you specify this option for set "Content-Encoding" HTTP header, please take care for RFC 2616.

144
src/curl.cpp
View File

@ -3780,6 +3780,11 @@ int S3fsMultiCurl::Request(void)
return 0; return 0;
} }
//-------------------------------------------------------------------
// Symbols
//-------------------------------------------------------------------
#define ADD_HEAD_REGEX "reg:"
//------------------------------------------------------------------- //-------------------------------------------------------------------
// Class AdditionalHeader // Class AdditionalHeader
//------------------------------------------------------------------- //-------------------------------------------------------------------
@ -3822,6 +3827,7 @@ bool AdditionalHeader::Load(const char* file)
// read file // read file
string line; string line;
PADDHEAD paddhead;
while(getline(AH, line)){ while(getline(AH, line)){
if('#' == line[0]){ if('#' == line[0]){
continue; continue;
@ -3854,26 +3860,44 @@ bool AdditionalHeader::Load(const char* file)
return false; return false;
} }
// set charcntlist paddhead = new ADDHEAD;
int keylen = key.size(); if(0 == strncasecmp(key.c_str(), ADD_HEAD_REGEX, strlen(ADD_HEAD_REGEX))){
charcnt_list_t::iterator iter; // regex
for(iter = charcntlist.begin(); iter != charcntlist.end(); ++iter){ if(key.size() <= strlen(ADD_HEAD_REGEX)){
if(keylen == (*iter)){ S3FS_PRN_ERR("file format error: %s key(suffix) does not have key string.", key.c_str());
break; continue;
} }
key = key.substr(strlen(ADD_HEAD_REGEX));
// compile
regex_t* preg = new regex_t;
int result;
char errbuf[256];
if(0 != (result = regcomp(preg, key.c_str(), REG_EXTENDED | REG_NOSUB))){ // we do not need matching info
regerror(result, preg, errbuf, sizeof(errbuf));
S3FS_PRN_ERR("failed to compile regex from %s key by %s.", key.c_str(), errbuf);
delete preg;
delete paddhead;
continue;
} }
if(iter == charcntlist.end()){
charcntlist.push_back(keylen); // set
} paddhead->pregex = preg;
// set addheader paddhead->basestring = key;
addheader_t::iterator aiter; paddhead->headkey = head;
if(addheader.end() == (aiter = addheader.find(key))){ paddhead->headvalue = value;
headerpair_t hpair;
hpair[head] = value;
addheader[key] = hpair;
}else{ }else{
aiter->second[head] = value; // not regex, directly comparing
paddhead->pregex = NULL;
paddhead->basestring = key;
paddhead->headkey = head;
paddhead->headvalue = value;
} }
// add list
addheadlist.push_back(paddhead);
// set flag // set flag
if(!is_enable){ if(!is_enable){
is_enable = true; is_enable = true;
@ -3885,8 +3909,17 @@ bool AdditionalHeader::Load(const char* file)
void AdditionalHeader::Unload(void) void AdditionalHeader::Unload(void)
{ {
is_enable = false; is_enable = false;
charcntlist.clear();
addheader.clear(); for(addheadlist_t::iterator iter = addheadlist.begin(); iter != addheadlist.end(); iter = addheadlist.erase(iter)){
PADDHEAD paddhead = *iter;
if(paddhead){
if(paddhead->pregex){
regfree(paddhead->pregex);
delete paddhead->pregex;
}
delete paddhead;
}
}
} }
bool AdditionalHeader::AddHeader(headers_t& meta, const char* path) const bool AdditionalHeader::AddHeader(headers_t& meta, const char* path) const
@ -3898,21 +3931,35 @@ bool AdditionalHeader::AddHeader(headers_t& meta, const char* path) const
S3FS_PRN_WARN("path is NULL."); S3FS_PRN_WARN("path is NULL.");
return false; return false;
} }
int nPathLen = strlen(path);
for(charcnt_list_t::const_iterator iter = charcntlist.begin(); iter != charcntlist.end(); ++iter){ size_t pathlength = strlen(path);
// get target character count
if(nPathLen < (*iter)){ // loop
for(addheadlist_t::const_iterator iter = addheadlist.begin(); iter != addheadlist.end(); ++iter){
const PADDHEAD paddhead = *iter;
if(!paddhead){
continue; continue;
} }
// make target suffix(same character count) & find
string suffix(&path[nPathLen - (*iter)]); if(paddhead->pregex){
addheader_t::const_iterator aiter; // regex
if(addheader.end() == (aiter = addheader.find(suffix))){ int result;
continue; regmatch_t match; // not use
if(0 == (result = regexec(paddhead->pregex, path, 1, &match, 0))){
// match -> adding header
meta[paddhead->headkey] = paddhead->headvalue;
break;
}
}else{
// directly comparing
if(paddhead->basestring.length() < pathlength){
if(0 == paddhead->basestring.length() || 0 == strcmp(&path[pathlength - paddhead->basestring.length()], paddhead->basestring.c_str())){
// match -> adding header
meta[paddhead->headkey] = paddhead->headvalue;
break;
}
} }
for(headerpair_t::const_iterator piter = aiter->second.begin(); piter != aiter->second.end(); ++piter){
// Adding header
meta[(*piter).first] = (*piter).second;
} }
} }
return true; return true;
@ -3939,26 +3986,31 @@ bool AdditionalHeader::Dump(void) const
if(!IS_S3FS_LOG_DBG()){ if(!IS_S3FS_LOG_DBG()){
return true; return true;
} }
// character count list
stringstream ssdbg;
ssdbg << "Character count list[" << charcntlist.size() << "] = {";
for(charcnt_list_t::const_iterator citer = charcntlist.begin(); citer != charcntlist.end(); ++citer){
ssdbg << " " << (*citer);
}
ssdbg << " }\n";
// additional header stringstream ssdbg;
ssdbg << "Additional Header list[" << addheader.size() << "] = {\n"; int cnt = 1;
for(addheader_t::const_iterator aiter = addheader.begin(); aiter != addheader.end(); ++aiter){
string key = (*aiter).first; ssdbg << "Additional Header list[" << addheadlist.size() << "] = {" << endl;
if(0 == key.size()){
key = "*"; for(addheadlist_t::const_iterator iter = addheadlist.begin(); iter != addheadlist.end(); ++iter, ++cnt){
const PADDHEAD paddhead = *iter;
ssdbg << " [" << cnt << "] = {" << endl;
if(paddhead){
if(paddhead->pregex){
ssdbg << " type\t\t--->\tregex" << endl;
}else{
ssdbg << " type\t\t--->\tsuffix matching" << endl;
} }
for(headerpair_t::const_iterator piter = (*aiter).second.begin(); piter != (*aiter).second.end(); ++piter){ ssdbg << " base string\t--->\t" << paddhead->basestring << endl;
ssdbg << " " << key << "\t--->\t" << (*piter).first << ": " << (*piter).second << "\n"; ssdbg << " add header\t--->\t" << paddhead->headkey << ": " << paddhead->headvalue << endl;
} }
ssdbg << " }" << endl;
} }
ssdbg << "}";
ssdbg << "}" << endl;
// print all // print all
S3FS_PRN_DBG("%s", ssdbg.str().c_str()); S3FS_PRN_DBG("%s", ssdbg.str().c_str());

23
src/curl.h
View File

@ -428,25 +428,32 @@ class S3fsMultiCurl
//---------------------------------------------- //----------------------------------------------
// class AdditionalHeader // class AdditionalHeader
//---------------------------------------------- //----------------------------------------------
typedef std::list<int> charcnt_list_t; #include <regex.h>
typedef std::map<std::string, std::string> headerpair_t;
typedef std::map<std::string, headerpair_t> addheader_t; typedef struct add_header{
regex_t* pregex; // not NULL means using regex, NULL means comparing suffix directly.
std::string basestring;
std::string headkey;
std::string headvalue;
}ADDHEAD, *PADDHEAD;
typedef std::vector<PADDHEAD> addheadlist_t;
class AdditionalHeader class AdditionalHeader
{ {
private: private:
static AdditionalHeader singleton; static AdditionalHeader singleton;
bool is_enable; bool is_enable;
charcnt_list_t charcntlist; addheadlist_t addheadlist;
addheader_t addheader;
protected:
AdditionalHeader();
~AdditionalHeader();
public: public:
// Reference singleton // Reference singleton
static AdditionalHeader* get(void) { return &singleton; } static AdditionalHeader* get(void) { return &singleton; }
AdditionalHeader();
~AdditionalHeader();
bool Load(const char* file); bool Load(const char* file);
void Unload(void); void Unload(void);

8
src/s3fs_util.cpp
View File

@ -954,9 +954,11 @@ void show_help (void)
" file is the additional HTTP header by file(object) extension.\n" " file is the additional HTTP header by file(object) extension.\n"
" The configuration file format is below:\n" " The configuration file format is below:\n"
" -----------\n" " -----------\n"
" line = [file suffix] HTTP-header [HTTP-values]\n" " line = [file suffix or regex] HTTP-header [HTTP-values]\n"
" file suffix = file(object) suffix, if this field is empty,\n" " file suffix = file(object) suffix, if this field is empty,\n"
" it means \"*\"(all object).\n" " it means \"reg:(.*)\".(=all object).\n"
" regex = regular expression to match the file(object) path.\n"
" this type starts with \"reg:\" prefix.\n"
" HTTP-header = additional HTTP header name\n" " HTTP-header = additional HTTP header name\n"
" HTTP-values = additional HTTP header value\n" " HTTP-values = additional HTTP header value\n"
" -----------\n" " -----------\n"
@ -964,7 +966,7 @@ void show_help (void)
" -----------\n" " -----------\n"
" .gz Content-Encoding gzip\n" " .gz Content-Encoding gzip\n"
" .Z Content-Encoding compress\n" " .Z Content-Encoding compress\n"
" X-S3FS-MYHTTPHEAD myvalue\n" " reg:^/MYDIR/(.*)[.]t2$ Content-Encoding text2\n"
" -----------\n" " -----------\n"
" A sample configuration file is uploaded in \"test\" directory.\n" " A sample configuration file is uploaded in \"test\" directory.\n"
" If you specify this option for set \"Content-Encoding\" HTTP \n" " If you specify this option for set \"Content-Encoding\" HTTP \n"

19
test/sample_ahbe.conf
View File

@ -4,21 +4,27 @@
# s3fs loads this file at starting. # s3fs loads this file at starting.
# #
# Format: # Format:
# line = [file suffix] HTTP-header [HTTP-header-values] # line = [file suffix or regex] HTTP-header [HTTP-header-values]
# file suffix = file(object) suffix, if this field is empty, # file suffix = file(object) suffix, if this field is empty,
# it means "*"(all object). # it means "reg:(.*)".(=all object).
# regex = regular expression to match the file(object) path.
# this type starts with "reg:" prefix.
# HTTP-header = additional HTTP header name # HTTP-header = additional HTTP header name
# HTTP-header-values = additional HTTP header value # HTTP-header-values = additional HTTP header value
# #
# <suffix(extension)> <HTTP header> <HTTP header values> # <suffix(extension)> <HTTP header> <HTTP header values>
# #
# Verification is done in the order in which they are described in the file.
# That order is very important.
#
# Example: # Example:
# " Content-Encoding gzip" --> all object # " Content-Encoding gzip" --> all object
# ".gz Content-Encoding gzip" --> only ".gz" extension file # ".gz Content-Encoding gzip" --> only ".gz" extension file
# "reg:^/DIR/(.*).t2$ Content-Encoding text2" --> "/DIR/*.t2" extension file
# #
# Notice: # Notice:
# If you need to set all object, you can specify without "suffix". # If you need to set all object, you can specify without "suffix" or regex
# Then all of object(file) is added additional header. # type "reg:(.*)". Then all of object(file) is added additional header.
# If you have this configuration file for Content-Encoding, you should # If you have this configuration file for Content-Encoding, you should
# know about RFC 2616. # know about RFC 2616.
# #
@ -27,6 +33,8 @@
# Encoding header, and SHOULD NOT be used in the Content-Encoding # Encoding header, and SHOULD NOT be used in the Content-Encoding
# header." # header."
# #
# file suffix type
.gz Content-Encoding gzip .gz Content-Encoding gzip
.Z Content-Encoding compress .Z Content-Encoding compress
.bz2 Content-Encoding bzip2 .bz2 Content-Encoding bzip2
@ -39,3 +47,6 @@
.tbz2 Content-Encoding gzip .tbz2 Content-Encoding gzip
gz.js Content-Encoding gzip gz.js Content-Encoding gzip
# regex type(test)
reg:^/MYDIR/(.*)[.]t2$ Content-Encoding text2