diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..bf3c10f --- /dev/null +++ b/AUTHORS @@ -0,0 +1,11 @@ +1. Randy Rizun + +Wrote from scratch the initial version of S3FS. + +2. Dan Moore + +Patches and improvements. + +3. Adrian Petrescu + +Converted the project to be autotools-based. diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..d511905 --- /dev/null +++ b/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/ChangeLog b/ChangeLog new file mode 100644 index 0000000..ecf8403 --- /dev/null +++ b/ChangeLog @@ -0,0 +1,10 @@ +ChangeLog for S3FS +------------------ + +Version 1.1 -- Mon Oct 18 2010 + +Dan Moore reopens the project and fixes various issues that had accumulated in the tracker. Adrian Petrescu converts the project to autotools and posts it to GitHub. + +Version 1.0 -- 2008 + +Randy Rizun releases a basic version of S3FS on Google Code. diff --git a/INSTALL b/INSTALL new file mode 100644 index 0000000..7d1c323 --- /dev/null +++ b/INSTALL @@ -0,0 +1,365 @@ +Installation Instructions +************************* + +Copyright (C) 1994, 1995, 1996, 1999, 2000, 2001, 2002, 2004, 2005, +2006, 2007, 2008, 2009 Free Software Foundation, Inc. + + Copying and distribution of this file, with or without modification, +are permitted in any medium without royalty provided the copyright +notice and this notice are preserved. This file is offered as-is, +without warranty of any kind. + +Basic Installation +================== + + Briefly, the shell commands `./configure; make; make install' should +configure, build, and install this package. The following +more-detailed instructions are generic; see the `README' file for +instructions specific to this package. Some packages provide this +`INSTALL' file but do not implement all of the features documented +below. The lack of an optional feature in a given package is not +necessarily a bug. More recommendations for GNU packages can be found +in *note Makefile Conventions: (standards)Makefile Conventions. + + The `configure' shell script attempts to guess correct values for +various system-dependent variables used during compilation. It uses +those values to create a `Makefile' in each directory of the package. +It may also create one or more `.h' files containing system-dependent +definitions. Finally, it creates a shell script `config.status' that +you can run in the future to recreate the current configuration, and a +file `config.log' containing compiler output (useful mainly for +debugging `configure'). + + It can also use an optional file (typically called `config.cache' +and enabled with `--cache-file=config.cache' or simply `-C') that saves +the results of its tests to speed up reconfiguring. Caching is +disabled by default to prevent problems with accidental use of stale +cache files. + + If you need to do unusual things to compile the package, please try +to figure out how `configure' could check whether to do them, and mail +diffs or instructions to the address given in the `README' so they can +be considered for the next release. If you are using the cache, and at +some point `config.cache' contains results you don't want to keep, you +may remove or edit it. + + The file `configure.ac' (or `configure.in') is used to create +`configure' by a program called `autoconf'. You need `configure.ac' if +you want to change it or regenerate `configure' using a newer version +of `autoconf'. + + The simplest way to compile this package is: + + 1. `cd' to the directory containing the package's source code and type + `./configure' to configure the package for your system. + + Running `configure' might take a while. While running, it prints + some messages telling which features it is checking for. + + 2. Type `make' to compile the package. + + 3. Optionally, type `make check' to run any self-tests that come with + the package, generally using the just-built uninstalled binaries. + + 4. Type `make install' to install the programs and any data files and + documentation. When installing into a prefix owned by root, it is + recommended that the package be configured and built as a regular + user, and only the `make install' phase executed with root + privileges. + + 5. Optionally, type `make installcheck' to repeat any self-tests, but + this time using the binaries in their final installed location. + This target does not install anything. Running this target as a + regular user, particularly if the prior `make install' required + root privileges, verifies that the installation completed + correctly. + + 6. You can remove the program binaries and object files from the + source code directory by typing `make clean'. To also remove the + files that `configure' created (so you can compile the package for + a different kind of computer), type `make distclean'. There is + also a `make maintainer-clean' target, but that is intended mainly + for the package's developers. If you use it, you may have to get + all sorts of other programs in order to regenerate files that came + with the distribution. + + 7. Often, you can also type `make uninstall' to remove the installed + files again. In practice, not all packages have tested that + uninstallation works correctly, even though it is required by the + GNU Coding Standards. + + 8. Some packages, particularly those that use Automake, provide `make + distcheck', which can by used by developers to test that all other + targets like `make install' and `make uninstall' work correctly. + This target is generally not run by end users. + +Compilers and Options +===================== + + Some systems require unusual options for compilation or linking that +the `configure' script does not know about. Run `./configure --help' +for details on some of the pertinent environment variables. + + You can give `configure' initial values for configuration parameters +by setting variables in the command line or in the environment. Here +is an example: + + ./configure CC=c99 CFLAGS=-g LIBS=-lposix + + *Note Defining Variables::, for more details. + +Compiling For Multiple Architectures +==================================== + + You can compile the package for more than one kind of computer at the +same time, by placing the object files for each architecture in their +own directory. To do this, you can use GNU `make'. `cd' to the +directory where you want the object files and executables to go and run +the `configure' script. `configure' automatically checks for the +source code in the directory that `configure' is in and in `..'. This +is known as a "VPATH" build. + + With a non-GNU `make', it is safer to compile the package for one +architecture at a time in the source code directory. After you have +installed the package for one architecture, use `make distclean' before +reconfiguring for another architecture. + + On MacOS X 10.5 and later systems, you can create libraries and +executables that work on multiple system types--known as "fat" or +"universal" binaries--by specifying multiple `-arch' options to the +compiler but only a single `-arch' option to the preprocessor. Like +this: + + ./configure CC="gcc -arch i386 -arch x86_64 -arch ppc -arch ppc64" \ + CXX="g++ -arch i386 -arch x86_64 -arch ppc -arch ppc64" \ + CPP="gcc -E" CXXCPP="g++ -E" + + This is not guaranteed to produce working output in all cases, you +may have to build one architecture at a time and combine the results +using the `lipo' tool if you have problems. + +Installation Names +================== + + By default, `make install' installs the package's commands under +`/usr/local/bin', include files under `/usr/local/include', etc. You +can specify an installation prefix other than `/usr/local' by giving +`configure' the option `--prefix=PREFIX', where PREFIX must be an +absolute file name. + + You can specify separate installation prefixes for +architecture-specific files and architecture-independent files. If you +pass the option `--exec-prefix=PREFIX' to `configure', the package uses +PREFIX as the prefix for installing programs and libraries. +Documentation and other data files still use the regular prefix. + + In addition, if you use an unusual directory layout you can give +options like `--bindir=DIR' to specify different values for particular +kinds of files. Run `configure --help' for a list of the directories +you can set and what kinds of files go in them. In general, the +default for these options is expressed in terms of `${prefix}', so that +specifying just `--prefix' will affect all of the other directory +specifications that were not explicitly provided. + + The most portable way to affect installation locations is to pass the +correct locations to `configure'; however, many packages provide one or +both of the following shortcuts of passing variable assignments to the +`make install' command line to change installation locations without +having to reconfigure or recompile. + + The first method involves providing an override variable for each +affected directory. For example, `make install +prefix=/alternate/directory' will choose an alternate location for all +directory configuration variables that were expressed in terms of +`${prefix}'. Any directories that were specified during `configure', +but not in terms of `${prefix}', must each be overridden at install +time for the entire installation to be relocated. The approach of +makefile variable overrides for each directory variable is required by +the GNU Coding Standards, and ideally causes no recompilation. +However, some platforms have known limitations with the semantics of +shared libraries that end up requiring recompilation when using this +method, particularly noticeable in packages that use GNU Libtool. + + The second method involves providing the `DESTDIR' variable. For +example, `make install DESTDIR=/alternate/directory' will prepend +`/alternate/directory' before all installation names. The approach of +`DESTDIR' overrides is not required by the GNU Coding Standards, and +does not work on platforms that have drive letters. On the other hand, +it does better at avoiding recompilation issues, and works well even +when some directory options were not specified in terms of `${prefix}' +at `configure' time. + +Optional Features +================= + + If the package supports it, you can cause programs to be installed +with an extra prefix or suffix on their names by giving `configure' the +option `--program-prefix=PREFIX' or `--program-suffix=SUFFIX'. + + Some packages pay attention to `--enable-FEATURE' options to +`configure', where FEATURE indicates an optional part of the package. +They may also pay attention to `--with-PACKAGE' options, where PACKAGE +is something like `gnu-as' or `x' (for the X Window System). The +`README' should mention any `--enable-' and `--with-' options that the +package recognizes. + + For packages that use the X Window System, `configure' can usually +find the X include and library files automatically, but if it doesn't, +you can use the `configure' options `--x-includes=DIR' and +`--x-libraries=DIR' to specify their locations. + + Some packages offer the ability to configure how verbose the +execution of `make' will be. For these packages, running `./configure +--enable-silent-rules' sets the default to minimal output, which can be +overridden with `make V=1'; while running `./configure +--disable-silent-rules' sets the default to verbose, which can be +overridden with `make V=0'. + +Particular systems +================== + + On HP-UX, the default C compiler is not ANSI C compatible. If GNU +CC is not installed, it is recommended to use the following options in +order to use an ANSI C compiler: + + ./configure CC="cc -Ae -D_XOPEN_SOURCE=500" + +and if that doesn't work, install pre-built binaries of GCC for HP-UX. + + On OSF/1 a.k.a. Tru64, some versions of the default C compiler cannot +parse its `' header file. The option `-nodtk' can be used as +a workaround. If GNU CC is not installed, it is therefore recommended +to try + + ./configure CC="cc" + +and if that doesn't work, try + + ./configure CC="cc -nodtk" + + On Solaris, don't put `/usr/ucb' early in your `PATH'. This +directory contains several dysfunctional programs; working variants of +these programs are available in `/usr/bin'. So, if you need `/usr/ucb' +in your `PATH', put it _after_ `/usr/bin'. + + On Haiku, software installed for all users goes in `/boot/common', +not `/usr/local'. It is recommended to use the following options: + + ./configure --prefix=/boot/common + +Specifying the System Type +========================== + + There may be some features `configure' cannot figure out +automatically, but needs to determine by the type of machine the package +will run on. Usually, assuming the package is built to be run on the +_same_ architectures, `configure' can figure that out, but if it prints +a message saying it cannot guess the machine type, give it the +`--build=TYPE' option. TYPE can either be a short name for the system +type, such as `sun4', or a canonical name which has the form: + + CPU-COMPANY-SYSTEM + +where SYSTEM can have one of these forms: + + OS + KERNEL-OS + + See the file `config.sub' for the possible values of each field. If +`config.sub' isn't included in this package, then this package doesn't +need to know the machine type. + + If you are _building_ compiler tools for cross-compiling, you should +use the option `--target=TYPE' to select the type of system they will +produce code for. + + If you want to _use_ a cross compiler, that generates code for a +platform different from the build platform, you should specify the +"host" platform (i.e., that on which the generated programs will +eventually be run) with `--host=TYPE'. + +Sharing Defaults +================ + + If you want to set default values for `configure' scripts to share, +you can create a site shell script called `config.site' that gives +default values for variables like `CC', `cache_file', and `prefix'. +`configure' looks for `PREFIX/share/config.site' if it exists, then +`PREFIX/etc/config.site' if it exists. Or, you can set the +`CONFIG_SITE' environment variable to the location of the site script. +A warning: not all `configure' scripts look for a site script. + +Defining Variables +================== + + Variables not defined in a site shell script can be set in the +environment passed to `configure'. However, some packages may run +configure again during the build, and the customized values of these +variables may be lost. In order to avoid this problem, you should set +them in the `configure' command line, using `VAR=value'. For example: + + ./configure CC=/usr/local2/bin/gcc + +causes the specified `gcc' to be used as the C compiler (unless it is +overridden in the site shell script). + +Unfortunately, this technique does not work for `CONFIG_SHELL' due to +an Autoconf bug. Until the bug is fixed you can use this workaround: + + CONFIG_SHELL=/bin/bash /bin/bash ./configure CONFIG_SHELL=/bin/bash + +`configure' Invocation +====================== + + `configure' recognizes the following options to control how it +operates. + +`--help' +`-h' + Print a summary of all of the options to `configure', and exit. + +`--help=short' +`--help=recursive' + Print a summary of the options unique to this package's + `configure', and exit. The `short' variant lists options used + only in the top level, while the `recursive' variant lists options + also present in any nested packages. + +`--version' +`-V' + Print the version of Autoconf used to generate the `configure' + script, and exit. + +`--cache-file=FILE' + Enable the cache: use and save the results of the tests in FILE, + traditionally `config.cache'. FILE defaults to `/dev/null' to + disable caching. + +`--config-cache' +`-C' + Alias for `--cache-file=config.cache'. + +`--quiet' +`--silent' +`-q' + Do not print messages saying which checks are being made. To + suppress all normal output, redirect it to `/dev/null' (any error + messages will still be shown). + +`--srcdir=DIR' + Look for the package's source code in directory DIR. Usually + `configure' can determine that directory automatically. + +`--prefix=DIR' + Use DIR as the installation prefix. *note Installation Names:: + for more details, including other options available for fine-tuning + the installation locations. + +`--no-create' +`-n' + Run the configure checks, but stop before creating any output + files. + +`configure' also accepts some other, not widely useful, options. Run +`configure --help' for more details. + diff --git a/Makefile.am b/Makefile.am new file mode 100644 index 0000000..94c4523 --- /dev/null +++ b/Makefile.am @@ -0,0 +1 @@ +SUBDIRS=src test diff --git a/NEWS b/NEWS new file mode 100644 index 0000000..e69de29 diff --git a/README b/README new file mode 100644 index 0000000..d474dcd --- /dev/null +++ b/README @@ -0,0 +1,61 @@ +S3FS-Fuse + +S3FS is FUSE (File System in User Space) based solution to mount/unmount an Amazon S3 storage buckets and use system commands with S3 just like it was another Hard Disk. + +In order to compile s3fs, You'll need the following requirements: + +* Kernel-devel packages (or kernel source) installed that is the SAME version of your running kernel +* LibXML2-devel packages +* CURL-devel packages (or compile curl from sources at: curl.haxx.se/ use 7.15.X) +* GCC, GCC-C++ +* pkgconfig +* FUSE (2.7.x) +* FUSE Kernel module installed and running (RHEL 4.x/CentOS 4.x users - read below) +* OpenSSL-devel (0.9.8) +* Subversion + +If you're using YUM or APT to install those packages, then it might require additional packaging, allow it to be installed. + +Downloading & Compiling: +------------------------ +In order to download s3fs, user the following command: +svn checkout http://s3fs.googlecode.com/svn/trunk/ s3fs-read-only + +Go inside the directory that has been created (s3fs-read-only/s3fs) and run: ./autogen.sh +This will generate a number of scripts in the project directory, including a configure script which you should run with: ./configure +If configure succeeded, you can now run: make. If it didn't, make sure you meet the dependencies above. +This should compile the code. If everything goes OK, you'll be greated with "ok!" at the end and you'll have a binary file called "s3fs" +in the src/ directory. + +As root (you can use su, su -, sudo) do: "make install" -this will copy the "s3fs" binary to /usr/bin. + +Congratulations. S3fs is now compiled and installed. + +Usage: +------ +In order to use s3fs, make sure you have the Access Key and the Secret Key handy. +First, create a directory where to mount the S3 bucket you want to use. +Example (as root): mkdir -p /mnt/s3 +Then run: s3fs mybucket -o accessKeyId=aaa -o secretAccessKey=bbb /mnt/s3 + +This will mount your bucket to /mnt/s3. You can do a simple "ls -l /mnt/s3" to see the content of your bucket. + +If you want to allow other people access the same bucket in the same machine, you can add "-o allow _other" to read/write/delete content of the bucket. + +You can add a fixed mount point in /etc/fstab, here's an example: + +s3fs#mybucket /mnt/s3 fuse allow_other,accessKeyId=XXX ,secretAccessKey=YYY 0 0 + +This will mount upon reboot (or by launching: mount -a) your bucket on your machine. + +All other options can be read at: http://code.google.com/p/s3fs/wiki/FuseOverAmazon + +Known Issues: +------------- +s3fs should be working fine with S3 storage. However, There are couple of limitations: + +* There is no full UID/GID support yet, everything looks as "root" and if you allow others to access the bucket, others can erase files. There is, however, permissions support built in. +* Currently s3fs could hang the CPU if you have lots of time-outs. This is *NOT* a fault of s3fs but rather libcurl. This happends when you try to copy thousands of files in 1 session, it doesn't happend when you upload hundreds of files or less. +* CentOS 4.x/RHEL 4.x users - if you use the kernel that shipped with your distribution and didn't upgrade to the latest kernel RedHat/CentOS gives, you might have a problem loading the "fuse" kernel. Please upgrade to the latest kernel (2.6.16 or above) and make sure "fuse" kernel module is compiled and loadable since FUSE requires this kernel module and s3fs requires it as well. +* Moving/renaming/erasing files takes time since the whole file needs to be accessed first. A workaround could be to use s3fs's cache support with the use_cache option. + diff --git a/autogen.sh b/autogen.sh new file mode 100755 index 0000000..b1e9c84 --- /dev/null +++ b/autogen.sh @@ -0,0 +1,24 @@ +#! /bin/sh + +# This file is part of S3FS. +# +# Copyright 2009, 2010 Free Software Foundation, Inc. +# +# S3FS is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or (at +# your option) any later version. +# +# S3FS is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see http://www.gnu.org/licenses/. +# +# See the file ChangeLog for a revision history. + +aclocal \ +&& automake --add-missing \ +&& autoconf diff --git a/configure.ac b/configure.ac new file mode 100644 index 0000000..830daa0 --- /dev/null +++ b/configure.ac @@ -0,0 +1,16 @@ +dnl Process this file with autoconf to produce a configure script. + +AC_PREREQ(2.59) +AC_INIT(s3fs, 1.12) + + +AC_CANONICAL_SYSTEM +AM_INIT_AUTOMAKE() + +AC_PROG_CXX + +PKG_CHECK_MODULES([DEPS], [fuse >= 2.7 libcurl >= 7.0 libxml-2.0 >= 2.6 libcrypto >= 0.9]) + +AC_CONFIG_FILES(Makefile src/Makefile test/Makefile) +AC_OUTPUT + diff --git a/src/Makefile.am b/src/Makefile.am new file mode 100644 index 0000000..224324e --- /dev/null +++ b/src/Makefile.am @@ -0,0 +1,7 @@ +bin_PROGRAMS=s3fs + +AM_CPPFLAGS = $(DEPS_CFLAGS) + +s3fs_SOURCES = s3fs.cpp s3fs.h string_util.cpp string_util.h +s3fs_LDADD = $(DEPS_LIBS) + diff --git a/src/s3fs.cpp b/src/s3fs.cpp new file mode 100644 index 0000000..265a2ff --- /dev/null +++ b/src/s3fs.cpp @@ -0,0 +1,2192 @@ +/* + * s3fs - FUSE-based file system backed by Amazon S3 + * + * Copyright 2007-2008 Randy Rizun + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "s3fs.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "string_util.h" + +using namespace std; + +class auto_fd { + public: + auto_fd(int fd): fd(fd) { } + ~auto_fd() { + close(fd); + } + + int get() { + return fd; + } + + private: + int fd; +}; + +class auto_lock { + public: + auto_lock(pthread_mutex_t& lock) : lock(lock) { + pthread_mutex_lock(&lock); + } + ~auto_lock() { + pthread_mutex_unlock(&lock); + } + + private: + pthread_mutex_t& lock; +}; + +// homegrown timeout mechanism +static int my_curl_progress( + void *clientp, double dltotal, double dlnow, double ultotal, double ulnow) { + CURL* curl = static_cast(clientp); + + time_t now = time(0); + progress_t p(dlnow, ulnow); + + //###cout << "/dlnow=" << dlnow << "/ulnow=" << ulnow << endl; + + auto_lock lock(curl_handles_lock); + + // any progress? + if (p != curl_progress[curl]) { + // yes! + curl_times[curl] = now; + curl_progress[curl] = p; + } else { + // timeout? + if (now - curl_times[curl] > readwrite_timeout) + return CURLE_ABORTED_BY_CALLBACK; + } + + return 0; +} + +static CURL* alloc_curl_handle() { + CURL* curl; + auto_lock lock(curl_handles_lock); + if (curl_handles.size() == 0) { + curl = curl_easy_init(); + } else { + curl = curl_handles.top(); + curl_handles.pop(); + } + curl_easy_reset(curl); + long signal = 1; + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, signal); + +// long timeout = 3600; +// curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout); + + //###long seconds = 10; + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, connect_timeout); + + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0); + curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, my_curl_progress); + curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, curl); + time_t now = time(0); + curl_times[curl] = now; + curl_progress[curl] = progress_t(-1, -1); + return curl; +} + +static void return_curl_handle(CURL* curl_handle) { + if (curl_handle != 0) { + auto_lock lock(curl_handles_lock); + curl_handles.push(curl_handle); + curl_times.erase(curl_handle); + curl_progress.erase(curl_handle); + } +} + +class auto_curl { + public: + auto_curl() : curl_handle(alloc_curl_handle()) { } + +// auto_curl(CURL* curl): curl(curl) { +//// auto_lock lock(curl_handles_lock); +//// if (curl_handles.size() == 0) +//// curl = curl_easy_init(); +//// else { +//// curl = curl_handles.top(); +//// curl_handles.pop(); +//// } +//// curl_easy_reset(curl); +//// long seconds = 10; +//// //###curl_easy_setopt(curl, CURLOPT_TIMEOUT, seconds); // bad idea +//// curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, seconds); +// } + ~auto_curl() { + if (curl_handle != 0) { + return_curl_handle(curl_handle); +// auto_lock lock(curl_handles_lock); +// curl_handles.push(curl); + } + } + + CURL* get() const { return curl_handle; } +// CURL* release() { +// CURL* tmp = curl; +// curl = 0; +// return tmp; +// } +// void reset(CURL* curl) { +// if (curl != 0) { +// auto_lock lock(curl_handles_lock); +// curl_handles.push(curl); +// } +// this->curl = curl; +// } + operator CURL*() const { return curl_handle; } + + private: + CURL* curl_handle; +}; + +struct curl_multi_remove_handle_functor { + CURLM* multi_handle; + curl_multi_remove_handle_functor(CURLM* multi_handle) : multi_handle(multi_handle) { } + + void operator()(CURL* curl_handle) { + curl_multi_remove_handle(multi_handle, curl_handle); + return_curl_handle(curl_handle); + } +}; + +class auto_curl_multi { + public: + auto_curl_multi(): multi_handle(curl_multi_init()) { } + ~auto_curl_multi() { + curl_multi_cleanup(for_each(curl_handles.begin(), curl_handles.end(), + curl_multi_remove_handle_functor(multi_handle)).multi_handle); + } + + CURLM* get() const { return multi_handle; } + + void add_curl(CURL* curl_handle) { + curl_handles.push_back(curl_handle); + curl_multi_add_handle(multi_handle, curl_handle); + } + + private: + CURLM* multi_handle; + vector curl_handles; +}; + +class auto_curl_slist { + public: + auto_curl_slist() : slist(0) { } + ~auto_curl_slist() { curl_slist_free_all(slist); } + + struct curl_slist* get() const { return slist; } + + void append(const string& s) { + slist = curl_slist_append(slist, s.c_str()); + } + + private: + struct curl_slist* slist; +}; + +static string prepare_url(const char* url) { + if(debug) syslog(LOG_DEBUG, "URL is %s", url); + + string url_str = str(url); + string token = str("/" + bucket); + int bucket_pos = url_str.find(token); + int bucket_size = token.size(); + + int clipBy = 7; + if(!strncasecmp(url_str.c_str(), "https://", 8)) { + clipBy = 8; + } + url_str = url_str.substr(0, clipBy) + bucket + "." + url_str.substr(clipBy, bucket_pos - clipBy) + + url_str.substr((bucket_pos + bucket_size)); + + if(debug) syslog(LOG_DEBUG, "URL changed is %s", url_str.c_str()); + + return str(url_str); +} + +/** + * @return fuse return code + */ +static int my_curl_easy_perform(CURL* curl, FILE* f = 0) { + char* url = new char[128]; + curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL , &url); + if(debug) syslog(LOG_DEBUG, "connecting to URL %s", url); + + // 1 attempt + retries... + int t = retries + 1; + while (t-- > 0) { + if (f) + rewind(f); + CURLcode curlCode = curl_easy_perform(curl); + if (curlCode == 0) + return 0; + if (curlCode == CURLE_OPERATION_TIMEDOUT) { + syslog(LOG_ERR, "###timeout"); + } else if (curlCode == CURLE_HTTP_RETURNED_ERROR) { + long responseCode; + if (curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &responseCode) != 0) + return -EIO; + if (responseCode == 404) + return -ENOENT; + syslog(LOG_ERR, "###response=%ld", responseCode); + + if (responseCode < 500) + return -EIO; + } else { + syslog(LOG_ERR, "###%s", curl_easy_strerror(curlCode));; + } + syslog(LOG_ERR, "###retrying..."); + } + syslog(LOG_ERR, "###giving up"); + return -EIO; +} + +/** + * urlEncode a fuse path, + * taking into special consideration "/", + * otherwise regular urlEncode. + */ +string urlEncode(const string &s) { + string result; + for (unsigned i = 0; i < s.length(); ++i) { + if (s[i] == '/') // Note- special case for fuse paths... + result += s[i]; + else if (isalnum(s[i])) + result += s[i]; + else if (s[i] == '.' || s[i] == '-' || s[i] == '*' || s[i] == '_') + result += s[i]; + else if (s[i] == ' ') + result += '+'; + else { + result += "%"; + result += hexAlphabet[static_cast(s[i]) / 16]; + result += hexAlphabet[static_cast(s[i]) % 16]; + } + } + return result; +} + +/** + * Returns the current date + * in a format suitable for a HTTP request header. + */ +string get_date() { + char buf[100]; + time_t t = time(NULL); + strftime(buf, sizeof(buf), "%a, %d %b %Y %H:%M:%S GMT", gmtime(&t)); + return buf; +} + +/** + * Returns the Amazon AWS signature for the given parameters. + * + * @param method e.g., "GET" + * @param content_type e.g., "application/x-directory" + * @param date e.g., get_date() + * @param resource e.g., "/pub" + */ +string calc_signature( + string method, string content_type, string date, curl_slist* headers, string resource) { + + string Signature; + string StringToSign; + StringToSign += method + "\n"; + StringToSign += "\n"; // md5 + StringToSign += content_type + "\n"; + StringToSign += date + "\n"; + int count = 0; + if (headers != 0) { + do { + //###cout << headers->data << endl; + if (strncmp(headers->data, "x-amz", 5) == 0) { + ++count; + StringToSign += headers->data; + StringToSign += 10; // linefeed + } + } while ((headers = headers->next) != 0); + } + StringToSign += resource; + const void* key = AWSSecretAccessKey.data(); + int key_len = AWSSecretAccessKey.size(); + const unsigned char* d = reinterpret_cast(StringToSign.data()); + int n = StringToSign.size(); + unsigned int md_len; + unsigned char md[EVP_MAX_MD_SIZE]; + + HMAC(evp_md, key, key_len, d, n, md, &md_len); + + BIO* b64 = BIO_new(BIO_f_base64()); + BIO* bmem = BIO_new(BIO_s_mem()); + b64 = BIO_push(b64, bmem); + BIO_write(b64, md, md_len); + BIO_flush(b64); + BUF_MEM *bptr; + BIO_get_mem_ptr(b64, &bptr); + + Signature.resize(bptr->length - 1); + memcpy(&Signature[0], bptr->data, bptr->length-1); + + BIO_free_all(b64); + + return Signature; +} + +// libcurl callback +static size_t writeCallback(void* data, size_t blockSize, size_t numBlocks, void* userPtr) { + string* userString = static_cast(userPtr); + (*userString).append(reinterpret_cast(data), blockSize*numBlocks); + return blockSize * numBlocks; +} + +static size_t header_callback(void *data, size_t blockSize, size_t numBlocks, void *userPtr) { + headers_t* headers = reinterpret_cast(userPtr); + string header(reinterpret_cast(data), blockSize * numBlocks); + string key; + stringstream ss(header); + if (getline(ss, key, ':')) { + string value; + getline(ss, value); + (*headers)[key] = trim(value); + } + return blockSize * numBlocks; +} + +// safe variant of dirname +static string mydirname(string path) { + // dirname clobbers path so let it operate on a tmp copy + return dirname(&path[0]); +} + +// safe variant of basename +static string mybasename(string path) { + // basename clobbers path so let it operate on a tmp copy + return basename(&path[0]); +} + +// mkdir --parents +static int mkdirp(const string& path, mode_t mode) { + string base; + string component; + stringstream ss(path); + while (getline(ss, component, '/')) { + base += "/" + component; + /*if (*/mkdir(base.c_str(), mode)/* == -1); + return -1*/; + } + return 0; +} + +/** + * @return fuse return code + * TODO return pair?!? + */ +int get_headers(const char* path, headers_t& meta) { + + string resource(urlEncode(service_path + bucket + path)); + string url(host + resource); + + auto_curl curl; + curl_easy_setopt(curl, CURLOPT_FAILONERROR, true); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true); + curl_easy_setopt(curl, CURLOPT_NOBODY, true); // HEAD + curl_easy_setopt(curl, CURLOPT_FILETIME, true); // Last-Modified + + headers_t responseHeaders; + curl_easy_setopt(curl, CURLOPT_HEADERDATA, &responseHeaders); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback); + + auto_curl_slist headers; + string date = get_date(); + headers.append("Date: " + date); + headers.append("Content-Type: "); + if (public_bucket.substr(0,1) != "1") { + headers.append("Authorization: AWS " + AWSAccessKeyId + ":" + + calc_signature("HEAD", "", date, headers.get(), resource)); + } + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get()); + + string my_url = prepare_url(url.c_str()); + curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str()); + + VERIFY(my_curl_easy_perform((curl.get()))); + + // at this point we know the file exists in s3 + + for (headers_t::iterator iter = responseHeaders.begin(); iter != responseHeaders.end(); ++iter) { + string key = (*iter).first; + string value = (*iter).second; + if (key == "Content-Type") + meta[key] = value; + if (key == "ETag") + meta[key] = value; + if (key.substr(0, 5) == "x-amz") + meta[key] = value; + } + + return 0; +} + +/** + * get_local_fd + */ +int get_local_fd(const char* path) { + string resource(urlEncode(service_path + bucket + path)); + string url(host + resource); + + string baseName = mybasename(path); + string resolved_path(use_cache + "/" + bucket); + + int fd = -1; + + string cache_path(resolved_path + path); + + headers_t responseHeaders; + + if (use_cache.size() > 0) { + VERIFY(get_headers(path, responseHeaders)); + + fd = open(cache_path.c_str(), O_RDWR); // ### TODO should really somehow obey flags here + + if (fd != -1) { + MD5_CTX c; + if (MD5_Init(&c) != 1) + Yikes(-EIO); + int count; + char buf[1024]; + while ((count = read(fd, buf, sizeof(buf))) > 0) { + if (MD5_Update(&c, buf, count) != 1) + Yikes(-EIO); + } + unsigned char md[MD5_DIGEST_LENGTH]; + if (MD5_Final(md, &c) != 1) + Yikes(-EIO); + + char localMd5[2 * MD5_DIGEST_LENGTH+1]; + sprintf(localMd5, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", + md[0], md[1], md[2], md[3], md[4], md[5], md[6], md[7], md[8], md[9], md[10], md[11], + md[12], md[13], md[14], md[15]); + + string remoteMd5(trim(responseHeaders["ETag"], "\"")); + + // md5 match? + if (string(localMd5) != remoteMd5) { + // no! prepare to download + if (close(fd) == -1) + Yikes(-errno); + fd = -1; + } + } + } + // need to download? + if (fd == -1) { + // yes! + if (use_cache.size() > 0) { + // only download files, not folders + mode_t mode = strtoul(responseHeaders["x-amz-meta-mode"].c_str(), (char **)NULL, 10); + if (S_ISREG(mode)) { + /*if (*/mkdirp(resolved_path + mydirname(path), 0777)/* == -1) + return -errno*/; + fd = open(cache_path.c_str(), O_CREAT|O_RDWR|O_TRUNC, mode); + } else { + // its a folder; do *not* create anything in local cache... (###TODO do this in a better way) + fd = fileno(tmpfile()); + } + } else { + fd = fileno(tmpfile()); + } + + if (fd == -1) + Yikes(-errno); + + auto_curl curl; + curl_easy_setopt(curl, CURLOPT_FAILONERROR, true); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true); + + FILE* f = fdopen(fd, "w+"); + if (f == 0) + Yikes(-errno); + curl_easy_setopt(curl, CURLOPT_FILE, f); + + auto_curl_slist headers; + string date = get_date(); + syslog(LOG_INFO, "LOCAL FD"); + headers.append("Date: " + date); + headers.append("Content-Type: "); + if (public_bucket.substr(0,1) != "1") { + headers.append("Authorization: AWS " + AWSAccessKeyId + ":" + + calc_signature("GET", "", date, headers.get(), resource)); + } + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get()); + + cout << "downloading[path=" << path << "][fd=" << fd << "]" << endl; + + string my_url = prepare_url(url.c_str()); + curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str()); + + VERIFY(my_curl_easy_perform(curl.get(), f)); + + //only one of these is needed... + fflush(f); + fsync(fd); + + if (fd == -1) + Yikes(-errno); + } + + return fd; +} + +/** + * create or update s3 meta + * @return fuse return code + */ +static int put_headers(const char* path, headers_t meta) { + string resource = urlEncode(service_path + bucket + path); + string url = host + resource; + + auto_curl curl; + curl_easy_setopt(curl, CURLOPT_FAILONERROR, true); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true); + + string responseText; + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &responseText); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback); + + curl_easy_setopt(curl, CURLOPT_UPLOAD, true); // HTTP PUT + curl_easy_setopt(curl, CURLOPT_INFILESIZE, 0); // Content-Length + + string ContentType = meta["Content-Type"]; + + auto_curl_slist headers; + string date = get_date(); + headers.append("Date: " + date); + + meta["x-amz-acl"] = default_acl; + + for (headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter) { + string key = (*iter).first; + string value = (*iter).second; + if (key == "Content-Type") + headers.append(key + ":" + value); + if (key.substr(0,9) == "x-amz-acl") + headers.append(key + ":" + value); + if (key.substr(0,10) == "x-amz-meta") + headers.append(key + ":" + value); + if (key == "x-amz-copy-source") + headers.append(key + ":" + value); + } + + if (use_rrs.substr(0,1) == "1") { + headers.append("x-amz-storage-class:REDUCED_REDUNDANCY"); + } + + if (public_bucket.substr(0,1) != "1") { + headers.append("Authorization: AWS " + AWSAccessKeyId + ":" + + calc_signature("PUT", ContentType, date, headers.get(), resource)); + } + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get()); + + //###rewind(f); + + syslog(LOG_INFO, "copy path=%s", path); + cout << "copying[path=" << path << "]" << endl; + + string my_url = prepare_url(url.c_str()); + curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str()); + + VERIFY(my_curl_easy_perform(curl.get())); + + return 0; +} + +/** + * create or update s3 object + * @return fuse return code + */ +static int put_local_fd(const char* path, headers_t meta, int fd) { + string resource = urlEncode(service_path + bucket + path); + string url = host + resource; + + struct stat st; + if (fstat(fd, &st) == -1) + Yikes(-errno); + + auto_curl curl; + curl_easy_setopt(curl, CURLOPT_FAILONERROR, true); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true); + + string responseText; + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &responseText); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback); + + curl_easy_setopt(curl, CURLOPT_UPLOAD, true); // HTTP PUT + curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, static_cast(st.st_size)); // Content-Length + + FILE* f = fdopen(fd, "rb"); + if (f == 0) + Yikes(-errno); + curl_easy_setopt(curl, CURLOPT_INFILE, f); + + string ContentType = meta["Content-Type"]; + + auto_curl_slist headers; + string date = get_date(); + headers.append("Date: " + date); + + meta["x-amz-acl"] = default_acl; + + for (headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter) { + string key = (*iter).first; + string value = (*iter).second; + if (key == "Content-Type") + headers.append(key + ":" + value); + if (key.substr(0,9) == "x-amz-acl") + headers.append(key + ":" + value); + if (key.substr(0,10) == "x-amz-meta") + headers.append(key + ":" + value); + } + + if (use_rrs.substr(0,1) == "1") { + headers.append("x-amz-storage-class:REDUCED_REDUNDANCY"); + } + + if (public_bucket.substr(0,1) != "1") { + headers.append("Authorization: AWS " + AWSAccessKeyId + ":" + + calc_signature("PUT", ContentType, date, headers.get(), resource)); + } + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get()); + + //###rewind(f); + + syslog(LOG_INFO, "upload path=%s size=%llu", path, st.st_size); + cout << "uploading[path=" << path << "][fd=" << fd << "][size="<st_nlink = 1; // see fuse faq + stbuf->st_mode = root_mode | S_IFDIR; + return 0; + } + + { + auto_lock lock(stat_cache_lock); + stat_cache_t::iterator iter = stat_cache.find(path); + if (iter != stat_cache.end()) { + *stbuf = (*iter).second; + stat_cache.erase(path); + return 0; + } + } + + string resource = urlEncode(service_path + bucket + path); + string url = host + resource; + + auto_curl curl; + curl_easy_setopt(curl, CURLOPT_FAILONERROR, true); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true); + curl_easy_setopt(curl, CURLOPT_NOBODY, true); // HEAD + curl_easy_setopt(curl, CURLOPT_FILETIME, true); // Last-Modified + + headers_t responseHeaders; + curl_easy_setopt(curl, CURLOPT_HEADERDATA, &responseHeaders); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback); + + auto_curl_slist headers; + string date = get_date(); + headers.append("Date: " + date); + headers.append("Content-Type: "); + if (public_bucket.substr(0,1) != "1") { + headers.append("Authorization: AWS " + AWSAccessKeyId + ":" + + calc_signature("HEAD", "", date, headers.get(), resource)); + } + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get()); + string my_url = prepare_url(url.c_str()); + curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str()); + + VERIFY(my_curl_easy_perform(curl.get())); + + stbuf->st_nlink = 1; // see fuse faq + + stbuf->st_mtime = strtoul(responseHeaders["x-amz-meta-mtime"].c_str(), (char **)NULL, 10); + if (stbuf->st_mtime == 0) { + long LastModified; + if (curl_easy_getinfo(curl, CURLINFO_FILETIME, &LastModified) == 0) + stbuf->st_mtime = LastModified; + } + + stbuf->st_mode = strtoul(responseHeaders["x-amz-meta-mode"].c_str(), (char **)NULL, 10); + char* ContentType = 0; + if (curl_easy_getinfo(curl, CURLINFO_CONTENT_TYPE, &ContentType) == 0) { + if (ContentType) + stbuf->st_mode |= strcmp(ContentType, "application/x-directory") == 0 ? S_IFDIR : S_IFREG; + } + + double ContentLength; + if (curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &ContentLength) == 0) + stbuf->st_size = static_cast(ContentLength); + + if (S_ISREG(stbuf->st_mode)) + stbuf->st_blocks = stbuf->st_size / 512 + 1; + + stbuf->st_uid = strtoul(responseHeaders["x-amz-meta-uid"].c_str(), (char **)NULL, 10); + stbuf->st_gid = strtoul(responseHeaders["x-amz-meta-gid"].c_str(), (char **)NULL, 10); + + return 0; +} + +static int s3fs_readlink(const char *path, char *buf, size_t size) { + if (size > 0) { + --size; // reserve nil terminator + + cout << "readlink[path=" << path << "]" << endl; + + auto_fd fd(get_local_fd(path)); + + struct stat st; + if (fstat(fd.get(), &st) == -1) + Yikes(-errno); + + if (st.st_size < size) + size = st.st_size; + + if (pread(fd.get(), buf, size, 0) == -1) + Yikes(-errno); + + buf[size] = 0; + } + + return 0; +} + +struct case_insensitive_compare_func { + bool operator ()(const string &a, const string &b) { + return strcasecmp(a.c_str(), b.c_str()) < 0; + } +}; + +typedef map mimes_t; + +static mimes_t mimeTypes; + +/** + * @param s e.g., "index.html" + * @return e.g., "text/html" + */ +string lookupMimeType(string s) { + string result("application/octet-stream"); + string::size_type last_pos = s.find_last_of('.'); + string::size_type first_pos = s.find_first_of('.'); + string prefix, ext, ext2; + + // No dots in name, just return + if (last_pos == string::npos) { + return result; + } + + // extract the last extension + if (last_pos != string::npos) { + ext = s.substr(1+last_pos, string::npos); + } + + + if (last_pos != string::npos) { + // one dot was found, now look for another + if (first_pos != string::npos && first_pos < last_pos) { + prefix = s.substr(0, last_pos); + // Now get the second to last file extension + string::size_type next_pos = prefix.find_last_of('.'); + if (next_pos != string::npos) { + ext2 = prefix.substr(1+next_pos, string::npos); + } + } + } + + // if we get here, then we have an extension (ext) + mimes_t::const_iterator iter = mimeTypes.find(ext); + // if the last extension matches a mimeType, then return + // that mime type + if (iter != mimeTypes.end()) { + result = (*iter).second; + return result; + } + + // return with the default result if there isn't a second extension + if (first_pos == last_pos) { + return result; + } + + // Didn't find a mime-type for the first extension + // Look for second extension in mimeTypes, return if found + iter = mimeTypes.find(ext2); + if (iter != mimeTypes.end()) { + result = (*iter).second; + return result; + } + + // neither the last extension nor the second-to-last extension + // matched a mimeType, return the default mime type + return result; +} + +static int s3fs_mknod(const char *path, mode_t mode, dev_t rdev) { + // see man 2 mknod + // If pathname already exists, or is a symbolic link, this call fails with an EEXIST error. + cout << "mknod[path=" << path << "][mode=" << mode << "]" << endl; + + string resource = urlEncode(service_path + bucket + path); + string url = host + resource; + + auto_curl curl; + curl_easy_setopt(curl, CURLOPT_FAILONERROR, true); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true); + curl_easy_setopt(curl, CURLOPT_UPLOAD, true); // HTTP PUT + curl_easy_setopt(curl, CURLOPT_INFILESIZE, 0); // Content-Length: 0 + + auto_curl_slist headers; + string date = get_date(); + headers.append("Date: " + date); + string contentType(lookupMimeType(path)); + headers.append("Content-Type: " + contentType); + // x-amz headers: (a) alphabetical order and (b) no spaces after colon + headers.append("x-amz-acl:" + default_acl); + headers.append("x-amz-meta-gid:" + str(getgid())); + headers.append("x-amz-meta-mode:" + str(mode)); + headers.append("x-amz-meta-mtime:" + str(time(NULL))); + headers.append("x-amz-meta-uid:" + str(getuid())); + if (public_bucket.substr(0,1) != "1") { + headers.append("Authorization: AWS " + AWSAccessKeyId + ":" + + calc_signature("PUT", contentType, date, headers.get(), resource)); + } + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get()); + + string my_url = prepare_url(url.c_str()); + curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str()); + + VERIFY(my_curl_easy_perform(curl.get())); + + return 0; +} + +static int s3fs_mkdir(const char *path, mode_t mode) { + cout << "mkdir[path=" << path << "][mode=" << mode << "]" << endl; + + string resource = urlEncode(service_path + bucket + path); + string url = host + resource; + + auto_curl curl; + curl_easy_setopt(curl, CURLOPT_FAILONERROR, true); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true); + curl_easy_setopt(curl, CURLOPT_UPLOAD, true); // HTTP PUT + curl_easy_setopt(curl, CURLOPT_INFILESIZE, 0); // Content-Length: 0 + + auto_curl_slist headers; + string date = get_date(); + headers.append("Date: " + date); + headers.append("Content-Type: application/x-directory"); + // x-amz headers: (a) alphabetical order and (b) no spaces after colon + headers.append("x-amz-acl:" + default_acl); + headers.append("x-amz-meta-gid:" + str(getgid())); + headers.append("x-amz-meta-mode:" + str(mode)); + headers.append("x-amz-meta-mtime:" + str(time(NULL))); + headers.append("x-amz-meta-uid:" + str(getuid())); + if (public_bucket.substr(0,1) != "1") { + headers.append("Authorization: AWS " + AWSAccessKeyId + ":" + + calc_signature("PUT", "application/x-directory", date, headers.get(), resource)); + } + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get()); + + string my_url = prepare_url(url.c_str()); + curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str()); + + VERIFY(my_curl_easy_perform(curl.get())); + + return 0; +} + +// aka rm +static int s3fs_unlink(const char *path) { + cout << "unlink[path=" << path << "]" << endl; + + string resource = urlEncode(service_path + bucket + path); + string url = host + resource; + + auto_curl curl; + curl_easy_setopt(curl, CURLOPT_FAILONERROR, true); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true); + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); + + auto_curl_slist headers; + string date = get_date(); + headers.append("Date: " + date); + headers.append("Content-Type: "); + if (public_bucket.substr(0,1) != "1") { + headers.append("Authorization: AWS " + AWSAccessKeyId + ":" + + calc_signature("DELETE", "", date, headers.get(), resource)); + } + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get()); + + string my_url = prepare_url(url.c_str()); + curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str()); + + VERIFY(my_curl_easy_perform(curl.get())); + + return 0; +} + +static int s3fs_rmdir(const char *path) { + cout << "rmdir[path=" << path << "]" << endl; + + // need to check if the directory is empty + { + string responseText; + string resource = urlEncode(service_path + bucket); + string query = "delimiter=/&prefix="; + + if (strcmp(path, "/") != 0) + query += urlEncode(string(path).substr(1) + "/"); + + query += "&max-keys=50"; + + string url = host + resource + "?"+ query; + + auto_curl curl; + string my_url = prepare_url(url.c_str()); + curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str()); + curl_easy_setopt(curl, CURLOPT_FAILONERROR, true); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &responseText); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback); + + auto_curl_slist headers; + string date = get_date(); + headers.append("Date: " + date); + headers.append("ContentType: "); + if (public_bucket.substr(0,1) != "1") { + headers.append("Authorization: AWS " + AWSAccessKeyId + ":" + + calc_signature("GET", "", date, headers.get(), resource + "/")); + } + + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get()); + + VERIFY(my_curl_easy_perform(curl.get())); + + cout << endl << responseText << endl; + if (responseText.find ("") != std::string::npos || + responseText.find ("") != std::string::npos ) { + // directory is not empty + cout << "[path=" << path << "] not empty" << endl; + return -ENOTEMPTY; + } + } + // delete the directory + string resource = urlEncode(service_path + bucket + path); + string url = host + resource; + + auto_curl curl; + curl_easy_setopt(curl, CURLOPT_FAILONERROR, true); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true); + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); + + auto_curl_slist headers; + string date = get_date(); + headers.append("Date: " + date); + headers.append("Content-Type: "); + if (public_bucket.substr(0,1) != "1") { + headers.append("Authorization: AWS " + AWSAccessKeyId + ":" + + calc_signature("DELETE", "", date, headers.get(), resource)); + } + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get()); + + string my_url = prepare_url(url.c_str()); + curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str()); + + VERIFY(my_curl_easy_perform(curl.get())); + + return 0; +} + +static int s3fs_symlink(const char *from, const char *to) { + cout << "symlink[from=" << from << "][to=" << to << "]" << endl; + + headers_t headers; + headers["x-amz-meta-mode"] = str(S_IFLNK); + headers["x-amz-meta-mtime"] = str(time(NULL)); + + auto_fd fd(fileno(tmpfile())); + + if (pwrite(fd.get(), from, strlen(from), 0) == -1) + Yikes(-errno); + + VERIFY(put_local_fd(to, headers, fd.get())); + + return 0; +} + +static int s3fs_rename(const char *from, const char *to) { + cout << "rename[from=" << from << "][to=" << to << "]" << endl; + + // preserve meta headers across rename + headers_t meta; + VERIFY(get_headers(from, meta)); + + meta["x-amz-copy-source"] = urlEncode("/" + bucket + from); + + meta["Content-Type"] = lookupMimeType(to); + meta["x-amz-metadata-directive"] = "REPLACE"; + + int result = put_headers(to, meta); + if (result != 0) + return result; + + return s3fs_unlink(from); +} + +static int s3fs_link(const char *from, const char *to) { + cout << "link[from=" << from << "][to=" << to << "]" << endl; + return -EPERM; +} + +static int s3fs_chmod(const char *path, mode_t mode) { + cout << "chmod[path=" << path << "][mode=" << mode << "]" << endl; + headers_t meta; + VERIFY(get_headers(path, meta)); + meta["x-amz-meta-mode"] = str(mode); + meta["x-amz-copy-source"] = urlEncode("/" + bucket + path); + meta["x-amz-metadata-directive"] = "REPLACE"; + return put_headers(path, meta); +} + + +static int s3fs_chown(const char *path, uid_t uid, gid_t gid) { + cout << "chown[path=" << path << "]" << endl; + + headers_t meta; + VERIFY(get_headers(path, meta)); + + struct passwd* aaa = getpwuid(uid); + if (aaa != 0) + meta["x-amz-meta-uid"] = str((*aaa).pw_uid); + + struct group* bbb = getgrgid(gid); + if (bbb != 0) + meta["x-amz-meta-gid"] = str((*bbb).gr_gid); + + meta["x-amz-copy-source"] = urlEncode("/" + bucket + path); + meta["x-amz-metadata-directive"] = "REPLACE"; + return put_headers(path, meta); +} + +static int s3fs_truncate(const char *path, off_t size) { + //###TODO honor size?!? + + cout << "truncate[path=" << path << "][size=" << size << "]" << endl; + + // preserve headers across truncate + headers_t meta; + VERIFY(get_headers(path, meta)); + auto_fd fd(fileno(tmpfile())); + //###verify fd here?!? + VERIFY(put_local_fd(path, meta, fd.get())); + + return 0; +} + +static int s3fs_open(const char *path, struct fuse_file_info *fi) { + cout << "open[path=" << path << "][flags=" << fi->flags << "]" << endl; + + headers_t meta; + //###TODO check fi->fh here... + fi->fh = get_local_fd(path); + + // remember flags and headers... + auto_lock lock(s3fs_descriptors_lock); + + s3fs_descriptors[fi->fh] = fi->flags; + + return 0; +} + +static int s3fs_read( + const char *path, char *buf, size_t size, off_t offset, struct fuse_file_info *fi) { + //###cout << "read: " << path << endl; + int res = pread(fi->fh, buf, size, offset); + if (res == -1) + Yikes(-errno); + return res; +} + +static int s3fs_write( + const char *path, const char *buf, size_t size, off_t offset, struct fuse_file_info *fi) { + + //###cout << "write: " << path << endl; + int res = pwrite(fi->fh, buf, size, offset); + if (res == -1) + Yikes(-errno); + return res; +} + +static int s3fs_statfs(const char *path, struct statvfs *stbuf) { + // 256T + stbuf->f_bsize = 0X1000000; + stbuf->f_blocks = 0X1000000; + stbuf->f_bfree = 0x1000000; + stbuf->f_bavail = 0x1000000; + return 0; +} + +static int get_flags(int fd) { + auto_lock lock(s3fs_descriptors_lock); + return s3fs_descriptors[fd]; +} + +static int s3fs_flush(const char *path, struct fuse_file_info *fi) { + int fd = fi->fh; + cout << "flush[path=" << path << "][fd=" << fd << "]" << endl; + // NOTE- fi->flags is not available here + int flags = get_flags(fd); + if ((flags & O_RDWR) || (flags & O_WRONLY)) { + headers_t meta; + VERIFY(get_headers(path, meta)); + meta["x-amz-meta-mtime"] = str(time(NULL)); + return put_local_fd(path, meta, fd); + } + return 0; +} + +static int s3fs_release(const char *path, struct fuse_file_info *fi) { + int fd = fi->fh; + cout << "release[path=" << path << "][fd=" << fd << "]" << endl; + if (close(fd) == -1) + Yikes(-errno); + return 0; +} + +time_t my_timegm (struct tm *tm) { + time_t ret; + char *tz; + + tz = getenv("TZ"); + setenv("TZ", "", 1); + tzset(); + ret = mktime(tm); + if (tz) + setenv("TZ", tz, 1); + else + unsetenv("TZ"); + tzset(); + return ret; +} + +// All this "stuff" stuff is kinda ugly... it works though... needs cleanup +struct stuff_t { + // default ctor works + string path; + string* url; + struct curl_slist* requestHeaders; + headers_t* responseHeaders; +}; +typedef map stuffMap_t; + +struct cleanup_stuff { + void operator()(pair qqq) { + stuff_t stuff = qqq.second; + delete stuff.url; + curl_slist_free_all(stuff.requestHeaders); + delete stuff.responseHeaders; + } +}; + +class auto_stuff { + public: + auto_stuff() { } + ~auto_stuff() { + for_each(stuffMap.begin(), stuffMap.end(), cleanup_stuff()); + } + + stuffMap_t& get() { return stuffMap; } + +private: + stuffMap_t stuffMap; +}; + +static int s3fs_readdir( + const char *path, void *buf, fuse_fill_dir_t filler, off_t offset, struct fuse_file_info *fi) { + //cout << "readdir:"<< " path="<< path << endl; + + string NextMarker; + string IsTruncated("true"); + + while (IsTruncated == "true") { + string responseText; + string resource = urlEncode(service_path + bucket); // this is what gets signed + string query = "delimiter=/&prefix="; + + if (strcmp(path, "/") != 0) + query += urlEncode(string(path).substr(1) + "/"); + + if (NextMarker.size() > 0) + query += "&marker=" + urlEncode(NextMarker); + + query += "&max-keys=50"; + + string url = host + resource + "?" + query; + + { + auto_curl curl; + string my_url = prepare_url(url.c_str()); + + curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str()); + curl_easy_setopt(curl, CURLOPT_FAILONERROR, true); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &responseText); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback); + +// headers_t responseHeaders; +// curl_easy_setopt(curl, CURLOPT_HEADERDATA, &responseHeaders); +// curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback); + + auto_curl_slist headers; + string date = get_date(); + headers.append("Date: " + date); + headers.append("ContentType: "); + if (public_bucket.substr(0,1) != "1") { + headers.append("Authorization: AWS " + AWSAccessKeyId + ":" + + calc_signature("GET", "", date, headers.get(), resource + "/")); + } + + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get()); + + + VERIFY(my_curl_easy_perform(curl.get())); + } + + auto_stuff curlMap; + auto_curl_multi multi_handle; + +// long max_connects = 5; +// curl_multi_setopt(multi_handle.get(), CURLMOPT_MAXCONNECTS, max_connects); + + { + xmlDocPtr doc = xmlReadMemory(responseText.c_str(), responseText.size(), "", NULL, 0); + if (doc != NULL && doc->children != NULL) { + for (xmlNodePtr cur_node = doc->children->children; + cur_node != NULL; + cur_node = cur_node->next) { + + string cur_node_name(reinterpret_cast(cur_node->name)); + if (cur_node_name == "IsTruncated") + IsTruncated = reinterpret_cast(cur_node->children->content); + if (cur_node_name == "NextMarker") + NextMarker = reinterpret_cast(cur_node->children->content); + if (cur_node_name == "Contents") { + if (cur_node->children != NULL) { + string Key; + string LastModified; + string Size; + for (xmlNodePtr sub_node = cur_node->children; + sub_node != NULL; + sub_node = sub_node->next) { + + if (sub_node->type == XML_ELEMENT_NODE) { + string elementName = reinterpret_cast(sub_node->name); + if (sub_node->children != NULL) { + if (sub_node->children->type == XML_TEXT_NODE) { + if (elementName == "Key") + Key = reinterpret_cast(sub_node->children->content); + if (elementName == "LastModified") + LastModified = reinterpret_cast(sub_node->children->content); + if (elementName == "Size") + Size = reinterpret_cast(sub_node->children->content); + } + } + } + } + + if (Key.size() > 0) { + if (filler(buf, mybasename(Key).c_str(), 0, 0)) + break; + + CURL* curl_handle = alloc_curl_handle(); + + string resource = urlEncode(service_path + bucket + "/" + Key); + string url = host + resource; + + stuff_t stuff; + stuff.path = "/"+Key; + + // libcurl 7.17 does deep copy of url... e.g., fc7 has libcurl 7.16... therefore, must deep copy "stable" url... + string my_url = prepare_url(url.c_str()); + stuff.url = new string(my_url.c_str()); + stuff.requestHeaders = 0; + stuff.responseHeaders = new headers_t; + + curl_easy_setopt(curl_handle, CURLOPT_URL, stuff.url->c_str()); + curl_easy_setopt(curl_handle, CURLOPT_FAILONERROR, true); + curl_easy_setopt(curl_handle, CURLOPT_FOLLOWLOCATION, true); + curl_easy_setopt(curl_handle, CURLOPT_NOBODY, true); // HEAD + curl_easy_setopt(curl_handle, CURLOPT_FILETIME, true); // Last-Modified + + // requestHeaders + string date = get_date(); + stuff.requestHeaders = curl_slist_append( + stuff.requestHeaders, string("Date: " + date).c_str()); + stuff.requestHeaders = curl_slist_append( + stuff.requestHeaders, string("Content-Type: ").c_str()); + if (public_bucket.substr(0,1) != "1") { + stuff.requestHeaders = curl_slist_append( + stuff.requestHeaders, string("Authorization: AWS " + AWSAccessKeyId + ":" + + calc_signature("HEAD", "", date, stuff.requestHeaders, resource)).c_str()); + } + curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, stuff.requestHeaders); + + // responseHeaders + curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, stuff.responseHeaders); + curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, header_callback); + + curlMap.get()[curl_handle] = stuff; + multi_handle.add_curl(curl_handle); + } + } + } + } + } + xmlFreeDoc(doc); + } + + int running_handles; + + while (curl_multi_perform(multi_handle.get(), &running_handles) == CURLM_CALL_MULTI_PERFORM); + + while (running_handles) { + fd_set read_fd_set; + fd_set write_fd_set; + fd_set exc_fd_set; + + FD_ZERO(&read_fd_set); + FD_ZERO(&write_fd_set); + FD_ZERO(&exc_fd_set); + + long milliseconds; + VERIFY(curl_multi_timeout(multi_handle.get(), &milliseconds)); + if (milliseconds < 0) + milliseconds = 50; + if (milliseconds > 0) { + struct timeval timeout; + timeout.tv_sec = 1000 * milliseconds / 1000000; + timeout.tv_usec = 1000 * milliseconds % 1000000; + + int max_fd; + VERIFY(curl_multi_fdset( + multi_handle.get(), &read_fd_set, &write_fd_set, &exc_fd_set, &max_fd)); + + if (select(max_fd + 1, &read_fd_set, &write_fd_set, &exc_fd_set, &timeout) == -1) + Yikes(-errno); + } + + while (curl_multi_perform(multi_handle.get(), &running_handles) == CURLM_CALL_MULTI_PERFORM); + } + + int remaining_msgs = 1; + while (remaining_msgs) { + // this next line pegs cpu for directories w/lotsa files + CURLMsg* msg = curl_multi_info_read(multi_handle.get(), &remaining_msgs); + if (msg != NULL) { + CURLcode code =msg->data.result; + if (code != 0) + syslog(LOG_ERR, "###%d %s", code, curl_easy_strerror(code)); + if (code == 0) { + CURL* curl_handle = msg->easy_handle; + stuff_t stuff = curlMap.get()[curl_handle]; + + struct stat st; + memset(&st, 0, sizeof(st)); + st.st_nlink = 1; // see fuse faq + // mode + st.st_mode = strtoul( + (*stuff.responseHeaders)["x-amz-meta-mode"].c_str(), (char **)NULL, 10); + char* ContentType = 0; + if (curl_easy_getinfo(curl_handle, CURLINFO_CONTENT_TYPE, &ContentType) == 0) { + if (ContentType) + st.st_mode |= strcmp(ContentType, "application/x-directory") == 0 ? S_IFDIR : S_IFREG; + } + // mtime + st.st_mtime = strtoul + ((*stuff.responseHeaders)["x-amz-meta-mtime"].c_str(), (char **)NULL, 10); + if (st.st_mtime == 0) { + long LastModified; + if (curl_easy_getinfo(curl_handle, CURLINFO_FILETIME, &LastModified) == 0) + st.st_mtime = LastModified; + } + // size + double ContentLength; + if (curl_easy_getinfo(curl_handle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &ContentLength) == 0) + st.st_size = static_cast(ContentLength); + // blocks + if (S_ISREG(st.st_mode)) + st.st_blocks = st.st_size / 512 + 1; + + st.st_uid = strtoul((*stuff.responseHeaders)["x-amz-meta-uid"].c_str(), (char **)NULL, 10); + st.st_gid = strtoul((*stuff.responseHeaders)["x-amz-meta-gid"].c_str(), (char **)NULL, 10); + + auto_lock lock(stat_cache_lock); + stat_cache[stuff.path] = st; + } + } + } + + } // IsTruncated + + return 0; +} + +/** + * OpenSSL locking function. + * + * @param mode lock mode + * @param n lock number + * @param file source file name + * @param line source file line number + * @return none + */ +static void locking_function(int mode, int n, const char *file, int line) { + if (mode & CRYPTO_LOCK) { + pthread_mutex_lock(&mutex_buf[n]); + } else { + pthread_mutex_unlock(&mutex_buf[n]); + } +} + +/** + * OpenSSL uniq id function. + * + * @return thread id + */ +static unsigned long id_function(void) +{ + return ((unsigned long) pthread_self()); +} + +static void* s3fs_init(struct fuse_conn_info *conn) { + syslog(LOG_INFO, "init $Rev$"); + // openssl + mutex_buf = static_cast(malloc(CRYPTO_num_locks() * sizeof(pthread_mutex_t))); + for (int i = 0; i < CRYPTO_num_locks(); i++) + pthread_mutex_init(&mutex_buf[i], NULL); + CRYPTO_set_locking_callback(locking_function); + CRYPTO_set_id_callback(id_function); + curl_global_init(CURL_GLOBAL_ALL); + pthread_mutex_init(&curl_handles_lock, NULL); + pthread_mutex_init(&s3fs_descriptors_lock, NULL); + pthread_mutex_init(&stat_cache_lock, NULL); + // + string line; + ifstream passwd("/etc/mime.types"); + while (getline(passwd, line)) { + if (line[0]=='#') + continue; + stringstream tmp(line); + string mimeType; + tmp >> mimeType; + while (tmp) { + string ext; + tmp >> ext; + if (ext.size() == 0) + continue; + mimeTypes[ext] = mimeType; + } + } + return 0; +} + +static void s3fs_destroy(void*) { + syslog(LOG_INFO, "destroy"); + // openssl + CRYPTO_set_id_callback(NULL); + CRYPTO_set_locking_callback(NULL); + for (int i = 0; i < CRYPTO_num_locks(); i++) + pthread_mutex_destroy(&mutex_buf[i]); + free(mutex_buf); + mutex_buf = NULL; + curl_global_cleanup(); + pthread_mutex_destroy(&curl_handles_lock); + pthread_mutex_destroy(&s3fs_descriptors_lock); + pthread_mutex_destroy(&stat_cache_lock); +} + +static int s3fs_access(const char *path, int mask) { + //###cout << "###access[path=" << path << "]" << endl; + return 0; +} + +// aka touch +static int s3fs_utimens(const char *path, const struct timespec ts[2]) { + cout << "utimens[path=" << path << "][mtime=" << str(ts[1].tv_sec) << "]" << endl; + headers_t meta; + VERIFY(get_headers(path, meta)); + meta["x-amz-meta-mtime"] = str(ts[1].tv_sec); + meta["x-amz-copy-source"] = urlEncode("/" + bucket + path); + meta["x-amz-metadata-directive"] = "REPLACE"; + return put_headers(path, meta); +} + +////////////////////////////////////////////////////////////////// +// check_passwd_file_perms +// +// expect that global passwd_file variable contains +// a non-empty value and is readable by the current user +// +// Check for too permissive access to the file +// help save users from themselves via a security hole +// +// only two options: return or error out +////////////////////////////////////////////////////////////////// +static void check_passwd_file_perms (void) { + + struct stat info; + + // let's get the file info + if (stat(passwd_file.c_str(), &info) != 0) { + fprintf (stderr, "%s: unexpected error from stat(%s, ) \n", + program_name.c_str(), passwd_file.c_str()); + exit(1); + } + + // return error if any file has others permissions + if ((info.st_mode & S_IROTH) || + (info.st_mode & S_IWOTH) || + (info.st_mode & S_IXOTH)) { + fprintf (stderr, "%s: credentials file %s should not have others permissions\n", + program_name.c_str(), passwd_file.c_str()); + exit(1); + } + + // Any local file should not have any group permissions + if (passwd_file != "/etc/passwd-s3fs") { + if ((info.st_mode & S_IRGRP) || + (info.st_mode & S_IWGRP) || + (info.st_mode & S_IXGRP)) { + fprintf (stderr, "%s: credentials file %s should not have group permissions\n", + program_name.c_str(), passwd_file.c_str()); + exit(1); + } + } + + // check for owner execute permissions? + + // /etc/passwd-s3fs can have group permissions + + return; +} + +////////////////////////////////////////////////////////////////// +// read_passwd_file +// +// Support for per bucket credentials +// +// Format for the credentials file: +// [bucket:]AccessKeyId:SecretAccessKey +// +// Lines beginning with # are considered comments +// and ignored, as are empty lines +// +// Uncommented lines without the ":" character are flagged as +// an error, so are lines with spaces or tabs +// +// only one default key pair is allowed, but not required +////////////////////////////////////////////////////////////////// +static void read_passwd_file (void) { + string line; + string field1, field2, field3; + size_t first_pos = string::npos; + size_t last_pos = string::npos; + bool default_found = 0; + + // if you got here, the password file + // exists and is readable by the + // current user, check for permissions + check_passwd_file_perms(); + + ifstream PF(passwd_file.c_str()); + if (PF.good()) { + while (getline(PF, line)) { + if (line[0]=='#') { + continue; + } + if (line.size() == 0) { + continue; + } + + first_pos = line.find_first_of(" \t"); + if (first_pos != string::npos) { + printf ("%s: invalid line in passwd file, found whitespace character\n", + program_name.c_str()); + exit(1); + } + + first_pos = line.find_first_of(":"); + if (first_pos == string::npos) { + printf ("%s: invalid line in passwd file, no \":\" separator found\n", + program_name.c_str()); + exit(1); + } + last_pos = line.find_last_of(":"); + + if (first_pos != last_pos) { + // bucket specified + field1 = line.substr(0,first_pos); + field2 = line.substr(first_pos + 1, last_pos - first_pos - 1); + field3 = line.substr(last_pos + 1, string::npos); + } else { + // no bucket specified - original style - found default key + if (default_found == 1) { + printf ("%s: more than one default key pair found in passwd file\n", + program_name.c_str()); + exit(1); + } + default_found = 1; + field1.assign(""); + field2 = line.substr(0,first_pos); + field3 = line.substr(first_pos + 1, string::npos); + AWSAccessKeyId = field2; + AWSSecretAccessKey = field3; + } + + // does the bucket we are mounting match this passwd file entry? + // if so, use that key pair, otherwise use the default key, if found, + // will be used + if (field1.size() != 0 && field1 == bucket) { + AWSAccessKeyId = field2; + AWSSecretAccessKey = field3; + break; + } + } + } + return; +} + +///////////////////////////////////////////////////////////// +// get_access_keys +// +// called only when were are not mounting a +// public bucket +// +// Here is the order precedence for getting the +// keys: +// +// 1 - from the command line (security risk) +// 2 - from a password file specified on the command line +// 3 - from environment variables +// 4 - from the users ~/.passwd-s3fs +// 5 - from /etc/passwd-s3fs +///////////////////////////////////////////////////////////// +static void get_access_keys (void) { + + // should be redundant + if (public_bucket.substr(0,1) == "1") { + return; + } + + // 1 - keys specified on the command line + if (AWSAccessKeyId.size() > 0 && AWSSecretAccessKey.size() > 0) { + return; + } + + // 2 - was specified on the command line + if (passwd_file.size() > 0) { + ifstream PF(passwd_file.c_str()); + if (PF.good()) { + PF.close(); + read_passwd_file(); + return; + } else { + fprintf(stderr, "%s: specified passwd_file is not readable\n", + program_name.c_str()); + exit(1); + } + } + + // 3 - environment variables + char * AWSACCESSKEYID; + char * AWSSECRETACCESSKEY; + + AWSACCESSKEYID = getenv("AWSACCESSKEYID"); + AWSSECRETACCESSKEY = getenv("AWSSECRETACCESSKEY"); + if (AWSACCESSKEYID != NULL || AWSSECRETACCESSKEY != NULL) { + if ((AWSACCESSKEYID == NULL && AWSSECRETACCESSKEY != NULL) || + (AWSACCESSKEYID != NULL && AWSSECRETACCESSKEY == NULL) ){ + + fprintf(stderr, "%s: if environment variable AWSACCESSKEYID is set then AWSSECRETACCESSKEY must be set too\n", + program_name.c_str()); + exit(1); + } + AWSAccessKeyId.assign(AWSACCESSKEYID); + AWSSecretAccessKey.assign(AWSSECRETACCESSKEY); + return; + } + + // 4 - from the default location in the users home directory + char * HOME; + HOME = getenv ("HOME"); + if (HOME != NULL) { + passwd_file.assign(HOME); + passwd_file.append("/.passwd-s3fs"); + ifstream PF(passwd_file.c_str()); + if (PF.good()) { + PF.close(); + read_passwd_file(); + // It is possible that the user's file was there but + // contained no key pairs i.e. commented out + // in that case, go look in the final location + if (AWSAccessKeyId.size() > 0 && AWSSecretAccessKey.size() > 0) { + return; + } + } + } + + // 5 - from the system default location + passwd_file.assign("/etc/passwd-s3fs"); + ifstream PF(passwd_file.c_str()); + if (PF.good()) { + PF.close(); + read_passwd_file(); + return; + } + + fprintf(stderr, "%s: could not determine how to establish security credentials\n", + program_name.c_str()); + exit(1); +} + +static void show_usage (void) { + printf("Usage: %s BUCKET MOUNTPOINT [OPTION]...\n", + program_name.c_str()); +} + +static void show_help (void) { + show_usage(); + printf( + "\n" + "Mount an Amazon S3 bucket as a file system.\n" + "\n" + " General forms for s3fs and FUSE/mount options:\n" + " -o opt[,opt...]\n" + " -o opt [-o opt] ...\n" + "\n" + "s3fs Options:\n" + "\n" + " All s3fs options must given in the form where \"opt\" is:\n" + "\n" + " =\n" + "\n" + " accessKeyId\n" + " secretAccessKey\n" + " - command line over-rides of these settings\n" + "\n" + " default_acl (default=\"private\")\n" + " - the default canned acl to apply to all written s3 objects\n" + " see http://aws.amazon.com/documentation/s3/ for the \n" + " full list of canned acls\n" + "\n" + " retries (default=\"2\")\n" + " - number of times to retry a failed s3 transaction\n" + "\n" + " use_cache (default=\"\" which means disabled)\n" + " - local folder to use for local file cache\n" + "\n" + " use_rrs (default=\"\" which means diabled)\n" + " - use Amazon's Reduced Redundancy Storage when set to 1\n" + "\n" + " public_bucket (default=\"\" which means disabled)\n" + " - anonymously mount a public bucket when set to 1\n" + "\n" + " passwd_file (default=\"\")\n" + " - specify which s3fs password file to use\n" + "\n" + " connect_timeout (default=\"2\" seconds)\n" + " - time to wait for connection before giving up\n" + "\n" + " readwrite_timeout (default=\"10\" seconds)\n" + " - time to wait between read/write activity before giving up\n" + "\n" + " url (default=\"http://s3.amazonaws.com\")\n" + " - sets the url to use to access amazon s3\n" + "\n" + "FUSE/mount Options:\n" + "\n" + " Most of the generic mount options described in 'man mount' are\n" + " supported (ro, rw, suid, nosuid, dev, nodev, exec, noexec, atime,\n" + " noatime, sync async, dirsync). Filesystems are mounted with\n" + " '-onodev,nosuid' by default, which can only be overridden by a\n" + " privileged user.\n" + " \n" + " There are many FUSE specific mount options that can be specified.\n" + " e.g. allow_other See the FUSE's README for the full set.\n" + "\n" + "Miscellaneous Options:\n" + "\n" + " -h, --help Output this help.\n" + " --version Output version info.\n" + " -d --debug Turn on DEBUG messages to syslog. Specifying -d\n" + " twice turns on FUSE debug messages to STDOUT.\n" + "\n" + "\n" + "Report bugs to \n" + "s3fs home page: \n" + ); + exit(0); +} + +static void show_version(void) { + printf( + "Amazon Simple Storage Service File System %s\n" + "Copyright (C) 2010 Randy Rizun \n" + "License GPL2: GNU GPL version 2 \n" + "This is free software: you are free to change and redistribute it.\n" + "There is NO WARRANTY, to the extent permitted by law.\n", + VERSION ); + exit(0); +} + +// This function gets called repeatedly by the +// fuse option parser +static int my_fuse_opt_proc(void *data, const char *arg, int key, struct fuse_args *outargs) { + + if (key == FUSE_OPT_KEY_NONOPT) { + // tricky way to set the bucket name + // the first plain option is assumed to be + // the bucket + if (bucket.size() == 0) { + bucket = arg; + return 0; + } else { + struct stat buf; + if (mountpoint.size() != 0) { + fprintf(stderr, "%s: argument MOUNTPOINT %s was all ready provided, %s is an invalid option\n", + program_name.c_str(), mountpoint.c_str(), arg); + show_usage(); + exit(1); + } + // record the mountpoint + mountpoint = arg; + // it is the mountpoint...do some error checking, fuse will do more for us later + if (stat(arg, &buf) != -1) { + int isempty = 1; + // record mode for later usage + root_mode = buf.st_mode; + + if (!(S_ISDIR( buf.st_mode ))) { + fprintf(stderr, "%s: MOUNTPOINT: %s is not a directory\n", + program_name.c_str(), mountpoint.c_str()); + exit(1); + } else { + struct dirent *ent; + DIR *dp = opendir(mountpoint.c_str()); + if (dp == NULL) { + fprintf(stderr, "%s: failed to open MOUNTPOINT: %s for reading, error: %s\n", + program_name.c_str(), mountpoint.c_str(), strerror(errno)); + exit(1); + } + while ((ent = readdir(dp)) != NULL) { + if (strcmp(ent->d_name, ".") != 0 && strcmp(ent->d_name, "..") != 0) { + isempty = 0; + break; + } + } + closedir(dp); + if (!isempty) { + fprintf(stderr, "%s: MOUNTPOINT directory %s is not empty\n", + program_name.c_str(), mountpoint.c_str()); + exit(1); + } + } + } else { + fprintf(stderr, "%s: accessing MOUNTPOINT %s had an error: %s\n", + program_name.c_str(), mountpoint.c_str(), strerror(errno)); + exit(1); + } + } + } + + if (key == FUSE_OPT_KEY_OPT) { + if (strstr(arg, "accessKeyId=") != 0) { + AWSAccessKeyId = strchr(arg, '=') + 1; + return 0; + } + if (strstr(arg, "secretAccessKey=") != 0) { + AWSSecretAccessKey = strchr(arg, '=') + 1; + return 0; + } + if (strstr(arg, "default_acl=") != 0) { + default_acl = strchr(arg, '=') + 1; + return 0; + } + // ### TODO: prefix + if (strstr(arg, "retries=") != 0) { + retries = atoi(strchr(arg, '=') + 1); + return 0; + } + if (strstr(arg, "use_cache=") != 0) { + use_cache = strchr(arg, '=') + 1; + return 0; + } + if (strstr(arg, "use_rrs=") != 0) { + use_rrs = strchr(arg, '=') + 1; + if (strcmp(use_rrs.c_str(), "1") == 0 || + strcmp(use_rrs.c_str(), "") == 0 ) { + return 0; + } else { + fprintf(stderr, "%s: poorly formed argument to option: use_rrs\n", + program_name.c_str()); + exit(1); + } + } + if (strstr(arg, "passwd_file=") != 0) { + passwd_file = strchr(arg, '=') + 1; + return 0; + } + if (strstr(arg, "public_bucket=") != 0) { + public_bucket = strchr(arg, '=') + 1; + if (strcmp(public_bucket.c_str(), "1") == 0 || + strcmp(public_bucket.c_str(), "") == 0 ) { + return 0; + } else { + fprintf(stderr, "%s: poorly formed argument to option: public_bucket\n", + program_name.c_str()); + exit(1); + } + } + if (strstr(arg, "host=") != 0) { + host = strchr(arg, '=') + 1; + return 0; + } + if (strstr(arg, "servicepath=") != 0) { + service_path = strchr(arg, '=') + 1; + return 0; + } + if (strstr(arg, "connect_timeout=") != 0) { + connect_timeout = strtol(strchr(arg, '=') + 1, 0, 10); + return 0; + } + if (strstr(arg, "readwrite_timeout=") != 0) { + readwrite_timeout = strtoul(strchr(arg, '=') + 1, 0, 10); + return 0; + } + if (strstr(arg, "url=") != 0) { + host = strchr(arg, '=') + 1; + return 0; + } + // debug option + // + // The first -d (or --debug) enables s3fs debug + // the second -d option is passed to fuse to turn on its + // debug output + if ( (strcmp(arg, "-d") == 0) || (strcmp(arg, "--debug") == 0) ) { + if (!debug) { + debug = 1; + return 0; + } else { + // fuse doesn't understand "--debug", but it + // understands -d, but we can't pass -d back + // to fuse, in this case just ignore the + // second --debug if is was provided. If we + // do not ignore this, fuse emits an error + if(strcmp(arg, "--debug") == 0) { + return 0; + } + } + } + } + return 1; +} + + + +int main(int argc, char *argv[]) { + + int ch; + int option_index = 0; + + static const struct option long_opts[] = { + {"help", no_argument, NULL, 'h'}, + {"version", no_argument, 0, 0}, + {"debug", no_argument, NULL, 'd'}, + {0, 0, 0, 0}}; + + // get progam name - emulate basename + size_t found; + program_name.assign(argv[0]); + found = program_name.find_last_of("/"); + if(found != string::npos) { + program_name.replace(0, found+1, ""); + } + + while ((ch = getopt_long(argc, argv, "dho:", long_opts, &option_index)) != -1) { + switch (ch) { + case 0: + if (strcmp(long_opts[option_index].name, "version") == 0) { + show_version(); + } + break; + + case 'h': + show_help(); + break; + + case 'o': + break; + + case 'd': + break; + + default: + exit(1); + } + } + + // clear this structure + memset(&s3fs_oper, 0, sizeof(s3fs_oper)); + + // This is the fuse-style parser for the arguments + // after which the bucket name and mountpoint names + // should have been set + struct fuse_args custom_args = FUSE_ARGS_INIT(argc, argv); + fuse_opt_parse(&custom_args, NULL, NULL, my_fuse_opt_proc); + + // The first plain argument is the bucket + if (bucket.size() == 0) { + fprintf(stderr, "%s: missing BUCKET argument\n", program_name.c_str()); + show_usage(); + exit(1); + } + + // bucket names cannot contain upper case characters + if (lower(bucket) != bucket) { + fprintf(stderr, "%s: BUCKET %s, upper case characters are not supported\n", + program_name.c_str(), bucket.c_str()); + exit(1); + } + + // The second plain argument is the mountpoint + // if the option was given, we all ready checked for a + // readable, non-empty directory, this checks determines + // if the mountpoint option was ever supplied + if (mountpoint.size() == 0) { + fprintf(stderr, "%s: missing MOUNTPOINT argument\n", program_name.c_str()); + show_usage(); + exit(1); + } + + // error checking of command line arguments for compatability + if ((AWSSecretAccessKey.size() > 0 && AWSAccessKeyId.size() == 0) || + (AWSSecretAccessKey.size() == 0 && AWSAccessKeyId.size() > 0)) { + fprintf(stderr, "%s: if one access key is specified, both keys need to be specified\n", + program_name.c_str()); + exit(1); + } + + if (public_bucket.substr(0,1) == "1" && + (AWSSecretAccessKey.size() > 0 || AWSAccessKeyId.size() > 0)) { + fprintf(stderr, "%s: specifying both public_bucket and the access keys options is invalid\n", + program_name.c_str()); + exit(1); + } + + if (passwd_file.size() > 0 && + (AWSSecretAccessKey.size() > 0 || AWSAccessKeyId.size() > 0)) { + fprintf(stderr, "%s: specifying both passwd_file and the access keys options is invalid\n", + program_name.c_str()); + exit(1); + } + + if (public_bucket.substr(0,1) != "1") { + get_access_keys(); + if(AWSSecretAccessKey.size() == 0 || AWSAccessKeyId.size() == 0) { + fprintf(stderr, "%s: could not establish security credentials, check documentation\n", + program_name.c_str()); + exit(1); + } + // More error checking on the access key pair can be done + // like checking for appropriate lengths and characters + } + + // There's room for more command line error checking + + s3fs_oper.getattr = s3fs_getattr; + s3fs_oper.readlink = s3fs_readlink; + s3fs_oper.mknod = s3fs_mknod; + s3fs_oper.mkdir = s3fs_mkdir; + s3fs_oper.unlink = s3fs_unlink; + s3fs_oper.rmdir = s3fs_rmdir; + s3fs_oper.symlink = s3fs_symlink; + s3fs_oper.rename = s3fs_rename; + s3fs_oper.link = s3fs_link; + s3fs_oper.chmod = s3fs_chmod; + s3fs_oper.chown = s3fs_chown; + s3fs_oper.truncate = s3fs_truncate; + s3fs_oper.open = s3fs_open; + s3fs_oper.read = s3fs_read; + s3fs_oper.write = s3fs_write; + s3fs_oper.statfs = s3fs_statfs; + s3fs_oper.flush = s3fs_flush; + s3fs_oper.release = s3fs_release; + s3fs_oper.readdir = s3fs_readdir; + s3fs_oper.init = s3fs_init; + s3fs_oper.destroy = s3fs_destroy; + s3fs_oper.access = s3fs_access; + s3fs_oper.utimens = s3fs_utimens; + + // now passing things off to fuse, fuse will finish evaluating the command line args + return fuse_main(custom_args.argc, custom_args.argv, &s3fs_oper, NULL); +} diff --git a/src/s3fs.h b/src/s3fs.h new file mode 100644 index 0000000..284c890 --- /dev/null +++ b/src/s3fs.h @@ -0,0 +1,116 @@ +#ifndef S3FS_S3_H_ +#define S3FS_S3_H_ + +#define FUSE_USE_VERSION 26 + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +using namespace std; + +#define VERIFY(s) if (true) { \ + int result = (s); \ + if (result != 0) \ + return result; \ +} + +#define Yikes(result) if (true) { \ + syslog(LOG_ERR, "%d###result=%d", __LINE__, result); \ + return result; \ +} + +typedef pair progress_t; + +static long connect_timeout = 2; +static time_t readwrite_timeout = 10; + +static stack curl_handles; +static pthread_mutex_t curl_handles_lock; +static map curl_times; +static map curl_progress; + +static int retries = 2; + +static string bucket; +static string mountpoint; +static string program_name; +static string AWSAccessKeyId; +static string AWSSecretAccessKey; +static string host = "http://s3.amazonaws.com"; +static mode_t root_mode = 0; +static string service_path = "/"; +static string passwd_file = ""; +static bool debug = 0; + +// if .size()==0 then local file cache is disabled +static string use_cache; +static string use_rrs; +static string public_bucket; + +// TODO(apetresc): make this an enum +// private, public-read, public-read-write, authenticated-read +static string default_acl("private"); + +// key=path +typedef map stat_cache_t; +static stat_cache_t stat_cache; +static pthread_mutex_t stat_cache_lock; + +static const char hexAlphabet[] = "0123456789ABCDEF"; + +// http headers +typedef map headers_t; + +static const EVP_MD* evp_md = EVP_sha1(); + +// fd -> flags +typedef map s3fs_descriptors_t; +static s3fs_descriptors_t s3fs_descriptors; +static pthread_mutex_t s3fs_descriptors_lock; + +static pthread_mutex_t *mutex_buf = NULL; + +static struct fuse_operations s3fs_oper; + +string urlEncode(const string &s); + +static int s3fs_getattr(const char *path, struct stat *stbuf); +static int s3fs_readlink(const char *path, char *buf, size_t size); +static int s3fs_mknod(const char* path, mode_t mode, dev_t rdev); +static int s3fs_mkdir(const char *path, mode_t mode); +static int s3fs_unlink(const char *path); +static int s3fs_rmdir(const char *path); +static int s3fs_symlink(const char *from, const char *to); +static int s3fs_rename(const char *from, const char *to); +static int s3fs_link(const char *from, const char *to); +static int s3fs_chmod(const char *path, mode_t mode); +static int s3fs_chown(const char *path, uid_t uid, gid_t gid); +static int s3fs_truncate(const char *path, off_t size); +static int s3fs_open(const char *path, struct fuse_file_info *fi); +static int s3fs_read( + const char *path, char *buf, size_t size, off_t offset, struct fuse_file_info *fi); +static int s3fs_write( + const char *path, const char *buf, size_t size, off_t offset, struct fuse_file_info *fi); +static int s3fs_statfs(const char *path, struct statvfs *stbuf); +static int s3fs_flush(const char *path, struct fuse_file_info *fi); +static int s3fs_release(const char *path, struct fuse_file_info *fi); +static int s3fs_readdir( + const char *path, void *buf, fuse_fill_dir_t filler, off_t offset, struct fuse_file_info *fi); +static int s3fs_access(const char *path, int mask); +static int s3fs_utimens(const char *path, const struct timespec ts[2]); +static void* s3fs_init(struct fuse_conn_info *conn); +static void s3fs_destroy(void*); + +#endif // S3FS_S3_H_ diff --git a/src/string_util.cpp b/src/string_util.cpp new file mode 100644 index 0000000..b47fdd7 --- /dev/null +++ b/src/string_util.cpp @@ -0,0 +1,30 @@ +/* + * s3fs - FUSE-based file system backed by Amazon S3 + * + * Copyright 2007-2008 Randy Rizun + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "string_util.h" + +string lower(string s) { + // change each character of the string to lower case + for (unsigned int i = 0; i < s.length(); i++) { + s[i] = tolower(s[i]); + } + return s; +} + diff --git a/src/string_util.h b/src/string_util.h new file mode 100644 index 0000000..a8731a3 --- /dev/null +++ b/src/string_util.h @@ -0,0 +1,41 @@ +#ifndef S3FS_STRING_UTIL_H_ +#define S3FS_STRING_UTIL_H_ + +/* + * A collection of string utilities for manipulating URLs and HTTP responses. + */ +#include +#include + +#define SPACES " \t\r\n" + +using namespace std; + +template string str(T value) { + stringstream tmp; + tmp << value; + return tmp.str(); +} + +inline string trim_left(const string &s, const string &t = SPACES) { + string d(s); + return d.erase(0, s.find_first_not_of(t)); +} + +inline string trim_right(const string &s, const string &t = SPACES) { + string d(s); + string::size_type i(d.find_last_not_of(t)); + if (i == string::npos) + return ""; + else + return d.erase(d.find_last_not_of(t) + 1); +} + +inline string trim(const string &s, const string &t = SPACES) { + string d(s); + return trim_left(trim_right(d, t), t); +} + +string lower(string s); + +#endif // S3FS_STRING_UTIL_H_ diff --git a/test/Makefile.am b/test/Makefile.am new file mode 100644 index 0000000..13000ea --- /dev/null +++ b/test/Makefile.am @@ -0,0 +1 @@ +TESTS=small-integration-test.sh diff --git a/test/integration-test-common.sh b/test/integration-test-common.sh new file mode 100644 index 0000000..cf2d0d1 --- /dev/null +++ b/test/integration-test-common.sh @@ -0,0 +1,14 @@ +#!/bin/bash -e + +S3FS=../src/s3fs + +S3FS_CREDENTIALS_FILE=$(eval echo ~${SUDO_USER}/.passwd-s3fs) + +TEST_BUCKET_1=${SUDO_USER}-s3fs-integration-test +TEST_BUCKET_MOUNT_POINT_1=/mnt/${TEST_BUCKET_1} + +if [ ! -f "$S3FS_CREDENTIALS_FILE" ] +then + echo "Missing credentials file: $S3FS_CREDENTIALS_FILE" + exit 1 +fi diff --git a/test/require-root.sh b/test/require-root.sh new file mode 100755 index 0000000..a16196e --- /dev/null +++ b/test/require-root.sh @@ -0,0 +1,7 @@ +#!/bin/bash -e + +if [[ $EUID -ne 0 ]] +then + echo "This test script must be run as root" 1>&2 + exit 1 +fi diff --git a/test/small-integration-test.sh b/test/small-integration-test.sh new file mode 100755 index 0000000..90f4a10 --- /dev/null +++ b/test/small-integration-test.sh @@ -0,0 +1,41 @@ +#!/bin/bash -e +COMMON=integration-test-common.sh +source $COMMON + +# Require root +REQUIRE_ROOT=require-root.sh +source $REQUIRE_ROOT + +# Configuration +TEST_TEXT="HELLO WORLD" +TEST_TEXT_FILE=test-s3fs.txt +TEST_TEXT_FILE_LENGTH=15 + +# Mount the bucket +if [ ! -d $TEST_BUCKET_MOUNT_POINT_1 ] +then + mkdir -p $TEST_BUCKET_MOUNT_POINT_1 +fi +$S3FS $TEST_BUCKET_1 $TEST_BUCKET_MOUNT_POINT_1 -o passwd_file=$S3FS_CREDENTIALS_FILE +CUR_DIR=`pwd` +cd $TEST_BUCKET_MOUNT_POINT_1 + +# Write a small test file +for x in `seq 1 $TEST_TEXT_FILE_LENGTH` +do + echo $TEST_TEXT >> $TEST_TEXT_FILE +done + +# Verify contents of file +FILE_LENGTH=`wc -l $TEST_TEXT_FILE | awk '{print $1}'` +if [ "$FILE_LENGTH" -ne "$TEST_TEXT_FILE_LENGTH" ] +then + exit 1 +fi + +# Delete the test file +rm $TEST_TEXT_FILE + +# Unmount the bucket +cd $CUR_DIR +umount $TEST_BUCKET_MOUNT_POINT_1