diff --git a/ChangeLog b/ChangeLog index e90e59b..b8ca850 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,9 +1,21 @@ -??-??-????: 2.1.0 +??-??-????: 2.0.6 fix: no longer stops syslogging on HUP signals fix: OSX event watcher no longer misses moves into and out of the watch tree fix: not refinding a relative path to the config file in case of HUP. fix: rsync doing error 13 and killing Lsyncd. see http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=659941 + fix: no event creation during shutdown (might loop before) + fix: no logging due to wrong log levels + fix: without-inotify compile option now works to compile on OSX + fix: in case of HUP-reset imply insist=true, since startup is known to be + configured correctly. + fix: a series of typos in comments, manpage etc. + change: complain if any "rsyncOps" is given + change: splitted the default configurations in their own files. + more cleanly seperated from the Lsyncd runner, and highlights it are just + Layer 1 configurations that happen to be provided by default. + change: Beautified the code, no extra spaces at line end, ' instead of ", + supposing 100 char width to view, change: Lsyncd now remembers the absolute path of its config file during HUPs 25-08-2011: 2.0.5 diff --git a/Makefile.am b/Makefile.am index 5f89620..03cf0d9 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,7 +1,8 @@ AUTOMAKE_OPTIONS = foreign -CFLAGS += -Wall $(LUA_CFLAGS) +CFLAGS += -Wall $(LUA_CFLAGS) bin_PROGRAMS = lsyncd -lsyncd_SOURCES = lsyncd.h lsyncd.c lsyncd.lua +lsyncd_SOURCES = lsyncd.h lsyncd.c lsyncd.lua default-rsync.lua + if INOTIFY lsyncd_SOURCES += inotify.c endif @@ -38,7 +39,7 @@ doc/lsyncd.1: doc/lsyncd.1.xml doc/lsyncd.1.xml: doc/lsyncd.1.txt asciidoc -o $@ -b docbook -d manpage $< -CLEANFILES = luac.out luac.c +CLEANFILES = runner.out defaults.out runner.c defaults.c if RUNNER # installs the runner script @@ -46,15 +47,22 @@ runnerdir = $(RUNNER_DIR) runner_DATA = lsyncd.lua else # or compiles it into the binary -lsyncd_LDADD += luac.o +lsyncd_LDADD += runner.o defaults.o -luac.o: luac.c +runner.o: runner.c +default.o: default.c -luac.c: luac.out bin2carray.lua - lua ./bin2carray.lua luac.out luac luac.c +runner.c: runner.out bin2carray.lua + lua ./bin2carray.lua $< runner $@ -luac.out: lsyncd.lua - luac $< +defaults.c: defaults.out bin2carray.lua + lua ./bin2carray.lua $< defaults $@ + +runner.out: lsyncd.lua + luac -o $@ $< + +defaults.out: default.lua default-rsync.lua default-rsyncssh.lua default-direct.lua + luac -o $@ $^ endif diff --git a/default-direct.lua b/default-direct.lua new file mode 100644 index 0000000..c06aad1 --- /dev/null +++ b/default-direct.lua @@ -0,0 +1,164 @@ +--================================================================================================== +-- default-rsyncssh.lua +-- +-- Keeps two directories with /bin/cp, /bin/rm and /bin/mv in sync. +-- Startup still uses rsync tough. +-- +-- A (Layer 1) configuration. +-- +-- Note: +-- this is infact just configuration using Layer 1 configuration, +-- like any other. It only gets compiled into the binary by default. +-- You can simply use a modified one, by copying everything into a +-- config file of yours and call it differently. +-- +-- License: GPLv2 (see COPYING) or any later version +-- Authors: Axel Kittenberger +-- +--================================================================================================== + +if not default then error('default not loaded'); end +if not default.rsync then error('default-direct (currently) needs default.rsync loaded'); end +if default.direct then error('default-direct already loaded'); end + +default.direct = { + ----- + -- Spawns rsync for a list of events + -- + action = function(inlet) + -- gets all events ready for syncing + local event, event2 = inlet.getEvent() + + if event.etype == 'Create' then + if event.isdir then + spawn( + event, + '/bin/mkdir', + '-p', + event.targetPath + ) + else + spawn( + event, + '/bin/cp', + '-t', + event.targetPathdir, + event.sourcePath + ) + end + elseif event.etype == 'Modify' then + if event.isdir then + error("Do not know how to handle 'Modify' on dirs") + end + spawn(event, + '/bin/cp', + '-t', + event.targetPathdir, + event.sourcePath + ) + elseif event.etype == 'Delete' then + local tp = event.targetPath + -- extra security check + if tp == '' or tp == '/' or not tp then + error('Refusing to erase your harddisk!') + end + spawn(event, '/bin/rm', '-rf', tp) + elseif event.etype == 'Move' then + local tp = event.targetPath + -- extra security check + if tp == '' or tp == '/' or not tp then + error('Refusing to erase your harddisk!') + end + spawnShell( + event, + '/bin/mv $1 $2 || /bin/rm -rf $1', + event.targetPath, + event2.targetPath) + else + log('Warn', 'ignored an event of type "',event.etype, '"') + inlet.discardEvent(event) + end + end, + + ----- + -- Called when collecting a finished child process + -- + collect = function(agent, exitcode) + local config = agent.config + + if not agent.isList and agent.etype == 'Init' then + local rc = config.rsyncExitCodes[exitcode] + if rc == 'ok' then + log('Normal', 'Startup of "',agent.source,'" finished: ', exitcode) + elseif rc == 'again' then + if settings.insist then + log('Normal', 'Retrying startup of "',agent.source,'": ', exitcode) + else + log('Error', 'Temporary or permanent failure on startup of "', + agent.source, '". Terminating since "insist" is not set.'); + terminate(-1) -- ERRNO + end + elseif rc == 'die' then + log('Error', 'Failure on startup of "',agent.source,'": ', exitcode) + else + log('Error', 'Unknown exitcode on startup of "', agent.source,': "',exitcode) + rc = 'die' + end + return rc + end + + -- everything else is just as it is, + -- there is no network to retry something. + return + end, + + ----- + -- Spawns the recursive startup sync + -- (currently) identical to default rsync. + -- + init = default.rsync.init, + + ----- + -- Checks the configuration. + -- + prepare = function(config) + if not config.target then + error('default.direct needs "target".', 4) + end + + if config.rsyncOps then + error('did you mean rsyncOpts with "t"?', 4) + end + end, + + ----- + -- Default delay is very short. + -- + delay = 1, + + ------ + -- Let the core not split move events. + -- + onMove = true, + + ----- + -- The rsync binary called. + -- + rsyncBinary = '/usr/bin/rsync', + + ----- + -- For startup sync + -- + rsyncOpts = '-lts', + + ----- + -- rsync exit codes + -- + rsyncExitCodes = default.rsyncExitCodes, + + ----- + -- On many system multiple disk operations just rather slow down + -- than speed up. + + maxProcesses = 1, +} diff --git a/default-rsync.lua b/default-rsync.lua new file mode 100644 index 0000000..a11b5cd --- /dev/null +++ b/default-rsync.lua @@ -0,0 +1,174 @@ +--================================================================================================== +-- default-rsync.lua +-- +-- Syncs with rsync ("classic" Lsyncd) +-- A (Layer 1) configuration. +-- +-- Note: +-- this is infact just configuration using Layer 1 configuration, +-- like any other. It only gets compiled into the binary by default. +-- You can simply use a modified one, by copying everything into a +-- config file of yours and call it differently. +-- +-- License: GPLv2 (see COPYING) or any later version +-- Authors: Axel Kittenberger +-- +--================================================================================================== + +if not default then error('default not loaded'); end +if default.rsync then error('default-rsync already loaded'); end + +default.rsync = { + ----- + -- Spawns rsync for a list of events + -- + action = function(inlet) + -- gets all events ready for syncing + local elist = inlet.getEvents( + function(event) + return event.etype ~= 'Init' and event.etype ~= 'Blanket' + end + ) + + ----- + -- replaces filter rule by literals + -- + local function sub(p) + if not p then + return + end + return p:gsub('%?', '\\?'): + gsub('%*', '\\*'): + gsub('%[', '\\['): + gsub('%]', '\\]') + end + + local paths = elist.getPaths( + function(etype, path1, path2) + if etype == 'Delete' and string.byte(path1, -1) == 47 then + return sub(path1)..'***', sub(path2) + else + return sub(path1), sub(path2) + end + end) + -- stores all filters with integer index + -- local filterI = inlet.getExcludes(); + local filterI = {} + -- stores all filters with path index + local filterP = {} + + -- adds one entry into the filter + -- @param path ... path to add + -- @param leaf ... true if this the original path + -- false if its a parent + local function addToFilter(path) + if filterP[path] then + return + end + filterP[path]=true + table.insert(filterI, path) + end + + -- adds a path to the filter, for rsync this needs + -- to have entries for all steps in the path, so the file + -- d1/d2/d3/f1 needs filters + -- 'd1/', 'd1/d2/', 'd1/d2/d3/' and 'd1/d2/d3/f1' + for _, path in ipairs(paths) do + if path and path ~= '' then + addToFilter(path) + local pp = string.match(path, '^(.*/)[^/]+/?') + while pp do + addToFilter(pp) + pp = string.match(pp, '^(.*/)[^/]+/?') + end + end + end + + local filterS = table.concat(filterI, '\n') + local filter0 = table.concat(filterI, '\000') + log('Normal', 'Calling rsync with filter-list of new/modified files/dirs\n', filterS) + local config = inlet.getConfig() + spawn(elist, config.rsyncBinary, + '<', filter0, + config.rsyncOpts, + '-r', + '--delete', + '--force', + '--from0', + '--include-from=-', + '--exclude=*', + config.source, + config.target) + end, + + ----- + -- Spawns the recursive startup sync + -- + init = function(event) + local config = event.config; + local inlet = event.inlet; + local excludes = inlet.getExcludes(); + if #excludes == 0 then + log('Normal', 'recursive startup rsync: ', config.source, ' -> ', config.target) + spawn(event, config.rsyncBinary, + '--delete', + config.rsyncOpts, + '-r', + config.source, + config.target) + else + local exS = table.concat(excludes, '\n') + log('Normal', 'recursive startup rsync: ',config.source, + ' -> ',config.target,' excluding\n',exS) + spawn(event, config.rsyncBinary, + '<', exS, + '--exclude-from=-', + '--delete', + config.rsyncOpts, '-r', + config.source, + config.target) + end + end, + + ----- + -- Checks the configuration. + -- + prepare = function(config) + if not config.target then + error('default.rsync needs "target" configured', 4) + end + + if config.rsyncOps then + error('did you mean rsyncOpts with "t"?', 4) + end + + -- appends a / to target if not present + if string.sub(config.target, -1) ~= '/' then + config.target = config.target..'/' + end + end, + + ----- + -- rsync uses default collect + ---- + + ----- + -- The rsync binary to be called. + -- + rsyncBinary = '/usr/bin/rsync', + + ----- + -- Calls rsync with this default short opts. + -- + rsyncOpts = '-lts', + + ----- + -- Exit codes for rsync. + -- + exitcodes = default.rsyncExitCodes, + + ----- + -- Default delay + -- + delay = 15, +} diff --git a/default-rsyncssh.lua b/default-rsyncssh.lua new file mode 100644 index 0000000..3d1c83b --- /dev/null +++ b/default-rsyncssh.lua @@ -0,0 +1,255 @@ +--================================================================================================== +-- default-rsyncssh.lua +-- +-- Improved rsync - sync with rsync, but moves and deletes executed over ssh. +-- A (Layer 1) configuration. +-- +-- Note: +-- this is infact just configuration using Layer 1 configuration, +-- like any other. It only gets compiled into the binary by default. +-- You can simply use a modified one, by copying everything into a +-- config file of yours and call it differently. +-- +-- License: GPLv2 (see COPYING) or any later version +-- Authors: Axel Kittenberger +-- +--================================================================================================== + +if not default then error('default not loaded'); end +if default.rsyncssh then error('default-rsyncssh already loaded'); end + +default.rsyncssh = { + ----- + -- Spawns rsync for a list of events + -- + action = function(inlet) + local event, event2 = inlet.getEvent() + local config = inlet.getConfig() + + -- makes move local on host + -- if fails deletes the source... + if event.etype == 'Move' then + log('Normal', 'Moving ',event.path,' -> ',event2.path) + spawn(event, '/usr/bin/ssh', + config.host, + 'mv', + '\"' .. config.targetdir .. event.path .. '\"', + '\"' .. config.targetdir .. event2.path .. '\"', + '||', 'rm', '-rf', + '\"' .. config.targetdir .. event.path .. '\"') + return + end + + -- uses ssh to delete files on remote host + -- instead of constructing rsync filters + if event.etype == 'Delete' then + local elist = inlet.getEvents( + function(e) + return e.etype == 'Delete' + end) + + local paths = elist.getPaths( + function(etype, path1, path2) + if path2 then + return config.targetdir..path1, config.targetdir..path2 + else + return config.targetdir..path1 + end + end) + + for _, v in pairs(paths) do + if string.match(v, '^%s*/+%s*$') then + log('Error', 'refusing to `rm -rf /` the target!') + terminate(-1) -- ERRNO + end + end + + local sPaths = table.concat(paths, '\n') + local zPaths = table.concat(paths, config.xargs.delimiter) + log('Normal', 'Deleting list\n', sPaths) + spawn(elist, '/usr/bin/ssh', + '<', zPaths, + config.host, + config.xargs.binary, config.xargs.xparams) + return + end + + -- for everything else spawn a rsync + local elist = inlet.getEvents( + function(e) + -- TODO use a table + return e.etype ~= 'Move' and + e.etype ~= 'Delete' and + e.etype ~= 'Init' and + e.etype ~= 'Blanket' + end) + local paths = elist.getPaths() + + -- removes trailing slashes from dirs. + for k, v in ipairs(paths) do + if string.byte(v, -1) == 47 then + paths[k] = string.sub(v, 1, -2) + end + end + local sPaths = table.concat(paths, '\n') + local zPaths = table.concat(paths, '\000') + log('Normal', 'Rsyncing list\n', sPaths) + spawn( + elist, config.rsyncBinary, + '<', zPaths, + config.rsyncOpts, + '--from0', + '--files-from=-', + config.source, + config.host .. ':' .. config.targetdir + ) + end, + + ----- + -- Called when collecting a finished child process + -- + collect = function(agent, exitcode) + local config = agent.config + + if not agent.isList and agent.etype == 'Init' then + local rc = config.rsyncExitCodes[exitcode] + if rc == 'ok' then + log('Normal', 'Startup of "',agent.source,'" finished: ', exitcode) + elseif rc == 'again' then + if settings.insist then + log('Normal', 'Retrying startup of "',agent.source,'": ', exitcode) + else + log('Error', 'Temporary or permanent failure on startup of "', + agent.source, '". Terminating since "insist" is not set.'); + terminate(-1) -- ERRNO + end + elseif rc == 'die' then + log('Error', 'Failure on startup of "',agent.source,'": ', exitcode) + else + log('Error', 'Unknown exitcode on startup of "', agent.source,': "',exitcode) + rc = 'die' + end + return rc + end + + if agent.isList then + local rc = config.rsyncExitCodes[exitcode] + if rc == 'ok' then log('Normal', 'Finished (list): ',exitcode) + elseif rc == 'again' then log('Normal', 'Retrying (list): ',exitcode) + elseif rc == 'die' then log('Error', 'Failure (list): ', exitcode) + else + log('Error', 'Unknown exitcode (list): ',exitcode) + rc = 'die' + end + return rc + else + local rc = config.sshExitCodes[exitcode] + if rc == 'ok' then + log('Normal', 'Finished ',agent.etype,' ',agent.sourcePath,': ',exitcode) + elseif rc == 'again' then + log('Normal', 'Retrying ',agent.etype,' ',agent.sourcePath,': ',exitcode) + elseif rc == 'die' then + log('Normal', 'Failure ',agent.etype,' ',agent.sourcePath,': ',exitcode) + else + log('Error', 'Unknown exitcode ',agent.etype,' ',agent.sourcePath,': ',exitcode) + rc = 'die' + end + return rc + end + end, + + ----- + -- Spawns the recursive startup sync + -- + init = function(event) + local config = event.config + local inlet = event.inlet + local excludes = inlet.getExcludes() + local target = config.host .. ':' .. config.targetdir + + if #excludes == 0 then + log('Normal', 'Recursive startup rsync: ',config.source,' -> ',target) + spawn( + event, config.rsyncBinary, + '--delete', + '-r', + config.rsyncOpts, + config.source, + target + ) + else + local exS = table.concat(excludes, '\n') + log('Normal', 'Recursive startup rsync: ',config.source, + ' -> ',target,' with excludes.') + spawn( + event, config.rsyncBinary, + '<', exS, + '--exclude-from=-', + '--delete', + '-r', + config.rsyncOpts, + config.source, + target + ) + end + end, + + ----- + -- Checks the configuration. + -- + prepare = function(config) + if not config.host then error('default.rsyncssh needs "host" configured', 4) end + if not config.targetdir then error('default.rsyncssh needs "targetdir" configured', 4) end + + if config.rsyncOps then + error('did you mean rsyncOpts with "t"?', 4) + end + + -- appends a slash to the targetdir if missing + if string.sub(config.targetdir, -1) ~= '/' then + config.targetdir = config.targetdir .. '/' + end + end, + + ----- + -- The rsync binary called. + -- + rsyncBinary = '/usr/bin/rsync', + + ----- + -- Calls rsync with this default short opts. + -- + rsyncOpts = '-lts', + + ----- + -- allow processes + -- + maxProcesses = 1, + + ------ + -- Let the core not split move events. + -- + onMove = true, + + ----- + -- Default delay. + -- + delay = 15, + + ----- + -- rsync exit codes + -- + rsyncExitCodes = default.rsyncExitCodes, + + ----- + -- Delimiter, the binary and the paramters passed to xargs + -- xargs is used to delete multiple remote files, when ssh access is + -- available this is simpler than to build filters for rsync for this. + -- Default uses '0' as limiter, you might override this for old systems. + -- + xargs = { + binary = '/usr/bin/xargs', + delimiter = '\000', + xparams = {'-0', 'rm -rf'} + } +} diff --git a/default.lua b/default.lua new file mode 100644 index 0000000..23d882d --- /dev/null +++ b/default.lua @@ -0,0 +1,166 @@ +--============================================================================ +-- default.lua Live (Mirror) Syncing Demon +-- +-- The default table for the user to access. +-- This default layer 1 functions provide the higher layer functionality. +-- +-- License: GPLv2 (see COPYING) or any later version +-- Authors: Axel Kittenberger +--============================================================================ + +if default then error('default already loaded'); end + +default = { + ----- + -- Default action calls user scripts on**** functions. + -- + action = function(inlet) + -- in case of moves getEvent returns the origin and dest of the move + local event, event2 = inlet.getEvent() + local config = inlet.getConfig() + local func = config['on'.. event.etype] + if func then + func(event, event2) + end + -- if function didnt change the wait status its not interested + -- in this event -> drop it. + if event.status == 'wait' then + inlet.discardEvent(event) + end + end, + + + ----- + -- Default collector. + -- + -- Called when collecting a finished child process + -- + collect = function(agent, exitcode) + local config = agent.config + local rc + if config.exitcodes then + rc = config.exitcodes[exitcode] + elseif exitcode == 0 then + rc = 'ok' + else + rc = 'die' + end + + -- TODO synchronize with similar code before + if not agent.isList and agent.etype == 'Init' then + if rc == 'ok' then + log('Normal', 'Startup of "',agent.source,'" finished.') + return 'ok' + elseif rc == 'again' then + log('Normal', 'Retrying startup of "',agent.source,'".') + return "again" + elseif rc == 'die' then + log('Error', 'Failure on startup of "',agent.source,'".') + terminate(-1) -- ERRNO + else + log('Error', 'Unknown exitcode "',exitcode,'" on startup of "',agent.source,'".') + return 'die' + end + end + + if agent.isList then + if rc == 'ok' then log('Normal', 'Finished a list = ',exitcode) + elseif rc == 'again' then log('Normal', 'Retrying a list on exitcode = ',exitcode) + elseif rc == 'die' then log('Error', 'Failure with a list on exitcode = ',exitcode) + else + log('Error', 'Unknown exitcode "',exitcode,'" with a list') + rc = 'die' + end + else + if rc == 'ok' then + log('Normal', 'Retrying ',agent.etype,' on ',agent.sourcePath,' = ',exitcode) + elseif rc == 'again' then + log('Normal', 'Finished ',agent.etype,' on ',agent.sourcePath,' = ',exitcode) + elseif rc == 'die' then + log('Error', 'Failure with ',agent.etype,' on ',agent.sourcePath,' = ',exitcode) + else + log('Normal', 'Unknown exitcode "',exitcode,'" with ', agent.etype, + ' on ',agent.sourcePath,' = ',exitcode) + rc = 'die' + end + end + return rc + end, + + ----- + -- called on (re)initialization of Lsyncd. + -- + init = function(event) + local config = event.config + local inlet = event.inlet + -- user functions + -- calls a startup if given by user script. + if type(config.onStartup) == 'function' then + local startup = config.onStartup(event) + -- TODO honor some return codes of startup like "warmstart". + end + + if event.status == 'wait' then + -- user script did not spawn anything + -- thus the blanket event is deleted again. + inlet.discardEvent(event) + end + end, + + ----- + -- The maximum number of processes Lsyncd will spawn simultanously for + -- one sync. + -- + maxProcesses = 1, + + ----- + -- Try not to have more than these delays. + -- not too large, since total calculation for stacking + -- events is n*log(n) or so.. + -- + maxDelays = 1000, + + ----- + -- a default configuration using /bin/cp|rm|mv. + -- + direct = default_direct, + + ------ + -- Exitcodes of rsync and what to do. + -- + rsyncExitCodes = { + [ 0] = 'ok', + [ 1] = 'die', + [ 2] = 'die', + [ 3] = 'again', + [ 4] = 'die', + [ 5] = 'again', + [ 6] = 'again', + [ 10] = 'again', + [ 11] = 'again', + [ 12] = 'again', + [ 14] = 'again', + [ 20] = 'again', + [ 21] = 'again', + [ 22] = 'again', + [ 23] = 'ok', -- partial transfers are ok, since Lsyncd has registered the event that + [ 24] = 'ok', -- caused the transfer to be partial and will recall rsync. + [ 25] = 'die', + [ 30] = 'again', + [ 35] = 'again', + [255] = 'again', + }, + + ----- + -- Exitcodes of ssh and what to do. + -- + sshExitCodes = { + [0] = 'ok', + [255] = 'again', + }, + + ----- + -- Minimum seconds between two writes of a status file. + -- + statusInterval = 10, +} diff --git a/lsyncd.c b/lsyncd.c index f81afb1..663d8fc 100644 --- a/lsyncd.c +++ b/lsyncd.c @@ -49,10 +49,13 @@ * The Lua part of lsyncd if compiled into the binary. */ #ifndef LSYNCD_DEFAULT_RUNNER_FILE - extern const char luac_out[]; - extern size_t luac_size; + extern const char runner_out[]; + extern size_t runner_size; #endif +extern const char defaults_out[]; +extern size_t defaults_size; + /** * Makes sure there is one monitor. */ @@ -68,15 +71,19 @@ * All monitors supported by this Lsyncd. */ static char *monitors[] = { + #ifdef LSYNCD_WITH_INOTIFY "inotify", #endif + #ifdef LSYNCD_WITH_FANOTIFY "fanotify", #endif + #ifdef LSYNCD_WITH_FSEVENTS "fsevents", #endif + NULL, }; @@ -96,7 +103,7 @@ struct settings settings = { * True when lsyncd daemonized itself. */ static bool is_daemon = false; - + /** * The config file loaded by Lsyncd. * Global so it is retained during HUPs @@ -1735,7 +1742,8 @@ main1(int argc, char *argv[]) printlogf(L, "Error", "%s --runner RUNNER_FILE CONFIG_FILE", argv[0]); exit(-1); // ERRNO } - /* loads the runner file */ + + // loads the runner file if (luaL_loadfile(L, lsyncd_runner_file)) { printlogf(L, "Error", "error loading '%s': %s", lsyncd_runner_file, lua_tostring(L, -1)); @@ -1744,10 +1752,8 @@ main1(int argc, char *argv[]) } else { #ifndef LSYNCD_DEFAULT_RUNNER_FILE // loads the runner from binary - if (luaL_loadbuffer(L, luac_out, luac_size, "lsyncd.lua")) { - printlogf(L, "Error", - "error loading precompiled lsyncd.lua runner: %s", - lua_tostring(L, -1)); + if (luaL_loadbuffer(L, runner_out, runner_size, "runner")) { + printlogf(L, "Error", "loading precompiled runner: %s", lua_tostring(L, -1)); exit(-1); // ERRNO } #else @@ -1761,10 +1767,7 @@ main1(int argc, char *argv[]) // place to store the lua runners functions // executes the runner defining all its functions if (lua_pcall(L, 0, LUA_MULTRET, 0)) { - printlogf(L, "Error", - "error preparing '%s': %s", - lsyncd_runner_file ? lsyncd_runner_file : "internal runner", - lua_tostring(L, -1)); + printlogf(L, "Error", "preparing runner: %s", lua_tostring(L, -1)); exit(-1); // ERRNO } lua_pushlightuserdata(L, (void *)&runner); @@ -1798,6 +1801,21 @@ main1(int argc, char *argv[]) lua_pop(L, 1); } + { + // loads the defaults from binary + if (luaL_loadbuffer(L, defaults_out, defaults_size, "defaults")) { + printlogf(L, "Error", "loading defaults: %s", lua_tostring(L, -1)); + exit(-1); // ERRNO + } + + // prepares the defaults + if (lua_pcall(L, 0, 0, 0)) { + printlogf(L, "Error", "preparing defaults: %s", lua_tostring(L, -1)); + exit(-1); // ERRNO + } + } + + { // checks if there is a "-help" or "--help" int i; @@ -1857,7 +1875,7 @@ main1(int argc, char *argv[]) } free(lsyncd_config_file); lsyncd_config_file = apath; - + if (stat(lsyncd_config_file, &st)) { printlogf(L, "Error", "Cannot find config file at '%s'.", lsyncd_config_file); exit(-1); // ERRNO @@ -1865,7 +1883,7 @@ main1(int argc, char *argv[]) // loads and executes the config file if (luaL_loadfile(L, lsyncd_config_file)) { - printlogf(L, "Error", + printlogf(L, "Error", "error loading %s: %s", lsyncd_config_file, lua_tostring(L, -1)); exit(-1); // ERRNO } diff --git a/lsyncd.lua b/lsyncd.lua index 866d8b7..dc8632e 100644 --- a/lsyncd.lua +++ b/lsyncd.lua @@ -3053,699 +3053,6 @@ function string.ends(String,End) return End=='' or string.sub(String,-#End)==End end - ---============================================================================ --- Lsyncd default settings ---============================================================================ - ------ --- Exitcodes to retry on network failures of rsync. --- -local rsync_exitcodes = { - [ 0] = 'ok', - [ 1] = 'die', - [ 2] = 'die', - [ 3] = 'again', - [ 4] = 'die', - [ 5] = 'again', - [ 6] = 'again', - [ 10] = 'again', - [ 11] = 'again', - [ 12] = 'again', - [ 14] = 'again', - [ 20] = 'again', - [ 21] = 'again', - [ 22] = 'again', - [ 23] = 'ok', -- partial transfers are ok, since Lsyncd has registered the event that - [ 24] = 'ok', -- caused the transfer to be partial and will recall rsync. - [ 25] = 'die', - [ 30] = 'again', - [ 35] = 'again', - [255] = 'again', -} - ------ --- Exitcodes to retry on network failures of rsync. --- -local ssh_exitcodes = { - [0] = 'ok', - [255] = 'again', -} - ------ --- Lsyncd classic - sync with rsync --- -local default_rsync = { - ----- - -- Spawns rsync for a list of events - -- - action = function(inlet) - -- gets all events ready for syncing - local elist = inlet.getEvents( - function(event) - return event.etype ~= 'Init' and event.etype ~= 'Blanket' - end - ) - - ----- - -- replaces filter rule by literals - -- - local function sub(p) - if not p then - return - end - return p:gsub('%?', '\\?'): - gsub('%*', '\\*'): - gsub('%[', '\\['): - gsub('%]', '\\]') - end - - local paths = elist.getPaths( - function(etype, path1, path2) - if etype == 'Delete' and string.byte(path1, -1) == 47 then - return sub(path1)..'***', sub(path2) - else - return sub(path1), sub(path2) - end - end) - -- stores all filters with integer index - -- local filterI = inlet.getExcludes(); - local filterI = {} - -- stores all filters with path index - local filterP = {} - - -- adds one entry into the filter - -- @param path ... path to add - -- @param leaf ... true if this the original path - -- false if its a parent - local function addToFilter(path) - if filterP[path] then - return - end - filterP[path]=true - table.insert(filterI, path) - end - - -- adds a path to the filter, for rsync this needs - -- to have entries for all steps in the path, so the file - -- d1/d2/d3/f1 needs filters - -- 'd1/', 'd1/d2/', 'd1/d2/d3/' and 'd1/d2/d3/f1' - for _, path in ipairs(paths) do - if path and path ~= '' then - addToFilter(path) - local pp = string.match(path, '^(.*/)[^/]+/?') - while pp do - addToFilter(pp) - pp = string.match(pp, '^(.*/)[^/]+/?') - end - end - end - - local filterS = table.concat(filterI, '\n') - local filter0 = table.concat(filterI, '\000') - log('Normal', 'Calling rsync with filter-list of new/modified files/dirs\n', filterS) - local config = inlet.getConfig() - spawn(elist, config.rsyncBinary, - '<', filter0, - config.rsyncOpts, - '-r', - '--delete', - '--force', - '--from0', - '--include-from=-', - '--exclude=*', - config.source, - config.target) - end, - - ----- - -- Spawns the recursive startup sync - -- - init = function(event) - local config = event.config; - local inlet = event.inlet; - local excludes = inlet.getExcludes(); - if #excludes == 0 then - log('Normal', 'recursive startup rsync: ', config.source, ' -> ', config.target) - spawn(event, config.rsyncBinary, - '--delete', - config.rsyncOpts, - '-r', - config.source, - config.target) - else - local exS = table.concat(excludes, '\n') - log('Normal', 'recursive startup rsync: ',config.source, - ' -> ',config.target,' excluding\n',exS) - spawn(event, config.rsyncBinary, - '<', exS, - '--exclude-from=-', - '--delete', - config.rsyncOpts, '-r', - config.source, - config.target) - end - end, - - ----- - -- Checks the configuration. - -- - prepare = function(config) - if not config.target then - error('default.rsync needs "target" configured', 4) - end - - -- appends a / to target if not present - if string.sub(config.target, -1) ~= '/' then - config.target = config.target..'/' - end - end, - - ----- - -- The rsync binary called. - -- - rsyncBinary = '/usr/bin/rsync', - - ----- - -- Calls rsync with this default short opts. - -- - rsyncOpts = '-lts', - - ----- - -- exit codes for rsync. - -- - exitcodes = rsync_exitcodes, - - ----- - -- Default delay - -- - delay = 15, -} - - ------ --- Lsyncd 2 improved rsync - sync with rsync but move over ssh. --- -local default_rsyncssh = { - ----- - -- Spawns rsync for a list of events - -- - action = function(inlet) - local event, event2 = inlet.getEvent() - local config = inlet.getConfig() - - -- makes move local on host - -- if fails deletes the source... - if event.etype == 'Move' then - log('Normal', 'Moving ',event.path,' -> ',event2.path) - spawn(event, '/usr/bin/ssh', - config.host, - 'mv', - '\"' .. config.targetdir .. event.path .. '\"', - '\"' .. config.targetdir .. event2.path .. '\"', - '||', 'rm', '-rf', - '\"' .. config.targetdir .. event.path .. '\"') - return - end - - -- uses ssh to delete files on remote host - -- instead of constructing rsync filters - if event.etype == 'Delete' then - local elist = inlet.getEvents( - function(e) - return e.etype == 'Delete' - end) - - local paths = elist.getPaths( - function(etype, path1, path2) - if path2 then - return config.targetdir..path1, config.targetdir..path2 - else - return config.targetdir..path1 - end - end) - - for _, v in pairs(paths) do - if string.match(v, '^%s*/+%s*$') then - log('Error', 'refusing to `rm -rf /` the target!') - terminate(-1) -- ERRNO - end - end - - local sPaths = table.concat(paths, '\n') - local zPaths = table.concat(paths, config.xargs.delimiter) - log('Normal', 'Deleting list\n', sPaths) - spawn(elist, '/usr/bin/ssh', - '<', zPaths, - config.host, - config.xargs.binary, config.xargs.xparams) - return - end - - -- for everything else spawn a rsync - local elist = inlet.getEvents( - function(e) - -- TODO use a table - return e.etype ~= 'Move' and - e.etype ~= 'Delete' and - e.etype ~= 'Init' and - e.etype ~= 'Blanket' - end) - local paths = elist.getPaths() - - -- removes trailing slashes from dirs. - for k, v in ipairs(paths) do - if string.byte(v, -1) == 47 then - paths[k] = string.sub(v, 1, -2) - end - end - local sPaths = table.concat(paths, '\n') - local zPaths = table.concat(paths, '\000') - log('Normal', 'Rsyncing list\n', sPaths) - spawn( - elist, config.rsyncBinary, - '<', zPaths, - config.rsyncOpts, - '--from0', - '--files-from=-', - config.source, - config.host .. ':' .. config.targetdir - ) - end, - - ----- - -- Called when collecting a finished child process - -- - collect = function(agent, exitcode) - if not agent.isList and agent.etype == 'Init' then - local rc = rsync_exitcodes[exitcode] - if rc == 'ok' then - log('Normal', 'Startup of "',agent.source,'" finished: ', exitcode) - elseif rc == 'again' then - if settings.insist then - log('Normal', 'Retrying startup of "',agent.source,'": ', exitcode) - else - log('Error', 'Temporary or permanent failure on startup of "', - agent.source, '". Terminating since "insist" is not set.'); - terminate(-1) -- ERRNO - end - elseif rc == 'die' then - log('Error', 'Failure on startup of "',agent.source,'": ', exitcode) - else - log('Error', 'Unknown exitcode on startup of "', agent.source,': "',exitcode) - rc = 'die' - end - return rc - end - - if agent.isList then - local rc = rsync_exitcodes[exitcode] - if rc == 'ok' then log('Normal', 'Finished (list): ',exitcode) - elseif rc == 'again' then log('Normal', 'Retrying (list): ',exitcode) - elseif rc == 'die' then log('Error', 'Failure (list): ', exitcode) - else - log('Error', 'Unknown exitcode (list): ',exitcode) - rc = 'die' - end - return rc - else - local rc = ssh_exitcodes[exitcode] - if rc == 'ok' then - log('Normal', 'Finished ',agent.etype,' ',agent.sourcePath,': ',exitcode) - elseif rc == 'again' then - log('Normal', 'Retrying ',agent.etype,' ',agent.sourcePath,': ',exitcode) - elseif rc == 'die' then - log('Normal', 'Failure ',agent.etype,' ',agent.sourcePath,': ',exitcode) - else - log('Error', 'Unknown exitcode ',agent.etype,' ',agent.sourcePath,': ',exitcode) - rc = 'die' - end - return rc - end - end, - - ----- - -- Spawns the recursive startup sync - -- - init = function(event) - local config = event.config - local inlet = event.inlet - local excludes = inlet.getExcludes() - local target = config.host .. ':' .. config.targetdir - - if #excludes == 0 then - log('Normal', 'Recursive startup rsync: ',config.source,' -> ',target) - spawn( - event, config.rsyncBinary, - '--delete', - '-r', - config.rsyncOpts, - config.source, - target - ) - else - local exS = table.concat(excludes, '\n') - log('Normal', 'Recursive startup rsync: ',config.source, - ' -> ',target,' with excludes.') - spawn( - event, config.rsyncBinary, - '<', exS, - '--exclude-from=-', - '--delete', - '-r', - config.rsyncOpts, - config.source, - target - ) - end - end, - - ----- - -- Checks the configuration. - -- - prepare = function(config) - if not config.host then error('default.rsyncssh needs "host" configured', 4) end - if not config.targetdir then error('default.rsyncssh needs "targetdir" configured', 4) end - - -- appends a slash to the targetdir if missing - if string.sub(config.targetdir, -1) ~= '/' then - config.targetdir = config.targetdir .. '/' - end - end, - - ----- - -- The rsync binary called. - -- - rsyncBinary = '/usr/bin/rsync', - - ----- - -- Calls rsync with this default short opts. - -- - rsyncOpts = '-lts', - - ----- - -- allow processes - -- - maxProcesses = 1, - - ------ - -- Let the core not split move events. - -- - onMove = true, - - ----- - -- Default delay. - -- - delay = 15, - - ----- - -- Delimiter, the binary and the paramters passed to xargs - -- xargs is used to delete multiple remote files, when ssh access is - -- available this is simpler than to build filters for rsync for this. - -- Default uses '0' as limiter, you might override this for old systems. - -- - xargs = { - binary = '/usr/bin/xargs', - delimiter = '\000', - xparams = {'-0', 'rm -rf'} - } -} - ------ --- Keeps two directories with /bin/cp, /bin/rm and /bin/mv in sync. --- Startup still uses rsync tough. --- -local default_direct = { - ----- - -- Spawns rsync for a list of events - -- - action = function(inlet) - -- gets all events ready for syncing - local event, event2 = inlet.getEvent() - - if event.etype == 'Create' then - if event.isdir then - spawn( - event, - '/bin/mkdir', - '-p', - event.targetPath - ) - else - spawn( - event, - '/bin/cp', - '-t', - event.targetPathdir, - event.sourcePath - ) - end - elseif event.etype == 'Modify' then - if event.isdir then - error("Do not know how to handle 'Modify' on dirs") - end - spawn(event, - '/bin/cp', - '-t', - event.targetPathdir, - event.sourcePath - ) - elseif event.etype == 'Delete' then - local tp = event.targetPath - -- extra security check - if tp == '' or tp == '/' or not tp then - error('Refusing to erase your harddisk!') - end - spawn(event, '/bin/rm', '-rf', tp) - elseif event.etype == 'Move' then - local tp = event.targetPath - -- extra security check - if tp == '' or tp == '/' or not tp then - error('Refusing to erase your harddisk!') - end - spawnShell( - event, - '/bin/mv $1 $2 || /bin/rm -rf $1', - event.targetPath, - event2.targetPath) - else - log('Warn', 'ignored an event of type "',event.etype, '"') - inlet.discardEvent(event) - end - end, - - ----- - -- Called when collecting a finished child process - -- - collect = function(agent, exitcode) - local config = agent.config - - if not agent.isList and agent.etype == 'Init' then - local rc = rsync_exitcodes[exitcode] - if rc == 'ok' then - log('Normal', 'Startup of "',agent.source,'" finished: ', exitcode) - elseif rc == 'again' then - if settings.insist then - log('Normal', 'Retrying startup of "',agent.source,'": ', exitcode) - else - log('Error', 'Temporary or permanent failure on startup of "', - agent.source, '". Terminating since "insist" is not set.'); - terminate(-1) -- ERRNO - end - elseif rc == 'die' then - log('Error', 'Failure on startup of "',agent.source,'": ', exitcode) - else - log('Error', 'Unknown exitcode on startup of "', agent.source,': "',exitcode) - rc = 'die' - end - return rc - end - - -- everything else is just as it is, - -- there is no network to retry something. - return - end, - - ----- - -- Spawns the recursive startup sync - -- identical to default rsync. - -- - init = default_rsync.init, - - ----- - -- Checks the configuration. - -- - prepare = function(config) - if not config.target then - error('default.direct needs "target".', 4) - end - end, - - ----- - -- Default delay is very short. - -- - delay = 1, - - ------ - -- Let the core not split move events. - -- - onMove = true, - - ----- - -- The rsync binary called. - -- - rsyncBinary = '/usr/bin/rsync', - - ----- - -- For startup sync - -- - rsyncOpts = '-lts', - - ----- - -- On many system multiple disk operations just rather slow down - -- than speed up. - - maxProcesses = 1, -} - - ------ --- The default table for the user to access. --- Provides all the default layer 1 functions. --- --- TODO make readonly --- -default = { - - ----- - -- Default action calls user scripts on**** functions. - -- - action = function(inlet) - -- in case of moves getEvent returns the origin and dest of the move - local event, event2 = inlet.getEvent() - local config = inlet.getConfig() - local func = config['on'.. event.etype] - if func then - func(event, event2) - end - -- if function didnt change the wait status its not interested - -- in this event -> drop it. - if event.status == 'wait' then - inlet.discardEvent(event) - end - end, - - - ----- - -- Default collector. - -- Called when collecting a finished child process - -- - collect = function(agent, exitcode) - local config = agent.config - local rc - if config.exitcodes then - rc = config.exitcodes[exitcode] - elseif exitcode == 0 then - rc = 'ok' - else - rc = 'die' - end - - -- TODO synchronize with similar code before - if not agent.isList and agent.etype == 'Init' then - if rc == 'ok' then - log('Normal', 'Startup of "',agent.source,'" finished.') - return 'ok' - elseif rc == 'again' then - log('Normal', 'Retrying startup of "',agent.source,'".') - return "again" - elseif rc == 'die' then - log('Error', 'Failure on startup of "',agent.source,'".') - terminate(-1) -- ERRNO - else - log('Error', 'Unknown exitcode "',exitcode,'" on startup of "',agent.source,'".') - return 'die' - end - end - - if agent.isList then - if rc == 'ok' then log('Normal', 'Finished a list = ',exitcode) - elseif rc == 'again' then log('Normal', 'Retrying a list on exitcode = ',exitcode) - elseif rc == 'die' then log('Error', 'Failure with a list on exitcode = ',exitcode) - else - log('Error', 'Unknown exitcode "',exitcode,'" with a list') - rc = 'die' - end - else - if rc == 'ok' then - log('Normal', 'Retrying ',agent.etype,' on ',agent.sourcePath,' = ',exitcode) - elseif rc == 'again' then - log('Normal', 'Finished ',agent.etype,' on ',agent.sourcePath,' = ',exitcode) - elseif rc == 'die' then - log('Error', 'Failure with ',agent.etype,' on ',agent.sourcePath,' = ',exitcode) - else - log('Normal', 'Unknown exitcode "',exitcode,'" with ', agent.etype, - ' on ',agent.sourcePath,' = ',exitcode) - rc = 'die' - end - end - return rc - end, - - ----- - -- called on (re)initialization of Lsyncd. - -- - init = function(event) - local config = event.config - local inlet = event.inlet - -- user functions - -- calls a startup if given by user script. - if type(config.onStartup) == 'function' then - local startup = config.onStartup(event) - -- TODO honor some return codes of startup like "warmstart". - end - - if event.status == 'wait' then - -- user script did not spawn anything - -- thus the blanket event is deleted again. - inlet.discardEvent(event) - end - end, - - ----- - -- The maximum number of processes Lsyncd will spawn simultanously for - -- one sync. - -- - maxProcesses = 1, - - ----- - -- Try not to have more than these delays. - -- not too large, since total calculation for stacking - -- events is n*log(n) or so.. - -- - maxDelays = 1000, - - ----- - -- a default rsync configuration for easy usage. - -- - rsync = default_rsync, - - ----- - -- a default rsync configuration with ssh'd move and rm actions - -- - rsyncssh = default_rsyncssh, - - ----- - -- a default configuration using /bin/cp|rm|mv. - -- - direct = default_direct, - - ----- - -- Minimum seconds between two writes of a status file. - -- - statusInterval = 10, -} - ----- -- provides a default empty settings table. --