From 770d1880d6e277e1915646fbd08d85cbbcd404ca Mon Sep 17 00:00:00 2001 From: Axel Kittenberger Date: Thu, 16 Feb 2012 08:28:40 +0100 Subject: [PATCH] Added 'delete=false' flag. if deleting set rsync to force it (otherwise this could result in some deletes not being transfered in the tests --- ChangeLog | 8 +++++++- README.md | 6 +++--- default-direct.lua | 14 +++++++++++++- default-rsync.lua | 25 ++++++++++++++++++------- default-rsyncssh.lua | 22 ++++++++++++++++++++-- lsyncd.c | 11 +++++++++-- lsyncd.lua | 7 ++++--- 7 files changed, 74 insertions(+), 19 deletions(-) diff --git a/ChangeLog b/ChangeLog index 68fc410..a718999 100644 --- a/ChangeLog +++ b/ChangeLog @@ -11,7 +11,9 @@ configured correctly. fix: a series of typos in comments, manpage etc. fix: moves to and from deleted directories (deleted when Lsyncd gets notified) - were not correctly translated + were not correctly translated + fix: added --ignore-errors to rsync when deleting files, so it will not miss a delete + even when another part of the tree made an IO-error. change: complain if any "rsyncOps" is given change: splitted the default configurations in their own files. more cleanly seperated from the Lsyncd runner, and highlights it are just @@ -19,6 +21,10 @@ change: Beautified the code, no extra spaces at line end, ' instead of ", supposing 100 char width to view, change: Lsyncd now remembers the absolute path of its config file during HUPs + enhancement: Defaults not respect a 'delete=false' flag when set as parameter to sync{} + default.rsync: does not add --delete to rsync + default.rsyncssh: does not add --delete to rsync, and does not use rm via ssh tunnel + default.direct: does not add --delete to startup rsync and does not use rm 25-08-2011: 2.0.5 fix: Lsyncd will now terminate if it inotify watching exceeds diff --git a/README.md b/README.md index e095d66..3e6c7a1 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Lsyncd -- Live Syncing (Mirror) Daemon ====================================== Description ----------- -Lsyncd watches a local directory trees event monitor interface (inotify). It aggregates and combines events for a few seconds and then spawns one (or more) process(es) to synchronize the changes. By default this is [rsync](http://rsync.samba.org/). Lsyncd is thus a light-weight live mirror solution that is comparatively easy to install not requiring new filesystems or blockdevices and does not hamper local filesystem performance. +Lsyncd watches a local directory trees event monitor interface (inotify or fsevents). It aggregates and combines events for a few seconds and then spawns one (or more) process(es) to synchronize the changes. By default this is [rsync](http://rsync.samba.org/). Lsyncd is thus a light-weight live mirror solution that is comparatively easy to install not requiring new filesystems or blockdevices and does not hamper local filesystem performance. Rsync+ssh is an advanced action configuration that uses a SSH to act file and directory moves directly on the target instead of retransmitting the move destination over the wire. @@ -16,7 +16,7 @@ Lsyncd is designed to synchronize a local directory tree with low profile of exp Other synchronization tools ------------------------ -[DRBD](http://www.drbd.org) operates on block device level. This makes it useful for synchronizing systems that are under heavy load. Lsyncd on the other hand does not require you to change block devices and/or mount points, allows you to change uid/gid of the transferred files, separates the receiver through the one-way nature of rsync. DRBD is likely the better option if you are syncing Databases. +[DRBD](http://www.drbd.org) operates on block device level. This makes it useful for synchronizing systems that are under heavy load. Lsyncd on the other hand does not require you to change block devices and/or mount points, allows you to change uid/gid of the transferred files, separates the receiver through the one-way nature of rsync. DRBD is likely the better option if you are syncing Databases. [GlusterFS](http://www.gluster.org) and [BindFS](http://www.cs.helsinki.fi/u/partel/bindfs/) use a FUSE-Filesystem to interject kernel/userspace filesystem events. @@ -36,4 +36,4 @@ Some more complicated examples, tips and tricks you can find in the [Lsyncd20Man Disclaimer ---------- Besides the usual disclaimer in the license, we want to specifically emphasize that neither the authors nor any organization the authors are associated with can and will hold responsible for data-loss caused by possible malfunctions of Lsyncd. - + diff --git a/default-direct.lua b/default-direct.lua index 2b65b55..1b837e8 100644 --- a/default-direct.lua +++ b/default-direct.lua @@ -28,6 +28,7 @@ default.direct = { action = function(inlet) -- gets all events ready for syncing local event, event2 = inlet.getEvent() + local config = inlet.getConfig() if event.etype == 'Create' then if event.isdir then @@ -57,6 +58,10 @@ default.direct = { event.sourcePath ) elseif event.etype == 'Delete' then + if not config.delete then + inlet.discardEvent(event) + end + local tp = event.targetPath -- extra security check if tp == '' or tp == '/' or not tp then @@ -69,9 +74,11 @@ default.direct = { if tp == '' or tp == '/' or not tp then error('Refusing to erase your harddisk!') end + local command = '/bin/mv $1 $2 || /bin/rm -rf $1' + if not config.delete then command = '/bin/mv $1 $2'; end spawnShell( event, - '/bin/mv $1 $2 || /bin/rm -rf $1', + command, event.targetPath, event2.targetPath) else @@ -151,6 +158,11 @@ default.direct = { -- rsyncOpts = '-lts', + ----- + -- By default do deletes. + -- + delete = true, + ----- -- rsync exit codes -- diff --git a/default-rsync.lua b/default-rsync.lua index 85f2e0f..5ff45d4 100644 --- a/default-rsync.lua +++ b/default-rsync.lua @@ -88,11 +88,13 @@ default.rsync = { local filter0 = table.concat(filterI, '\000') log('Normal', 'Calling rsync with filter-list of new/modified files/dirs\n', filterS) local config = inlet.getConfig() + local delete = nil + if config.delete then delete = { '--delete', '--ignore-errors' }; end spawn(elist, config.rsyncBinary, '<', filter0, config.rsyncOpts, '-r', - '--delete', + delete, '--force', '--from0', '--include-from=-', @@ -105,13 +107,16 @@ default.rsync = { -- Spawns the recursive startup sync -- init = function(event) - local config = event.config; - local inlet = event.inlet; - local excludes = inlet.getExcludes(); + local config = event.config + local inlet = event.inlet + local excludes = inlet.getExcludes() + local delete = nil + if config.delete then delete = { '--delete', '--ignore-errors' }; end + if #excludes == 0 then log('Normal', 'recursive startup rsync: ', config.source, ' -> ', config.target) spawn(event, config.rsyncBinary, - '--delete', + delete, config.rsyncOpts, '-r', config.source, @@ -123,8 +128,9 @@ default.rsync = { spawn(event, config.rsyncBinary, '<', exS, '--exclude-from=-', - '--delete', - config.rsyncOpts, '-r', + delete, + config.rsyncOpts, + '-r', config.source, config.target) end @@ -152,6 +158,11 @@ default.rsync = { -- rsync uses default collect ---- + ----- + -- By default do deletes. + -- + delete = true, + ----- -- The rsync binary to be called. -- diff --git a/default-rsyncssh.lua b/default-rsyncssh.lua index c32f0fe..f2a1af7 100644 --- a/default-rsyncssh.lua +++ b/default-rsyncssh.lua @@ -43,6 +43,11 @@ default.rsyncssh = { -- uses ssh to delete files on remote host -- instead of constructing rsync filters if event.etype == 'Delete' then + if not config.delete then + inlet.discardEvent(event) + return + end + local elist = inlet.getEvents( function(e) return e.etype == 'Delete' @@ -166,12 +171,14 @@ default.rsyncssh = { local inlet = event.inlet local excludes = inlet.getExcludes() local target = config.host .. ':' .. config.targetdir + local delete = nil + if config.delete then delete = { '--delete', '--ignore-errors' }; end if #excludes == 0 then log('Normal', 'Recursive startup rsync: ',config.source,' -> ',target) spawn( event, config.rsyncBinary, - '--delete', + delete, '-r', config.rsyncOpts, config.source, @@ -185,7 +192,7 @@ default.rsyncssh = { event, config.rsyncBinary, '<', exS, '--exclude-from=-', - '--delete', + delete, '-r', config.rsyncOpts, config.source, @@ -236,11 +243,22 @@ default.rsyncssh = { -- delay = 15, + + ----- + -- By default do deletes. + -- + delete = true, + ----- -- rsync exit codes -- rsyncExitCodes = default.rsyncExitCodes, + ----- + -- ssh exit codes + -- + sshExitCodes = default.sshExitCodes, + ----- -- Delimiter, the binary and the paramters passed to xargs -- xargs is used to delete multiple remote files, when ssh access is diff --git a/lsyncd.c b/lsyncd.c index 3bed58d..2c69290 100644 --- a/lsyncd.c +++ b/lsyncd.c @@ -831,13 +831,20 @@ l_exec(lua_State *L) int pipefd[2]; // pipe file descriptors int i; - // expands tables if there are any + // expands tables if there are any, removes nils for(i = 1; i <= lua_gettop(L); i++) { + if (lua_isnil(L, i)) { + lua_remove(L, i); + i--; + argc--; + continue; + } + if (lua_istable(L, i)) { int tlen; int it; - // table is now on top of stack lua_checkstack(L, lua_gettop(L) + lua_objlen(L, i) + 1); + // move table to top of stack lua_pushvalue(L, i); lua_remove(L, i); argc--; diff --git a/lsyncd.lua b/lsyncd.lua index 93d1822..75181b7 100644 --- a/lsyncd.lua +++ b/lsyncd.lua @@ -2962,17 +2962,18 @@ function spawn(agent, binary, ...) if agent == nil or type(agent) ~= 'table' then error('spawning with an invalid agent', 2) end + if lsyncdStatus == 'fade' then log('Normal', 'ignored process spawning while fading') return end + if type(binary) ~= 'string' then error('calling spawn(agent, binary, ...), binary is not a string', 2) end + local dol = InletFactory.getDelayOrList(agent) - if not dol then - error('spawning with an unknown agent', 2) - end + if not dol then error('spawning with an unknown agent', 2) end -- checks if spawn is called on already active event if dol.status then