From 3b015f8c05f6c8c9adebde926036b2929a64dadd Mon Sep 17 00:00:00 2001 From: Birger Schmidt Date: Wed, 10 Oct 2012 11:30:50 +0200 Subject: [PATCH 1/3] lsyncd/csync2 config: initial commit with ugly glob hack --- examples/lsyncd-csync2.conf.lua | 120 ++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 examples/lsyncd-csync2.conf.lua diff --git a/examples/lsyncd-csync2.conf.lua b/examples/lsyncd-csync2.conf.lua new file mode 100644 index 0000000..1b4ccc7 --- /dev/null +++ b/examples/lsyncd-csync2.conf.lua @@ -0,0 +1,120 @@ +----- +-- User configuration file for lsyncd. +-- +-- This example synchronizes one specific directory through multiple nodes, +-- by combining csync2 and lsyncd as monitoring tools. +-- It avoids any race conditions generated by lsyncd, while detecting and +-- processing multiple inotify events in batch, on each node monitored by +-- csync2 daemon. +-- +-- @author Floren Munteanu +-- @link http://www.axivo.com/community/threads/lightning-fast-synch-with-csync2-and-lsyncd.121/ +----- +settings = { + logident = "lsyncd", + logfacility = "user", + logfile = "/var/log/lsyncd/lsyncd.log", + statusFile = "/var/log/lsyncd/status.log", + statusInterval = 1 +} + +initSync = { + delay = 1, + maxProcesses = 1, + action = function(inlet) + local config = inlet.getConfig() + local count = 0; + local elist = inlet.getEvents(function(event) + if string.starts(event.pathname, "/.") then + -- log("Normal", "discard ",event.pathname) + inlet.discardEvent(event) + return + end + count = count + 1; + return event.etype ~= 'Init' and event.etype ~= "Blanket" + end) + local directory = string.sub(config.source, 1, -2) + local paths = elist.getPaths(function(etype, path) + return directory .. path + end) + if count > 0 then + log("Normal", "Processing syncing list of ",#paths, " files.") + local zpaths = table.concat(paths, ('\000')) + spawn(elist, "/usr/bin/xargs", "<", zpaths, "-0", "/usr/sbin/csync2", "-C", config.syncid, "-Amv") + local sync = inlet.createBlanketEvent() + spawn(sync, "/usr/bin/xargs", "<", zpaths, "-0", "/usr/sbin/csync2", "-C", config.syncid, "-ABuv") + end + end, + collect = function(agent, exitcode) + local config = agent.config + if not agent.isList and agent.etype == "Init" then + if exitcode == 0 then + log("Normal", "Startup of '", config.syncid, ":", config.source, "' instance finished.") + elseif config.exitcodes and config.exitcodes[exitcode] == "again" then + log("Normal", "Retrying startup of '", config.syncid, ":", config.source, "' instance.") + return "again" + else + log("Error", "Failure on startup of '", config.syncid, ":", config.source, "' instance.") + terminate(-1) + end + return + end + local rc = config.exitcodes and config.exitcodes[exitcode] + if rc == "die" then + return rc + end + if agent.isList then + if rc == "again" then + log("Normal", "Retrying events list on exitcode = ", exitcode) + else + -- log("Normal", "Finished events list = ", exitcode) + end + else + if rc == "again" then + log("Normal", "Retrying ", agent.etype, " on ", agent.sourcePath, " = ", exitcode) + else + log("Normal", "Finished ", agent.etype, " on ", agent.sourcePath, " = ", exitcode) + end + end + return rc + end, + init = function(event) + local inlet = event.inlet; + local config = inlet.getConfig() + log("Normal", "started up in dirty sync state: ", config.syncid, ":", config.source) + spawn(event, "/bin/true") + end, + prepare = function(config) + -- log("Normal", "prepare to sync: ", config.syncid, ":", config.source) + if not config.syncid then + error("Missing 'syncid' parameter.", 4) + end + local c = "csync2_" .. config.syncid .. ".cfg" + local f, err = io.open("/etc/" .. c, "r") + if not f then + error("Invalid 'syncid' parameter: " .. err, 4) + end + f:close() + end +} + +local sources = { + ["/mnt/assets/uploads/2011/profiles/"] = "0mnt0assets0uploads020110profiles0", + ["/mnt/assets/uploads/2011/profiles/1*"] = "0mnt0assets0uploads020110profiles01", + ["/mnt/assets/uploads/2011/profiles/2*"] = "0mnt0assets0uploads020110profiles02", + ["/mnt/assets/uploads/2011/profiles/3*"] = "0mnt0assets0uploads020110profiles03", +} +for key, value in pairs(sources) do + -- very ugly hack to handle well known subdirs, because we have no glob + if string.ends(key, "*") then + local directory = string.sub(key, 1, -2) + for _,i in pairs({"","0","1","2","3","4","5","6","7","8","9","a","b","c","d","e","f"}) do + for _,j in pairs({"","0","1","2","3","4","5","6","7","8","9","a","b","c","d","e","f"}) do + os.execute("install -o www-data -g www-data -d " .. directory .. i .. j) + sync {initSync, source=directory .. i .. j, syncid=value} + end + end + else + sync {initSync, source=key, syncid=value} + end +end From e116ce0a6e93bd7599c5a29d819d5fb0aae01d38 Mon Sep 17 00:00:00 2001 From: Birger Schmidt Date: Wed, 24 Oct 2012 17:46:18 +1100 Subject: [PATCH 2/3] fix parameter handling for rsync options --- default-rsync.lua | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/default-rsync.lua b/default-rsync.lua index 13f65f2..ba45535 100644 --- a/default-rsync.lua +++ b/default-rsync.lua @@ -50,6 +50,7 @@ rsync.checkgauge = { rsync = { -- rsync binary binary = true, + _extra = true, -- rsync shortflags verbose = true, @@ -461,17 +462,17 @@ rsync.prepare = function( end if crsync.rsh then - computed[ computedN ] = '--rsh=' + crsync.rsh + computed[ computedN ] = '--rsh='..crsync.rsh computedN = computedN + 1 end if crsync.rsync_path then - computed[ computedN ] = '--rsync-path=' + crsync.rsync_path + computed[ computedN ] = '--rsync-path='..crsync.rsync_path computedN = computedN + 1 end if crsync.temp_dir then - computed[ computedN ] = '--temp-dir=' + crsync.temp_dir + computed[ computedN ] = '--temp-dir='..crsync.temp_dir computedN = computedN + 1 end From 33c8f88bd3a367389176cc54576863603d90e41f Mon Sep 17 00:00:00 2001 From: Birger Schmidt Date: Wed, 24 Oct 2012 17:51:25 +1100 Subject: [PATCH 3/3] delete bad (in the big scale) non working example --- examples/lsyncd-csync2.conf.lua | 120 -------------------------------- 1 file changed, 120 deletions(-) delete mode 100644 examples/lsyncd-csync2.conf.lua diff --git a/examples/lsyncd-csync2.conf.lua b/examples/lsyncd-csync2.conf.lua deleted file mode 100644 index 1b4ccc7..0000000 --- a/examples/lsyncd-csync2.conf.lua +++ /dev/null @@ -1,120 +0,0 @@ ------ --- User configuration file for lsyncd. --- --- This example synchronizes one specific directory through multiple nodes, --- by combining csync2 and lsyncd as monitoring tools. --- It avoids any race conditions generated by lsyncd, while detecting and --- processing multiple inotify events in batch, on each node monitored by --- csync2 daemon. --- --- @author Floren Munteanu --- @link http://www.axivo.com/community/threads/lightning-fast-synch-with-csync2-and-lsyncd.121/ ------ -settings = { - logident = "lsyncd", - logfacility = "user", - logfile = "/var/log/lsyncd/lsyncd.log", - statusFile = "/var/log/lsyncd/status.log", - statusInterval = 1 -} - -initSync = { - delay = 1, - maxProcesses = 1, - action = function(inlet) - local config = inlet.getConfig() - local count = 0; - local elist = inlet.getEvents(function(event) - if string.starts(event.pathname, "/.") then - -- log("Normal", "discard ",event.pathname) - inlet.discardEvent(event) - return - end - count = count + 1; - return event.etype ~= 'Init' and event.etype ~= "Blanket" - end) - local directory = string.sub(config.source, 1, -2) - local paths = elist.getPaths(function(etype, path) - return directory .. path - end) - if count > 0 then - log("Normal", "Processing syncing list of ",#paths, " files.") - local zpaths = table.concat(paths, ('\000')) - spawn(elist, "/usr/bin/xargs", "<", zpaths, "-0", "/usr/sbin/csync2", "-C", config.syncid, "-Amv") - local sync = inlet.createBlanketEvent() - spawn(sync, "/usr/bin/xargs", "<", zpaths, "-0", "/usr/sbin/csync2", "-C", config.syncid, "-ABuv") - end - end, - collect = function(agent, exitcode) - local config = agent.config - if not agent.isList and agent.etype == "Init" then - if exitcode == 0 then - log("Normal", "Startup of '", config.syncid, ":", config.source, "' instance finished.") - elseif config.exitcodes and config.exitcodes[exitcode] == "again" then - log("Normal", "Retrying startup of '", config.syncid, ":", config.source, "' instance.") - return "again" - else - log("Error", "Failure on startup of '", config.syncid, ":", config.source, "' instance.") - terminate(-1) - end - return - end - local rc = config.exitcodes and config.exitcodes[exitcode] - if rc == "die" then - return rc - end - if agent.isList then - if rc == "again" then - log("Normal", "Retrying events list on exitcode = ", exitcode) - else - -- log("Normal", "Finished events list = ", exitcode) - end - else - if rc == "again" then - log("Normal", "Retrying ", agent.etype, " on ", agent.sourcePath, " = ", exitcode) - else - log("Normal", "Finished ", agent.etype, " on ", agent.sourcePath, " = ", exitcode) - end - end - return rc - end, - init = function(event) - local inlet = event.inlet; - local config = inlet.getConfig() - log("Normal", "started up in dirty sync state: ", config.syncid, ":", config.source) - spawn(event, "/bin/true") - end, - prepare = function(config) - -- log("Normal", "prepare to sync: ", config.syncid, ":", config.source) - if not config.syncid then - error("Missing 'syncid' parameter.", 4) - end - local c = "csync2_" .. config.syncid .. ".cfg" - local f, err = io.open("/etc/" .. c, "r") - if not f then - error("Invalid 'syncid' parameter: " .. err, 4) - end - f:close() - end -} - -local sources = { - ["/mnt/assets/uploads/2011/profiles/"] = "0mnt0assets0uploads020110profiles0", - ["/mnt/assets/uploads/2011/profiles/1*"] = "0mnt0assets0uploads020110profiles01", - ["/mnt/assets/uploads/2011/profiles/2*"] = "0mnt0assets0uploads020110profiles02", - ["/mnt/assets/uploads/2011/profiles/3*"] = "0mnt0assets0uploads020110profiles03", -} -for key, value in pairs(sources) do - -- very ugly hack to handle well known subdirs, because we have no glob - if string.ends(key, "*") then - local directory = string.sub(key, 1, -2) - for _,i in pairs({"","0","1","2","3","4","5","6","7","8","9","a","b","c","d","e","f"}) do - for _,j in pairs({"","0","1","2","3","4","5","6","7","8","9","a","b","c","d","e","f"}) do - os.execute("install -o www-data -g www-data -d " .. directory .. i .. j) - sync {initSync, source=directory .. i .. j, syncid=value} - end - end - else - sync {initSync, source=key, syncid=value} - end -end