diff --git a/CMakeLists.txt b/CMakeLists.txt index 4c0ccc5..38a8024 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -115,6 +115,7 @@ add_custom_target( tests COMMAND echo " * have lua-posix installed" COMMAND ${LUA_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/setup.lua COMMAND ${CMAKE_BINARY_DIR}/lsyncd -log all -script ${CMAKE_SOURCE_DIR}/tests/utils_test.lua + COMMAND ${LUA_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/cron-rsync.lua COMMAND ${LUA_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/schedule.lua COMMAND ${LUA_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/l4rsyncdata.lua COMMAND ${LUA_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/filter-rsync.lua diff --git a/default-rsync.lua b/default-rsync.lua index dfcfb17..a0229a1 100644 --- a/default-rsync.lua +++ b/default-rsync.lua @@ -246,7 +246,7 @@ local getBatchSize = if event.status == 'active' then return false end - if event.etype == 'Init' or event.etype == 'Blanket' then + if event.etype == 'Init' or event.etype == 'Blanket' or event.etype == 'Full' then return false end -- moves and deletes go always into batch @@ -274,6 +274,9 @@ rsync.action = function else -- spawn all files under the size limit/deletes/moves in batch mode local eventInBatch = function(event) + if event.etype == "Full" then + return false + end local size = getBatchSize(event) if type(size) == "boolean" then return size @@ -290,6 +293,9 @@ rsync.action = function local single_returned = false -- grab all events for seperate transfers local eventNoBatch = function(event) + if event.etype == "Full" then + return false + end local size = getBatchSize(event) if type(size) ~= "number" or size == nil then return false @@ -303,10 +309,11 @@ rsync.action = function end return false end + local extralist = inlet.getEvents(eventInBatch) - local elist = inlet.getEvents(eventInBatch) - if elist.size() > 0 then - run_action(inlet, elist) + -- get all batched events + if extralist.size() > 0 then + run_action(inlet, extralist) end while true do @@ -450,7 +457,7 @@ rsync.full = function -- starts rsync without any filters or excludes log( 'Normal', - 'recursive startup rsync: ', + 'recursive full rsync: ', config.source, ' -> ', target @@ -474,7 +481,7 @@ rsync.full = function log( 'Normal', - 'recursive startup rsync: ', + 'recursive full rsync: ', config.source, ' -> ', target, @@ -500,7 +507,7 @@ rsync.full = function log( 'Normal', - 'recursive startup rsync: ', + 'recursive full rsync: ', config.source, ' -> ', target, diff --git a/lsyncd.lua b/lsyncd.lua index ce1f935..d9340fd 100644 --- a/lsyncd.lua +++ b/lsyncd.lua @@ -1751,6 +1751,19 @@ local InletFactory = ( function return #dlist end end, + + -- + -- Returns the list of events + -- + getList = function( elist ) + local dlist = e2d[ elist ] + + if not dlist then + return {} + else + return dlist + end + end, } -- @@ -1979,6 +1992,19 @@ local InletFactory = ( function return d2e( sync:addBlanketDelay( ) ) end, + -- + -- Creates a blanketEvent that blocks everything + -- and is blocked by everything. + -- + createFullEvent = function + ( + sync, -- the sync of the inlet + path -- path for the full event + ) + -- return d2e( sync:delay("Full", timestamp, path, nil) ) + return d2e( sync:addFullDelay(path) ) + end, + -- -- Discards a waiting event. -- @@ -2860,7 +2886,7 @@ local Sync = ( function -- new delay local nd = Delay.new( etype, self, alarm, path, path2 ) - if nd.etype == 'Init' or nd.etype == 'Blanket' + if nd.etype == 'Init' or nd.etype == 'Blanket' or nd.etype == 'Full' then -- always stack init or blanket events on the last event log( @@ -3126,8 +3152,7 @@ local Sync = ( function if self.nextCronAlarm ~= false and self.nextCronAlarm < timestamp then -- time fo a full sync log('Info', 'Crontab triggered full sync') - -- TODO - self:delay("Full", timestamp, "/", nil) + self.inlet.createFullEvent("/") updateNextCronAlarm(self, timestamp) end @@ -3156,11 +3181,12 @@ local Sync = ( function if d.status == 'wait' then -- found a waiting delay - if d.etype ~= 'Init' - then - self.config.action( self.inlet ) - else + if d.etype == 'Init' then self.config.init( InletFactory.d2e( d ) ) + elseif d.etype == 'Full' and self.config.full then + self.config.full( InletFactory.d2e( d ) ) + else + self.config.action( self.inlet ) end if self.processes:size( ) >= self.config.maxProcesses @@ -3216,6 +3242,21 @@ local Sync = ( function return newd end + -- + -- Adds an full delay that will initiate a full transfer. + -- + local function addFullDelay + ( + self, + path + ) + local newd = Delay.new( 'Full', self, true, path ) + + newd.dpos = self.delays:push( newd ) + + return newd + end + -- -- Adds and returns a blanket delay thats blocks all. -- Used as startup marker to call init asap. @@ -3340,6 +3381,7 @@ local Sync = ( function -- functions addBlanketDelay = addBlanketDelay, + addFullDelay = addFullDelay, addExclude = addExclude, addInitDelay = addInitDelay, appendFilter = appendFilter, diff --git a/tests/cron-rsync.lua b/tests/cron-rsync.lua new file mode 100644 index 0000000..7843fd4 --- /dev/null +++ b/tests/cron-rsync.lua @@ -0,0 +1,170 @@ +require( 'posix' ) +dofile( 'tests/testlib.lua' ) + +cwriteln( '****************************************************************' ) +cwriteln( ' Testing crontab (rsync)' ) +cwriteln( '****************************************************************' ) + +local tdir, srcdir, trgdir = mktemps( ) +local logfile = tdir .. "log" +local cfgfile = tdir .. "config.lua" +local range = 5 +local log = {"-log", "all"} + +writefile(cfgfile, [[ +settings { + logfile = "]]..logfile..[[", + nodaemon = true, +} + +sync { + default.rsync, + crontab = { + -- trigger full sync every 1 minute + "*/10 * * * * *", + }, + source = "]]..srcdir..[[", + action = function (inlet) + local e = inlet.getEvent( ); + print("inhibit action ".. e.path.. " = " .. e.etype); + if e.etype ~= "Full" then + inlet.discardEvent(e); + return; + end + return default.rsync.action(inlet); + end, + target = "]]..trgdir..[[", + delay = 2, + delete = true, + rsync = { + verbose = true, + inplace = true, + _extra = { + "-vv", + "--info=progress2" + } + }, + filter = { + '- /ab**', + '+ /a**', + '- /**', + }, +} +]]) + +-- writes all files +local function writefiles +( ) + writefile( srcdir .. 'abc', 'abc' ) + writefile( srcdir .. 'acc', 'acc' ) + writefile( srcdir .. 'baa', 'baa' ) + posix.mkdir( srcdir .. 'abx' ) + writefile( srcdir .. 'abx/a', 'abxa' ) + posix.mkdir( srcdir .. 'acx' ) + writefile( srcdir .. 'acx/x', 'acxx' ) +end + +-- +-- Tests if the filename exists +-- fails if this is different to expect. +-- +local function testfile +( + filename, + expect +) + local stat, err = posix.stat( filename ) + + if stat and not expect + then + cwriteln( 'failure: ', filename, ' should be filtered') + + os.exit( 1 ) + end + + if not stat and expect + then + cwriteln( 'failure: ', filename, ' should not be filtered' ) + os.exit( 1 ) + end +end + +-- test all files +local function testfiles +( ) + testfile( trgdir .. 'abc', false ) + testfile( trgdir .. 'acc', true ) + testfile( trgdir .. 'baa', false ) + testfile( trgdir .. 'abx/a', false ) + testfile( trgdir .. 'acx/x', true ) +end + + +cwriteln( 'testing crontab' ) + +writefiles( ) + +cwriteln( 'starting Lsyncd' ) + +local pid = spawn( './lsyncd', cfgfile, '-log', 'all' ) + +cwriteln( 'waiting for Lsyncd to start' ) + +posix.sleep( 3 ) + +cwriteln( 'testing filters after startup' ) + +testfiles( ) + +cwriteln( 'ok, removing sources' ) + +if srcdir:sub( 1,4 ) ~= '/tmp' +then + -- just to make sure before rm -rf + cwriteln( 'exit before drama, srcdir is "', srcdir, '"' ) + + os.exit( 1 ) +end + +os.execute( 'rm -rf '..srcdir..'/*' ) + +cwriteln( 'waiting for Lsyncd to remove destination' ) + +posix.sleep( 20 ) + +result, code = execute( 'diff -urN ' .. srcdir .. ' ' .. trgdir ) + +if result ~= 'exit' or code ~= 0 +then + cwriteln( 'fail, target directory not empty!' ) + posix.kill( pid ) + + os.exit( 1 ) +end + +cwriteln( 'writing files after startup' ) + +writefiles( ) + +cwriteln( 'waiting for Lsyncd to transmit changes' ) + +posix.sleep( 5 ) + +testfiles( ) + +cwriteln( 'killing started Lsyncd' ) + +posix.kill( pid ) +local _, exitmsg, exitcode = posix.wait( lpid ) + +cwriteln( 'Exitcode of Lsyncd = ', exitmsg, ' ', exitcode ); + +if exitcode == 143 +then + cwriteln( 'OK' ) + os.exit( 0 ) +else + os.exit( 1 ) +end + +-- TODO remove temp