code cleanups, fixing exclusion test

This commit is contained in:
Axel Kittenberger 2016-12-06 11:11:48 +01:00
parent a4481c98e6
commit ccc492cac3
2 changed files with 162 additions and 107 deletions

View File

@ -1537,14 +1537,12 @@ local Sync = ( function( )
Queue.remove( self.delays, delay.dpos ) Queue.remove( self.delays, delay.dpos )
-- free all delays blocked by this one. -- free all delays blocked by this one.
if delay.blocks then if delay.blocks
then
for i, vd in pairs( delay.blocks ) do for i, vd in pairs( delay.blocks )
do
vd.status = 'wait' vd.status = 'wait'
end end
end end
end end
@ -1554,26 +1552,21 @@ local Sync = ( function( )
local function concerns( self, path ) local function concerns( self, path )
-- not concerned if watch rootdir doesnt match -- not concerned if watch rootdir doesnt match
if not path:starts( self.source ) then if not path:starts( self.source )
then
return false return false
end end
-- a sub dir and not concerned about subdirs -- a sub dir and not concerned about subdirs
if self.config.subdirs == false and if
self.config.subdirs == false
path:sub( #self.source, -1 ):match( '[^/]+/?' ) and path:sub( #self.source, -1 ):match( '[^/]+/?' )
then then
return false return false
end end
-- concerned if not excluded -- concerned if not excluded
return not self.excludes:test( path:sub( #self.source ) ) return not self.excludes:test( path:sub( #self.source ) )
end end
-- --
@ -1583,16 +1576,18 @@ local Sync = ( function( )
local delay = self.processes[ pid ] local delay = self.processes[ pid ]
if not delay then if not delay
then
-- not a child of this sync. -- not a child of this sync.
return return
end end
if delay.status then if delay.status
then
log( 'Delay', 'collected an event' ) log( 'Delay', 'collected an event' )
if delay.status ~= 'active' then if delay.status ~= 'active'
then
error('collecting a non-active process') error('collecting a non-active process')
end end
@ -1628,7 +1623,8 @@ local Sync = ( function( )
local alarm = self.config.delay local alarm = self.config.delay
-- delays at least 1 second -- delays at least 1 second
if alarm < 1 then if alarm < 1
then
alarm = 1 alarm = 1
end end
@ -1645,31 +1641,39 @@ local Sync = ( function( )
exitcode exitcode
) )
if rc == 'die' then if rc == 'die'
then
log( 'Error', 'Critical exitcode.' ); log( 'Error', 'Critical exitcode.' );
terminate( -1 ) terminate( -1 )
end end
if rc == 'again' then if rc == 'again'
then
-- sets the delay on wait again -- sets the delay on wait again
delay.status = 'wait' delay.status = 'wait'
local alarm = self.config.delay local alarm = self.config.delay
-- delays at least 1 second -- delays at least 1 second
if alarm < 1 then if alarm < 1
then
alarm = 1 alarm = 1
end end
alarm = now() + alarm alarm = now() + alarm
for _, d in ipairs( delay ) do for _, d in ipairs( delay )
do
d.alarm = alarm d.alarm = alarm
d.status = 'wait' d.status = 'wait'
end end
end end
for _, d in ipairs( delay ) do for _, d in ipairs( delay )
if rc ~= 'again' then do
if rc ~= 'again'
then
removeDelay( self, d ) removeDelay( self, d )
else else
d.status = 'wait' d.status = 'wait'
@ -1693,7 +1697,8 @@ local Sync = ( function( )
newDelay.status = 'block' newDelay.status = 'block'
if not oldDelay.blocks then if not oldDelay.blocks
then
oldDelay.blocks = { } oldDelay.blocks = { }
end end
@ -1719,17 +1724,18 @@ local Sync = ( function( )
-- TODO -- TODO
local function recurse( ) local function recurse( )
if etype == 'Create' and path:byte( -1 ) == 47 then if etype == 'Create' and path:byte( -1 ) == 47
then
local entries = lsyncd.readdir( self.source .. path ) local entries = lsyncd.readdir( self.source .. path )
if entries then if entries
then
for dirname, isdir in pairs(entries) do for dirname, isdir in pairs( entries )
do
local pd = path .. dirname local pd = path .. dirname
if isdir then if isdir
then
pd = pd..'/' pd = pd..'/'
end end
@ -1738,20 +1744,19 @@ local Sync = ( function( )
'Create creates Create on ', 'Create creates Create on ',
pd pd
) )
delay( self, 'Create', time, pd, nil ) delay( self, 'Create', time, pd, nil )
end end
end end
end end
end end
-- exclusion tests -- exclusion tests
if not path2 then if not path2
then
-- simple test for single path events -- simple test for single path events
if self.excludes:test(path) then if self.excludes:test(path)
then
log( log(
'Exclude', 'Exclude',
'excluded ', 'excluded ',
@ -1763,12 +1768,13 @@ local Sync = ( function( )
return return
end end
else else
-- for double paths (move) it might result into a split -- for double paths ( move ) it might result into a split
local ex1 = self.excludes:test( path ) local ex1 = self.excludes:test( path )
local ex2 = self.excludes:test( path2 ) local ex2 = self.excludes:test( path2 )
if ex1 and ex2 then if ex1 and ex2
then
log( log(
'Exclude', 'Exclude',
'excluded "', 'excluded "',
@ -1782,8 +1788,8 @@ local Sync = ( function( )
return return
elseif not ex1 and ex2 then elseif not ex1 and ex2
then
-- splits the move if only partly excluded -- splits the move if only partly excluded
log( log(
'Exclude', 'Exclude',
@ -1802,8 +1808,8 @@ local Sync = ( function( )
) )
return return
elseif ex1 and not ex2
elseif ex1 and not ex2 then then
-- splits the move if only partly excluded -- splits the move if only partly excluded
log( log(
'Exclude', 'Exclude',
@ -1825,22 +1831,26 @@ local Sync = ( function( )
end end
end end
if etype == 'Move' and not self.config.onMove then if etype == 'Move' and not self.config.onMove
then
-- if there is no move action defined, -- if there is no move action defined,
-- split a move as delete/create -- split a move as delete/create
-- layer 1 scripts which want moves events have to -- layer 1 scripts which want moves events have to
-- set onMove simply to 'true' -- set onMove simply to 'true'
log( 'Delay', 'splitting Move into Delete & Create' ) log( 'Delay', 'splitting Move into Delete & Create' )
delay( self, 'Delete', time, path, nil )
delay( self, 'Create', time, path2, nil )
return
delay( self, 'Delete', time, path, nil )
delay( self, 'Create', time, path2, nil )
return
end end
-- creates the new action -- creates the new action
local alarm local alarm
if time and self.config.delay then
if time and self.config.delay
then
alarm = time + self.config.delay alarm = time + self.config.delay
else else
alarm = now( ) alarm = now( )
@ -1855,8 +1865,8 @@ local Sync = ( function( )
path2 path2
) )
if nd.etype == 'Init' or nd.etype == 'Blanket' then if nd.etype == 'Init' or nd.etype == 'Blanket'
then
-- always stack init or blanket events on the last event -- always stack init or blanket events on the last event
log( log(
'Delay', 'Delay',
@ -1865,7 +1875,8 @@ local Sync = ( function( )
' event.' ' event.'
) )
if self.delays.size > 0 then if self.delays.size > 0
then
stack( self.delays[ self.delays.last ], nd ) stack( self.delays[ self.delays.last ], nd )
end end
@ -1873,30 +1884,36 @@ local Sync = ( function( )
recurse( ) recurse( )
return return
end end
-- detects blocks and combos by working from back until -- detects blocks and combos by working from back until
-- front through the fifo -- front through the fifo
for il, od in Queue.qpairsReverse( self.delays ) do for il, od in Queue.qpairsReverse( self.delays )
do
-- asks Combiner what to do -- asks Combiner what to do
local ac = Combiner.combine( od, nd ) local ac = Combiner.combine( od, nd )
if ac then if ac
if ac == 'remove' then then
if ac == 'remove'
then
Queue.remove( self.delays, il ) Queue.remove( self.delays, il )
elseif ac == 'stack' then elseif ac == 'stack'
then
stack( od, nd ) stack( od, nd )
nd.dpos = Queue.push( self.delays, nd ) nd.dpos = Queue.push( self.delays, nd )
elseif ac == 'absorb' then elseif ac == 'absorb'
then
-- nada -- nada
elseif ac == 'replace' then elseif ac == 'replace'
then
od.etype = nd.etype od.etype = nd.etype
od.path = nd.path od.path = nd.path
od.path2 = nd.path2 od.path2 = nd.path2
elseif ac == 'split' then elseif ac == 'split'
then
delay( self, 'Delete', time, path, nil ) delay( self, 'Delete', time, path, nil )
delay( self, 'Create', time, path2, nil ) delay( self, 'Create', time, path2, nil )
else else
error( 'unknown result of combine()' ) error( 'unknown result of combine()' )
@ -1908,7 +1925,8 @@ local Sync = ( function( )
il = il - 1 il = il - 1
end end
if nd.path2 then if nd.path2
then
log( 'Delay','New ',nd.etype,':',nd.path,'->',nd.path2 ) log( 'Delay','New ',nd.etype,':',nd.path,'->',nd.path2 )
else else
log( 'Delay','New ',nd.etype,':',nd.path ) log( 'Delay','New ',nd.etype,':',nd.path )
@ -1916,6 +1934,7 @@ local Sync = ( function( )
-- no block or combo -- no block or combo
nd.dpos = Queue.push( self.delays, nd ) nd.dpos = Queue.push( self.delays, nd )
recurse( ) recurse( )
end end
@ -1924,16 +1943,21 @@ local Sync = ( function( )
-- --
local function getAlarm( self ) local function getAlarm( self )
if self.processes:size( ) >= self.config.maxProcesses then if self.processes:size( ) >= self.config.maxProcesses
then
return false return false
end end
-- first checks if more processes could be spawned -- first checks if more processes could be spawned
if self.processes:size( ) < self.config.maxProcesses then if self.processes:size( ) < self.config.maxProcesses
then
-- finds the nearest delay waiting to be spawned -- finds the nearest delay waiting to be spawned
for _, d in Queue.qpairs( self.delays ) do for _, d in Queue.qpairs( self.delays )
if d.status == 'wait' then return d.alarm end do
if d.status == 'wait'
then
return d.alarm
end
end end
end end
@ -1948,29 +1972,39 @@ local Sync = ( function( )
-- @param test function to test each delay -- @param test function to test each delay
-- --
local function getDelays( self, test ) local function getDelays( self, test )
local dlist = { sync = self}
local dlist = { sync = self }
local dlistn = 1 local dlistn = 1
local blocks = { } local blocks = { }
-- --
-- inheritly transfers all blocks from delay -- inheritly transfers all blocks from delay
-- --
local function getBlocks( delay ) local function getBlocks( delay )
blocks[ delay ] = true blocks[ delay ] = true
if delay.blocks then
for i, d in ipairs( delay.blocks ) do if delay.blocks
then
for i, d in ipairs( delay.blocks )
do
getBlocks( d ) getBlocks( d )
end end
end end
end end
for i, d in Queue.qpairs( self.delays ) do for i, d in Queue.qpairs( self.delays )
do
if d.status == 'active' or if d.status == 'active' or
( test and not test( InletFactory.d2e( d ) ) ) ( test and not test( InletFactory.d2e( d ) ) )
then then
getBlocks( d ) getBlocks( d )
elseif not blocks[ d ] then elseif not blocks[ d ]
then
dlist[ dlistn ] = d dlist[ dlistn ] = d
dlistn = dlistn + 1 dlistn = dlistn + 1
end end
end end
@ -2005,28 +2039,33 @@ local Sync = ( function( )
and processCount >= uSettings.maxProcesses and processCount >= uSettings.maxProcesses
then then
log('Alarm', 'at global process limit.') log('Alarm', 'at global process limit.')
return return
end end
if self.delays.size < self.config.maxDelays then if self.delays.size < self.config.maxDelays
then
-- time constrains are only concerned if not maxed -- time constrains are only concerned if not maxed
-- the delay FIFO already. -- the delay FIFO already.
if d.alarm ~= true and timestamp < d.alarm then if d.alarm ~= true and timestamp < d.alarm
then
-- reached point in stack where delays are in future -- reached point in stack where delays are in future
return return
end end
end end
if d.status == 'wait' then if d.status == 'wait'
then
-- found a waiting delay -- found a waiting delay
if d.etype ~= 'Init' then if d.etype ~= 'Init'
then
self.config.action( self.inlet ) self.config.action( self.inlet )
else else
self.config.init( InletFactory.d2e( d ) ) self.config.init( InletFactory.d2e( d ) )
end end
if self.processes:size( ) >= self.config.maxProcesses then if self.processes:size( ) >= self.config.maxProcesses
then
-- no further processes -- no further processes
return return
end end
@ -2039,18 +2078,21 @@ local Sync = ( function( )
-- --
local function getNextDelay( self, timestamp ) local function getNextDelay( self, timestamp )
for i, d in Queue.qpairs( self.delays ) do for i, d in Queue.qpairs( self.delays )
do
if self.delays.size < self.config.maxDelays then if self.delays.size < self.config.maxDelays
then
-- time constrains are only concerned if not maxed -- time constrains are only concerned if not maxed
-- the delay FIFO already. -- the delay FIFO already.
if d.alarm ~= true and timestamp < d.alarm then if d.alarm ~= true and timestamp < d.alarm
then
-- reached point in stack where delays are in future -- reached point in stack where delays are in future
return nil return nil
end end
end end
if d.status == 'wait' then if d.status == 'wait'
then
-- found a waiting delay -- found a waiting delay
return d return d
end end
@ -2063,8 +2105,11 @@ local Sync = ( function( )
-- Used as custom marker. -- Used as custom marker.
-- --
local function addBlanketDelay( self ) local function addBlanketDelay( self )
local newd = Delay.new( 'Blanket', self, true, '' ) local newd = Delay.new( 'Blanket', self, true, '' )
newd.dpos = Queue.push( self.delays, newd ) newd.dpos = Queue.push( self.delays, newd )
return newd return newd
end end
@ -2089,15 +2134,18 @@ local Sync = ( function( )
local spaces = ' ' local spaces = ' '
f:write( self.config.name, ' source=', self.source, '\n' ) f:write( self.config.name, ' source=', self.source, '\n' )
f:write( 'There are ', self.delays.size, ' delays\n') f:write( 'There are ', self.delays.size, ' delays\n')
for i, vd in Queue.qpairs( self.delays ) do for i, vd in Queue.qpairs( self.delays )
do
local st = vd.status local st = vd.status
f:write( st, string.sub( spaces, 1, 7 - #st ) ) f:write( st, string.sub( spaces, 1, 7 - #st ) )
f:write( vd.etype, ' ' ) f:write( vd.etype, ' ' )
f:write( vd.path ) f:write( vd.path )
if vd.path2 then if vd.path2
then
f:write( ' -> ',vd.path2 ) f:write( ' -> ',vd.path2 )
end end
@ -2109,11 +2157,14 @@ local Sync = ( function( )
local nothing = true local nothing = true
for t, p in pairs( self.excludes.list ) do for t, p in pairs( self.excludes.list )
do
nothing = false nothing = false
f:write( t,'\n' ) f:write( t,'\n' )
end end
if nothing then
if nothing
then
f:write(' nothing.\n') f:write(' nothing.\n')
end end
@ -2125,7 +2176,8 @@ local Sync = ( function( )
-- --
local function new( config ) local function new( config )
local s = { local s =
{
-- fields -- fields
config = config, config = config,
@ -2154,7 +2206,8 @@ local Sync = ( function( )
s.inlet = InletFactory.newInlet( s ) s.inlet = InletFactory.newInlet( s )
-- provides a default name if needed -- provides a default name if needed
if not config.name then if not config.name
then
config.name = 'Sync' .. nextDefaultName config.name = 'Sync' .. nextDefaultName
end end
@ -2163,13 +2216,15 @@ local Sync = ( function( )
nextDefaultName = nextDefaultName + 1 nextDefaultName = nextDefaultName + 1
-- loads exclusions -- loads exclusions
if config.exclude then if config.exclude
then
local te = type( config.exclude ) local te = type( config.exclude )
if te == 'table' then if te == 'table'
then
s.excludes:addList( config.exclude ) s.excludes:addList( config.exclude )
elseif te == 'string' then elseif te == 'string'
then
s.excludes:add( config.exclude ) s.excludes:add( config.exclude )
else else
error( 'type for exclude must be table or string', 2 ) error( 'type for exclude must be table or string', 2 )
@ -2180,17 +2235,16 @@ local Sync = ( function( )
if if
config.delay ~= nil and config.delay ~= nil and
( (
type(config.delay) ~= 'number' or type( config.delay ) ~= 'number'
config.delay < 0 or config.delay < 0
) )
then then
error( 'delay must be a number and >= 0', 2 ) error( 'delay must be a number and >= 0', 2 )
end end
if config.excludeFrom then if config.excludeFrom
then
s.excludes:loadFile( config.excludeFrom ) s.excludes:loadFile( config.excludeFrom )
end end
return s return s
@ -2231,7 +2285,8 @@ local Syncs = ( function( )
round = round + 1; round = round + 1;
if round > #syncsList then if round > #syncsList
then
round = 1 round = 1
end end

View File

@ -13,16 +13,16 @@ local range = 5
local log = {"-log", "all"} local log = {"-log", "all"}
writefile(cfgfile, [[ writefile(cfgfile, [[
settings = { settings {
logfile = "]]..logfile..[[", logfile = "]]..logfile..[[",
nodaemon = true, nodaemon = true,
delay = 3,
} }
sync { sync {
default.rsync, default.rsync,
source = "]]..srcdir..[[", source = "]]..srcdir..[[",
target = "]]..trgdir..[[", target = "]]..trgdir..[[",
delay = 3,
exclude = { exclude = {
"erf", "erf",
"/eaf", "/eaf",