diff --git a/ChangeLog b/ChangeLog index 39d3ce6..64138c3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,5 @@ 2018-??-??: 2.2.3 + enhaencement: supporting includes with new filter and filterFrom options change: if the target/targetdir ands with a ':' do not append a trailing '/' to it, since that would change it from homedir to rootdir! add: example for Amazon S3 Bucket (Daniel Miranda) diff --git a/default-rsync.lua b/default-rsync.lua index 7222faf..5f04796 100644 --- a/default-rsync.lua +++ b/default-rsync.lua @@ -49,6 +49,8 @@ rsync.checkgauge = { delete = true, exclude = true, excludeFrom = true, + filter = true, + filterFrom = true, target = true, rsync = { @@ -117,28 +119,6 @@ local eventNotInitBlank = end --- --- Replaces what rsync would consider filter rules by literals. --- -local replaceRsyncFilter = - function -( - path -) - if not path - then - return - end - - return( - path - :gsub( '%?', '\\?' ) - :gsub( '%*', '\\*' ) - :gsub( '%[', '\\[' ) - ) -end - - -- -- Spawns rsync for a list of events -- @@ -161,10 +141,11 @@ rsync.action = function -- -- Replaces what rsync would consider filter rules by literals -- - local function sub( p ) - if not p then - return - end + local function sub + ( + p -- pattern + ) + if not p then return end return p: gsub( '%?', '\\?' ): @@ -179,8 +160,14 @@ rsync.action = function -- Deletes create multi match patterns -- local paths = elist.getPaths( - function( etype, path1, path2 ) - if string.byte( path1, -1 ) == 47 and etype == 'Delete' then + function + ( + etype, -- event type + path1, -- path + path2 -- path to for move events + ) + if string.byte( path1, -1 ) == 47 and etype == 'Delete' + then return sub( path1 )..'***', sub( path2 ) else return sub( path1 ), sub( path2 ) @@ -195,11 +182,12 @@ rsync.action = function local filterP = { } -- adds one path to the filter - local function addToFilter( path ) + local function addToFilter + ( + path + ) - if filterP[ path ] then - return - end + if filterP[ path ] then return end filterP[ path ] = true @@ -211,21 +199,21 @@ rsync.action = function -- rsync needs to have entries for all steps in the path, -- so the file for example d1/d2/d3/f1 needs following filters: -- 'd1/', 'd1/d2/', 'd1/d2/d3/' and 'd1/d2/d3/f1' - for _, path in ipairs( paths ) do - - if path and path ~= '' then - - addToFilter(path) + for _, path in ipairs( paths ) + do + if path and path ~= '' + then + addToFilter( path ) local pp = string.match( path, '^(.*/)[^/]+/?' ) - while pp do - addToFilter(pp) + while pp + do + addToFilter( pp ) + pp = string.match( pp, '^(.*/)[^/]+/?' ) end - end - end log( @@ -334,6 +322,8 @@ rsync.init = function local excludes = inlet.getExcludes( ) + local filters = inlet.hasFilters( ) and inlet.getFilters( ) + local delete = nil local target = config.target @@ -354,9 +344,9 @@ rsync.init = function delete = { '--delete', '--ignore-errors' } end - if #excludes == 0 + if not filters and #excludes == 0 then - -- starts rsync without any excludes + -- starts rsync without any filters or excludes log( 'Normal', 'recursive startup rsync: ', @@ -375,7 +365,8 @@ rsync.init = function target ) - else + elseif not filters + then -- starts rsync providing an exclusion list -- on stdin local exS = table.concat( excludes, '\n' ) @@ -401,6 +392,32 @@ rsync.init = function config.source, target ) + else + -- starts rsync providing a filter list + -- on stdin + local fS = table.concat( filters, '\n' ) + + log( + 'Normal', + 'recursive startup rsync: ', + config.source, + ' -> ', + target, + ' filtering\n', + fS + ) + + spawn( + event, + config.rsync.binary, + '<', fS, + '--filter=. -', + delete, + config.rsync._computed, + '-r', + config.source, + target + ) end end diff --git a/lsyncd.lua b/lsyncd.lua index e050451..1ee1670 100644 --- a/lsyncd.lua +++ b/lsyncd.lua @@ -1650,6 +1650,19 @@ local InletFactory = ( function sync:addExclude( pattern ) end, + + -- + -- Appens a filter. + -- + appendFilter = function + ( + sync, -- the sync of the inlet + rule, -- '+' or '-' + pattern -- exlusion pattern to add + ) + sync:appendFilter( rule, pattern ) + end, + -- -- Removes an exclude. -- @@ -1663,7 +1676,7 @@ local InletFactory = ( function -- -- Gets the list of excludes in their - -- rsynlike patterns form. + -- rsync-like patterns form. -- getExcludes = function ( @@ -1682,6 +1695,48 @@ local InletFactory = ( function return e; end, + -- + -- Gets the list of filters and excldues + -- as rsync-like filter/patterns form. + -- + getFilters = function + ( + sync -- the sync of the inlet + ) + -- creates a copy + local e = { } + local en = 1; + + -- first takes the filters + if sync.filters + then + for _, entry in ipairs( sync.filters.list ) + do + e[ en ] = entry.rule .. ' ' .. entry.pattern; + en = en + 1; + end + end + + -- then the excludes + for k, _ in pairs( sync.excludes.list ) + do + e[ en ] = '- ' .. k; + en = en + 1; + end + + return e; + end, + + -- + -- Returns true if the sync has filters + -- + hasFilters = function + ( + sync -- the sync of the inlet + ) + return not not sync.filters + end, + -- -- Creates a blanketEvent that blocks everything -- and is blocked by everything. @@ -1892,8 +1947,9 @@ local Excludes = ( function( ) self, -- self pattern -- the pattern to remove ) + -- already in the list? if not self.list[ pattern ] - then -- already in the list? + then log( 'Normal', 'Removing not excluded exclude "' .. pattern .. '"' @@ -2016,6 +2072,180 @@ local Excludes = ( function( ) end )( ) +-- +-- A set of filter patterns. +-- +-- Filters allow excludes and includes +-- +local Filters = ( function( ) + + -- + -- Turns a rsync like file pattern to a lua pattern. + -- ( at best it can ) + -- + local function toLuaPattern + ( + p -- the rsync like pattern + ) + local o = p + + p = string.gsub( p, '%%', '%%%%' ) + p = string.gsub( p, '%^', '%%^' ) + p = string.gsub( p, '%$', '%%$' ) + p = string.gsub( p, '%(', '%%(' ) + p = string.gsub( p, '%)', '%%)' ) + p = string.gsub( p, '%.', '%%.' ) + p = string.gsub( p, '%[', '%%[' ) + p = string.gsub( p, '%]', '%%]' ) + p = string.gsub( p, '%+', '%%+' ) + p = string.gsub( p, '%-', '%%-' ) + p = string.gsub( p, '%?', '[^/]' ) + p = string.gsub( p, '%*', '[^/]*' ) + -- this was a ** before + p = string.gsub( p, '%[%^/%]%*%[%^/%]%*', '.*' ) + p = string.gsub( p, '^/', '^/' ) + + if p:sub( 1, 2 ) ~= '^/' + then + -- if does not begin with '^/' + -- then all matches should begin with '/'. + p = '/' .. p; + end + + log( 'Filter', 'toLuaPattern "', o, '" = "', p, '"' ) + + return p + end + + -- + -- Appends a filter pattern + -- + local function append + ( + self, -- the filters object + line -- filter line + ) + local rule, pattern = string.match( line, '%s*([+|-])%s*(.*)' ) + + if not rule or not pattern + then + log( 'Error', 'Unknown filter rule: "', line, '"' ) + terminate( -1 ) + end + + local lp = toLuaPattern( pattern ) + + table.insert( self. list, { rule = rule, pattern = pattern, lp = lp } ) + end + + -- + -- Adds a list of patterns to exclude. + -- + local function appendList + ( + self, + plist + ) + for _, v in ipairs( plist ) + do + append( self, v ) + end + end + + -- + -- Loads the filters from a file. + -- + local function loadFile + ( + self, -- self + file -- filename to load from + ) + f, err = io.open( file ) + + if not f + then + log( 'Error', 'Cannot open filter file "', file, '": ', err ) + + terminate( -1 ) + end + + for line in f:lines( ) + do + if string.match( line, '^%s*#' ) + or string.match( line, '^%s*$' ) + then + -- a comment or empty line: ignore + else + append( self, line ) + end + end + + f:close( ) + end + + -- + -- Tests if 'path' is excluded. + -- + local function test + ( + self, -- self + path -- the path to test + ) + if path:byte( 1 ) ~= 47 + then + error( 'Paths for exlusion tests must start with \'/\'' ) + end + + for _, entry in ipairs( self.list ) + do + local rule = entry.rule + local lp = entry.lp -- lua pattern + + if lp:byte( -1 ) == 36 + then + -- ends with $ + if path:match( lp ) + then + return rule == '-' + end + else + -- ends either end with / or $ + if path:match( lp .. '/' ) + or path:match( lp .. '$' ) + then + return rule == '-' + end + end + end + + return true + end + + -- + -- Cretes a new filter set. + -- + local function new + ( ) + return { + list = { }, + -- functions + append = append, + appendList = appendList, + loadFile = loadFile, + test = test, + } + end + + + -- + -- Public interface. + -- + return { new = new } + +end )( ) + + + -- -- Holds information about one observed directory including subdirs. -- @@ -2038,6 +2268,17 @@ local Sync = ( function return self.excludes:add( pattern ) end + local function appendFilter + ( + self, + rule, + pattern + ) + if not self.filters then self.filters = Filters.new( ) end + + return self.filters:append( rule, pattern ) + end + -- -- Removes an exclude. -- @@ -2280,14 +2521,8 @@ local Sync = ( function -- simple test for single path events if self.excludes:test( path ) then - log( - 'Exclude', - 'excluded ', - etype, - ' on "', - path, - '"' - ) + log( 'Exclude', 'excluded ', etype, ' on "', path, '"' ) + return end else @@ -2298,16 +2533,7 @@ local Sync = ( function if ex1 and ex2 then - log( - 'Exclude', - 'excluded "', - etype, - ' on "', - path, - '" -> "', - path2, - '"' - ) + log( 'Exclude', 'excluded "', etype, ' on "', path, '" -> "', path2, '"' ) return elseif not ex1 and ex2 @@ -2321,13 +2547,7 @@ local Sync = ( function path ) - delay( - self, - 'Delete', - time, - path, - nil - ) + delay( self, 'Delete', time, path, nil ) return elseif ex1 and not ex2 @@ -2341,13 +2561,7 @@ local Sync = ( function path2 ) - delay( - self, - 'Create', - time, - path2, - nil - ) + delay( self, 'Create', time, path2, nil ) return end @@ -2379,13 +2593,7 @@ local Sync = ( function end -- new delay - local nd = Delay.new( - etype, - self, - alarm, - path, - path2 - ) + local nd = Delay.new( etype, self, alarm, path, path2 ) if nd.etype == 'Init' or nd.etype == 'Blanket' then @@ -2562,10 +2770,7 @@ local Sync = ( function tr = test( InletFactory.d2e( d ) ) end - if tr == 'break' - then - break - end + if tr == 'break' then break end if d.status == 'active' or not tr then @@ -2767,12 +2972,14 @@ local Sync = ( function source = config.source, processes = CountArray.new( ), excludes = Excludes.new( ), + filters = nil, -- functions addBlanketDelay = addBlanketDelay, addExclude = addExclude, addInitDelay = addInitDelay, + appendFilter = appendFilter, collect = collect, concerns = concerns, delay = delay, @@ -2797,6 +3004,25 @@ local Sync = ( function -- so Sync{n} will be the n-th call to sync{} nextDefaultName = nextDefaultName + 1 + -- loads filters + if config.filter + then + local te = type( config.filter ) + + s.filters = Filters.new( ) + + if te == 'table' + then + s.filters:appendList( config.filter ) + elseif te == 'string' + then + s.filters:append( config.filter ) + else + error( 'type for filter must be table or string', 2 ) + end + + end + -- loads exclusions if config.exclude then @@ -2814,16 +3040,19 @@ local Sync = ( function end - if - config.delay ~= nil and - ( - type( config.delay ) ~= 'number' - or config.delay < 0 - ) + if config.delay ~= nil + and ( type( config.delay ) ~= 'number' or config.delay < 0 ) then error( 'delay must be a number and >= 0', 2 ) end + if config.filterFrom + then + if not s.filters then s.filters = Filters.new( ) end + + s.filters:loadFile( config.filterFrom ) + end + if config.excludeFrom then s.excludes:loadFile( config.excludeFrom ) @@ -3847,13 +4076,12 @@ local functionWriter = ( function( ) local as = '' local first = true - for _, v in ipairs( a ) do + for _, v in ipairs( a ) + do + if not first then as = as..' .. ' end - if not first then - as = as..' .. ' - end - - if v[ 1 ] then + if v[ 1 ] + then as = as .. '"' .. v[ 2 ] .. '"' else as = as .. v[ 2 ] @@ -3866,6 +4094,7 @@ local functionWriter = ( function( ) end local ft + if not haveEvent2 then ft = 'function( event )\n'