diff --git a/CMakeLists.txt b/CMakeLists.txt index b7ac556..0286cdb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ # preamble project( Lsyncd ) cmake_minimum_required( VERSION 2.8 ) -set( LSYNCD_VERSION 2.2.1 ) +set( LSYNCD_VERSION 2.2.2 ) set( CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/" ) diff --git a/ChangeLog b/ChangeLog index 9e05495..0de566b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,8 +1,12 @@ -????-??-??: 2.2.2 +2017-02-16: 2.2.2 fix: checkgauge 'insist' fix: no partial path exlusion tests fix: write pid of forked process in pidfile fix: crash on not reachable target + workaround: + changed back to filter style rsync calling + until https://bugzilla.samba.org/show_bug.cgi?id=12569 + is fixed and released. 2017-01-05: 2.2.1 enhancement: now always using filter lists with rysnc diff --git a/default-rsync.lua b/default-rsync.lua index b8392e5..4c1b4a8 100644 --- a/default-rsync.lua +++ b/default-rsync.lua @@ -158,44 +158,169 @@ rsync.action = function -- deletes create multi match patterns local paths = elist.getPaths( ) - -- removes trailing slashes from dirs. - for k, v in ipairs( paths ) - do - if string.byte( v, -1 ) == 47 - then - paths[ k ] = string.sub( v, 1, -2 ) + -- + -- Replaces what rsync would consider filter rules by literals + -- + local function sub( p ) + if not p then + return end + + return p: + gsub( '%?', '\\?' ): + gsub( '%*', '\\*' ): + gsub( '%[', '\\[' ): + gsub( '%]', '\\]' ) + end + + -- + -- Gets the list of paths for the event list + -- + -- Deletes create multi match patterns + -- + local paths = elist.getPaths( + function( etype, path1, path2 ) + if string.byte( path1, -1 ) == 47 and etype == 'Delete' then + return sub( path1 )..'***', sub( path2 ) + else + return sub( path1 ), sub( path2 ) + end + end + ) + + -- stores all filters by integer index + local filterI = { } + + -- stores all filters with path index + local filterP = { } + + -- adds one path to the filter + local function addToFilter( path ) + + if filterP[ path ] then + return + end + + filterP[ path ] = true + + table.insert( filterI, path ) + end + + -- adds a path to the filter. + -- + -- rsync needs to have entries for all steps in the path, + -- so the file for example d1/d2/d3/f1 needs following filters: + -- 'd1/', 'd1/d2/', 'd1/d2/d3/' and 'd1/d2/d3/f1' + for _, path in ipairs( paths ) do + + if path and path ~= '' then + + addToFilter(path) + + local pp = string.match( path, '^(.*/)[^/]+/?' ) + + while pp do + addToFilter(pp) + pp = string.match( pp, '^(.*/)[^/]+/?' ) + end + + end + end log( 'Normal', 'Calling rsync with filter-list of new/modified files/dirs\n', - table.concat( paths, '\n' ) + table.concat( filterI, '\n' ) ) + local config = inlet.getConfig( ) + local delete = nil - if config.delete == true - or config.delete == 'running' + if config.delete == true or config.delete == 'running' then - delete = { '--delete-missing-args', '--ignore-errors' } + delete = { '--delete', '--ignore-errors' } end spawn( elist, config.rsync.binary, - '<', table.concat( paths, '\000' ), + '<', table.concat( filterI, '\000' ), config.rsync._computed, + '-r', delete, '--force', '--from0', - '--files-from=-', + '--include-from=-', + '--exclude=*', config.source, config.target ) end +---- +---- NOTE: This optimized version can be used once +---- https://bugzilla.samba.org/show_bug.cgi?id=12569 +---- is fixed. +---- +---- Spawns rsync for a list of events +---- +---- Exclusions are already handled by not having +---- events for them. +---- +--rsync.action = function +--( +-- inlet +--) +-- local config = inlet.getConfig( ) +-- +-- -- gets all events ready for syncing +-- local elist = inlet.getEvents( eventNotInitBlank ) +-- +-- -- gets the list of paths for the event list +-- -- deletes create multi match patterns +-- local paths = elist.getPaths( ) +-- +-- -- removes trailing slashes from dirs. +-- for k, v in ipairs( paths ) +-- do +-- if string.byte( v, -1 ) == 47 +-- then +-- paths[ k ] = string.sub( v, 1, -2 ) +-- end +-- end +-- +-- log( +-- 'Normal', +-- 'Calling rsync with filter-list of new/modified files/dirs\n', +-- table.concat( paths, '\n' ) +-- ) +-- +-- local delete = nil +-- +-- if config.delete == true +-- or config.delete == 'running' +-- then +-- delete = { '--delete-missing-args', '--ignore-errors' } +-- end +-- +-- spawn( +-- elist, +-- config.rsync.binary, +-- '<', table.concat( paths, '\000' ), +-- config.rsync._computed, +-- delete, +-- '--force', +-- '--from0', +-- '--files-from=-', +-- config.source, +-- config.target +-- ) +--end + + -- -- Spawns the recursive startup sync. -- diff --git a/default-rsyncssh.lua b/default-rsyncssh.lua index f30200c..750e3cb 100644 --- a/default-rsyncssh.lua +++ b/default-rsyncssh.lua @@ -157,44 +157,206 @@ rsyncssh.action = function -- deletes create multi match patterns local paths = elist.getPaths( ) - -- removes trailing slashes from dirs. - for k, v in ipairs( paths ) - do - if string.byte( v, -1 ) == 47 - then - paths[ k ] = string.sub( v, 1, -2 ) + -- + -- Replaces what rsync would consider filter rules by literals + -- + local function sub( p ) + if not p then + return end + + return p: + gsub( '%?', '\\?' ): + gsub( '%*', '\\*' ): + gsub( '%[', '\\[' ): + gsub( '%]', '\\]' ) + end + + -- + -- Gets the list of paths for the event list + -- + -- Deletes create multi match patterns + -- + local paths = elist.getPaths( + function( etype, path1, path2 ) + if string.byte( path1, -1 ) == 47 and etype == 'Delete' then + return sub( path1 )..'***', sub( path2 ) + else + return sub( path1 ), sub( path2 ) + end + end + ) + + -- stores all filters by integer index + local filterI = { } + + -- stores all filters with path index + local filterP = { } + + -- adds one path to the filter + local function addToFilter( path ) + + if filterP[ path ] + then + return + end + + filterP[ path ] = true + + table.insert( filterI, path ) + end + + -- adds a path to the filter. + -- + -- rsync needs to have entries for all steps in the path, + -- so the file for example d1/d2/d3/f1 needs following filters: + -- 'd1/', 'd1/d2/', 'd1/d2/d3/' and 'd1/d2/d3/f1' + for _, path in ipairs( paths ) + do + if path and path ~= '' + then + addToFilter(path) + + local pp = string.match( path, '^(.*/)[^/]+/?' ) + + while pp + do + addToFilter(pp) + pp = string.match( pp, '^(.*/)[^/]+/?' ) + end + + end + end log( 'Normal', - 'Rsyncing list\n', - table.concat( paths, '\n' ) + 'Calling rsync with filter-list of new/modified files/dirs\n', + table.concat( filterI, '\n' ) ) + local config = inlet.getConfig( ) + local delete = nil - if config.delete == true - or config.delete == 'running' + if config.delete == true or config.delete == 'running' then - delete = { '--delete-missing-args', '--ignore-errors' } + delete = { '--delete', '--ignore-errors' } end spawn( elist, config.rsync.binary, - '<', table.concat( paths, '\000' ), + '<', table.concat( filterI, '\000' ), config.rsync._computed, + '-r', delete, '--force', '--from0', - '--files-from=-', + '--include-from=-', + '--exclude=*', config.source, config.host .. ':' .. config.targetdir ) end ------ + +---- +---- NOTE: This optimized version can be used once +---- https://bugzilla.samba.org/show_bug.cgi?id=12569 +---- is fixed. +---- +-- +-- Spawns rsync for a list of events +-- +--rsyncssh.action = function +--( +-- inlet +--) +-- local config = inlet.getConfig( ) +-- +-- local event, event2 = inlet.getEvent( ) +-- +-- -- makes move local on target host +-- -- if the move fails, it deletes the source +-- if event.etype == 'Move' +-- then +-- local path1 = config.targetdir .. event.path +-- +-- local path2 = config.targetdir .. event2.path +-- +-- path1 = "'" .. path1:gsub ('\'', '\'"\'"\'') .. "'" +-- path2 = "'" .. path2:gsub ('\'', '\'"\'"\'') .. "'" +-- +-- log( +-- 'Normal', +-- 'Moving ', +-- event.path, +-- ' -> ', +-- event2.path +-- ) +-- +-- spawn( +-- event, +-- config.ssh.binary, +-- config.ssh._computed, +-- config.host, +-- 'mv', +-- path1, +-- path2, +-- '||', 'rm', '-rf', +-- path1 +-- ) +-- +-- return +-- end +-- +-- -- otherwise a rsync is spawned +-- local elist = inlet.getEvents( eventNotInitBlankMove ) +-- +-- -- gets the list of paths for the event list +-- -- deletes create multi match patterns +-- local paths = elist.getPaths( ) +-- +-- -- removes trailing slashes from dirs. +-- for k, v in ipairs( paths ) +-- do +-- if string.byte( v, -1 ) == 47 +-- then +-- paths[ k ] = string.sub( v, 1, -2 ) +-- end +-- end +-- +-- log( +-- 'Normal', +-- 'Rsyncing list\n', +-- table.concat( paths, '\n' ) +-- ) +-- +-- local delete = nil +-- +-- if config.delete == true +-- or config.delete == 'running' +-- then +-- delete = { '--delete-missing-args', '--ignore-errors' } +-- end +-- +-- spawn( +-- elist, +-- config.rsync.binary, +-- '<', table.concat( paths, '\000' ), +-- config.rsync._computed, +-- delete, +-- '--force', +-- '--from0', +-- '--files-from=-', +-- config.source, +-- config.host .. ':' .. config.targetdir +-- ) +--end + + +-- -- Called when collecting a finished child process -- rsyncssh.collect = function diff --git a/lsyncd.lua b/lsyncd.lua index 7a75743..949d3d9 100644 --- a/lsyncd.lua +++ b/lsyncd.lua @@ -32,7 +32,7 @@ then lsyncd.terminate( -1 ) end -lsyncd_version = '2.2.1' +lsyncd_version = '2.2.2' -- @@ -1012,7 +1012,7 @@ local Combiner = ( function ) end - + -- -- The new delay splits on the old one. -- @@ -1065,8 +1065,8 @@ local Combiner = ( function ) end end - - + + -- -- The new delay turns the old one (a move) into a delete and is blocked. -- diff --git a/tests/exclude-rsync.lua b/tests/exclude-rsync.lua index 0fac8cc..bab760a 100755 --- a/tests/exclude-rsync.lua +++ b/tests/exclude-rsync.lua @@ -3,7 +3,7 @@ require("posix") dofile("tests/testlib.lua") cwriteln('****************************************************************' ) -cwriteln(' Testing excludes' ) +cwriteln(' Testing excludes (rsync)' ) cwriteln('****************************************************************' ) local tdir, srcdir, trgdir = mktemps( ) diff --git a/tests/exclude-rsyncssh.lua b/tests/exclude-rsyncssh.lua index e1c6b1c..45388b5 100755 --- a/tests/exclude-rsyncssh.lua +++ b/tests/exclude-rsyncssh.lua @@ -4,7 +4,7 @@ require( 'posix' ) dofile( 'tests/testlib.lua' ) cwriteln( '****************************************************************' ); -cwriteln( ' Testing excludes' ); +cwriteln( ' Testing excludes (rsyncssh)' ); cwriteln( '****************************************************************' ); cwriteln( ' (this test needs passwordless ssh localhost access ' ); cwriteln( ' for current user)' );