mirror of
https://github.com/octoleo/lsyncd.git
synced 2025-01-05 16:12:32 +00:00
reverting to rsync filter useage, until rysnc bug is fixed
This commit is contained in:
Axel Kittenberger
parent
24d58f7741
commit
ea068b4014
@ -1,7 +1,7 @@
|
||||
# preamble
|
||||
project( Lsyncd )
|
||||
cmake_minimum_required( VERSION 2.8 )
|
||||
set( LSYNCD_VERSION 2.2.1 )
|
||||
set( LSYNCD_VERSION 2.2.2 )
|
||||
set( CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/" )
|
||||
|
||||
|
||||
|
||||
@ -1,8 +1,12 @@
|
||||
????-??-??: 2.2.2
|
||||
2017-02-16: 2.2.2
|
||||
fix: checkgauge 'insist'
|
||||
fix: no partial path exlusion tests
|
||||
fix: write pid of forked process in pidfile
|
||||
fix: crash on not reachable target
|
||||
workaround:
|
||||
changed back to filter style rsync calling
|
||||
until https://bugzilla.samba.org/show_bug.cgi?id=12569
|
||||
is fixed and released.
|
||||
|
||||
2017-01-05: 2.2.1
|
||||
enhancement: now always using filter lists with rysnc
|
||||
|
||||
@ -158,44 +158,169 @@ rsync.action = function
|
||||
-- deletes create multi match patterns
|
||||
local paths = elist.getPaths( )
|
||||
|
||||
-- removes trailing slashes from dirs.
|
||||
for k, v in ipairs( paths )
|
||||
do
|
||||
if string.byte( v, -1 ) == 47
|
||||
then
|
||||
paths[ k ] = string.sub( v, 1, -2 )
|
||||
--
|
||||
-- Replaces what rsync would consider filter rules by literals
|
||||
--
|
||||
local function sub( p )
|
||||
if not p then
|
||||
return
|
||||
end
|
||||
|
||||
return p:
|
||||
gsub( '%?', '\\?' ):
|
||||
gsub( '%*', '\\*' ):
|
||||
gsub( '%[', '\\[' ):
|
||||
gsub( '%]', '\\]' )
|
||||
end
|
||||
|
||||
--
|
||||
-- Gets the list of paths for the event list
|
||||
--
|
||||
-- Deletes create multi match patterns
|
||||
--
|
||||
local paths = elist.getPaths(
|
||||
function( etype, path1, path2 )
|
||||
if string.byte( path1, -1 ) == 47 and etype == 'Delete' then
|
||||
return sub( path1 )..'***', sub( path2 )
|
||||
else
|
||||
return sub( path1 ), sub( path2 )
|
||||
end
|
||||
end
|
||||
)
|
||||
|
||||
-- stores all filters by integer index
|
||||
local filterI = { }
|
||||
|
||||
-- stores all filters with path index
|
||||
local filterP = { }
|
||||
|
||||
-- adds one path to the filter
|
||||
local function addToFilter( path )
|
||||
|
||||
if filterP[ path ] then
|
||||
return
|
||||
end
|
||||
|
||||
filterP[ path ] = true
|
||||
|
||||
table.insert( filterI, path )
|
||||
end
|
||||
|
||||
-- adds a path to the filter.
|
||||
--
|
||||
-- rsync needs to have entries for all steps in the path,
|
||||
-- so the file for example d1/d2/d3/f1 needs following filters:
|
||||
-- 'd1/', 'd1/d2/', 'd1/d2/d3/' and 'd1/d2/d3/f1'
|
||||
for _, path in ipairs( paths ) do
|
||||
|
||||
if path and path ~= '' then
|
||||
|
||||
addToFilter(path)
|
||||
|
||||
local pp = string.match( path, '^(.*/)[^/]+/?' )
|
||||
|
||||
while pp do
|
||||
addToFilter(pp)
|
||||
pp = string.match( pp, '^(.*/)[^/]+/?' )
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
log(
|
||||
'Normal',
|
||||
'Calling rsync with filter-list of new/modified files/dirs\n',
|
||||
table.concat( paths, '\n' )
|
||||
table.concat( filterI, '\n' )
|
||||
)
|
||||
|
||||
local config = inlet.getConfig( )
|
||||
|
||||
local delete = nil
|
||||
|
||||
if config.delete == true
|
||||
or config.delete == 'running'
|
||||
if config.delete == true or config.delete == 'running'
|
||||
then
|
||||
delete = { '--delete-missing-args', '--ignore-errors' }
|
||||
delete = { '--delete', '--ignore-errors' }
|
||||
end
|
||||
|
||||
spawn(
|
||||
elist,
|
||||
config.rsync.binary,
|
||||
'<', table.concat( paths, '\000' ),
|
||||
'<', table.concat( filterI, '\000' ),
|
||||
config.rsync._computed,
|
||||
'-r',
|
||||
delete,
|
||||
'--force',
|
||||
'--from0',
|
||||
'--files-from=-',
|
||||
'--include-from=-',
|
||||
'--exclude=*',
|
||||
config.source,
|
||||
config.target
|
||||
)
|
||||
end
|
||||
|
||||
|
||||
----
|
||||
---- NOTE: This optimized version can be used once
|
||||
---- https://bugzilla.samba.org/show_bug.cgi?id=12569
|
||||
---- is fixed.
|
||||
----
|
||||
---- Spawns rsync for a list of events
|
||||
----
|
||||
---- Exclusions are already handled by not having
|
||||
---- events for them.
|
||||
----
|
||||
--rsync.action = function
|
||||
--(
|
||||
-- inlet
|
||||
--)
|
||||
-- local config = inlet.getConfig( )
|
||||
--
|
||||
-- -- gets all events ready for syncing
|
||||
-- local elist = inlet.getEvents( eventNotInitBlank )
|
||||
--
|
||||
-- -- gets the list of paths for the event list
|
||||
-- -- deletes create multi match patterns
|
||||
-- local paths = elist.getPaths( )
|
||||
--
|
||||
-- -- removes trailing slashes from dirs.
|
||||
-- for k, v in ipairs( paths )
|
||||
-- do
|
||||
-- if string.byte( v, -1 ) == 47
|
||||
-- then
|
||||
-- paths[ k ] = string.sub( v, 1, -2 )
|
||||
-- end
|
||||
-- end
|
||||
--
|
||||
-- log(
|
||||
-- 'Normal',
|
||||
-- 'Calling rsync with filter-list of new/modified files/dirs\n',
|
||||
-- table.concat( paths, '\n' )
|
||||
-- )
|
||||
--
|
||||
-- local delete = nil
|
||||
--
|
||||
-- if config.delete == true
|
||||
-- or config.delete == 'running'
|
||||
-- then
|
||||
-- delete = { '--delete-missing-args', '--ignore-errors' }
|
||||
-- end
|
||||
--
|
||||
-- spawn(
|
||||
-- elist,
|
||||
-- config.rsync.binary,
|
||||
-- '<', table.concat( paths, '\000' ),
|
||||
-- config.rsync._computed,
|
||||
-- delete,
|
||||
-- '--force',
|
||||
-- '--from0',
|
||||
-- '--files-from=-',
|
||||
-- config.source,
|
||||
-- config.target
|
||||
-- )
|
||||
--end
|
||||
|
||||
|
||||
--
|
||||
-- Spawns the recursive startup sync.
|
||||
--
|
||||
|
||||
@ -157,44 +157,206 @@ rsyncssh.action = function
|
||||
-- deletes create multi match patterns
|
||||
local paths = elist.getPaths( )
|
||||
|
||||
-- removes trailing slashes from dirs.
|
||||
for k, v in ipairs( paths )
|
||||
do
|
||||
if string.byte( v, -1 ) == 47
|
||||
then
|
||||
paths[ k ] = string.sub( v, 1, -2 )
|
||||
--
|
||||
-- Replaces what rsync would consider filter rules by literals
|
||||
--
|
||||
local function sub( p )
|
||||
if not p then
|
||||
return
|
||||
end
|
||||
|
||||
return p:
|
||||
gsub( '%?', '\\?' ):
|
||||
gsub( '%*', '\\*' ):
|
||||
gsub( '%[', '\\[' ):
|
||||
gsub( '%]', '\\]' )
|
||||
end
|
||||
|
||||
--
|
||||
-- Gets the list of paths for the event list
|
||||
--
|
||||
-- Deletes create multi match patterns
|
||||
--
|
||||
local paths = elist.getPaths(
|
||||
function( etype, path1, path2 )
|
||||
if string.byte( path1, -1 ) == 47 and etype == 'Delete' then
|
||||
return sub( path1 )..'***', sub( path2 )
|
||||
else
|
||||
return sub( path1 ), sub( path2 )
|
||||
end
|
||||
end
|
||||
)
|
||||
|
||||
-- stores all filters by integer index
|
||||
local filterI = { }
|
||||
|
||||
-- stores all filters with path index
|
||||
local filterP = { }
|
||||
|
||||
-- adds one path to the filter
|
||||
local function addToFilter( path )
|
||||
|
||||
if filterP[ path ]
|
||||
then
|
||||
return
|
||||
end
|
||||
|
||||
filterP[ path ] = true
|
||||
|
||||
table.insert( filterI, path )
|
||||
end
|
||||
|
||||
-- adds a path to the filter.
|
||||
--
|
||||
-- rsync needs to have entries for all steps in the path,
|
||||
-- so the file for example d1/d2/d3/f1 needs following filters:
|
||||
-- 'd1/', 'd1/d2/', 'd1/d2/d3/' and 'd1/d2/d3/f1'
|
||||
for _, path in ipairs( paths )
|
||||
do
|
||||
if path and path ~= ''
|
||||
then
|
||||
addToFilter(path)
|
||||
|
||||
local pp = string.match( path, '^(.*/)[^/]+/?' )
|
||||
|
||||
while pp
|
||||
do
|
||||
addToFilter(pp)
|
||||
pp = string.match( pp, '^(.*/)[^/]+/?' )
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
log(
|
||||
'Normal',
|
||||
'Rsyncing list\n',
|
||||
table.concat( paths, '\n' )
|
||||
'Calling rsync with filter-list of new/modified files/dirs\n',
|
||||
table.concat( filterI, '\n' )
|
||||
)
|
||||
|
||||
local config = inlet.getConfig( )
|
||||
|
||||
local delete = nil
|
||||
|
||||
if config.delete == true
|
||||
or config.delete == 'running'
|
||||
if config.delete == true or config.delete == 'running'
|
||||
then
|
||||
delete = { '--delete-missing-args', '--ignore-errors' }
|
||||
delete = { '--delete', '--ignore-errors' }
|
||||
end
|
||||
|
||||
spawn(
|
||||
elist,
|
||||
config.rsync.binary,
|
||||
'<', table.concat( paths, '\000' ),
|
||||
'<', table.concat( filterI, '\000' ),
|
||||
config.rsync._computed,
|
||||
'-r',
|
||||
delete,
|
||||
'--force',
|
||||
'--from0',
|
||||
'--files-from=-',
|
||||
'--include-from=-',
|
||||
'--exclude=*',
|
||||
config.source,
|
||||
config.host .. ':' .. config.targetdir
|
||||
)
|
||||
end
|
||||
|
||||
-----
|
||||
|
||||
----
|
||||
---- NOTE: This optimized version can be used once
|
||||
---- https://bugzilla.samba.org/show_bug.cgi?id=12569
|
||||
---- is fixed.
|
||||
----
|
||||
--
|
||||
-- Spawns rsync for a list of events
|
||||
--
|
||||
--rsyncssh.action = function
|
||||
--(
|
||||
-- inlet
|
||||
--)
|
||||
-- local config = inlet.getConfig( )
|
||||
--
|
||||
-- local event, event2 = inlet.getEvent( )
|
||||
--
|
||||
-- -- makes move local on target host
|
||||
-- -- if the move fails, it deletes the source
|
||||
-- if event.etype == 'Move'
|
||||
-- then
|
||||
-- local path1 = config.targetdir .. event.path
|
||||
--
|
||||
-- local path2 = config.targetdir .. event2.path
|
||||
--
|
||||
-- path1 = "'" .. path1:gsub ('\'', '\'"\'"\'') .. "'"
|
||||
-- path2 = "'" .. path2:gsub ('\'', '\'"\'"\'') .. "'"
|
||||
--
|
||||
-- log(
|
||||
-- 'Normal',
|
||||
-- 'Moving ',
|
||||
-- event.path,
|
||||
-- ' -> ',
|
||||
-- event2.path
|
||||
-- )
|
||||
--
|
||||
-- spawn(
|
||||
-- event,
|
||||
-- config.ssh.binary,
|
||||
-- config.ssh._computed,
|
||||
-- config.host,
|
||||
-- 'mv',
|
||||
-- path1,
|
||||
-- path2,
|
||||
-- '||', 'rm', '-rf',
|
||||
-- path1
|
||||
-- )
|
||||
--
|
||||
-- return
|
||||
-- end
|
||||
--
|
||||
-- -- otherwise a rsync is spawned
|
||||
-- local elist = inlet.getEvents( eventNotInitBlankMove )
|
||||
--
|
||||
-- -- gets the list of paths for the event list
|
||||
-- -- deletes create multi match patterns
|
||||
-- local paths = elist.getPaths( )
|
||||
--
|
||||
-- -- removes trailing slashes from dirs.
|
||||
-- for k, v in ipairs( paths )
|
||||
-- do
|
||||
-- if string.byte( v, -1 ) == 47
|
||||
-- then
|
||||
-- paths[ k ] = string.sub( v, 1, -2 )
|
||||
-- end
|
||||
-- end
|
||||
--
|
||||
-- log(
|
||||
-- 'Normal',
|
||||
-- 'Rsyncing list\n',
|
||||
-- table.concat( paths, '\n' )
|
||||
-- )
|
||||
--
|
||||
-- local delete = nil
|
||||
--
|
||||
-- if config.delete == true
|
||||
-- or config.delete == 'running'
|
||||
-- then
|
||||
-- delete = { '--delete-missing-args', '--ignore-errors' }
|
||||
-- end
|
||||
--
|
||||
-- spawn(
|
||||
-- elist,
|
||||
-- config.rsync.binary,
|
||||
-- '<', table.concat( paths, '\000' ),
|
||||
-- config.rsync._computed,
|
||||
-- delete,
|
||||
-- '--force',
|
||||
-- '--from0',
|
||||
-- '--files-from=-',
|
||||
-- config.source,
|
||||
-- config.host .. ':' .. config.targetdir
|
||||
-- )
|
||||
--end
|
||||
|
||||
|
||||
--
|
||||
-- Called when collecting a finished child process
|
||||
--
|
||||
rsyncssh.collect = function
|
||||
|
||||
@ -32,7 +32,7 @@ then
|
||||
lsyncd.terminate( -1 )
|
||||
end
|
||||
|
||||
lsyncd_version = '2.2.1'
|
||||
lsyncd_version = '2.2.2'
|
||||
|
||||
|
||||
--
|
||||
@ -1012,7 +1012,7 @@ local Combiner = ( function
|
||||
)
|
||||
end
|
||||
|
||||
|
||||
|
||||
--
|
||||
-- The new delay splits on the old one.
|
||||
--
|
||||
@ -1065,8 +1065,8 @@ local Combiner = ( function
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
|
||||
|
||||
--
|
||||
-- The new delay turns the old one (a move) into a delete and is blocked.
|
||||
--
|
||||
|
||||
@ -3,7 +3,7 @@ require("posix")
|
||||
dofile("tests/testlib.lua")
|
||||
|
||||
cwriteln('****************************************************************' )
|
||||
cwriteln(' Testing excludes' )
|
||||
cwriteln(' Testing excludes (rsync)' )
|
||||
cwriteln('****************************************************************' )
|
||||
|
||||
local tdir, srcdir, trgdir = mktemps( )
|
||||
|
||||
@ -4,7 +4,7 @@ require( 'posix' )
|
||||
dofile( 'tests/testlib.lua' )
|
||||
|
||||
cwriteln( '****************************************************************' );
|
||||
cwriteln( ' Testing excludes' );
|
||||
cwriteln( ' Testing excludes (rsyncssh)' );
|
||||
cwriteln( '****************************************************************' );
|
||||
cwriteln( ' (this test needs passwordless ssh localhost access ' );
|
||||
cwriteln( ' for current user)' );
|
||||
|
||||
Loading…
Reference in New Issue
Block a user