preparing 2.2.3

ChangeLog

@@ -1,11 +1,18 @@
+2018-??-??: 2.2.3
+	enhaencement: supporting includes with new filter and filterFrom options
+	change: if the target/targetdir ands with a ':' do not append
+			a trailing '/' to it, since that would change it from homedir to rootdir!
+	add: example for Amazon S3 Bucket (Daniel Miranda)
+	fix: setting stdout/stderr to linebuffer mode.
+
 2017-02-16: 2.2.2
-	 fix: checkgauge 'insist'
+	fix: checkgauge 'insist'
-	 fix: no partial path exclusion tests
+	fix: no partial path exlusion tests
-	 fix: write pid of forked process in pidfile
+	fix: write pid of forked process in pidfile
-	 fix: crash on not reachable target
+	fix: crash on not reachable target
-	 workaround:
+	workaround:
-	    changed back to filter style rsync calling
+		changed back to filter style rsync calling
-	    until https://bugzilla.samba.org/show_bug.cgi?id=12569
+		until https://bugzilla.samba.org/show_bug.cgi?id=12569
 		is fixed and released.
 
 2017-01-05: 2.2.1

default-rsync.lua

@@ -16,16 +16,9 @@
 --~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 
-if not default
+if not default then error( 'default not loaded' ) end
-then
-	error( 'default not loaded' )
-end
 
-
+if default.rsync then error( 'default-rsync already loaded' ) end
-if default.rsync
-then
-	error( 'default-rsync already loaded' )
-end
 
 
 local rsync = { }
@@ -49,6 +42,8 @@ rsync.checkgauge = {
 	delete      =  true,
 	exclude     =  true,
 	excludeFrom =  true,
+	filter      =  true,
+	filterFrom  =  true,
 	target      =  true,
 
 	rsync  = {
@@ -117,28 +112,6 @@ local eventNotInitBlank =
 end
 
 
---
--- Replaces what rsync would consider filter rules by literals.
---
-local replaceRsyncFilter =
-	function
-(
-	path
-)
-	if not path
-	then
-		return
-	end
-
-	return(
-		path
-		:gsub( '%?', '\\?' )
-		:gsub( '%*', '\\*' )
-		:gsub( '%[', '\\[' )
-	)
-end
-
-
 --
 -- Spawns rsync for a list of events
 --
@@ -161,10 +134,11 @@ rsync.action = function
 	--
 	-- Replaces what rsync would consider filter rules by literals
 	--
-	local function sub( p )
+	local function sub
-		if not p then
+	(
-			return
+		p  -- pattern
-		end
+	)
+		if not p then return end
 
 		return p:
 			gsub( '%?', '\\?' ):
@@ -179,8 +153,14 @@ rsync.action = function
 	-- Deletes create multi match patterns
 	--
 	local paths = elist.getPaths(
-		function( etype, path1, path2 )
+		function
-			if string.byte( path1, -1 ) == 47 and etype == 'Delete' then
+		(
+			etype,  -- event type
+			path1,  -- path
+			path2   -- path to for move events
+		)
+			if string.byte( path1, -1 ) == 47 and etype == 'Delete'
+			then
 				return sub( path1 )..'***', sub( path2 )
 			else
 				return sub( path1 ), sub( path2 )
@@ -195,11 +175,11 @@ rsync.action = function
 	local filterP = { }
 
 	-- adds one path to the filter
-	local function addToFilter( path )
+	local function addToFilter
-
+	(
-		if filterP[ path ] then
+		path
-			return
+	)
-		end
+		if filterP[ path ] then return end
 
 		filterP[ path ] = true
 
@@ -211,21 +191,21 @@ rsync.action = function
 	-- rsync needs to have entries for all steps in the path,
 	-- so the file for example d1/d2/d3/f1 needs following filters:
 	-- 'd1/', 'd1/d2/', 'd1/d2/d3/' and 'd1/d2/d3/f1'
-	for _, path in ipairs( paths ) do
+	for _, path in ipairs( paths )
-
+	do
-		if path and path ~= '' then
+		if path and path ~= ''
-
+		then
-			addToFilter(path)
+			addToFilter( path )
 
 			local pp = string.match( path, '^(.*/)[^/]+/?' )
 
-			while pp do
+			while pp
-				addToFilter(pp)
+			do
+				addToFilter( pp )
+
 				pp = string.match( pp, '^(.*/)[^/]+/?' )
 			end
-
 		end
-
 	end
 
 	log(
@@ -334,6 +314,8 @@ rsync.init = function
 
 	local excludes = inlet.getExcludes( )
 
+	local filters = inlet.hasFilters( ) and inlet.getFilters( )
+
 	local delete   = nil
 
 	local target   = config.target
@@ -354,9 +336,9 @@ rsync.init = function
 		delete = { '--delete', '--ignore-errors' }
 	end
 
-	if #excludes == 0
+	if not filters and #excludes == 0
 	then
-		-- starts rsync without any excludes
+		-- starts rsync without any filters or excludes
 		log(
 			'Normal',
 			'recursive startup rsync: ',
@@ -375,7 +357,8 @@ rsync.init = function
 			target
 		)
 
-	else
+	elseif not filters
+	then
 		-- starts rsync providing an exclusion list
 		-- on stdin
 		local exS = table.concat( excludes, '\n' )
@@ -401,6 +384,32 @@ rsync.init = function
 			config.source,
 			target
 		)
+	else
+		-- starts rsync providing a filter list
+		-- on stdin
+		local fS = table.concat( filters, '\n' )
+
+		log(
+			'Normal',
+			'recursive startup rsync: ',
+			config.source,
+			' -> ',
+			target,
+			' filtering\n',
+			fS
+		)
+
+		spawn(
+			event,
+			config.rsync.binary,
+			'<', fS,
+			'--filter=. -',
+			delete,
+			config.rsync._computed,
+			'-r',
+			config.source,
+			target
+		)
 	end
 end
 
@@ -647,11 +656,13 @@ rsync.prepare = function
 	end
 
 	-- appends a / to target if not present
-	if not skipTarget and string.sub(config.target, -1) ~= '/'
+	-- and not a ':' for home dir.
+	if not skipTarget
+	and string.sub( config.target, -1 ) ~= '/'
+	and string.sub( config.target, -1 ) ~= ':'
 	then
 		config.target = config.target..'/'
 	end
-
 end
 
 

default-rsyncssh.lua

@@ -91,10 +91,7 @@ local replaceRsyncFilter =
 (
 	path
 )
-	if not path
+	if not path then return end
-	then
-		return
-	end
 
 	return(
 		path
@@ -161,9 +158,7 @@ rsyncssh.action = function
 	-- Replaces what rsync would consider filter rules by literals
 	--
 	local function sub( p )
-		if not p then
+		if not p then return end
-			return
-		end
 
 		return p:
 			gsub( '%?', '\\?' ):
@@ -195,11 +190,7 @@ rsyncssh.action = function
 
 	-- adds one path to the filter
 	local function addToFilter( path )
-
+		if filterP[ path ] then return end
-		if filterP[ path ]
-		then
-			return
-		end
 
 		filterP[ path ] = true
 
@@ -372,7 +363,7 @@ rsyncssh.collect = function
 
 		if rc == 'ok'
 		then
-			log('Normal', 'Startup of "', agent.source, '" finished: ', exitcode)
+			log( 'Normal', 'Startup of "', agent.source, '" finished: ', exitcode )
 		elseif rc == 'again'
 		then
 			if settings('insist')
@@ -382,16 +373,16 @@ rsyncssh.collect = function
 				log(
 					'Error',
 					'Temporary or permanent failure on startup of "',
-					agent.source, '". Terminating since "insist" is not set.' 
+					agent.source, '". Terminating since "insist" is not set.'
 				)
 
 				terminate( -1 ) -- ERRNO
 			end
 		elseif rc == 'die'
 		then
-			log( 'Error', 'Failure on startup of "',agent.source,'": ', exitcode )
+			log( 'Error', 'Failure on startup of "', agent.source, '": ', exitcode )
 		else
-			log( 'Error', 'Unknown exitcode on startup of "', agent.source,': "',exitcode )
+			log( 'Error', 'Unknown exitcode on startup of "', agent.source, ': "', exitcode )
 
 			rc = 'die'
 		end
@@ -431,7 +422,7 @@ rsyncssh.collect = function
 		then
 			log( 'Normal', 'Failure ', agent.etype, ' ', agent.sourcePath, ': ', exitcode )
 		else
-			log( 'Error', 'Unknown exitcode ',agent.etype,' ',agent.sourcePath,': ',exitcode )
+			log( 'Error', 'Unknown exitcode ', agent.etype,' ', agent.sourcePath,': ', exitcode )
 
 			rc = 'die'
 		end
@@ -453,18 +444,12 @@ rsyncssh.prepare = function
 
 	if not config.host
 	then
-		error(
+		error( 'default.rsyncssh needs "host" configured', level )
-			'default.rsyncssh needs "host" configured',
-			level
-		)
 	end
 
 	if not config.targetdir
 	then
-		error(
+		error( 'default.rsyncssh needs "targetdir" configured', level )
-			'default.rsyncssh needs "targetdir" configured',
-			level
-		)
 	end
 
 	--
@@ -472,18 +457,12 @@ rsyncssh.prepare = function
 	--
 	if config.ssh._computed
 	then
-		error(
+		error( 'please do not use the internal rsync._computed parameter', level )
-			'please do not use the internal rsync._computed parameter',
-			level
-		)
 	end
 
 	if config.maxProcesses ~= 1
 	then
-		error(
+		error( 'default.rsyncssh must have maxProcesses set to 1.', level )
-			'default.rsyncssh must have maxProcesses set to 1.',
-			level
-		)
 	end
 
 	local cssh = config.ssh;
@@ -578,10 +557,11 @@ rsyncssh.prepare = function
 	end
 
 	-- appends a slash to the targetdir if missing
+	-- and is not ':' for home dir
 	if string.sub( config.targetdir, -1 ) ~= '/'
+	and string.sub( config.targetdir, -1 ) ~= ':'
 	then
-		config.targetdir =
+		config.targetdir = config.targetdir .. '/'
-			config.targetdir .. '/'
 	end
 
 end

default.lua

@@ -388,13 +388,10 @@ local function check
 )
 	for k, v in pairs( config )
 	do
-		if not gauge[k]
+		if not gauge[ k ]
 		then
 			error(
-				'Parameter "'
+				'Parameter "' .. subtable .. k .. '" unknown.'
-				.. subtable
-				.. k
-				.. '" unknown.'
 				.. ' ( if this is not a typo add it to checkgauge )',
 				level
 			);
@@ -405,10 +402,7 @@ local function check
 			if type( v ) ~= 'table'
 			then
 				error(
-					'Parameter "'
+					'Parameter "' .. subtable .. k .. '" must be a table.',
-					.. subtable
-					.. k
-					.. '" must be a table.',
 					level
 				)
 			end
@@ -432,10 +426,7 @@ default.prepare = function
 
 	local gauge = config.checkgauge
 
-	if not gauge
+	if not gauge then return end
-	then
-		return
-	end
 
 	check( config, gauge, '', level + 1 )
 end

lsyncd.h

@@ -12,7 +12,7 @@
 #define LSYNCD_H
 
 // some older machines need this to see pselect
-#define _BSD_SOURCE 1
+#define _DEFAULT_SOURCE 1
 #define _XOPEN_SOURCE 700
 #define _DARWIN_C_SOURCE 1