introducing checkgauge, code beautifications

default-rsync.lua

@@ -28,6 +28,64 @@ end
 
 default.rsync = { }
 local rsync = default.rsync
+-- uses default collect
+
+--
+-- used to ensure there aren't typos in the keys
+--
+rsync.checkgauge = {
+
+	default.checkgauge,
+
+	-- unsets default user action handlers
+	onCreate   =  false,
+	onModify   =  false,
+	onDelete   =  false,
+	onStartup  =  false,
+	onMove     =  false,
+
+	delete     =  true,
+	exclude    =  true,
+	source     =  true,
+	target     =  true,
+
+	rsync  = {
+		-- rsync binary
+		binary            =  true,
+
+		-- rsync shortflags
+		verbose           =  true,
+		quiet             =  true,
+		checksum          =  true,
+		update            =  true,
+		links             =  true,
+		copy_links        =  true,
+		hard_links        =  true,
+		perms             =  true,
+		executability     =  true,
+		acls              =  true,
+		xattrs            =  true,
+		owner             =  true,
+        group             =  true,
+        times             =  true,
+		sparse            =  true,
+		dry_run           =  true,
+        whole_file        =  true,
+		one_file_system   =  true,
+		prune_empty_dirs  =  true,
+		ignore_times      =  true,
+		compress          =  true,
+		cvs_exclude       =  true,
+		protect_args      =  true,
+		ipv4              =  true,
+		ipv6              =  true,
+
+		-- further rsync options
+		rsh               =  true,
+		rsync_path        =  true,
+	},
+}
+
 
 --
 -- Spawns rsync for a list of events
@@ -167,6 +225,15 @@ rsync.init = function(event)
 	local inlet    = event.inlet
 	local excludes = inlet.getExcludes( )
 	local delete   = nil
+	local target   = config.target
+
+	if not target then
+		if not config.host then
+			error('Internal fail, Neither target nor host is configured')
+		end
+
+		target = config.host .. ':' .. config.targetdir
+	end
 
 	if config.delete then
 		delete = { '--delete', '--ignore-errors' }
@@ -179,7 +246,7 @@ rsync.init = function(event)
 			'recursive startup rsync: ',
 			config.source,
 			' -> ',
-			config.target
+			target
 		)
 
 		spawn(
@@ -189,7 +256,7 @@ rsync.init = function(event)
 			config.rsync._computed,
 			'-r',
 			config.source,
-			config.target
+			target
 		)
 
 	else
@@ -202,7 +269,7 @@ rsync.init = function(event)
 			'recursive startup rsync: ',
 			config.source,
 			' -> ',
-			config.target,
+			target,
 			' excluding\n',
 			exS
 		)
@@ -216,7 +283,7 @@ rsync.init = function(event)
 			config.rsync._computed,
 			'-r',
 			config.source,
-			config.target
+			target
 		)
 	end
 end
@@ -225,19 +292,30 @@ end
 --
 -- Prepares and checks a syncs configuration on startup.
 --
-rsync.prepare = function( config )
+rsync.prepare = function(
+	config,    -- the configuration
+	level,     -- additional error level for inherited use ( by rsyncssh )
+	skipTarget -- used by rsyncssh, do not check for target
+)
 
-	if not config.target then
+	level = level or 4
+
+	--
+	-- First let default.prepare test the checkgauge
+	--
+	default.prepare( config, level + 6 )
+
+	if not skipTarget and not config.target then
 		error(
 			'default.rsync needs "target" configured',
-			4
+			level
 		)
 	end
 
 	if config.rsyncOps then
 		error(
 			'"rsyncOps" is outdated please use the new rsync = { ... } syntax.',
-			4
+			level
 		)
 	end
 
@@ -246,7 +324,7 @@ rsync.prepare = function( config )
 			'"rsyncOpts" is outdated in favor of the new rsync = { ... } syntax\n"' +
 			'for which you provided the _extra attribute as well.\n"' +
 			'Please remove rsyncOpts from your config.',
-			4
+			level
 		)
 	end
 
@@ -265,7 +343,7 @@ rsync.prepare = function( config )
 			'"rsyncBinary is outdated in favor of the new rsync = { ... } syntax\n"'+
 			'for which you provided the binary attribute as well.\n"' +
 			"Please remove rsyncBinary from your config.'",
-			4
+			level
 		)
 	end
 
@@ -283,57 +361,95 @@ rsync.prepare = function( config )
 	if config.rsync._computed then
 		error(
 			'please do not use the internal rsync._computed parameter',
-			4
+			level
 		)
 	end
 
 	-- computes the rsync arguments into one list
 	local rsync = config.rsync;
+
 	rsync._computed = { true }
 	local computed = rsync._computed
+	local computedN = 1
+
+	local shortFlags = {
+		verbose            = 'v',
+		quiet              = 'q',
+		checksum           = 'c',
+		update             = 'u',
+		links              = 'l',
+		copy_links         = 'L',
+		hard_links         = 'H',
+		perms              = 'p',
+		executability      = 'E',
+		acls               = 'A',
+		xattrs             = 'X',
+		owner              = 'o',
+        group              = 'g',
+        times              = 't',
+		sparse             = 'S',
+		dry_run            = 'n',
+        whole_file         = 'W',
+		one_file_system    = 'x',
+		prune_empty_dirs   = 'm',
+		ignore_times       = 'I',
+		compress           = 'z',
+		cvs_exclude        = 'C',
+		protect_args       = 's',
+		ipv4               = '4',
+		ipv6               = '6'
+	}
+
 	local shorts = { '-' }
+	local shortsN = 2
 
 	if config.rsync._extra then
 		for k, v in ipairs( config.rsync._extra ) do
-			computed[ k + 1 ] = v
+			computed[ computedN ] = v
+			computedN = computedN  + 1
 		end
 	end
 
-	if rsync.links then
+	for k, flag in pairs( shortFlags ) do
-		shorts[ #shorts + 1 ] = 'l'
+		if config.rsync[k] then
+			shorts[ shortsN ] = flag
+			shortsN = shortsN + 1
+		end
 	end
 
-	if rsync.times then
+	if config.rsync.rsh then
-		shorts[ #shorts + 1 ] = 't'
+		computed[ computedN ] = '--rsh=' + config.rsync.rsh
+		computedN = computedN  + 1
 	end
 
-	if rsync.protectArgs then
+	if config.rsync.rsync_path then
-		shorts[ #shorts + 1 ] = 's'
+		computed[ computedN ] = '--rsync-path=' + config.rsync.rsync_path
+		computedN = computedN  + 1
 	end
 
-	if #shorts ~= 1 then
+	if shortsN ~= 2 then
 		computed[ 1 ] = table.concat( shorts, '' )
 	else
 		computed[ 1 ] = { }
 	end
 
 	-- appends a / to target if not present
-	if string.sub(config.target, -1) ~= '/' then
+	if not skipTarget and string.sub(config.target, -1) ~= '/' then
 		config.target = config.target..'/'
 	end
+
 end
 
 
---
--- rsync uses default collect
---
-
-
 --
 -- By default do deletes.
 --
 rsync.delete = true
 
+--
+-- Rsyncd exitcodes
+--
+rsync.exitcodes  = default.rsyncExitCodes
 
 --
 -- Calls rsync with this default options
@@ -343,15 +459,10 @@ rsync.rsync = {
 	binary        = '/usr/bin/rsync',
 	links         = true,
 	times         = true,
-	protectArgs  = true
+	protect_args  = true
 }
 
 
---
--- Exit codes for rsync.
---
-rsync.exitcodes = default.rsyncExitCodes
-
 --
 -- Default delay
 --

default-rsyncssh.lua

@@ -19,25 +19,65 @@ if not default then
 	error( 'default not loaded' );
 end
 
+if not default.rsync then
+	error( 'default.rsync not loaded' );
+end
+
 if default.rsyncssh then
 	error( 'default-rsyncssh already loaded' );
 end
 
-default.rsyncssh = {
+--
+-- rsyncssh extends default.rsync
+--
+local rsyncssh = { default.rsync }
+default.rsyncssh = rsyncssh
+
+--
+-- used to ensure there aren't typos in the keys
+--
+rsyncssh.checkgauge = {
+
+	-- unsets the inherited value of from default.rsync
+	target     =  false,
+	onMove     =  true,
+
+	-- rsyncssh users host and targetdir
+	host       =  true,
+	targetdir  =  true,
+
+	-- ssh settings
+	ssh = {
+		binary  =  true,
+		port    =  true,
+		_extra  =  true
+	},
+
+	-- xargs settings
+	xargs = {
+		binary    = true,
+		delimiter = true,
+		_extra    = true
+	}
+}
 
 --
 -- Spawns rsync for a list of events
 --
-	action = function(inlet)
+rsyncssh.action = function( inlet )
 
 	local event, event2 = inlet.getEvent()
 	local config = inlet.getConfig()
 
-		-- makes move local on host
+	-- makes move local on target host
-		-- if fails deletes the source...
+	-- if the move fails, it deletes the source
 	if event.etype == 'Move' then
 		log('Normal', 'Moving ',event.path,' -> ',event2.path)
-			spawn(event, '/usr/bin/ssh',
+
+		spawn(
+			event,
+			config.ssh.binary,
+--			config.ssh._computed, TODO XXX
 			config.host,
 			'mv',
 			'\"' .. config.targetdir .. event.path .. '\"',
@@ -51,18 +91,20 @@ default.rsyncssh = {
 	-- instead of constructing rsync filters
 
 	if event.etype == 'Delete' then
-
 		if not config.delete then
 			inlet.discardEvent(event)
 			return
 		end
 
+		-- gets all other deletes ready to be
+		-- executed
 		local elist = inlet.getEvents(
 			function( e )
 				return e.etype == 'Delete'
 			end
 		)
 
+		-- returns the paths of the delete list
 		local paths = elist.getPaths(
 			function( etype, path1, path2 )
 				if path2 then
@@ -73,6 +115,7 @@ default.rsyncssh = {
 			end
 		)
 
+		-- ensures none of the paths is '/'
 		for _, v in pairs( paths ) do
 			if string.match(v, '^%s*/+%s*$') then
 				log('Error', 'refusing to `rm -rf /` the target!')
@@ -80,21 +123,20 @@ default.rsyncssh = {
 			end
 		end
 
-			log('Normal', 'Deleting list\n', table.concat(paths, '\n'))
+		log(
+			'Normal',
+			'Deleting list\n',
+			table.concat( paths, '\n' )
+		)
 
 		local params = { }
 
-			if config.port then
-				params[#params + 1] = 'p'
-				params[#params + 1] = config.port
-			end
-
 		spawn(
 			elist,
 			config.ssh.binary,
 			'<', table.concat(paths, config.xargs.delimiter),
 			params,
-				config.ssh._extra,
+			-- config.ssh._computed, TODO XXX
 			config.host,
 			config.xargs.binary,
 			config.xargs._extra
@@ -103,7 +145,9 @@ default.rsyncssh = {
 		return
 	end
 
-		-- for everything else spawn a rsync
+	--
+	-- for everything else a rsync is spawned
+	--
 	local elist = inlet.getEvents(
 		function(e)
 			-- TODO use a table
@@ -116,38 +160,41 @@ default.rsyncssh = {
 
 	local paths = elist.getPaths( )
 
+	--
 	-- removes trailing slashes from dirs.
+	--
 	for k, v in ipairs( paths ) do
 		if string.byte(v, -1) == 47 then
 			paths[k] = string.sub(v, 1, -2)
 		end
+
 	end
 
 	local sPaths = table.concat(paths, '\n')
 	local zPaths = table.concat(paths, '\000')
+
 	log('Normal', 'Rsyncing list\n', sPaths)
 
 	spawn(
 		elist,
-			config.rsyncBinary,
+		config.rsync.binary,
 		'<', zPaths,
-			config.rsyncOpts,
+		config.rsync._computed,
 		'--from0',
 		'--files-from=-',
 		config.source,
 		config.host .. ':' .. config.targetdir
 	)
-	end,
+end
 
 -----
 -- Called when collecting a finished child process
 --
-	collect = function(agent, exitcode)
+rsyncssh.collect = function( agent, exitcode )
 
 	local config = agent.config
 
 	if not agent.isList and agent.etype == 'Init' then
-
 		local rc = config.rsyncExitCodes[exitcode]
 
 		if rc == 'ok' then
@@ -160,6 +207,7 @@ default.rsyncssh = {
 				agent.source, '". Terminating since "insist" is not set.');
 				terminate(-1) -- ERRNO
 			end
+
 		elseif rc == 'die' then
 			log('Error', 'Failure on startup of "',agent.source,'": ', exitcode)
 		else
@@ -172,9 +220,7 @@ default.rsyncssh = {
 	end
 
 	if agent.isList then
-
 		local rc = config.rsyncExitCodes[exitcode]
-
 		if rc == 'ok' then
 			log('Normal', 'Finished (list): ',exitcode)
 		elseif rc == 'again' then
@@ -185,11 +231,8 @@ default.rsyncssh = {
 			log('Error', 'Unknown exitcode (list): ',exitcode)
 			rc = 'die'
 		end
-
 		return rc
-
 	else
-
 		local rc = config.sshExitCodes[exitcode]
 
 		if rc == 'ok' then
@@ -204,57 +247,16 @@ default.rsyncssh = {
 		end
 
 		return rc
-
 	end
 
-	end,
-
-	--
-	-- spawns the recursive startup sync
-	--
-	init = function(event)
-
-		local config   = event.config
-		local inlet    = event.inlet
-		local excludes = inlet.getExcludes()
-		local target   = config.host .. ':' .. config.targetdir
-		local delete   = nil
-
-		if config.delete then
-			delete = { '--delete', '--ignore-errors' };
 end
 
-		if #excludes == 0 then
-			log('Normal', 'Recursive startup rsync: ',config.source,' -> ',target)
-			spawn(
-				event, config.rsyncBinary,
-				delete,
-				'-r',
-				config.rsyncOpts,
-				config.source,
-				target
-			)
-		else
-			local exS = table.concat(excludes, '\n')
-			log('Normal', 'Recursive startup rsync: ',config.source,
-				' -> ',target,' with excludes.')
-			spawn(
-				event, config.rsyncBinary,
-				'<', exS,
-				'--exclude-from=-',
-				delete,
-				'-r',
-				config.rsyncOpts,
-				config.source,
-				target
-			)
-		end
-	end,
-
 --
 -- checks the configuration.
 --
-	prepare = function(config)
+rsyncssh.prepare = function( config )
+
+	default.rsync.prepare( config, 5, true )
 
 	if not config.host then
 		error('default.rsyncssh needs "host" configured', 4)
@@ -272,48 +274,39 @@ default.rsyncssh = {
 	if string.sub(config.targetdir, -1) ~= '/' then
 		config.targetdir = config.targetdir .. '/'
 	end
-	end,
 
-	--
+end
-	-- the rsync binary called
-	--
-	rsyncBinary = '/usr/bin/rsync',
-
-	--
-	-- calls rsync with this default short opts
-	--
-	rsyncOpts = '-lts',
 
 --
 -- allow processes
 --
-	maxProcesses = 1,
+rsyncssh.maxProcesses = 1
 
 --
 -- The core should not split move events
 --
-	onMove = true,
+rsyncssh.onMove = true
 
 --
 -- default delay
 --
-	delay = 15,
+rsyncssh.delay = 15
 
 
 --
-	-- by default do deletes
+-- no default exit codes
 --
-	delete = true,
+rsyncssh.exitcodes = false
 
 --
 -- rsync exit codes
 --
-	rsyncExitCodes = default.rsyncExitCodes,
+rsyncssh.rsyncExitCodes = default.rsyncExitCodes
 
 --
 -- ssh exit codes
 --
-	sshExitCodes = default.sshExitCodes,
+rsyncssh.sshExitCodes = default.sshExitCodes
 
 --
 -- xargs calls configuration
@@ -321,7 +314,7 @@ default.rsyncssh = {
 -- xargs is used to delete multiple remote files, when ssh access is
 -- available this is simpler than to build filters for rsync for this.
 --
-	xargs = {
+rsyncssh.xargs = {
 
 	--
 	-- the binary called (on target host)
@@ -335,25 +328,28 @@ default.rsyncssh = {
 	--
 	-- extra parameters
 	_extra = { '-0', 'rm -rf' }
-	},
+}
 
 --
 -- ssh calls configuration
 --
 -- ssh is used to move and delete files on the target host
 --
-	ssh = {
+rsyncssh.ssh = {
+
 	--
 	-- the binary called
+	--
 	binary = '/usr/bin/ssh',
 
 	--
 	-- if set connect to this port
+	--
 	port = nil,
 
 	--
 	-- extra parameters
+	--
 	_extra = { }
 }
 
-}

default.lua

@@ -12,32 +12,61 @@ if default then
 	error( 'default already loaded' )
 end
 
-default = {
+default = { }
-	-----
+
-	-- Default action calls user scripts on**** functions.
+
 --
-	action = function(inlet)
+-- used to ensure there aren't typos in the keys
+--
+default.checkgauge = {
+	action          =  true,
+	checkgauge      =  true,
+	collect         =  true,
+	delay           =  true,
+	exitcodes       =  true,
+	init            =  true,
+	maxDelays       =  true,
+	maxProcesses    =  true,
+	onCreate        =  true,
+	onModify        =  true,
+	onDelete        =  true,
+	onStartup       =  true,
+	onMove          =  true,
+	prepare         =  true,
+	rsyncExitCodes  =  true, -- TODO
+	sshExitCodes    =  true  -- TODO
+}
+
+--
+-- On default action the user's on*** scripts are called.
+--
+default.action = function( inlet )
+
 	-- in case of moves getEvent returns the origin and dest of the move
 	local event, event2 = inlet.getEvent( )
 	local config = inlet.getConfig( )
+
 	local func = config[ 'on'.. event.etype ]
+
 	if func then
 		func( event, event2 )
 	end
+
 	-- if function didnt change the wait status its not interested
 	-- in this event -> drop it.
 	if event.status == 'wait' then
 		inlet.discardEvent( event )
 	end
-	end,
+
+end
 
 
-	-----
+--
 -- Default collector.
 --
 -- Called when collecting a finished child process
 --
-	collect = function(agent, exitcode)
+default.collect = function( agent, exitcode )
 
 	local config = agent.config
 	local rc
@@ -140,14 +169,17 @@ default = {
 	end
 
 	return rc
-	end,
+end
+
 
-	-----
-	-- called on (re)initialization of Lsyncd.
 --
-	init = function(event)
+-- Called on the Init event sent
+-- on (re)initialization of Lsyncd for every sync
+--
+default.init = function(event)
 	local config = event.config
 	local inlet = event.inlet
+
 	-- user functions
 	-- calls a startup if given by user script.
 	if type(config.onStartup) == 'function' then
@@ -160,30 +192,36 @@ default = {
 		-- thus the blanket event is deleted again.
 		inlet.discardEvent(event)
 	end
-	end,
+end
+
 
-	-----
-	-- The maximum number of processes Lsyncd will spawn simultanously for
-	-- one sync.
 --
-	maxProcesses = 1,
+-- The maximum number of processes Lsyncd will
-
+-- simultanously spawn for this sync.
-	-----
-	-- Try not to have more than these delays.
-	-- not too large, since total calculation for stacking
-	-- events is n*log(n) or so..
 --
-	maxDelays = 1000,
+default.maxProcesses = 1
 
-	-----
+
+--
+-- The collapsor tries not to have more than these delays.
+-- So it dealy stack does not grow too large,
+-- since calculation for stacking events is n*log(n) (or so)
+--
+default.maxDelays = 1000
+
+
+---
 -- a default configuration using /bin/cp|rm|mv.
+-- TODO huh?
 --
-	direct = default_direct,
+default.direct = default_direct
+
 
-	------
-	-- Exitcodes of rsync and what to do.
 --
-	rsyncExitCodes = {
+-- Exitcodes of rsync and what to do.
+-- TODO move to rsync
+--
+default.rsyncExitCodes = {
 
 	--
 	-- if another config provides the same table
@@ -209,20 +247,24 @@ default = {
 	[  20 ] = 'again',
 	[  21 ] = 'again',
 	[  22 ] = 'again',
+
 	-- partial transfers are ok, since Lsyncd has registered the event that
 	-- caused the transfer to be partial and will recall rsync.
 	[  23 ] = 'ok',
 	[  24 ] = 'ok',
+
 	[  25 ] = 'die',
 	[  30 ] = 'again',
 	[  35 ] = 'again',
-		[ 255 ] = 'again',
-	},
 
-	-----
+	[ 255 ] = 'again',
+}
+
+
+--
 -- Exitcodes of ssh and what to do.
 --
-	sshExitCodes = {
+default.sshExitCodes = {
 
 	--
 	-- if another config provides the same table
@@ -236,10 +278,63 @@ default = {
 
 	[   0 ] = 'ok',
 	[ 255 ] = 'again',
-	},
+}
 
-	-----
+
+--
 -- Minimum seconds between two writes of a status file.
 --
-	statusInterval = 10,
+default.statusInterval = 10
-}
+
+
+--
+-- checks all keys to be in the checkgauge
+--
+
+local function check(
+	config,
+	gauge,
+	level
+)
+	for k, v in pairs( config ) do
+
+		if not gauge[k] then
+			error(
+				'Parameter "'
+				.. k
+				.. '" unknown.'
+				.. ' (if this is not a typo add it to checkgauge)',
+				level
+			);
+		end
+
+		if type( gauge [ k ] ) == 'table' then
+
+			if type( v ) ~= 'table' then
+
+				error(
+					'Parameter "'
+					.. k
+					.. '" must be a table.',
+					level
+				)
+
+			end
+
+			check( config[ k ], gauge[ k ], level + 1 )
+
+		end
+	end
+end
+
+default.prepare = function( config, level )
+
+	local gauge = config.checkgauge
+
+	if not gauge then
+		return
+	end
+
+	check( config, gauge, level or 2 )
+end
+