further structering mantle

This commit is contained in:
Axel Kittenberger 2018-03-16 09:36:32 +01:00
parent 9170557760
commit 5771779e28
8 changed files with 541 additions and 389 deletions

View File

@ -46,6 +46,9 @@ set( LUA_CODE
${PROJECT_SOURCE_DIR}/mantle/filter.lua
${PROJECT_SOURCE_DIR}/mantle/sync.lua
${PROJECT_SOURCE_DIR}/mantle/syncmaster.lua
${PROJECT_SOURCE_DIR}/mantle/monitor.lua
${PROJECT_SOURCE_DIR}/mantle/fwriter.lua
${PROJECT_SOURCE_DIR}/mantle/status.lua
${PROJECT_SOURCE_DIR}/mantle/lsyncd.lua
${PROJECT_SOURCE_DIR}/mantle/wrapup.lua
${PROJECT_SOURCE_DIR}/default/default.lua

View File

@ -6,9 +6,9 @@ Lsyncd watches a local directory trees event monitor interface (inotify or fseve
Rsync+ssh is an advanced action configuration that uses a SSH to act file and directory moves directly on the target instead of re-transmitting the move destination over the wire.
Fine-grained customization can be achieved through the config file. Custom action configs can even be written from scratch in cascading layers ranging from shell scripts to code written in the [Lua language](http://www.lua.org/). This way simple, powerful and flexible configurations can be acheived. See [the manual](https://axkibe.github.io/lsyncd/) for details.
Fine-grained customization can be achieved through the config file. Custom action configs can even be written from scratch in cascading layers ranging from shell scripts to code written in the [Lua language](http://www.lua.org/). This way simple, powerful and flexible configurations can be achieved. See [the manual](https://axkibe.github.io/lsyncd/) for details.
Lsyncd 2.2.1 requires rsync >= 3.1 on all source and target machines.
Lsyncd 3.0 requires rsync >= 3.1 on all source and target machines.
License: [GPLv2](http://www.fsf.org/licensing/licenses/info/GPLv2.html) or any later GPL version.
@ -26,16 +26,9 @@ Other synchronization tools
Lsyncd usage examples
---------------------
```lsyncd -rsync /home remotehost.org::share/```
TODO make new examples
This watches and rsyncs the local directory /home with all sub-directories and
transfers them to 'remotehost' using the rsync-share 'share'.
```lsyncd -rsyncssh /home remotehost.org backup-home/```
This will also rsync/watch '/home', but it uses a ssh connection to make moves local on the remotehost instead of re-transmitting the moved file over the wire.
Some more complicated examples, tips and tricks you can find in the [manual](https://axkibe.github.io/lsyncd/).
You can find more examples in the [manual](https://axkibe.github.io/lsyncd/).
Disclaimer
----------

View File

@ -1,47 +1,56 @@
#!/usr/bin/lua
--============================================================================
-- bin2carray.lua
-- bin2carray.lua
--
-- License: GPLv2 (see COPYING) or any later version
--
-- Authors: Axel Kittenberger <axkibe@gmail.com>
--
-- Transforms a binary file (the compiled lsyncd runner script) in a c array
-- Transforms a binary file (the compiled lsyncd luacode) in a c array
-- so it can be included into the executable in a portable way.
--
--============================================================================
if #arg < 3 then
error("Usage: "..arg[0].." [infile] [varname] [outfile]")
if #arg < 3
then
error( 'Usage: '..arg[ 0 ]..' [infile] [varname] [outfile]' )
end
fin, err = io.open(arg[1], "rb")
if fin == nil then
error("Cannot open '"..arg[1].."' for reading: "..err)
fin, err = io.open( arg[ 1 ], 'rb' )
if fin == nil
then
error( 'Cannot open "'..arg[ 1 ]..'" for reading: '..err )
end
fout, err = io.open(arg[3], "w")
if fout == nil then
error("Cannot open '"..arg[3].."'for writing: "..err)
fout, err = io.open( arg[ 3 ], 'w' )
if fout == nil
then
error( 'Cannot open "'..arg[ 3 ]..'"for writing: '..err )
end
fout:write("/* created by "..arg[0].." from file "..arg[1].." */\n")
fout:write("#include <stddef.h>\n")
fout:write("const char "..arg[2].."_out[] = {\n")
while true do
local block = fin:read(16)
if block == nil then
break
fout:write( '/* created by '..arg[ 0 ]..' from file '..arg[ 1 ]..' */\n')
fout:write( '#include <stddef.h>\n' )
fout:write( 'const char '..arg[ 2 ]..'_out[] = {\n' )
while true
do
local block = fin:read( 16 )
if block == nil then break end
for i = 1, #block
do
local val = string.format( '%x', block:byte( i ) )
if #val < 2 then val = "0" ..val end
fout:write( "0x", val, "," )
end
for i = 1, #block do
local val = string.format("%x", block:byte(i))
if #val < 2 then
val = "0" ..val
end
fout:write("0x",val,",")
end
fout:write("\n")
end
fout:write("};\n\nsize_t "..arg[2].."_size = sizeof("..arg[2].."_out);\n");
fin:close();
fout:close();
fout:write( '\n' )
end
fout:write( '};\n\nsize_t '..arg[ 2 ]..'_size = sizeof('..arg[ 2 ]..'_out);\n' );
fin:close( );
fout:close( );

310
mantle/fwriter.lua Normal file
View File

@ -0,0 +1,310 @@
--~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-- lsyncd.lua Live (Mirror) Syncing Demon
--
--
-- Writes functions for the user for layer 3 configurations.
--
--
-- This code assumes your editor is at least 100 chars wide.
--
-- License: GPLv2 (see COPYING) or any later version
-- Authors: Axel Kittenberger <axkibe@gmail.com>
--
--~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
if mantle
then
print( 'Error, Lsyncd mantle already loaded' )
os.exit( -1 )
end
--
-- All variables known to layer 3 configs.
--
transVars = {
{ '%^pathname', 'event.pathname', 1 },
{ '%^pathdir', 'event.pathdir', 1 },
{ '%^path', 'event.path', 1 },
{ '%^sourcePathname', 'event.sourcePathname', 1 },
{ '%^sourcePathdir', 'event.sourcePathdir', 1 },
{ '%^sourcePath', 'event.sourcePath', 1 },
{ '%^source', 'event.source', 1 },
{ '%^targetPathname', 'event.targetPathname', 1 },
{ '%^targetPathdir', 'event.targetPathdir', 1 },
{ '%^targetPath', 'event.targetPath', 1 },
{ '%^target', 'event.target', 1 },
{ '%^o%.pathname', 'event.pathname', 1 },
{ '%^o%.path', 'event.path', 1 },
{ '%^o%.sourcePathname', 'event.sourcePathname', 1 },
{ '%^o%.sourcePathdir', 'event.sourcePathdir', 1 },
{ '%^o%.sourcePath', 'event.sourcePath', 1 },
{ '%^o%.targetPathname', 'event.targetPathname', 1 },
{ '%^o%.targetPathdir', 'event.targetPathdir', 1 },
{ '%^o%.targetPath', 'event.targetPath', 1 },
{ '%^d%.pathname', 'event2.pathname', 2 },
{ '%^d%.path', 'event2.path', 2 },
{ '%^d%.sourcePathname', 'event2.sourcePathname', 2 },
{ '%^d%.sourcePathdir', 'event2.sourcePathdir', 2 },
{ '%^d%.sourcePath', 'event2.sourcePath', 2 },
{ '%^d%.targetPathname', 'event2.targetPathname', 2 },
{ '%^d%.targetPathdir', 'event2.targetPathdir', 2 },
{ '%^d%.targetPath', 'event2.targetPath', 2 },
}
--
-- Splits a user string into its arguments.
-- Returns a table of arguments
--
local function splitStr(
str -- a string where parameters are seperated by spaces.
)
local args = { }
while str ~= ''
do
-- break where argument stops
local bp = #str
-- in a quote
local inQuote = false
-- tests characters to be space and not within quotes
for i = 1, #str
do
local c = string.sub( str, i, i )
if c == '"'
then
inQuote = not inQuote
elseif c == ' ' and not inQuote
then
bp = i - 1
break
end
end
local arg = string.sub( str, 1, bp )
arg = string.gsub( arg, '"', '\\"' )
table.insert( args, arg )
str = string.sub( str, bp + 1, -1 )
str = string.match( str, '^%s*(.-)%s*$' )
end
return args
end
--
-- Translates a call to a binary to a lua function.
-- TODO this has a little too blocking.
--
local function translateBinary
(
str
)
-- splits the string
local args = splitStr( str )
-- true if there is a second event
local haveEvent2 = false
for ia, iv in ipairs( args )
do
-- a list of arguments this arg is being split into
local a = { { true, iv } }
-- goes through all translates
for _, v in ipairs( transVars )
do
local ai = 1
while ai <= #a
do
if a[ ai ][ 1 ]
then
local pre, post =
string.match( a[ ai ][ 2 ], '(.*)'..v[1]..'(.*)' )
if pre
then
if v[3] > 1
then
haveEvent2 = true
end
if pre ~= ''
then
table.insert( a, ai, { true, pre } )
ai = ai + 1
end
a[ ai ] = { false, v[ 2 ] }
if post ~= ''
then
table.insert( a, ai + 1, { true, post } )
end
end
end
ai = ai + 1
end
end
-- concats the argument pieces into a string.
local as = ''
local first = true
for _, v in ipairs( a )
do
if not first then as = as..' .. ' end
if v[ 1 ]
then
as = as .. '"' .. v[ 2 ] .. '"'
else
as = as .. v[ 2 ]
end
first = false
end
args[ ia ] = as
end
local ft
if not haveEvent2
then
ft = 'function( event )\n'
else
ft = 'function( event, event2 )\n'
end
ft = ft ..
" log('Normal', 'Event ', event.etype, \n" ..
" ' spawns action \"".. str.."\"')\n" ..
" spawn( event"
for _, v in ipairs( args )
do
ft = ft .. ',\n ' .. v
end
ft = ft .. ')\nend'
return ft
end
--
-- Translates a call using a shell to a lua function
--
local function translateShell
(
str
)
local argn = 1
local args = { }
local cmd = str
local lc = str
-- true if there is a second event
local haveEvent2 = false
for _, v in ipairs( transVars )
do
local occur = false
cmd = string.gsub(
cmd,
v[ 1 ],
function
( )
occur = true
return '"$' .. argn .. '"'
end
)
lc = string.gsub( lc, v[1], ']]..' .. v[2] .. '..[[' )
if occur
then
argn = argn + 1
table.insert( args, v[ 2 ] )
if v[ 3 ] > 1
then
haveEvent2 = true
end
end
end
local ft
if not haveEvent2
then
ft = 'function( event )\n'
else
ft = 'function( event, event2 )\n'
end
-- TODO do array joining instead
ft = ft..
" log('Normal', 'Event ',event.etype,\n"..
" [[ spawns shell \""..lc.."\"]])\n"..
" spawnShell(event, [["..cmd.."]]"
for _, v in ipairs( args )
do
ft = ft..',\n '..v
end
ft = ft .. ')\nend'
return ft
end
--
-- Writes a lua function for a layer 3 user script.
--
local function translate
(
str
)
-- trims spaces
str = string.match( str, '^%s*(.-)%s*$' )
local ft
if string.byte( str, 1, 1 ) == 47
then
-- starts with /
ft = translateBinary( str )
elseif string.byte( str, 1, 1 ) == 94
then
-- starts with ^
ft = translateShell( str:sub( 2, -1 ) )
else
ft = translateShell( str )
end
log( 'FWrite', 'translated "', str, '" to \n', ft )
return ft
end
--
-- Exporter interface.
--
FWriter = { translate = translate }

View File

@ -85,346 +85,6 @@ uSettings = { }
--============================================================================
--
-- Holds information about the event monitor capabilities
-- of the core.
--
Monitors = ( function
( )
--
-- The cores monitor list
--
local list = { }
--
-- The default event monitor.
--
local function default
( )
return list[ 1 ]
end
--
-- Initializes with info received from core
--
local function initialize( clist )
for k, v in ipairs( clist )
do
list[ k ] = v
end
end
--
-- Public interface
--
return {
default = default,
list = list,
initialize = initialize
}
end)( )
--
-- Writes functions for the user for layer 3 configurations.
--
local functionWriter = ( function( )
--
-- All variables known to layer 3 configs.
--
transVars = {
{ '%^pathname', 'event.pathname', 1 },
{ '%^pathdir', 'event.pathdir', 1 },
{ '%^path', 'event.path', 1 },
{ '%^sourcePathname', 'event.sourcePathname', 1 },
{ '%^sourcePathdir', 'event.sourcePathdir', 1 },
{ '%^sourcePath', 'event.sourcePath', 1 },
{ '%^source', 'event.source', 1 },
{ '%^targetPathname', 'event.targetPathname', 1 },
{ '%^targetPathdir', 'event.targetPathdir', 1 },
{ '%^targetPath', 'event.targetPath', 1 },
{ '%^target', 'event.target', 1 },
{ '%^o%.pathname', 'event.pathname', 1 },
{ '%^o%.path', 'event.path', 1 },
{ '%^o%.sourcePathname', 'event.sourcePathname', 1 },
{ '%^o%.sourcePathdir', 'event.sourcePathdir', 1 },
{ '%^o%.sourcePath', 'event.sourcePath', 1 },
{ '%^o%.targetPathname', 'event.targetPathname', 1 },
{ '%^o%.targetPathdir', 'event.targetPathdir', 1 },
{ '%^o%.targetPath', 'event.targetPath', 1 },
{ '%^d%.pathname', 'event2.pathname', 2 },
{ '%^d%.path', 'event2.path', 2 },
{ '%^d%.sourcePathname', 'event2.sourcePathname', 2 },
{ '%^d%.sourcePathdir', 'event2.sourcePathdir', 2 },
{ '%^d%.sourcePath', 'event2.sourcePath', 2 },
{ '%^d%.targetPathname', 'event2.targetPathname', 2 },
{ '%^d%.targetPathdir', 'event2.targetPathdir', 2 },
{ '%^d%.targetPath', 'event2.targetPath', 2 },
}
--
-- Splits a user string into its arguments.
-- Returns a table of arguments
--
local function splitStr(
str -- a string where parameters are seperated by spaces.
)
local args = { }
while str ~= ''
do
-- break where argument stops
local bp = #str
-- in a quote
local inQuote = false
-- tests characters to be space and not within quotes
for i = 1, #str
do
local c = string.sub( str, i, i )
if c == '"'
then
inQuote = not inQuote
elseif c == ' ' and not inQuote
then
bp = i - 1
break
end
end
local arg = string.sub( str, 1, bp )
arg = string.gsub( arg, '"', '\\"' )
table.insert( args, arg )
str = string.sub( str, bp + 1, -1 )
str = string.match( str, '^%s*(.-)%s*$' )
end
return args
end
--
-- Translates a call to a binary to a lua function.
-- TODO this has a little too blocking.
--
local function translateBinary
(
str
)
-- splits the string
local args = splitStr( str )
-- true if there is a second event
local haveEvent2 = false
for ia, iv in ipairs( args )
do
-- a list of arguments this arg is being split into
local a = { { true, iv } }
-- goes through all translates
for _, v in ipairs( transVars )
do
local ai = 1
while ai <= #a
do
if a[ ai ][ 1 ]
then
local pre, post =
string.match( a[ ai ][ 2 ], '(.*)'..v[1]..'(.*)' )
if pre
then
if v[3] > 1
then
haveEvent2 = true
end
if pre ~= ''
then
table.insert( a, ai, { true, pre } )
ai = ai + 1
end
a[ ai ] = { false, v[ 2 ] }
if post ~= ''
then
table.insert( a, ai + 1, { true, post } )
end
end
end
ai = ai + 1
end
end
-- concats the argument pieces into a string.
local as = ''
local first = true
for _, v in ipairs( a )
do
if not first then as = as..' .. ' end
if v[ 1 ]
then
as = as .. '"' .. v[ 2 ] .. '"'
else
as = as .. v[ 2 ]
end
first = false
end
args[ ia ] = as
end
local ft
if not haveEvent2
then
ft = 'function( event )\n'
else
ft = 'function( event, event2 )\n'
end
ft = ft ..
" log('Normal', 'Event ', event.etype, \n" ..
" ' spawns action \"".. str.."\"')\n" ..
" spawn( event"
for _, v in ipairs( args )
do
ft = ft .. ',\n ' .. v
end
ft = ft .. ')\nend'
return ft
end
--
-- Translates a call using a shell to a lua function
--
local function translateShell
(
str
)
local argn = 1
local args = { }
local cmd = str
local lc = str
-- true if there is a second event
local haveEvent2 = false
for _, v in ipairs( transVars )
do
local occur = false
cmd = string.gsub(
cmd,
v[ 1 ],
function
( )
occur = true
return '"$' .. argn .. '"'
end
)
lc = string.gsub( lc, v[1], ']]..' .. v[2] .. '..[[' )
if occur
then
argn = argn + 1
table.insert( args, v[ 2 ] )
if v[ 3 ] > 1
then
haveEvent2 = true
end
end
end
local ft
if not haveEvent2
then
ft = 'function( event )\n'
else
ft = 'function( event, event2 )\n'
end
-- TODO do array joining instead
ft = ft..
" log('Normal', 'Event ',event.etype,\n"..
" [[ spawns shell \""..lc.."\"]])\n"..
" spawnShell(event, [["..cmd.."]]"
for _, v in ipairs( args )
do
ft = ft..',\n '..v
end
ft = ft .. ')\nend'
return ft
end
--
-- Writes a lua function for a layer 3 user script.
--
local function translate
(
str
)
-- trims spaces
str = string.match( str, '^%s*(.-)%s*$' )
local ft
if string.byte( str, 1, 1 ) == 47
then
-- starts with /
ft = translateBinary( str )
elseif string.byte( str, 1, 1 ) == 94
then
-- starts with ^
ft = translateShell( str:sub( 2, -1 ) )
else
ft = translateShell( str )
end
log( 'FWrite', 'translated "', str, '" to \n', ft )
return ft
end
--
-- Public interface.
--
return { translate = translate }
end )( )
--
-- Writes a status report file at most every 'statusintervall' seconds.
--
@ -806,9 +466,11 @@ end
--
-- terminates on invalid arguments.
--
function mci.configure( args, monitors )
Monitors.initialize( monitors )
function mci.configure(
args, -- arguments given by user
monitors -- list of monitors the core can do
)
Monitor.initialize( monitors )
--
-- a list of all valid options
@ -1031,7 +693,7 @@ function mci.initialize( firstTime )
do
if type(config[fn]) == 'string'
then
local ft = functionWriter.translate( config[ fn ] )
local ft = FWriter.translate( config[ fn ] )
config[ fn ] = assert( load( 'return '..ft ) )( )
end

63
mantle/monitor.lua Normal file
View File

@ -0,0 +1,63 @@
--~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-- monitor.lua
--
--
-- Holds information about the event monitor capabilities
-- of the core.
--
--
-- After the removal of /dev/events this a mood point since all
-- it can do is only inotify again. But this might improve again.
--
--
-- This code assumes your editor is at least 100 chars wide.
--
-- License: GPLv2 (see COPYING) or any later version
-- Authors: Axel Kittenberger <axkibe@gmail.com>
--
--~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
if mantle
then
print( 'Error, Lsyncd mantle already loaded' )
os.exit( -1 )
end
-- The cores monitor list
--
local list = { }
--
-- The default event monitor.
--
local function default
( )
return list[ 1 ]
end
--
-- Initializes with info received from core
--
local function initialize
( clist )
for k, v in ipairs( clist )
do
list[ k ] = v
end
end
--
-- Exported interface.
--
Monitor =
{
default = default,
list = list,
initialize = initialize
}

115
mantle/status.lua Normal file
View File

@ -0,0 +1,115 @@
--~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-- status.lua Live (Mirror) Syncing Demon
--
--
-- Writes a status report file at most every 'statusintervall' seconds.
--
--
-- This code assumes your editor is at least 100 chars wide.
--
-- License: GPLv2 (see COPYING) or any later version
-- Authors: Axel Kittenberger <axkibe@gmail.com>
--
--~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
if mantle
then
print( 'Error, Lsyncd mantle already loaded' )
os.exit( -1 )
end
--
-- Timestamp when the status file has been written.
--
local lastWritten = false
--
-- Timestamp when a status file should be written.
--
local alarm = false
--
-- Returns the alarm when the status file should be written-
--
local function getAlarm
( )
return alarm
end
--
-- Called to check if to write a status file.
--
local function write
(
timestamp
)
log( 'Function', 'write( ', timestamp, ' )' )
--
-- takes care not to write too often
--
if uSettings.statusInterval > 0
then
-- already waiting?
if alarm and timestamp < alarm
then
log( 'Statusfile', 'waiting(', timestamp, ' < ', alarm, ')' )
return
end
-- determines when a next write will be possible
if not alarm
then
local nextWrite = lastWritten and timestamp + uSettings.statusInterval
if nextWrite and timestamp < nextWrite
then
log( 'Statusfile', 'setting alarm: ', nextWrite )
alarm = nextWrite
return
end
end
lastWritten = timestamp
alarm = false
end
log( 'Statusfile', 'writing now' )
local f, err = io.open( uSettings.statusFile, 'w' )
if not f
then
log(
'Error',
'Cannot open status file "' ..
uSettings.statusFile ..
'" :' ..
err
)
return
end
f:write( 'Lsyncd status report at ', os.date( ), '\n\n' )
for i, s in SyncMaster.iwalk( )
do
s:statusReport( f )
f:write( '\n' )
end
Inotify.statusReport( f )
f:close( )
end
--
-- Exported interface.
--
StatusFile = { write = write, getAlarm = getAlarm }

View File

@ -281,10 +281,7 @@ local function add
end
-- the monitor to use
config.monitor =
uSettings.monitor
or config.monitor
or Monitors.default( )
config.monitor = uSettings.monitor or config.monitor or Monitor.default( )
if config.monitor ~= 'inotify'
then