Compare commits

...

27 Commits

Author SHA1 Message Date
Daniel Poelzleithner e1e13503b2 Use github discussions 2023-03-03 14:37:43 +01:00
Daniel Poelzleithner 95a2c2fca2 Update docs and build manpage on build process 2023-03-03 14:25:46 +01:00
Daniel Poelzleithner 5d3e0fe417
Merge pull request #697 from jandsu/patch-1
Added the 'filter' parameter to the manual
2023-03-03 13:07:56 +01:00
Jan Vorwerk 77f5a61bab Added the 'filter' parameter to the manual
According to
https://github.com/lsyncd/lsyncd/issues/510#issuecomment-390921153
2023-03-03 08:29:42 +01:00
Daniel Poelzleithner 3872ca77ac Check ssh arguments and issue warning 2023-03-02 18:45:58 +01:00
Daniel Poelzleithner a4556b835f Fix spacing/wording 2023-03-02 17:22:01 +01:00
Daniel Poelzleithner d4fc88ba10 Update README
- write about 2 way sync
- more alternatives
2023-02-28 19:22:00 +01:00
Daniel Poelzleithner dd48284cd0 fix changelog dates 2022-11-17 15:43:48 +01:00
Daniel Poelzleithner 6d59f16140 release 2.3.1 2022-11-17 02:22:35 +01:00
Daniel Poelzleithner e831ad6945 check alarm for type first. fixes #679 2022-11-11 20:05:09 +01:00
Daniel Poelzleithner d779eb434f Allow lsyncd to be build in the source folder without breaking. Not recommended 2022-11-11 19:32:09 +01:00
Daniel Poelzleithner 818fd4115f also test executable files 2022-11-11 19:00:26 +01:00
Daniel Poelzleithner f4c15496dc fix cron test 2022-11-11 18:26:57 +01:00
Daniel Poelzleithner 96a6276440 disable lua 5.4.4 due segfault on luac invocation 2022-11-11 18:05:50 +01:00
Daniel Poelzleithner 5096f27bbd Fix more warnings from static code analysis 2022-11-11 15:27:21 +01:00
Daniel Poelzleithner 945b57d8fb Add Cron test 2022-11-11 15:11:09 +01:00
Daniel Poelzleithner 787b2b0015 fix static code analysis warnings, typos 2022-11-10 04:21:17 +01:00
Daniel Poelzleithner e09b58b721 Fix wrong arguments for Sync:appendFilter proxy function 2022-11-09 22:44:04 +01:00
Daniel Poelzleithner 9b81bb1785 fix dump for more complex structures 2022-11-09 22:41:04 +01:00
Daniel Poelzleithner cda98d6ba9 add Queue.inject, test Queue, bugfixes in remove() 2022-11-09 15:22:58 +01:00
Daniel Poelzleithner daa8abb4cf nix: extract version from lsyncd.lua 2022-11-08 13:27:20 +01:00
Daniel Poelzleithner 7604dd0e16 cmake: extract lsyncd version from lsyncd.lua 2022-10-31 14:36:39 +01:00
Daniel Poelzleithner e371078222 update nix locks 2022-10-28 15:12:54 +02:00
Daniel Poelzleithner a2f7df504c bump version 2022-10-28 15:12:41 +02:00
Daniel Poelzleithner e08685cfd6 Search crontab library with different names 2022-10-28 14:08:00 +02:00
Daniel Poelzleithner 1eb8a83500 Print version in debug and help messages 2022-10-28 14:07:17 +02:00
Daniel Poelzleithner e6f3427c5f reduce required cmake version to 3.5 2022-06-30 11:43:12 +02:00
33 changed files with 547 additions and 125 deletions

View File

@ -10,7 +10,8 @@ jobs:
matrix:
version:
- lsyncd_lua5_3
- lsyncd_lua5_4
# broken with lua 5.4.4. luac segfault
# - lsyncd_lua5_4
- lsyncd_lua5_1
steps:
- uses: actions/checkout@v2.4.0

2
.gitignore vendored
View File

@ -10,7 +10,7 @@ lsyncd
AdditionalInfo.txt
config.h
Makefile
build/
build*/
CMakeCache.txt
CMakeFiles/
cmake_install.cmake

View File

@ -1,7 +1,13 @@
# preamble
project( Lsyncd )
cmake_minimum_required( VERSION 3.10 )
set( LSYNCD_VERSION 2.3.0 )
cmake_minimum_required( VERSION 3.5 )
# extract version
file(STRINGS "lsyncd.lua" LSYNCD_VERSION_RAW REGEX "lsyncd_version = '.*'")
string(REGEX REPLACE "lsyncd_version = \'(.*)\'"
"\\1" LSYNCD_VERSION
${LSYNCD_VERSION_RAW})
set( CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/" )
@ -85,7 +91,8 @@ add_custom_command( OUTPUT defaults.out
# the manpage
add_custom_target( manpage
COMMAND ${CMAKE_COMMAND} -E echo "Updating the manpage"
COMMAND a2x --format=manpage docs/manpage/lsyncd.1.txt
COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/man
COMMAND a2x --format=manpage ${PROJECT_SOURCE_DIR}/docs/manpage/lsyncd.1.txt -D ${CMAKE_CURRENT_BINARY_DIR}/man
DEPENDS docs/manpage/lsyncd.1.txt
)
@ -97,18 +104,21 @@ add_custom_target( docs-html
DEPENDS ${CMAKE_SOURCE_DIR}/docs
)
add_custom_command(
OUTPUT tests
COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_SOURCE_DIR}/tests tests)
# create_symlink( ${CMAKE_SOURCE_DIR}/tests tests)
ADD_CUSTOM_TARGET(prepare_tests ALL
COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_SOURCE_DIR}/tests tests
DEPENDS tests/
)
add_custom_target( tests
add_custom_target( run-tests
COMMAND echo "Running the tests"
COMMAND echo "Note you are expected to:"
COMMAND echo " * have lua-posix installed"
COMMAND ${LUA_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/setup.lua
COMMAND ${CMAKE_BINARY_DIR}/lsyncd -log all -script ${CMAKE_SOURCE_DIR}/tests/utils_test.lua
COMMAND ${LUA_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/cron-rsync.lua
COMMAND ${LUA_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/schedule.lua
COMMAND ${LUA_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/l4rsyncdata.lua
COMMAND ${LUA_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/filter-rsync.lua
@ -127,6 +137,6 @@ add_executable( lsyncd ${LSYNCD_SRC} )
target_link_libraries( lsyncd ${LUA_LIBRARIES} )
install( TARGETS lsyncd RUNTIME DESTINATION bin )
install( FILES docs/manpage/lsyncd.1 DESTINATION ${CMAKE_INSTALL_MANDIR}/man1 COMPONENT man )
install( FILES ${CMAKE_CURRENT_BINARY_DIR}/man/lsyncd.1 DESTINATION ${CMAKE_INSTALL_MANDIR}/man1 COMPONENT man )
install( DIRECTORY examples DESTINATION doc )
install( DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/html DESTINATION doc OPTIONAL)

View File

@ -1,4 +1,7 @@
2021-03-09: 2.3.0
2022-11-17: 2.3.1
change: multiple bugfixes, style fixes
2022-03-09: 2.3.0
add: nix flake support
add: support for tunnel commands
add: support for batchSizeLimit

View File

@ -16,6 +16,13 @@ When to use
-----------
Lsyncd is designed to synchronize a local directory tree with low profile of expected changes to a remote mirror. Lsyncd is especially useful to sync data from a secure area to a not-so-secure area.
2-way/bidirection synchronization
---------------------------------
It is not possible to use lsyncd to synchronize for example `folder1` with `folder2` and vice versa. Only one source to one target. Two way synchronization is a very hard problem that needs specialized tools.
Imagine you start writing a very large file to `folder1`, lsyncd will start synchronizing this file to `folder2`, which might be on a different machine. The lsyncd on that machine will see a new file, and try to synchronize it back to `folder1`. If at the same time, you change bytes in this file, those changes will be overwritten with old data. Using lsyncd in such a way might work in practice, but data corruption is easily possible if you write into files afterwards.
`git-annex` is a good way to do this, if you don't mind working with git repositories. It stores each change as a revision that can be rolled back.
Other synchronization tools
------------------------
[DRBD](http://www.drbd.org) operates on block device level. This makes it useful for synchronizing systems that are under heavy load. Lsyncd on the other hand does not require you to change block devices and/or mount points, allows you to change uid/gid of the transferred files, separates the receiver through the one-way nature of rsync. DRBD is likely the better option if you are syncing databases.
@ -24,6 +31,12 @@ Other synchronization tools
[Mirror](https://github.com/stephenh/mirror) is an asynchronous synchronisation tool that takes use of the inotify notifications much like Lsyncd. The main differences are: it is developed specifically for master-master use, thus running on a daemon on both systems, uses its own transportation layer instead of rsync and is Java instead of Lsyncd's C core with Lua scripting.
[git-annex](https://git-annex.branchable.com/) allows managing large files with git, without storing the file contents in git. It can sync, backup, and archive your data, offline and online. Checksums and encryption keep your data safe and secure. Bring the power and distributed nature of git to bear on your large files with git-annex.
git-annex is designed for git users who love the command line. For everyone else, the git-annex assistant turns git-annex into an easy to use folder synchroniser.
[Wikipedia](https://en.wikipedia.org/wiki/Comparison_of_file_synchronization_software) Comparison of file synchronization software
Lsyncd usage examples
---------------------
```lsyncd -rsync /home remotehost.org::share/```

View File

@ -246,7 +246,7 @@ local getBatchSize =
if event.status == 'active' then
return false
end
if event.etype == 'Init' or event.etype == 'Blanket' then
if event.etype == 'Init' or event.etype == 'Blanket' or event.etype == 'Full' then
return false
end
-- moves and deletes go always into batch
@ -274,6 +274,9 @@ rsync.action = function
else
-- spawn all files under the size limit/deletes/moves in batch mode
local eventInBatch = function(event)
if event.etype == "Full" then
return false
end
local size = getBatchSize(event)
if type(size) == "boolean" then
return size
@ -290,6 +293,9 @@ rsync.action = function
local single_returned = false
-- grab all events for seperate transfers
local eventNoBatch = function(event)
if event.etype == "Full" then
return false
end
local size = getBatchSize(event)
if type(size) ~= "number" or size == nil then
return false
@ -303,10 +309,11 @@ rsync.action = function
end
return false
end
local extralist = inlet.getEvents(eventInBatch)
local elist = inlet.getEvents(eventInBatch)
if elist.size() > 0 then
run_action(inlet, elist)
-- get all batched events
if extralist.size() > 0 then
run_action(inlet, extralist)
end
while true do
@ -450,7 +457,7 @@ rsync.full = function
-- starts rsync without any filters or excludes
log(
'Normal',
'recursive startup rsync: ',
'recursive full rsync: ',
config.source,
' -> ',
target
@ -474,7 +481,7 @@ rsync.full = function
log(
'Normal',
'recursive startup rsync: ',
'recursive full rsync: ',
config.source,
' -> ',
target,
@ -500,7 +507,7 @@ rsync.full = function
log(
'Normal',
'recursive startup rsync: ',
'recursive full rsync: ',
config.source,
' -> ',
target,
@ -585,6 +592,7 @@ rsync.prepare = function
crsync._computed = { true }
--- @type any
local computed = crsync._computed
local computedN = 2

View File

@ -102,6 +102,41 @@ local replaceRsyncFilter =
end
---
--- Check if _extra arguments are of correct form
---
local checkSSH = function (args)
local SINGLES = "46AaCfGgKkMNnqsTtVvXxYy"
local needs_more = true
local is_data = false
local rv = true
for index, value in ipairs(args) do
if is_data == false then
if string.sub(value, 1, 1) ~= "-" then
log('Warn', "_extra argument does not start with -")
rv = false
end
needs_more = true
end
for i = 1, #SINGLES do
if string.sub(SINGLES, i, i) == string.sub(value, 2, 2) then
needs_more = false
break
end
end
if needs_more == false then
is_data = false
else
is_data = true
end
end
if needs_more == true then
log('Warn', "passend argument requires more arguments")
rv = false
end
return rv
end
--
-- Spawns rsync for a list of events
--
@ -548,6 +583,9 @@ rsyncssh.prepare = function
if cssh._extra
then
if checkSSH(cssh._extra) == false then
log( 'Warn', 'The ssh._extra parameter is a list of arguments, ensure it is written correctly')
end
for k, v in ipairs( cssh._extra )
do
computed[ computedN ] = v

View File

@ -13,6 +13,7 @@ then
error( 'default already loaded' )
end
--- @diagnostic disable-next-line: lowercase-global
default = { }

View File

@ -5,11 +5,11 @@ short: "Help"
---
Discussion Group
----------------
There is a [discussion group to all things related Lsyncd](https://groups.google.com/forum/#!forum/lsyncd/join)
There is a [discussion group to all things related Lsyncd](https://github.com/lsyncd/lsyncd/discussions)
Issues
------
For cases Lsyncd is missbehaving there are [Issues](https://github.com/axkibe/lsyncd/issues) on Github.
For cases Lsyncd is missbehaving there are [Issues](https://github.com/lsyncd/lsyncd/issues) on Github.
Please check:
@ -19,4 +19,4 @@ Please check:
Source & Improvement
--------------------
[Fork Lsyncd at Github](https://github.com/axkibe/lsyncd)
[Fork Lsyncd at Github](https://github.com/lsyncd/lsyncd)

View File

@ -1,7 +1,7 @@
'\" t
.\" Title: lsyncd
.\" Author: [see the "AUTHOR" section]
.\" Generator: DocBook XSL Stylesheets v1.79.1 <http://docbook.sf.net/>
.\" Generator: DocBook XSL Stylesheets vsnapshot <http://docbook.sf.net/>
.\" Date: January 2017
.\" Manual: Lsyncd
.\" Source: Lsyncd 2.2.1
@ -149,13 +149,13 @@ Failure (syntax, unrecoverable error condition, internal failure)
.RE
.SH "SEE ALSO"
.sp
Online Manual: https://axkibe\&.github\&.io/lsyncd/
Online Manual: https://lsyncd\&.github\&.io/lsyncd/
.SH "VERSION"
.sp
This man page is for lsyncd(1) version 2\&.2\&.0
.SH "AUTHOR"
.sp
Axel Kittenberger, <axkibe@gmail\&.com> 2010\-2017
Axel Kittenberger, <axkibe@gmail\&.com> 2010\-2017 Daniel Poelzleithner, <poelzleithner@b1\-systems\&.de> 2021\-2023
.SH "COPYING"
.sp
Copyright (C) 2010\-2017 Axel Kittenberger\&. Free use of this software is granted under the terms of the GNU General Public License (GPL) version 2, or any later version\&. Free redistrubition of this Documentation (/doc directory) is granted under the terms of the Creative Commons 3\&.0 Attribution License (CC\-3\&.0\-BY)\&.

View File

@ -41,7 +41,7 @@ Fine-grained customization can be achieved through the CONFIG-FILE. Custom
action configs can even be written from scratch in cascading layers ranging
from shell scripts to code written in the LUA(1) language. This way simplicity
can be balanced with powerfulness. See the online manual for details on the
CONFIG-FILE https://axkibe.github.io/lsyncd/manual/config/file/ .
CONFIG-FILE https://lsyncd.github.io/lsyncd/manual/config/file/ .
Note that under normal configuration Lsyncd will delete pre-existing files in
the target directories that are not present in the respective source directory.
@ -69,6 +69,9 @@ OPTIONS
*-nodaemon*::
Lsyncd will not detach from the invoker and log as well to stdout/err.
*-onepass*::
Sync once and exit
*-pidfile* 'FILE'::
Lsyncd will write its process ID in 'FILE'.
@ -89,7 +92,7 @@ EXIT STATUS
SEE ALSO
--------
Online Manual: https://axkibe.github.io/lsyncd/
Online Manual: https://lsyncd.github.io/lsyncd/
VERSION
------
@ -98,6 +101,7 @@ This man page is for lsyncd(1) version 2.2.0
AUTHOR
------
Axel Kittenberger, <axkibe@gmail.com> 2010-2017
Daniel Poelzleithner, <poelzleithner@b1-systems.de> 2021-2023
COPYING
-------

View File

@ -219,6 +219,13 @@ Below is a table of options for the ```rsync``` parameter. Please have a look at
</td><td> (Lsyncd >= 2.2.0)
</td></tr>
<tr><td> filter
</td><td> =
</td><td> TABLE of STRINGS
</td><td>
</td><td> (Lsyncd >= 2.2.3)
</td></tr>
<tr><td> group
</td><td> =
</td><td> BOOL

View File

@ -15,12 +15,12 @@ settings {
-- for testing purposes. prefix can be used to slow commands down.
-- prefix = "sleep 5 && "
--
prefix = ""
local prefix = ""
-----
-- for testing purposes. uses bash command to hold local dirs in sync.
--
bash = {
local bash = {
delay = 0,
maxProcesses = 1,

View File

@ -7,7 +7,7 @@
-----
-- for testing purposes. just echos what is happening.
--
echo = {
local echo = {
maxProcesses = 1,
delay = 1,
onStartup = "/bin/echo telling about ^source",

View File

@ -22,12 +22,12 @@
----
-- forces this group.
--
fgroup = "staff"
local fgroup = "staff"
-----
-- script for all changes.
--
command =
local command =
-- checks if the group is the one enforced and sets them if not
[[
perm=`stat -c %A ^sourcePathname`
@ -59,13 +59,13 @@ fi
-- the carret as first char tells Lsycnd to call a shell altough it
-- starts with a slash otherwisw
--
startup =
local startup =
[[^/bin/chgrp -R ]]..fgroup..[[ ^source || /bin/true &&
/bin/chmod -R g+rw ^source || /bin/true &&
/usr/bin/find ^source -type d | xargs chmod g+x
]]
gforce = {
local gforce = {
maxProcesses = 99,
delay = 1,
onStartup = startup,

View File

@ -12,7 +12,7 @@
--
local formats = { jpg=true, gif=true, png=true, }
convert = {
local convert = {
delay = 0,
maxProcesses = 99,

View File

@ -19,7 +19,7 @@
--
-- Author: Daniel Miranda <danielkza2@gmail.com>
--
s3 = {}
local s3 = {}
s3.checkgauge = {
onCreate = false,

View File

@ -21,15 +21,16 @@ require("socket")
-- For demo reasons, do not detach
settings.nodaemon = true
hostname = "irc.freenode.org"
local hostname = "irc.freenode.org"
--hostname = "127.0.0.1"
port = 6667
nick = "lbot01"
chan = "##lfile01"
local port = 6667
local nick = "lbot01"
local chan = "##lfile01"
-- this blocks until the connection is established
-- for once lets say this ok since Lsyncd didnt yet actually
-- start.
--- @diagnostic disable-next-line: undefined-global
local ircSocket, err = socket.connect(hostname, port)
if not ircSocket then
log("Error", "Cannot connect to IRC: ", err)

View File

@ -2,11 +2,11 @@
"nodes": {
"flake-utils": {
"locked": {
"lastModified": 1653893745,
"narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=",
"lastModified": 1659877975,
"narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1",
"rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0",
"type": "github"
},
"original": {
@ -17,11 +17,11 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1654215692,
"narHash": "sha256-Ek74jQrAjTd4VP+33dxgpKf3kN0oq+7fQ8mneKXhzUs=",
"lastModified": 1666940097,
"narHash": "sha256-spcDvKqQU9iSMAjh5uj9gfu8Gu7vIFFkOKH2CCWvOVY=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "0f4c6596500078018e1797544a43ee8b50635dff",
"rev": "7269939a5d5610f2eec933607dc4d646394b29b8",
"type": "github"
},
"original": {

View File

@ -15,19 +15,22 @@
allowBroken = true;
};}); #.legacyPackages.${system};
defaultDeps = with pkgs; [
gcc
cmake
gnumake
glib
rsync
openssh
];
nativeDeps = with pkgs; [
curl
asciidoc
jekyll
gcc
cmake
gnumake
];
version = builtins.elemAt
(builtins.match ''.*set\(.LSYNCD_VERSION ([0-9\.]*).*''
(builtins.substring 0 500
(builtins.readFile ./CMakeLists.txt))) 0;
(builtins.match ''.*lsyncd_version = '([0-9\.]*)'.*''
(builtins.substring 0 1200
(builtins.readFile ./lsyncd.lua))) 0;
# mylua5_4 = pkgs.lua5_4.override({
# packageOverrides = luaself: luaprev: {
# luarocks = luaprev.luarocks-3_7;
@ -138,10 +141,11 @@
src = ./.;
buildPhase = ''
make all docs-html
make all manpage docs-html
'';
buildInputs = defaultDeps ++ luaPackages;
nativeBuildInputs = nativeDeps;
});
mkDev = extras: pkgs.mkShell {
propagatedBuildInputs = defaultDeps ++ extras;

View File

@ -25,38 +25,51 @@ then
lsyncd.terminate( -1 )
end
lsyncd_version = '2.3.0'
---@diagnostic disable-next-line: lowercase-global
lsyncd_version = '2.3.1'
-- compatibility with 5.1
if table.unpack == nil then
--- @diagnostic disable-next-line deprecated
table.unpack = unpack
end
local lver = string.gmatch(_VERSION, "%w (%d).(%d)")
local _LUA_VERSION_MAJOR, _LUA_VERSION_MINOR = lver()
local _LUA_VERSION_MAJOR, _LUA_VERSION_MINOR = string.gmatch(
_VERSION, "%w (%d).(%d)")()
---@type any
_LUA_VERSION_MAJOR = tonumber(_LUA_VERSION_MAJOR)
_LUA_VERSION_MINOR = tonumber(_LUA_VERSION_MINOR)
--
-- Shortcuts (which user is supposed to be able to use them as well)
--
---@diagnostic disable: lowercase-global
log = lsyncd.log
terminate = lsyncd.terminate
now = lsyncd.now
readdir = lsyncd.readdir
---@diagnostic enable: lowercase-global
--
-- Debug helper. Prints contents of `tbl`, with indentation.
-- `indent` sets the initial level of indentation.
--
---@diagnostic disable-next-line: lowercase-global
function dump(tbl, indent)
if not indent then indent = 0 end
for k, v in pairs(tbl) do
formatting = string.rep(" ", indent) .. k .. ": "
if type(k) ~= "string" then
k = tostring(k)
end
local formatting = string.rep(" ", indent) .. k .. ": "
if type(v) == "table" then
print(formatting)
dump(v, indent+1)
elseif type(v) == 'boolean' then
if indent > 3 then
print("dump: Already 3 deep, skip")
else
dump(v, indent+1)
end
elseif type(v) ~= 'string' then
print(formatting .. tostring(v))
else
print(formatting .. v)
@ -184,7 +197,7 @@ inheritKV =
end
end
function alarm2string(a)
local function alarm2string(a)
if type(a) == 'userdata' then
return a.string
end
@ -215,11 +228,15 @@ local processCount = 0
local crontab = nil
local function loadCrontab()
local ok, mod = pcall(require, "lua-crontab")
if ok then
-- print update crontab
crontab = mod
return true
local names = {"crontab.crontab", "crontab", "lua-crontab"}
for _,lname in ipairs(names) do
local ok, mod = pcall(require, lname)
if ok then
-- print update crontab
log('Debug', "Using crontab: "..lname)
crontab = mod
return true
end
end
return false
end
@ -574,31 +591,30 @@ Queue = ( function
error( 'Removing nonexisting item in Queue', 2 )
end
nt[ pos ] = nil
-- if removing first or last element,
-- the queue limits are adjusted.
if pos == nt.first
then
local last = nt.last
while nt[ pos ] == nil and pos <= last
do
pos = pos + 1
end
nt.first = pos
nt[ pos ] = nil
nt.first = pos + 1
elseif pos == nt.last
then
nt[ pos ] = nil
nt.last = nt.last - 1
else
local first = nt.first
local last = nt.first
while nt[ pos ] == nil and pos >= first
do
pos = pos - 1
local i = pos
while i < last do
if i == pos then
nt[ i ] = nt[ i + 1 ]
i = i + 1
end
i = i + 1
end
nt.last = pos
nt.last = last - 1
end
-- reset the indizies if the queue is empty
@ -612,6 +628,28 @@ Queue = ( function
nt.size = nt.size - 1
end
--
-- Injects a value in front of the Queue.
--
local function inject
(
self, -- the queue
value -- position to remove
)
local nt = self[ k_nt ]
local pos = nt.last
while pos >= nt.first
do
nt[pos + 1] = nt[pos]
pos = pos - 1
end
nt[nt.first] = value
nt.size = nt.size + 1
nt.last = nt.last + 1
end
--
-- Replaces a value.
--
@ -717,6 +755,7 @@ Queue = ( function
first = first,
last = last,
push = push,
inject = inject,
qpairs = qpairs,
qpairsReverse = qpairsReverse,
remove = remove,
@ -1712,6 +1751,19 @@ local InletFactory = ( function
return #dlist
end
end,
--
-- Returns the list of events
--
getList = function( elist )
local dlist = e2d[ elist ]
if not dlist then
return {}
else
return dlist
end
end,
}
--
@ -1940,6 +1992,19 @@ local InletFactory = ( function
return d2e( sync:addBlanketDelay( ) )
end,
--
-- Creates a blanketEvent that blocks everything
-- and is blocked by everything.
--
createFullEvent = function
(
sync, -- the sync of the inlet
path -- path for the full event
)
-- return d2e( sync:delay("Full", timestamp, path, nil) )
return d2e( sync:addFullDelay(path) )
end,
--
-- Discards a waiting event.
--
@ -2189,13 +2254,13 @@ local Excludes = ( function
self, -- self
file -- filename to load from
)
f, err = io.open( file )
local f, err = io.open( file )
if not f
then
log( 'Error', 'Cannot open exclude file "', file,'": ', err )
terminate( -1 )
return terminate( -1 )
end
for line in f:lines()
@ -2365,13 +2430,13 @@ local Filters = ( function
self, -- self
file -- filename to load from
)
f, err = io.open( file )
local f, err = io.open( file )
if not f
then
log( 'Error', 'Cannot open filter file "', file, '": ', err )
terminate( -1 )
return terminate( -1 )
end
for line in f:lines( )
@ -2476,15 +2541,17 @@ local Sync = ( function
return self.excludes:add( pattern )
end
--
-- Appends a filter line to the Sync
--
local function appendFilter
(
self,
rule,
pattern
line
)
if not self.filters then self.filters = Filters.new( ) end
return self.filters:append( rule, pattern )
return self.filters:append( line )
end
--
@ -2819,7 +2886,7 @@ local Sync = ( function
-- new delay
local nd = Delay.new( etype, self, alarm, path, path2 )
if nd.etype == 'Init' or nd.etype == 'Blanket'
if nd.etype == 'Init' or nd.etype == 'Blanket' or nd.etype == 'Full'
then
-- always stack init or blanket events on the last event
log(
@ -3085,8 +3152,7 @@ local Sync = ( function
if self.nextCronAlarm ~= false and self.nextCronAlarm < timestamp then
-- time fo a full sync
log('Info', 'Crontab triggered full sync')
-- TODO
self:delay("Full", timestamp, "/", nil)
self.inlet.createFullEvent("/")
updateNextCronAlarm(self, timestamp)
end
@ -3115,11 +3181,12 @@ local Sync = ( function
if d.status == 'wait'
then
-- found a waiting delay
if d.etype ~= 'Init'
then
self.config.action( self.inlet )
else
if d.etype == 'Init' then
self.config.init( InletFactory.d2e( d ) )
elseif d.etype == 'Full' and self.config.full then
self.config.full( InletFactory.d2e( d ) )
else
self.config.action( self.inlet )
end
if self.processes:size( ) >= self.config.maxProcesses
@ -3175,6 +3242,21 @@ local Sync = ( function
return newd
end
--
-- Adds an full delay that will initiate a full transfer.
--
local function addFullDelay
(
self,
path
)
local newd = Delay.new( 'Full', self, true, path )
newd.dpos = self.delays:push( newd )
return newd
end
--
-- Adds and returns a blanket delay thats blocks all.
-- Used as startup marker to call init asap.
@ -3299,6 +3381,7 @@ local Sync = ( function
-- functions
addBlanketDelay = addBlanketDelay,
addFullDelay = addFullDelay,
addExclude = addExclude,
addInitDelay = addInitDelay,
appendFilter = appendFilter,
@ -3402,7 +3485,7 @@ local Sync = ( function
if #cdata then
s.cron = cdata
else
error("Can't parse crontab data: "..cron, 0)
error("Can't parse crontab data: "..cdata, 0)
end
end
@ -3423,7 +3506,7 @@ local functionWriter = ( function( )
--
-- All variables known to layer 3 configs.
--
transVars = {
local transVars = {
{ '%^pathname', 'event.pathname', 1 },
{ '%^pathdir', 'event.pathdir', 1 },
{ '%^path', 'event.path', 1 },
@ -4027,6 +4110,7 @@ Tunnel = (function()
'Start tunnel command ',
cmd
)
--- @diagnostic disable-next-line: param-type-mismatch
local pid = lsyncd.exec(bin, table.unpack(cmd, 2))
--local pid = spawn(bin, table.unpack(self.options.command, 2))
if pid and pid > 0 then
@ -4102,7 +4186,7 @@ Tunnel = (function()
local good = false
if type(self.options.checkExitCodes) == 'table' then
for _,i in iwalk(self.options.checkExitCodes) do
for _,i in ipairs(self.options.checkExitCodes) do
if exitcode == i then
good = true
end
@ -4113,7 +4197,7 @@ Tunnel = (function()
end
end
if good then
if self.isReady() == false then
if self:isReady() == false then
log(
'Info',
self.options.name,
@ -4262,7 +4346,7 @@ local Tunnels = ( function
( )
return #tunnelList
end
--- @type any
local nextCycle = true
--
-- Cycle through all tunnels and call their invoke function
@ -4320,14 +4404,14 @@ local Tunnels = ( function
size = size,
invoke = invoke,
getAlarm = getAlarm,
killAll = killAll,
statusReport = statusReport
killAll = killAll
}
end )( )
--
-- create a new tunnel from the passed options and registers the tunnel
--- @diagnostic disable-next-line: lowercase-global
tunnel = function (options)
log(
'Debug',
@ -4621,7 +4705,7 @@ local function splitQuotedString
(
text
)
local spat, epat, buf, quoted = [=[^(['"])]=], [=[(['"])$]=]
local spat, epat, buf, quoted = [=[^(['"])]=], [=[(['"])$]=], nil, nil
local rv = {}
for str in text:gmatch("%S+") do
local squoted = str:match(spat)
@ -4648,6 +4732,7 @@ end
lsyncd.splitQuotedString = splitQuotedString
--- @diagnostic disable-next-line: lowercase-global
function substitudeCommands(cmd, data)
assert(type(data) == "table")
@ -4856,6 +4941,7 @@ local Inotify = ( function
end
-- looks up the watch descriptor id
--- @type any
local path = wdpaths[ wd ]
if path
@ -4863,6 +4949,7 @@ local Inotify = ( function
path = path..filename
end
--- @type any
local path2 = wd2 and wdpaths[ wd2 ]
if path2 and filename2
@ -5563,10 +5650,12 @@ end
-- Called by core if '-help' or '--help' is in
-- the arguments.
--
function runner.help( )
function runner.help(_arg0)
io.stdout:write(
'lsyncd version: ' .. lsyncd_version ..
[[
USAGE:
runs a config file:
lsyncd [OPTIONS] [CONFIG-FILE]
@ -5593,7 +5682,7 @@ OPTIONS:
-pidfile FILE Writes Lsyncds PID into FILE
-runner FILE Loads Lsyncds lua part from FILE
-script FILE Script to load before execting runner (ADVANCED)
-sshopts Additional ssh command options when using rsyncssh
-sshopts Additional ssh command options when using rsyncssh
-version Prints versions and exits
LICENSE:
@ -5867,6 +5956,9 @@ function runner.configure( args, monitors )
i = i + 1
end
log( 'Debug', 'lsyncd version: '.. lsyncd_version .. ' starting.' )
log( 'Debug', 'module search path: '.. package.path)
if clSettings.scripts then
for _, file in ipairs(clSettings.scripts) do
log( 'Info', 'Run addition script: ' .. file )
@ -6054,6 +6146,7 @@ function runner.initialize( firstTime )
local ft = functionWriter.translate( config[ fn ] )
if _LUA_VERSION_MAJOR <= 5 and _LUA_VERSION_MINOR < 2 then
-- lua 5.1 and older
--- @diagnostic disable-next-line deprecated
config[ fn ] = assert( loadstring( 'return '..ft ) )( )
else
config[ fn ] = assert( load( 'return '..ft ) )( )
@ -6114,7 +6207,7 @@ function runner.getAlarm
)
if a == nil then error( 'got nil alarm' ) end
if alarm == true or not a
if (type(alarm) == "boolean" and alarm == true) or not a
then
-- 'alarm' is already immediate or
-- a not a new alarm
@ -6249,6 +6342,7 @@ end
--
-- Returns an Inlet to that sync.
--
--- @diagnostic disable-next-line: lowercase-global
function sync
(
opts
@ -6265,6 +6359,7 @@ end
--
-- Spawns a new child process.
--
--- @diagnostic disable-next-line: lowercase-global
function spawn(
agent, -- the reason why a process is spawned.
-- a delay or delay list for a sync
@ -6358,6 +6453,7 @@ end
--
-- Spawns a child process using the default shell.
--
--- @diagnostic disable-next-line: lowercase-global
function spawnShell
(
agent, -- the delay(list) to spawn the command for
@ -6371,6 +6467,7 @@ end
--
-- Observes a filedescriptor.
--
--- @diagnostic disable-next-line: lowercase-global
function observefd
(
fd, -- file descriptor
@ -6384,6 +6481,7 @@ end
--
-- Stops observeing a filedescriptor.
--
--- @diagnostic disable-next-line: lowercase-global
function nonobservefd
(
fd -- file descriptor
@ -6398,6 +6496,7 @@ end
-- Use now() to receive current timestamp
-- add seconds with '+' to it
--
--- @diagnostic disable-next-line: lowercase-global
alarm = UserAlarms.alarm
@ -6430,6 +6529,7 @@ end
--
-- The settings call
--
--- @diagnostic disable-next-line: lowercase-global
function settings
(
a1 -- a string for getting a setting

View File

@ -12,7 +12,7 @@ cwriteln( '****************************************************************' )
local tdir, srcdir, trgdir = mktemps( )
-- makes some startup data
churn( srcdir, 10, init )
churn( srcdir, 10, false )
local logs = { }
--local logs = {'-log', 'Exec', '-log', 'Delay' }
@ -34,10 +34,10 @@ posix.sleep( 10 )
cwriteln( 'killing the Lsyncd daemon' )
posix.kill( pid )
local _, exitmsg, lexitcode = posix.wait( lpid )
local _, exitmsg, lexitcode = posix.wait( pid )
cwriteln( 'Exitcode of Lsyncd = ', exitmsg, ' ', lexitcode )
result, code = execute( 'diff -r ' .. srcdir .. ' ' .. trgdir )
local result, code = execute( 'diff -r ' .. srcdir .. ' ' .. trgdir )
if result == 'exit'
then

View File

@ -36,10 +36,10 @@ posix.sleep( 10 )
cwriteln( 'killing the Lsyncd daemon' )
posix.kill( pid )
local _, exitmsg, lexitcode = posix.wait( lpid )
local _, exitmsg, lexitcode = posix.wait( pid )
cwriteln( 'Exitcode of Lsyncd = ', exitmsg, ' ', lexitcode )
result, code = execute( 'diff -r ' .. srcdir .. ' ' .. trgdir )
local result, code = execute( 'diff -r ' .. srcdir .. ' ' .. trgdir )
if result == 'exit'
then

View File

@ -43,11 +43,11 @@ cwriteln( 'killing the Lsyncd daemon' )
posix.kill(pid)
local _, exitmsg, lexitcode = posix.wait( lpid )
local _, exitmsg, lexitcode = posix.wait( pid )
cwriteln( 'Exitcode of Lsyncd = ', exitmsg, ' ', lexitcode )
result, code = execute( 'diff -r ' .. srcdir .. ' ' .. trgdir )
local result, code = execute( 'diff -r ' .. srcdir .. ' ' .. trgdir )
if result == 'exit'
then

View File

@ -17,5 +17,5 @@ echo "Build folder: $BUILD_FOLDER"
cd $BUILD_FOLDER
cmake $SRC
make VERBOSE=1
make tests
make run-tests
rm -rf $BUILD_FOLDER

149
tests/cron-rsync.lua Normal file
View File

@ -0,0 +1,149 @@
require( 'posix' )
dofile( 'tests/testlib.lua' )
cwriteln( '****************************************************************' )
cwriteln( ' Testing crontab (rsync)' )
cwriteln( '****************************************************************' )
local tdir, srcdir, trgdir = mktemps( )
local logfile = tdir .. "log"
local cfgfile = tdir .. "config.lua"
local range = 5
local log = {"-log", "all"}
writefile(cfgfile, [[
settings {
logfile = "]]..logfile..[[",
nodaemon = true,
}
sync {
default.rsync,
crontab = {
-- trigger full sync every 1 minute
"*/10 * * * * *",
},
source = "]]..srcdir..[[",
action = function (inlet)
local e = inlet.getEvent( );
print("inhibit action ".. e.path.. " = " .. e.etype);
if e.etype ~= "Full" then
inlet.discardEvent(e);
return;
end
return default.rsync.action(inlet);
end,
target = "]]..trgdir..[[",
delay = 2,
delete = true,
rsync = {
verbose = true,
inplace = true,
_extra = {
"-vv",
"--info=progress2"
}
},
filter = {
'- /xb**',
'+ /x**',
'- /**',
},
}
]])
-- writes all files
local function writefiles
( )
writefile( srcdir .. 'xbc', 'xbc' )
writefile( srcdir .. 'xcc', 'xcc' )
writefile( srcdir .. 'yaa', 'yaa' )
posix.mkdir( srcdir .. 'xbx' )
writefile( srcdir .. 'xbx/a', 'xbxa' )
posix.mkdir( srcdir .. 'xcx' )
writefile( srcdir .. 'xcx/x', 'xcxx' )
writefile( srcdir .. 'xda', 'xda', '700' )
writefile( srcdir .. 'xdb', 'xdb', '755' )
end
-- test all files
local function testfiles
( )
testfile( trgdir .. 'xbc', false )
testfile( trgdir .. 'xcc', true )
testfile( trgdir .. 'yaa', false )
testfile( trgdir .. 'xbx/a', false )
testfile( trgdir .. 'xcx/x', true )
testfile( trgdir .. 'xda', true )
testfile( trgdir .. 'xdb', true )
end
cwriteln( 'testing crontab' )
writefiles( )
cwriteln( 'starting Lsyncd' )
local pid = spawn( './lsyncd', cfgfile, '-log', 'all' )
cwriteln( 'waiting for Lsyncd to start' )
posix.sleep( 3 )
cwriteln( 'testing filters after startup' )
testfiles( )
cwriteln( 'ok, removing sources' )
if srcdir:sub( 1,4 ) ~= '/tmp'
then
-- just to make sure before rm -rf
cwriteln( 'exit before drama, srcdir is "', srcdir, '"' )
os.exit( 1 )
end
os.execute( 'rm -rf '..srcdir..'/*' )
cwriteln( 'waiting for Lsyncd to remove destination' )
posix.sleep( 20 )
local result, code = execute( 'diff -urN ' .. srcdir .. ' ' .. trgdir )
if result ~= 'exit' or code ~= 0
then
cwriteln( 'fail, target directory not empty!' )
posix.kill( pid )
os.exit( 1 )
end
cwriteln( 'writing files after startup' )
writefiles( )
cwriteln( 'waiting for Lsyncd to transmit changes' )
posix.sleep( 20 )
testfiles( )
cwriteln( 'killing started Lsyncd' )
posix.kill( pid )
local _, exitmsg, exitcode = posix.wait( pid )
cwriteln( 'Exitcode of Lsyncd = ', exitmsg, ' ', exitcode );
if exitcode == 143
then
cwriteln( 'OK' )
os.exit( 0 )
else
os.exit( 1 )
end
-- TODO remove temp

View File

@ -115,7 +115,7 @@ cwriteln( 'waiting for Lsyncd to remove destination' )
posix.sleep( 5 )
result, code = execute( 'diff -urN ' .. srcdir .. ' ' .. trgdir )
local result, code = execute( 'diff -urN ' .. srcdir .. ' ' .. trgdir )
if result ~= 'exit' or code ~= 0
then
@ -137,7 +137,7 @@ testfiles( )
cwriteln( 'killing started Lsyncd' )
posix.kill( pid )
local _, exitmsg, exitcode = posix.wait( lpid )
local _, exitmsg, exitcode = posix.wait( pid )
cwriteln( 'Exitcode of Lsyncd = ', exitmsg, ' ', exitcode );

View File

@ -113,7 +113,7 @@ os.execute( 'rm -rf ' .. srcdir .. '/*' )
cwriteln( 'waiting for Lsyncd to remove destination' )
posix.sleep( 5 )
result, code = execute( 'diff -urN '..srcdir..' '..trgdir )
local result, code = execute( 'diff -urN '..srcdir..' '..trgdir )
if result ~= 'exit' or code ~= 0
then
@ -129,7 +129,7 @@ testfiles( )
cwriteln( 'killing started Lsyncd' )
posix.kill( pid )
local _, exitmsg, lexitcode = posix.wait( lpid )
local _, exitmsg, lexitcode = posix.wait( pid )
cwriteln( 'Exitcode of Lsyncd = ', exitmsg, ' ', lexitcode )
posix.sleep( 1 )

View File

@ -109,7 +109,7 @@ cwriteln( 'waiting for Lsyncd to remove destination' )
posix.sleep( 5 )
result, code = execute( 'diff -urN ' .. srcdir .. ' ' .. trgdir )
local result, code = execute( 'diff -urN ' .. srcdir .. ' ' .. trgdir )
if result ~= 'exit' or code ~= 0
then
@ -131,7 +131,7 @@ testfiles( )
cwriteln( 'killing started Lsyncd' )
posix.kill( pid )
local _, exitmsg, exitcode = posix.wait( lpid )
local _, exitmsg, exitcode = posix.wait( pid )
cwriteln( 'Exitcode of Lsyncd = ', exitmsg, ' ', exitcode );

View File

@ -19,7 +19,7 @@ then
end
cwriteln( 'starting Lsyncd' )
logs = { }
local logs = { }
local pid =
spawn(
'./lsyncd',
@ -55,12 +55,12 @@ posix.sleep( 10 )
cwriteln( '* killing Lsyncd' )
posix.kill( pid )
local _, exitmsg, lexitcode = posix.wait(lpid)
local _, exitmsg, lexitcode = posix.wait(pid)
cwriteln( 'Exitcode of Lsyncd = ', exitmsg, ' ', lexitcode)
posix.sleep( 1 )
cwriteln( '* differences:' )
result, code = execute( 'diff -urN ' .. srcdir .. ' ' .. trgdir )
local result, code = execute( 'diff -urN ' .. srcdir .. ' ' .. trgdir )
if result == 'exit'
then

View File

@ -69,7 +69,7 @@ testfile( srcdir..'b' )
testfile( srcdir..'c' )
cwriteln( 'killing started Lsyncd' )
posix.kill( pid )
local _, exitmsg, lexitcode = posix.wait( lpid )
local _, exitmsg, lexitcode = posix.wait( pid )
cwriteln( 'Exitcode of Lsyncd = ', exitmsg, ' ', lexitcode)
posix.sleep(1);

View File

@ -1,3 +1,5 @@
--- @diagnostic disable: lowercase-global, need-check-nil
-- common testing environment
posix = require( 'posix' )
string = require( 'string' )
@ -11,6 +13,7 @@ local c0='\027[0m'
-- compatibility with 5.1
if table.unpack == nil then
--- @diagnostic disable-next-line: deprecated
table.unpack = unpack
end
@ -46,8 +49,7 @@ end
-- If environment variable 'SEED' is set,
-- that one is used seed.
--
local seed = os.getenv( 'SEED') or os.time( )
local seed = tonumber(os.getenv( 'SEED')) or os.time( )
math.randomseed( seed )
cwriteln( 'random seed: ', seed )
@ -61,6 +63,10 @@ function mktempd
( )
local f = io.popen( 'mktemp -td ltest.XXX', 'r' )
if f == nil then
return error("can't create testing directory")
end
local s = f:read( '*a' )
f:close( )
@ -98,7 +104,8 @@ end
function writefile
(
filename,
text
text,
mode
)
local f = io.open( filename, 'w' )
@ -112,6 +119,10 @@ function writefile
f:write( '\n' )
f:close( )
if mode ~= nil then
posix.chmod(filename, mode)
end
return true
end
@ -668,3 +679,27 @@ function isTableEqual(o1, o2, ignore_mt)
return true
end
--
-- Tests if the filename exists
-- fails if this is different to expect.
--
function testfile
(
filename,
expect
)
local stat, err = posix.stat( filename )
if stat and not expect
then
cwriteln( 'failure: ', filename, ' should be filtered')
os.exit( 1 )
end
if not stat and expect
then
cwriteln( 'failure: ', filename, ' should not be filtered' )
os.exit( 1 )
end
end

View File

@ -28,4 +28,52 @@ assert(
assert(type(lsyncd.get_free_port()) == "number")
local function testQueue()
local q = Queue.new()
q:push(1)
q:push(2)
q:push(3)
q:push(4)
assert(q:size(), 4)
assert(q[1], 1)
assert(q[4], 4)
q:remove(4)
assert(q:size(), 3)
assert(q[3], 3)
assert(q[1], 1)
q:remove(1)
assert(q:size(), 2)
assert(q[3], 3)
assert(q[2], 2)
assert(q.first, 2)
assert(q.last, 3)
q:push(5)
assert(q:size(), 3)
assert(q.last, 4)
assert(q.first, 2)
assert(q[4], 5)
assert(q[3], 3)
assert(q[2], 2)
q:remove(3)
assert(q:size(), 2)
assert(q.last, 3)
assert(q.first, 2)
assert(q[2], 2)
assert(q[3], 5)
q:inject(23)
assert(q:size(), 3)
assert(q.last, 3)
assert(q.first, 1)
assert(q[1], 23)
assert(q[2], 2)
assert(q[3], 5)
end
testQueue()
os.exit(0)