octoleo/lsyncd
1
0
Fork 0
mirror of https://github.com/octoleo/lsyncd.git synced 2024-12-12 14:17:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'tunnel' into periodic-full

Browse Source
This commit is contained in:
Daniel Poelzleithner 2022-03-24 13:41:13 +01:00
commit e2a4dc0e87
13 changed files with 1228 additions and 178 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.editorconfig Normal file
View File

@ -0,0 +1,8 @@
# 4 tab indentation
indent_style = tab
indent_size = 4
[*.nix]
indent_style = space
indent_size = 2

4
CMakeLists.txt
View File

@ -1,7 +1,7 @@
# preamble
project( Lsyncd )
cmake_minimum_required( VERSION 3.10 )
set( LSYNCD_VERSION 2.2.3 )
set( LSYNCD_VERSION 2.3.0-beta1 )
set( CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/" )
@ -118,6 +118,6 @@ add_executable( lsyncd ${LSYNCD_SRC} )
target_link_libraries( lsyncd ${LUA_LIBRARIES} )
install( TARGETS lsyncd RUNTIME DESTINATION bin )
install( FILES doc/manpage/lsyncd.1 DESTINATION man )
install( FILES doc/manpage/lsyncd.1 DESTINATION ${CMAKE_INSTALL_MANDIR}/man1 COMPONENT man )
install( DIRECTORY examples DESTINATION doc )

2
README.md
View File

@ -6,7 +6,7 @@ Lsyncd watches a local directory trees event monitor interface (inotify or fseve
Rsync+ssh is an advanced action configuration that uses a SSH to act file and directory moves directly on the target instead of re-transmitting the move destination over the wire.
Fine-grained customization can be achieved through the config file. Custom action configs can even be written from scratch in cascading layers ranging from shell scripts to code written in the [Lua language](http://www.lua.org/). This way simple, powerful and flexible configurations can be acheived. See [the manual](https://axkibe.github.io/lsyncd/) for details.
Fine-grained customization can be achieved through the config file. Custom action configs can even be written from scratch in cascading layers ranging from shell scripts to code written in the [Lua language](http://www.lua.org/). This way simple, powerful and flexible configurations can be achieved. See [the manual](https://axkibe.github.io/lsyncd/) for details.
Lsyncd 2.2.1 requires rsync >= 3.1 on all source and target machines.

5
cmake/FindLua.cmake
View File

@ -16,6 +16,7 @@
#=============================================================================
# Copyright 2007-2009 Kitware, Inc.
# Modified to support Lua 5.2 by LuaDist 2012
# Modified to support Lua 5.4 by LuaDist 2022
#
# Distributed under the OSI-approved BSD License (the "License");
# see accompanying file Copyright.txt for details.
@ -27,7 +28,7 @@
# (To distribute this file outside of CMake, substitute the full
# License text for the above reference.)
#
# This module will try to find the newest Lua version down to 5.2
# This module will try to find the newest Lua version down to 5.4
# Always search for non-versioned lua first (recommended)
SET(_POSSIBLE_LUA_INCLUDE include include/lua)
@ -36,7 +37,7 @@ SET(_POSSIBLE_LUA_INCLUDE include include/lua)
#SET(_POSSIBLE_LUA_LIBRARY lua)
# Determine possible naming suffixes (there is no standard for this)
SET(_POSSIBLE_SUFFIXES "52" "5.2" "-5.2" "53" "5.3" "-5.3" "")
SET(_POSSIBLE_SUFFIXES "54" "5.4" "-5.4" "53" "5.3" "-5.3" "52" "5.2" "-5.2" "")
# Set up possible search names and locations
FOREACH(_SUFFIX IN LISTS _POSSIBLE_SUFFIXES)

16
default-rsync.lua
View File

@ -63,6 +63,7 @@ rsync.checkgauge = {
copy_links = true,
copy_unsafe_links = true,
cvs_exclude = true,
delete_excluded = true,
dry_run = true,
executability = true,
existing = true,
@ -128,6 +129,9 @@ rsync.action = function
-- gets all events ready for syncing
local elist = inlet.getEvents( eventNotInitBlank )
local substitudes = inlet.getSubstitutionData(elist, {})
local target = substitudeCommands(config.target, substitudes)
-- gets the list of paths for the event list
-- deletes create multi match patterns
local paths = elist.getPaths( )
@ -236,7 +240,7 @@ rsync.action = function
'--include-from=-',
'--exclude=*',
config.source,
config.target
target
)
end
@ -317,7 +321,7 @@ rsync.init = function
local filters = inlet.hasFilters( ) and inlet.getFilters( )
local delete = nil
local delete = {}
local target = config.target
@ -331,12 +335,20 @@ rsync.init = function
target = config.host .. ':' .. config.targetdir
end
local substitudes = inlet.getSubstitutionData(event, {})
target = substitudeCommands(target, substitudes)
if config.delete == true
or config.delete == 'startup'
then
delete = { '--delete', '--ignore-errors' }
end
if config.rsync.delete_excluded == true
then
table.insert( delete, '--delete-excluded' )
end
if not filters and #excludes == 0
then
-- starts rsync without any filters or excludes

10
default.lua
View File

@ -52,6 +52,7 @@ default.checkgauge = {
prepare = true,
source = true,
target = true,
tunnel = true,
}
--
@ -120,7 +121,14 @@ default.collect = function
agent.target,
' finished.'
)
if settings('onepass')
then
log(
'Normal',
'onepass config set, exiting'
)
terminate( 0 )
end
return 'ok'
elseif rc == 'again'
then

8
flake.lock
View File

@ -17,16 +17,16 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1637709854,
"narHash": "sha256-y98gkOBUEiPAmwRhZPzTQ0YayZKPS2loNgA0GcNewMM=",
"lastModified": 1639488789,
"narHash": "sha256-Ey12CBni1jlEGoW4eH4X0hugWs25MxHMcNH4N8VVX0U=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "9c43581935a23d56734bd02da0ba8e7fda21e747",
"rev": "ce635e9dca8f7e2bfab19a3667d7e697c019c68b",
"type": "github"
},
"original": {
"owner": "nixos",
"ref": "release-21.05",
"ref": "release-21.11",
"repo": "nixpkgs",
"type": "github"
}

69
flake.nix
View File

@ -1,27 +1,37 @@
{
description = "Lsyncd (Live Syncing Daemon)";
inputs.nixpkgs.url = "github:nixos/nixpkgs/release-21.05";
inputs.nixpkgs.url = "github:nixos/nixpkgs/release-21.11";
inputs.flake-utils.url = "github:numtide/flake-utils";
outputs = { self, nixpkgs, flake-utils }:
flake-utils.lib.eachDefaultSystem
(system:
let
pkgs = nixpkgs.legacyPackages.${system};
pkgs = (import nixpkgs {
inherit system;
# Makes the config pure as well. See <nixpkgs>/top-level/impure.nix:
config = {
allowBroken = true;
};}); #.legacyPackages.${system};
defaultDeps = with pkgs; [
gcc
cmake
gnumake
glib
rsync
openssh
curl
];
version = builtins.elemAt
(builtins.match ''.*set\(.LSYNCD_VERSION ([0-9\.]*).*''
(builtins.substring 0 500
(builtins.readFile ./CMakeLists.txt))) 0;
mylua5_4 = pkgs.lua5_4.override({
packageOverrides = luaself: luaprev: {
luarocks = luaprev.luarocks-3_7;
};
});
buildExtensions = luapkgs: (
let
@ -82,11 +92,34 @@
}
);
buildTypes = {
lua5_1 = [(pkgs.lua5_1.withPackages (ps: [ps.luaposix ps.penlight (buildExtensions pkgs.lua51Packages)]))];
lua5_2 = [(pkgs.lua5_2.withPackages (ps: [ps.luaposix ps.penlight (buildExtensions pkgs.lua52Packages)]))];
lua5_3 = [(pkgs.lua5_3.withPackages (ps: [ps.luaposix ps.penlight (buildExtensions pkgs.lua53Packages)]))];
lua5_4 = [(pkgs.lua5_4.withPackages (ps: [ps.luaposix ps.penlight (buildExtensions pkgs.lua54Packages)]))];
luaposix35 = mylua: mylua.pkgs.buildLuarocksPackage {
pname = "luaposix";
lua = mylua;
version = "35.1-1";
knownRockspec = (pkgs.fetchurl {
url = "https://luarocks.org/luaposix-35.1-1.rockspec";
sha256 = "1n6c7qyabj2y95jmbhf8fxbrp9i73kphmwalsam07f9w9h995xh1";
}).outPath;
src = pkgs.fetchurl {
url = "http://github.com/luaposix/luaposix/archive/v35.1.zip";
sha256 = "1c03chkzwr2p1wd0hs1bafl2890fqbrfc3qk0wxbd202gc6128zi";
};
#
propagatedBuildInputs = [ mylua ];
meta = {
homepage = "http://github.com/luaposix/luaposix/";
description = "Lua bindings for POSIX";
license.fullName = "MIT/X11";
};
};
buildTypes = {
lua5_1 = [pkgs.lua5_1 pkgs.lua51Packages.luaposix (buildExtensions pkgs.lua51Packages)];
lua5_2 = [pkgs.lua5_2 pkgs.lua52Packages.luaposix (buildExtensions pkgs.lua51Packages)];
lua5_3 = [pkgs.lua5_3 pkgs.lua53Packages.luaposix (buildExtensions pkgs.lua51Packages)];
lua5_4 = [pkgs.lua5_3 (luaposix35 mylua5_4) (buildExtensions mylua5_4)];
};
in
let
@ -95,9 +128,12 @@
name = "lsyncd";
src = ./.;
buildInputs = defaultDeps ++ luaPackages;
});
});
mkDev = packages: pkgs.mkShell {
propagatedBuildInputs = defaultDeps ++ packages;
};
in
{
packages = {
@ -108,10 +144,15 @@
lsyncd_lua5_4 = mkLsync buildTypes.lua5_4;
};
devShells = {
lsyncd = mkDev buildTypes.lua5_3;
lsyncd_lua5_1 = mkDev buildTypes.lua5_1;
lsyncd_lua5_2 = mkDev buildTypes.lua5_2;
lsyncd_lua5_3 = mkDev buildTypes.lua5_3;
lsyncd_lua5_4 = mkDev buildTypes.lua5_4;
};
defaultPackage = self.packages.${system}.lsyncd;
# devShell = pkgs.mkShell {
# buildInputs = defaultDeps ++ buildTypes.lua5_3;
# };
}
);
}

213
lsyncd.c
View File

@ -19,6 +19,8 @@
#define SYSLOG_NAMES 1
#include <sys/socket.h>
#include <netinet/in.h>
#include <sys/select.h>
#include <sys/stat.h>
#include <sys/times.h>
@ -46,6 +48,11 @@
#include <lualib.h>
#include <lauxlib.h>
#if defined(LUA_VERSION_NUM) && LUA_VERSION_NUM >= 504
#define lua_objlen lua_rawlen
#endif
/*
| The Lua part of Lsyncd
*/
@ -135,6 +142,22 @@ int pidfile_fd = 0;
static long clocks_per_sec;
/*
| Dummy variable of which it's address is used as
| the cores index in the lua registry to
| the lua runners function table in the lua registry.
*/
static int runner;
/*
| Dummy variable of which it's address is used as
| the cores index n the lua registry to
| the lua runners error handler.
*/
static int callError;
/**
* signal handler
*/
@ -438,6 +461,40 @@ printlogf0(lua_State *L,
}
/*
| Print a traceback of the error
*/
static int l_traceback (lua_State *L) {
// runner.callError
lua_getglobal(L, "debug");
lua_getfield(L, -1, "traceback");
lua_pushvalue(L, 1);
lua_pushinteger(L, 2);
lua_call(L, 2, 1);
printlogf( L, "traceback", "%s", lua_tostring(L, -1) );
return 1;
}
/*
| Call runners terminate function and exit with given exit code
*/
static void safeexit (lua_State *L, int exitcode) {
// load_runner_func(L, "teardown");
// pushes the function
lua_pushlightuserdata( L, (void *) &runner );
lua_gettable( L, LUA_REGISTRYINDEX );
lua_pushstring( L, "teardown" );
lua_gettable( L, -2 );
lua_remove( L, -2 );
lua_pushinteger(L, exitcode);
lua_call(L, 2, 1);
if (lua_isinteger(L, -1)) {
exitcode = luaL_checkinteger(L, -1);
}
exit(exitcode);
}
/*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*
( Simple memory management )
*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/
@ -605,23 +662,6 @@ pipe_tidy( struct observance * observance )
( Helper Routines )
*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/
/*
| Dummy variable of which it's address is used as
| the cores index in the lua registry to
| the lua runners function table in the lua registry.
*/
static int runner;
/*
| Dummy variable of which it's address is used as
| the cores index n the lua registry to
| the lua runners error handler.
*/
static int callError;
/*
| Sets the close-on-exit flag of a file descriptor.
*/
@ -697,7 +737,7 @@ write_pidfile
pidfile
);
exit( -1 );
safeexit(L, -1 );
}
int rc = lockf( pidfile_fd, F_TLOCK, 0 );
@ -710,7 +750,7 @@ write_pidfile
pidfile
);
exit( -1 );
safeexit(L, -1 );
}
snprintf( buf, sizeof( buf ), "%i\n", getpid( ) );
@ -918,7 +958,7 @@ user_obs_ready(
// calls the user function
if( lua_pcall( L, 1, 0, -3 ) )
{
exit( -1 );
safeexit(L, -1 );
}
lua_pop( L, 2 );
@ -954,7 +994,7 @@ user_obs_writey(
// calls the user function
if( lua_pcall( L, 1, 0, -3 ) )
{
exit(-1);
safeexit(L, -1);
}
lua_pop( L, 2 );
@ -1078,6 +1118,82 @@ l_now(lua_State *L)
return 1;
}
/*
| Sends a signal to proceess pid
|
| Params on Lua stack:
| 1: pid
| 2: signal
|
| Returns on Lua stack:
| return value of kill
*/
static int
l_kill( lua_State *L )
{
pid_t pid = luaL_checkinteger( L, 1 );
int sig = luaL_checkinteger( L, 2 );
int rv = kill(pid, sig );
lua_pushinteger( L, rv );
return 1;
}
/*
| Returns a free port of host
|
| Params on Lua stack:
| (not yet) 1: hostname or ip for bind
|
| Returns on Lua stack:
| return integer of free port
|
*/
static int
l_free_port(lua_State *L) {
int sock = socket(AF_INET, SOCK_STREAM, 0);
if(sock < 0) {
printf("error opening socket\n");
goto error;
}
struct sockaddr_in serv_addr;
memset((char *) &serv_addr, 0, sizeof(serv_addr));
serv_addr.sin_family = AF_INET;
serv_addr.sin_addr.s_addr = INADDR_ANY;
serv_addr.sin_port = 0;
if (bind(sock, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) {
if(errno == EADDRINUSE) {
printf("the port is not available. already to other process\n");
goto error;
} else {
printf("could not bind to process (%d) %s\n", errno, strerror(errno));
goto error;
}
}
socklen_t len = sizeof(serv_addr);
if (getsockname(sock, (struct sockaddr *)&serv_addr, &len) == -1) {
goto error;
}
lua_pushinteger(L, ntohs(serv_addr.sin_port));
if (close (sock) < 0 ) {
printf("did not close: %s\n", strerror(errno));
lua_pop ( L, 1 );
goto error;
}
return 1;
error:
lua_pushnil(L);
return 1;
}
/*
| Executes a subprocess. Does not wait for it to return.
@ -1203,7 +1319,7 @@ l_exec( lua_State *L )
"in spawn(), expected a string after pipe '<'"
);
exit( -1 );
safeexit(L, -1 );
}
pipe_text = lua_tolstring( L, 3, &pipe_len );
@ -1215,7 +1331,7 @@ l_exec( lua_State *L )
{
logstring( "Error", "cannot create a pipe!" );
exit( -1 );
safeexit(L, -1 );
}
// always closes the write end for child processes
@ -1839,6 +1955,8 @@ static const luaL_Reg lsyncdlib[] =
{ "exec", l_exec },
{ "log", l_log },
{ "now", l_now },
{ "kill", l_kill },
{ "get_free_port", l_free_port },
{ "nonobserve_fd", l_nonobserve_fd },
{ "observe_fd", l_observe_fd },
{ "readdir", l_readdir },
@ -1861,7 +1979,7 @@ l_jiffies_add( lua_State *L )
if( p1 && p2 )
{
logstring( "Error", "Cannot add two timestamps!" );
exit( -1 );
safeexit(L, -1 );
}
{
@ -1881,6 +1999,33 @@ l_jiffies_add( lua_State *L )
}
}
/*
| Adds a number in seconds to a jiffy timestamp.
*/
static int
l_jiffies_concat( lua_State *L )
{
char buf[1024];
clock_t *p1 = ( clock_t * ) lua_touserdata( L, 1 );
clock_t *p2 = ( clock_t * ) lua_touserdata( L, 2 );
if( p1 && p2 )
{
logstring( "Error", "Cannot add two timestamps!" );
safeexit(L, -1 );
}
{
if (p1) {
snprintf( buf, sizeof(buf), "%Lf", (long double)(*p1));
lua_pushfstring(L, "%s%s", &buf, luaL_checkstring( L, 2));
} else {
snprintf( buf, sizeof(buf), "%Lf", (long double)(*p2));
lua_pushfstring(L, "%s%s", luaL_checkstring( L, 1), &buf);
}
return 1;
}
}
/*
| Subracts two jiffy timestamps resulting in a number in seconds
@ -1987,6 +2132,9 @@ register_lsyncd( lua_State *L )
lua_pushcfunction( L, l_jiffies_eq );
lua_setfield( L, mt, "__eq" );
lua_pushcfunction( L, l_jiffies_concat );
lua_setfield( L, mt, "__concat" );
lua_pop( L, 1 ); // pop(mt)
#ifdef WITH_INOTIFY
@ -2162,7 +2310,7 @@ masterloop(lua_State *L)
if( lua_pcall( L, 0, 1, -2 ) )
{
exit( -1 );
safeexit(L, -1 );
}
if( lua_type( L, -1 ) == LUA_TBOOLEAN)
@ -2348,7 +2496,9 @@ masterloop(lua_State *L)
lua_pushinteger( L, WEXITSTATUS( status ) );
if ( lua_pcall( L, 2, 0, -4 ) )
{ exit(-1); }
{
safeexit(L, -1);
}
lua_pop( L, 1 );
}
@ -2360,7 +2510,7 @@ masterloop(lua_State *L)
if( lua_pcall( L, 0, 0, -2 ) )
{
exit( -1 );
safeexit( L, -1 );
}
lua_pop( L, 1 );
@ -2377,7 +2527,7 @@ masterloop(lua_State *L)
if( lua_pcall( L, 1, 0, -3 ) )
{
exit( -1 );
safeexit(L, -1 );
}
lua_pop( L, 1 );
@ -2393,7 +2543,7 @@ masterloop(lua_State *L)
if( lua_pcall( L, 1, 1, -3 ) )
{
exit( -1 );
safeexit(L, -1 );
}
if( !lua_toboolean( L, -1 ) )
@ -2411,7 +2561,7 @@ masterloop(lua_State *L)
"internal, stack is dirty."
);
l_stackdump( L );
exit( -1 );
safeexit(L, -1 );
}
}
}
@ -2791,8 +2941,9 @@ main1( int argc, char *argv[] )
lsyncd_config_file,
lua_tostring( L, -1 )
);
l_traceback(L);
exit( -1 );
safeexit(L, -1 );
}
}

1
lsyncd.h
View File

@ -57,6 +57,7 @@ extern struct settings {
int log_facility; // The syslog facility
int log_level; // -1 logs everything, 0 normal mode, LOG_ERROR errors only.
bool nodaemon; // True if Lsyncd shall not daemonize.
bool onepass; // True if Lsyncd should exit after first sync pass
char * pidfile; // If not NULL Lsyncd writes its pid into this file.
} settings;

1014
lsyncd.lua
View File

File diff suppressed because it is too large Load Diff

41
tests/testlib.lua
View File

@ -1,8 +1,7 @@
-- common testing environment
posix = require( 'posix' )
string = require( 'string' )
path = require( 'pl.path' )
stringx = require( 'pl.stringx' )
local sys_stat = require "posix.sys.stat"
-- escape codes to colorize output on terminal
@ -94,10 +93,39 @@ function writefile
return true
end
function splitpath(P)
local i = #P
local ch = P:sub(i,i)
while i > 0 and ch ~= "/" do
i = i - 1
ch = P:sub(i,i)
end
if i == 0 then
return '',P
else
return P:sub(1,i-1), P:sub(i+1)
end
end
function isabs(p)
return string.sub(p, 1, 2) == "/"
end
function abspath(P,pwd)
P = P:gsub('[\\/]$','')
if not isabs(P) then
local rv = posix.unistd.getcwd() .. "/" .. P
return rv
end
return P
end
function script_path()
-- local str = debug.getinfo(2, "S").source:sub(2)
-- return str:match("(.*/)")
return path.dirname(path.abspath(debug.getinfo(1).short_src))
local dir, file = splitpath(abspath(debug.getinfo(1).short_src))
return dir
end
function which(exec)
@ -159,6 +187,11 @@ function startSshd()
return true
end
function strip(s)
return s:match "^%s*(.-)%s*$"
end
--
-- Stop test ssh server
--
@ -169,7 +202,7 @@ function stopSshd()
then
return false
end
pid = stringx.strip(f:read("*a"))
pid = strip(f:read("*a"))
posix.kill(tonumber(pid))
end

15
tests/utils_test.lua
View File

@ -9,4 +9,19 @@ assert(isTableEqual(
{"-p", "22", "-i", "/home/test/bla blu/id_rsa"}
))
-- test string replacement
local testData = {
localPort = 1234,
localHost = "localhorst"
}
assert(substitudeCommands("echo ssh ${localHost}:${localPort}", testData) ==
"echo ssh localhorst:1234")
assert(isTableEqual(
substitudeCommands({"-p${doesNotExist}", "2${localHost}2", "-i '${localPort}'"}, testData),
{"-p", "2localhorst2", "-i '1234'"}
))
assert(type(lsyncd.get_free_port()) == "number")
os.exit(0)