Add example config for syncing with an AWS S3 bucket

It requires the official AWS CLI to be available, and that credentials
be set up through some external method, such as environment variables,
IAM profiles, the AWS SDK configuratin.

The AWS CLI sync exclude rules are not as powerful as the ones supported by
lsyncd. Hence, some of the do not translate perfectly. For example, '*'
(asterisk) matches slashes, while it does not in lsyncd. Hence it is a good
idea to only use exclude patterns for full directories, either by using a
trailing / (slash) or ** (double asterisk), as those will be correctly
translated.

An initialSync options is provided as a convenience, since it's not easy to
make sure exclusion rules match when doing it manually. It will *pull* from
the target bucket to the local dir (the opposite of the regular behavior)
then exit immediately.
This commit is contained in:
Daniel Miranda 2017-08-21 23:19:05 -03:00
parent fb4301d515
commit 1dfff8d13a
No known key found for this signature in database
GPG Key ID: 07909306308D7399

220
examples/ls3.lua Normal file
View File

@ -0,0 +1,220 @@
----
-- Example lsyncd configuration for syncing with an Amazon S3 bucket
--
-- This requires the official AWS CLI to be available, and that credentials
-- bet set up through some external method, such as environment variables,
-- IAM profiles or the AWS SDK configuration.
--
-- The AWS CLI sync exclude rules are not as powerful as the ones supported by
-- lsyncd. Hence, some of the do not translate perfectly. For example, '*'
-- (asterisk) matches slashes, while it does not in lsyncd. Hence it is a good
-- idea to only use exclude patterns for full directories, either by using a
-- trailing / (slash) or ** (double asterisk), as those will be correctly
-- translated.
--
-- An initialSync options is provided as a convenience, since it's not easy to
-- make sure exclusion rules match when doing it manually. It will *pull* from
-- the target bucket to the local dir (the opposite of the regular behavior)
-- then exit immediately.
s3 = {}
s3.checkgauge = {
onCreate = false,
onModify = false,
onDelete = false,
onStartup = false,
onMove = false,
delete = true,
exclude = true,
excludeFrom = true,
target = true,
s3 = {
-- Path to the AWS CLI binary
awscliBinary = true,
-- Extra options to pass to the AWS CLI (as a list)
awscliOptions = true,
-- Whether to do a dry-run, and not make any real changes
dryrun = true,
-- Do an initial pull from the bucket and exit immediately.
initialSync = true
}
}
-- Generate a list of exclude flags for the AWS CLI based on the lsyncd
-- patterns provided. Cache it to avoid re-generating it every time.
local s3Excludes = function(config, excludes)
if config.s3._excludes == nil then
config.s3._excludes = {}
for _, pat in ipairs(excludes) do
pat = pat:gsub('%*%*', '[[ANY]]')
pat = pat:gsub('%?', '[[ANY_BUT_SLASH_ONCE]]')
pat = pat:gsub('/$', '/*')
pat = pat:gsub('%[%[ANY%]%]', '*')
pat = pat:gsub('%[%[ANY_BUT_SLASH_ONCE%]%]', '[^/]')
if pat:match('^/') then
pat = pat:sub(2, -1)
else
pat = '*/' .. pat
end
table.insert(config.s3._excludes, '--exclude')
table.insert(config.s3._excludes, pat)
end
log('s3Excludes', table.concat(config.s3._excludes, '\n'))
end
return config.s3._excludes
end
-- Generates a command line to call the AWS CLI as configured, with the provided
-- S3 action (such as cp, mv, rm or sync).
-- Returns a tuple of (binaryPath, arguments)
local awscliCommand = function(verb, config)
local bin = config.s3.awscliBinary
local args = {'s3', verb, '--only-show-errors'}
if config.s3.dryrun then
table.insert(args, '--dryrun')
end
if verb == 'sync'
and (config.delete == true or config.delete == 'startup')
then
table.insert(args, '--delete')
end
for _, opt in ipairs(config.s3.awscliOptions) do
table.insert(args, opt)
end
return bin, args
end
s3.action = function(inlet)
local event, event2 = inlet.getEvent()
-- S3 never actually deals with directories - they are just an illusion
-- created based on the common prefixes of objects. Hence discard any events
-- that do not concern files.
if event.isdir then
inlet.discardEvent(event)
return
end
local config = inlet.getConfig()
if event.etype == 'Create' or event.etype == 'Modify' then
local bin, args = awscliCommand('cp', config)
spawn(
event,
bin,
args,
event.sourcePath,
event.targetPath
)
elseif event.etype == 'Delete' then
if config.delete ~= true and config.delete ~= 'running' then
inlet.discardEvent(event)
return
end
local bin, args = awscliCommand('rm', config)
spawn(
event,
bin,
args,
event.targetPath
)
elseif event.etype == 'Move' then
local bin, args = awscliCommand('mv', config)
spawn(
event,
bin,
args,
event.targetPath,
event2.targetPath
)
else
log('Warn', 'ignored an event of type "', event.etype, '"')
inlet.discardEvent(event)
end
end
s3.init = function(event)
local config = event.config
local inlet = event.inlet
local excludes = s3Excludes(config, inlet.getExcludes())
local bin, args = awscliCommand('sync', config)
-- Do a pull when initialSync is enabled.
if config.s3.initialSync then
spawn(
event,
bin,
args,
excludes,
config.target,
event.sourcePath
)
-- And a push, as usual, otherwise
else
spawn(
event,
bin,
args,
excludes,
event.sourcePath,
config.target
)
end
end
-- Define a collect callback so we can terminate immediately when initialSync
-- is enabled
s3.collect = function(agent, exitcode)
local config = agent.config
if not agent.isList and agent.etype == 'Init' and config.s3.initialSync then
terminate(exitcode == 0 and 0 or -1)
end
return
end
s3.prepare = function(config, level)
default.prepare(config, level + 1)
config.target = config.target:gsub('/+$', '')
if not config.target:match('^s3://') then
config.target = 's3://' .. config.target
end
end
s3.s3 = {
awscliBinary = '/usr/bin/aws',
awscliOptions = {},
dryrun = false,
initialSync = false
}
s3.delete = false
s3.delay = 10
s3.maxProcesses = 1
sync {
s3,
source = '/my/dir',
target = 's3://my-bucket/my-path',
delay = 30,
delete = true,
maxProcesses = 2,
exclude = {
'/sub/folder/',
},
s3 = {
awscliBinary = '/usr/local/bin/aws',
awscliOptions = {'--acl', 'public-read'},
dryrun = false
}
}