mirror of https://github.com/octoleo/lsyncd.git
Merge pull request #463 from danielkza/aws-s3-example
Add example config for syncing with an AWS S3 bucket
This commit is contained in:
commit
bbe9c5f38c
|
@ -0,0 +1,222 @@
|
|||
----
|
||||
-- Example lsyncd configuration for syncing with an Amazon S3 bucket
|
||||
--
|
||||
-- This requires the official AWS CLI to be available, and that credentials
|
||||
-- bet set up through some external method, such as environment variables,
|
||||
-- IAM profiles or the AWS SDK configuration.
|
||||
--
|
||||
-- The AWS CLI sync exclude rules are not as powerful as the ones supported by
|
||||
-- lsyncd. Hence, some of the do not translate perfectly. For example, '*'
|
||||
-- (asterisk) matches slashes, while it does not in lsyncd. Hence it is a good
|
||||
-- idea to only use exclude patterns for full directories, either by using a
|
||||
-- trailing / (slash) or ** (double asterisk), as those will be correctly
|
||||
-- translated.
|
||||
--
|
||||
-- An initialSync options is provided as a convenience, since it's not easy to
|
||||
-- make sure exclusion rules match when doing it manually. It will *pull* from
|
||||
-- the target bucket to the local dir (the opposite of the regular behavior)
|
||||
-- then exit immediately.
|
||||
--
|
||||
-- Author: Daniel Miranda <danielkza2@gmail.com>
|
||||
--
|
||||
s3 = {}
|
||||
|
||||
s3.checkgauge = {
|
||||
onCreate = false,
|
||||
onModify = false,
|
||||
onDelete = false,
|
||||
onStartup = false,
|
||||
onMove = false,
|
||||
|
||||
delete = true,
|
||||
exclude = true,
|
||||
excludeFrom = true,
|
||||
target = true,
|
||||
|
||||
s3 = {
|
||||
-- Path to the AWS CLI binary
|
||||
awscliBinary = true,
|
||||
-- Extra options to pass to the AWS CLI (as a list)
|
||||
awscliOptions = true,
|
||||
-- Whether to do a dry-run, and not make any real changes
|
||||
dryrun = true,
|
||||
-- Do an initial pull from the bucket and exit immediately.
|
||||
initialSync = true
|
||||
}
|
||||
}
|
||||
|
||||
-- Generate a list of exclude flags for the AWS CLI based on the lsyncd
|
||||
-- patterns provided. Cache it to avoid re-generating it every time.
|
||||
|
||||
local s3Excludes = function(config, excludes)
|
||||
if config.s3._excludes == nil then
|
||||
config.s3._excludes = {}
|
||||
for _, pat in ipairs(excludes) do
|
||||
pat = pat:gsub('%*%*', '[[ANY]]')
|
||||
pat = pat:gsub('%?', '[[ANY_BUT_SLASH_ONCE]]')
|
||||
pat = pat:gsub('/$', '/*')
|
||||
pat = pat:gsub('%[%[ANY%]%]', '*')
|
||||
pat = pat:gsub('%[%[ANY_BUT_SLASH_ONCE%]%]', '[^/]')
|
||||
|
||||
if pat:match('^/') then
|
||||
pat = pat:sub(2, -1)
|
||||
else
|
||||
pat = '*/' .. pat
|
||||
end
|
||||
|
||||
table.insert(config.s3._excludes, '--exclude')
|
||||
table.insert(config.s3._excludes, pat)
|
||||
end
|
||||
|
||||
log('s3Excludes', table.concat(config.s3._excludes, '\n'))
|
||||
end
|
||||
|
||||
return config.s3._excludes
|
||||
end
|
||||
|
||||
-- Generates a command line to call the AWS CLI as configured, with the provided
|
||||
-- S3 action (such as cp, mv, rm or sync).
|
||||
-- Returns a tuple of (binaryPath, arguments)
|
||||
local awscliCommand = function(verb, config)
|
||||
local bin = config.s3.awscliBinary
|
||||
local args = {'s3', verb, '--only-show-errors'}
|
||||
if config.s3.dryrun then
|
||||
table.insert(args, '--dryrun')
|
||||
end
|
||||
|
||||
if verb == 'sync'
|
||||
and (config.delete == true or config.delete == 'startup')
|
||||
then
|
||||
table.insert(args, '--delete')
|
||||
end
|
||||
|
||||
for _, opt in ipairs(config.s3.awscliOptions) do
|
||||
table.insert(args, opt)
|
||||
end
|
||||
|
||||
return bin, args
|
||||
end
|
||||
|
||||
s3.action = function(inlet)
|
||||
local event, event2 = inlet.getEvent()
|
||||
-- S3 never actually deals with directories - they are just an illusion
|
||||
-- created based on the common prefixes of objects. Hence discard any events
|
||||
-- that do not concern files.
|
||||
if event.isdir then
|
||||
inlet.discardEvent(event)
|
||||
return
|
||||
end
|
||||
|
||||
local config = inlet.getConfig()
|
||||
if event.etype == 'Create' or event.etype == 'Modify' then
|
||||
local bin, args = awscliCommand('cp', config)
|
||||
spawn(
|
||||
event,
|
||||
bin,
|
||||
args,
|
||||
event.sourcePath,
|
||||
event.targetPath
|
||||
)
|
||||
elseif event.etype == 'Delete' then
|
||||
if config.delete ~= true and config.delete ~= 'running' then
|
||||
inlet.discardEvent(event)
|
||||
return
|
||||
end
|
||||
|
||||
local bin, args = awscliCommand('rm', config)
|
||||
spawn(
|
||||
event,
|
||||
bin,
|
||||
args,
|
||||
event.targetPath
|
||||
)
|
||||
elseif event.etype == 'Move' then
|
||||
local bin, args = awscliCommand('mv', config)
|
||||
spawn(
|
||||
event,
|
||||
bin,
|
||||
args,
|
||||
event.targetPath,
|
||||
event2.targetPath
|
||||
)
|
||||
else
|
||||
log('Warn', 'ignored an event of type "', event.etype, '"')
|
||||
inlet.discardEvent(event)
|
||||
end
|
||||
end
|
||||
|
||||
s3.init = function(event)
|
||||
local config = event.config
|
||||
local inlet = event.inlet
|
||||
local excludes = s3Excludes(config, inlet.getExcludes())
|
||||
local bin, args = awscliCommand('sync', config)
|
||||
|
||||
-- Do a pull when initialSync is enabled.
|
||||
if config.s3.initialSync then
|
||||
spawn(
|
||||
event,
|
||||
bin,
|
||||
args,
|
||||
excludes,
|
||||
config.target,
|
||||
event.sourcePath
|
||||
)
|
||||
-- And a push, as usual, otherwise
|
||||
else
|
||||
spawn(
|
||||
event,
|
||||
bin,
|
||||
args,
|
||||
excludes,
|
||||
event.sourcePath,
|
||||
config.target
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
-- Define a collect callback so we can terminate immediately when initialSync
|
||||
-- is enabled
|
||||
s3.collect = function(agent, exitcode)
|
||||
local config = agent.config
|
||||
if not agent.isList and agent.etype == 'Init' and config.s3.initialSync then
|
||||
terminate(exitcode == 0 and 0 or -1)
|
||||
end
|
||||
|
||||
return
|
||||
end
|
||||
|
||||
s3.prepare = function(config, level)
|
||||
default.prepare(config, level + 1)
|
||||
|
||||
config.target = config.target:gsub('/+$', '')
|
||||
if not config.target:match('^s3://') then
|
||||
config.target = 's3://' .. config.target
|
||||
end
|
||||
end
|
||||
|
||||
s3.s3 = {
|
||||
awscliBinary = '/usr/bin/aws',
|
||||
awscliOptions = {},
|
||||
dryrun = false,
|
||||
initialSync = false
|
||||
}
|
||||
s3.delete = false
|
||||
s3.delay = 10
|
||||
s3.maxProcesses = 1
|
||||
|
||||
sync {
|
||||
s3,
|
||||
source = '/my/dir',
|
||||
target = 's3://my-bucket/my-path',
|
||||
delay = 30,
|
||||
delete = true,
|
||||
maxProcesses = 2,
|
||||
exclude = {
|
||||
'/sub/folder/',
|
||||
},
|
||||
s3 = {
|
||||
awscliBinary = '/usr/local/bin/aws',
|
||||
awscliOptions = {'--acl', 'public-read'},
|
||||
dryrun = false
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue