From 1dfff8d13a04871ff9504646fe0fac3cee1e115d Mon Sep 17 00:00:00 2001 From: Daniel Miranda Date: Mon, 21 Aug 2017 23:19:05 -0300 Subject: [PATCH] Add example config for syncing with an AWS S3 bucket It requires the official AWS CLI to be available, and that credentials be set up through some external method, such as environment variables, IAM profiles, the AWS SDK configuratin. The AWS CLI sync exclude rules are not as powerful as the ones supported by lsyncd. Hence, some of the do not translate perfectly. For example, '*' (asterisk) matches slashes, while it does not in lsyncd. Hence it is a good idea to only use exclude patterns for full directories, either by using a trailing / (slash) or ** (double asterisk), as those will be correctly translated. An initialSync options is provided as a convenience, since it's not easy to make sure exclusion rules match when doing it manually. It will *pull* from the target bucket to the local dir (the opposite of the regular behavior) then exit immediately. --- examples/ls3.lua | 220 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 examples/ls3.lua diff --git a/examples/ls3.lua b/examples/ls3.lua new file mode 100644 index 0000000..30a7259 --- /dev/null +++ b/examples/ls3.lua @@ -0,0 +1,220 @@ +---- +-- Example lsyncd configuration for syncing with an Amazon S3 bucket +-- +-- This requires the official AWS CLI to be available, and that credentials +-- bet set up through some external method, such as environment variables, +-- IAM profiles or the AWS SDK configuration. +-- +-- The AWS CLI sync exclude rules are not as powerful as the ones supported by +-- lsyncd. Hence, some of the do not translate perfectly. For example, '*' +-- (asterisk) matches slashes, while it does not in lsyncd. Hence it is a good +-- idea to only use exclude patterns for full directories, either by using a +-- trailing / (slash) or ** (double asterisk), as those will be correctly +-- translated. +-- +-- An initialSync options is provided as a convenience, since it's not easy to +-- make sure exclusion rules match when doing it manually. It will *pull* from +-- the target bucket to the local dir (the opposite of the regular behavior) +-- then exit immediately. + +s3 = {} + +s3.checkgauge = { + onCreate = false, + onModify = false, + onDelete = false, + onStartup = false, + onMove = false, + + delete = true, + exclude = true, + excludeFrom = true, + target = true, + + s3 = { + -- Path to the AWS CLI binary + awscliBinary = true, + -- Extra options to pass to the AWS CLI (as a list) + awscliOptions = true, + -- Whether to do a dry-run, and not make any real changes + dryrun = true, + -- Do an initial pull from the bucket and exit immediately. + initialSync = true + } +} + +-- Generate a list of exclude flags for the AWS CLI based on the lsyncd +-- patterns provided. Cache it to avoid re-generating it every time. + +local s3Excludes = function(config, excludes) + if config.s3._excludes == nil then + config.s3._excludes = {} + for _, pat in ipairs(excludes) do + pat = pat:gsub('%*%*', '[[ANY]]') + pat = pat:gsub('%?', '[[ANY_BUT_SLASH_ONCE]]') + pat = pat:gsub('/$', '/*') + pat = pat:gsub('%[%[ANY%]%]', '*') + pat = pat:gsub('%[%[ANY_BUT_SLASH_ONCE%]%]', '[^/]') + + if pat:match('^/') then + pat = pat:sub(2, -1) + else + pat = '*/' .. pat + end + + table.insert(config.s3._excludes, '--exclude') + table.insert(config.s3._excludes, pat) + end + + log('s3Excludes', table.concat(config.s3._excludes, '\n')) + end + + return config.s3._excludes +end + +-- Generates a command line to call the AWS CLI as configured, with the provided +-- S3 action (such as cp, mv, rm or sync). +-- Returns a tuple of (binaryPath, arguments) +local awscliCommand = function(verb, config) + local bin = config.s3.awscliBinary + local args = {'s3', verb, '--only-show-errors'} + if config.s3.dryrun then + table.insert(args, '--dryrun') + end + + if verb == 'sync' + and (config.delete == true or config.delete == 'startup') + then + table.insert(args, '--delete') + end + + for _, opt in ipairs(config.s3.awscliOptions) do + table.insert(args, opt) + end + + return bin, args +end + +s3.action = function(inlet) + local event, event2 = inlet.getEvent() + -- S3 never actually deals with directories - they are just an illusion + -- created based on the common prefixes of objects. Hence discard any events + -- that do not concern files. + if event.isdir then + inlet.discardEvent(event) + return + end + + local config = inlet.getConfig() + if event.etype == 'Create' or event.etype == 'Modify' then + local bin, args = awscliCommand('cp', config) + spawn( + event, + bin, + args, + event.sourcePath, + event.targetPath + ) + elseif event.etype == 'Delete' then + if config.delete ~= true and config.delete ~= 'running' then + inlet.discardEvent(event) + return + end + + local bin, args = awscliCommand('rm', config) + spawn( + event, + bin, + args, + event.targetPath + ) + elseif event.etype == 'Move' then + local bin, args = awscliCommand('mv', config) + spawn( + event, + bin, + args, + event.targetPath, + event2.targetPath + ) + else + log('Warn', 'ignored an event of type "', event.etype, '"') + inlet.discardEvent(event) + end +end + +s3.init = function(event) + local config = event.config + local inlet = event.inlet + local excludes = s3Excludes(config, inlet.getExcludes()) + local bin, args = awscliCommand('sync', config) + + -- Do a pull when initialSync is enabled. + if config.s3.initialSync then + spawn( + event, + bin, + args, + excludes, + config.target, + event.sourcePath + ) + -- And a push, as usual, otherwise + else + spawn( + event, + bin, + args, + excludes, + event.sourcePath, + config.target + ) + end +end + +-- Define a collect callback so we can terminate immediately when initialSync +-- is enabled +s3.collect = function(agent, exitcode) + local config = agent.config + if not agent.isList and agent.etype == 'Init' and config.s3.initialSync then + terminate(exitcode == 0 and 0 or -1) + end + + return +end + +s3.prepare = function(config, level) + default.prepare(config, level + 1) + + config.target = config.target:gsub('/+$', '') + if not config.target:match('^s3://') then + config.target = 's3://' .. config.target + end +end + +s3.s3 = { + awscliBinary = '/usr/bin/aws', + awscliOptions = {}, + dryrun = false, + initialSync = false +} +s3.delete = false +s3.delay = 10 +s3.maxProcesses = 1 + +sync { + s3, + source = '/my/dir', + target = 's3://my-bucket/my-path', + delay = 30, + delete = true, + maxProcesses = 2, + exclude = { + '/sub/folder/', + }, + s3 = { + awscliBinary = '/usr/local/bin/aws', + awscliOptions = {'--acl', 'public-read'}, + dryrun = false + } +}