lsyncd/examples/ls3.lua

----
-- Example lsyncd configuration for syncing with an Amazon S3 bucket
--
-- This requires the official AWS CLI to be available, and that credentials
-- bet set up through some external method, such as environment variables,
-- IAM profiles or the AWS SDK configuration.
--
-- The AWS CLI sync exclude rules are not as powerful as the ones supported by
-- lsyncd. Hence, some of the do not translate perfectly. For example, '*'
-- (asterisk) matches slashes, while it does not in lsyncd. Hence it is a good
-- idea to only use exclude patterns for full directories, either by using a
-- trailing / (slash) or ** (double asterisk), as those will be correctly
-- translated.
--
-- An initialSync options is provided as a convenience, since it's not easy to
-- make sure exclusion rules match when doing it manually. It will *pull* from
-- the target bucket to the local dir (the opposite of the regular behavior)
-- then exit immediately.
--
-- Author: Daniel Miranda <danielkza2@gmail.com>
--
local s3 = {}

s3.checkgauge = {
    onCreate  = false,
    onModify  = false,
    onDelete  = false,
    onStartup = false,
    onMove    = false,

    delete      = true,
    exclude     = true,
    excludeFrom = true,
    target      = true,

    s3 = {
        -- Path to the AWS CLI binary
        awscliBinary  = true,
        -- Extra options to pass to the AWS CLI (as a list)
        awscliOptions = true,
        -- Whether to do a dry-run, and not make any real changes
        dryrun        = true,
        -- Do an initial pull from the bucket and exit immediately.
        initialSync   = true
    }
}

-- Generate a list of exclude flags for the AWS CLI  based on the lsyncd
-- patterns provided. Cache it to avoid re-generating it every time.

local s3Excludes = function(config, excludes)
    if config.s3._excludes == nil then
        config.s3._excludes = {}
        for _, pat in ipairs(excludes) do
            pat = pat:gsub('%*%*', '[[ANY]]')
            pat = pat:gsub('%?',   '[[ANY_BUT_SLASH_ONCE]]')
            pat = pat:gsub('/$',   '/*')
            pat = pat:gsub('%[%[ANY%]%]',                '*')
            pat = pat:gsub('%[%[ANY_BUT_SLASH_ONCE%]%]', '[^/]')

            if pat:match('^/') then
                pat = pat:sub(2, -1)
            else
                pat = '*/' .. pat
            end

            table.insert(config.s3._excludes, '--exclude')
            table.insert(config.s3._excludes, pat)
        end

        log('s3Excludes', table.concat(config.s3._excludes, '\n'))
    end

    return config.s3._excludes
end

-- Generates a command line to call the AWS CLI as configured, with the provided
-- S3 action (such as cp, mv, rm or sync).
-- Returns a tuple of (binaryPath, arguments)
local awscliCommand = function(verb, config)
    local bin = config.s3.awscliBinary
    local args = {'s3', verb, '--only-show-errors'}
    if config.s3.dryrun then
        table.insert(args, '--dryrun')
    end

    if verb == 'sync'
       and (config.delete == true or config.delete == 'startup')
    then
        table.insert(args, '--delete')
    end

    for _, opt in ipairs(config.s3.awscliOptions) do
        table.insert(args, opt)
    end

    return bin, args
end

s3.action = function(inlet)
    local event, event2 = inlet.getEvent()
    -- S3 never actually deals with directories - they are just an illusion
    -- created based on the common prefixes of objects. Hence discard any events
    -- that do not concern files.
    if event.isdir then
        inlet.discardEvent(event)
        return
    end

    local config = inlet.getConfig()
    if event.etype == 'Create' or event.etype == 'Modify' then
        local bin, args = awscliCommand('cp', config)
        spawn(
            event,
            bin,
            args,
            event.sourcePath,
            event.targetPath
        )
    elseif event.etype == 'Delete' then
        if config.delete ~= true and config.delete ~= 'running' then
            inlet.discardEvent(event)
            return
        end

        local bin, args = awscliCommand('rm', config)
        spawn(
            event,
            bin,
            args,
            event.targetPath
        )
    elseif event.etype == 'Move' then
        local bin, args = awscliCommand('mv', config)
        spawn(
            event,
            bin,
            args,
            event.targetPath,
            event2.targetPath
        )
    else
        log('Warn', 'ignored an event of type "', event.etype, '"')
        inlet.discardEvent(event)
    end
end

s3.init = function(event)
    local config = event.config
    local inlet = event.inlet
    local excludes = s3Excludes(config, inlet.getExcludes())
    local bin, args = awscliCommand('sync', config)

    -- Do a pull when initialSync is enabled.
    if config.s3.initialSync then
        spawn(
            event,
            bin,
            args,
            excludes,
            config.target,
            event.sourcePath
        )
    -- And a push, as usual, otherwise
    else
        spawn(
            event,
            bin,
            args,
            excludes,
            event.sourcePath,
            config.target
        )
    end
end

-- Define a collect callback so we can terminate immediately when initialSync
-- is enabled
s3.collect = function(agent, exitcode)
    local config = agent.config
    if not agent.isList and agent.etype == 'Init' and config.s3.initialSync then
        terminate(exitcode == 0 and 0 or -1)
    end

    return
end

s3.prepare = function(config, level)
    default.prepare(config, level + 1)

    config.target = config.target:gsub('/+$', '')
    if not config.target:match('^s3://') then
        config.target = 's3://' .. config.target
    end
end

s3.s3 = {
    awscliBinary  = '/usr/bin/aws',
    awscliOptions = {},
    dryrun       = false,
    initialSync  = false
}
s3.delete       = false
s3.delay        = 10
s3.maxProcesses = 1

sync {
    s3,
    source       = '/my/dir',
    target       = 's3://my-bucket/my-path',
    delay        = 30,
    delete       = true,
    maxProcesses = 2,
    exclude = {
        '/sub/folder/',
    },
    s3 = {
        awscliBinary  = '/usr/local/bin/aws',
        awscliOptions = {'--acl', 'public-read'},
        dryrun        = false
    }
}