From cf489dc1bfb6a99323bf6edaa244462f1c66ef22 Mon Sep 17 00:00:00 2001 From: Axel Kittenberger Date: Sat, 22 May 2010 15:25:31 +0000 Subject: [PATCH] added delays\! --- ChangeLog | 7 + NEWS | 1 + TODO | 1 - doc/manpage.lsyncd.conf.xml | 3 + doc/manpage.lsyncd.xml | 10 ++ lsyncd.c | 252 ++++++++++++++++++++++++++++++++---- 6 files changed, 248 insertions(+), 26 deletions(-) diff --git a/ChangeLog b/ChangeLog index 3d9771f..d24ae76 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,10 @@ +??-05-2009: 1.33 + fix: exlude file argument passing to rsync + fix: allow exlude files specified for individual sources + fix/enhancement: exlusions will be compared with extended + path files allowing sub dirs to be excluded. + enhancement: allow delays and call aggregation + 05-01-2009: Release of lsyncd 1.26 fix: segfault on multitargets changed meaning of "version" tag in lsyncd.conf.xml diff --git a/NEWS b/NEWS index 8312798..07cdf2d 100644 --- a/NEWS +++ b/NEWS @@ -1 +1,2 @@ +2010-05-22: Finally calls to rsync are aggregated. 2007-12-05: Lsyncd has been created. Hurray! diff --git a/TODO b/TODO index cfeac5a..d6c1d8b 100644 --- a/TODO +++ b/TODO @@ -6,5 +6,4 @@ * Allow more complicated exclusion masks. * Do not call rsync at all if a file was edited that falls under the exclusion mask (currently it is called, and rsync drops it. -* When several things happen at once, wait if calls to rsync can be optimized * improve config.h.in generation diff --git a/doc/manpage.lsyncd.conf.xml b/doc/manpage.lsyncd.conf.xml index 9066371..235be37 100644 --- a/doc/manpage.lsyncd.conf.xml +++ b/doc/manpage.lsyncd.conf.xml @@ -87,6 +87,9 @@ man(1), man(7), http://www.tldp.org/HOWTO/Man-Page/ If a node is present all debug messages will be logged. Default is a normal verbosity. <debug/> + + If is greater than zero, lsyncd waits this amount of seconds between an event and calling the binary. All events for a directory in this timeframe will be aggregated to one call. Default is 5 seconds. + <delay/> If a node is present only error messages will be logged. Default is a normal verbosity. <scarce/> diff --git a/doc/manpage.lsyncd.xml b/doc/manpage.lsyncd.xml index bb75091..3743496 100644 --- a/doc/manpage.lsyncd.xml +++ b/doc/manpage.lsyncd.xml @@ -94,6 +94,7 @@ man(1), man(7), http://www.tldp.org/HOWTO/Man-Page/ + @@ -162,6 +163,15 @@ man(1), man(7), http://www.tldp.org/HOWTO/Man-Page/ Log debug messages + + + + + If is greater than zero, lsyncd waits this amount of seconds between an event and calling the binary. All events for a directory in this timeframe will be aggregated to one call. Default is 5 seconds. + <delay/> + + + diff --git a/lsyncd.c b/lsyncd.c index 13dfa8f..58ec65e 100644 --- a/lsyncd.c +++ b/lsyncd.c @@ -12,6 +12,7 @@ #include #include #include +#include #ifdef HAVE_SYS_INOTIFY_H # include @@ -165,12 +166,23 @@ struct dir_watch { */ int parent; + /** + * On a delay to be handled. + */ + bool tackled; + + /** + * Point in time when rsync should be called. + */ + clock_t alarm; + /** * The applicable configuration for this directory */ struct dir_conf * dir_conf; }; + /** * Structure to store strings for the diversve inotfy masked events. * Actually used for comfortable debugging only. @@ -272,6 +284,21 @@ int tosync_size = 0; */ int tosync_pos = 0; +/** + * List of directories on a delay. + */ +int *tackles; + +/** + * Size allocated of the tackle list. + */ +int tackle_size = 0; + +/** + * Actual list of tackles. + */ +int tackle_len = 0; + /** * A constant that assigns every inotify mask a printable string. * Used for debugging. @@ -325,6 +352,11 @@ char * logfile = NULL; */ int inotf; +/** + * Seconds of delay between event and action + */ +clock_t delay = 5; + /** * Array of strings of directory names to include. * This is limited to MAX_EXCLUDES. @@ -617,6 +649,58 @@ void dir_conf_add_target(struct dir_conf * dir_conf, char *target) dir_conf->targets[target_n + 1] = NULL; } +/*--------------------------------------------------------------------------* + * Tackle list handling. + *--------------------------------------------------------------------------*/ + +/** + * Adds a directory on the tackle len (on a delay) + * + * @param watch the index in dir_watches to the directory. + * @param alarm times() when the directory should be acted. + */ +bool append_tackle(int watch, clock_t alarm) { + printlogf(DEBUG, "add tackle(%d)", watch); + if (dir_watches[watch].tackled) { + printlogf(DEBUG, "ignored since already tackled.", watch); + return false; + } + dir_watches[watch].tackled = true; + dir_watches[watch].alarm = alarm; + + if (tackle_len + 1 >= tackle_size) { + tackle_size *= 2; + tackles = s_realloc(tackles, tackle_size*sizeof(int)); + } + tackles[tackle_len++] = watch; + return true; +} + +/** + * Removes the first directory on the tackle list. + */ +void remove_first_tackle() { + int tw = tackles[0]; + memmove(tackles, tackles + 1, (--tackle_len) * sizeof(int)); + dir_watches[tw].tackled = false; +} + +/** + * Removes a tosync entry in the stack at the position p. + */ +//bool remove_tosync_pos(int p) { +// int i; +// assert(p < tosync_pos); +// +// //TODO improve performance by using memcpy. +// for(i = p; i < tosync_pos; i++) { +// tosync[i] = tosync[i + 1]; +// } +// tosync_pos--; +// return true; +//} + + /*--------------------------------------------------------------------------* * ToSync Stack handling. *--------------------------------------------------------------------------*/ @@ -650,17 +734,17 @@ bool append_tosync_watch(int watch) { /** * Removes a tosync entry in the stack at the position p. */ -bool remove_tosync_pos(int p) { - int i; - assert(p < tosync_pos); - - //TODO improve performance by using memcpy. - for(i = p; i < tosync_pos; i++) { - tosync[i] = tosync[i + 1]; - } - tosync_pos--; - return true; -} +//bool remove_tosync_pos(int p) { +// int i; +// assert(p < tosync_pos); +// +// //TODO improve performance by using memcpy. +// for(i = p; i < tosync_pos; i++) { +// tosync[i] = tosync[i + 1]; +// } +// tosync_pos--; +// return true; +//} /** * Parses an option text, replacing all '%' specifiers with @@ -857,6 +941,8 @@ int add_watch(char const * pathname, dir_watches[newdw].parent = parent; dir_watches[newdw].dirname = s_strdup(dirname); dir_watches[newdw].dir_conf = dir_conf; + dir_watches[newdw].alarm = 0; // not needed, just to be clear + dir_watches[newdw].tackled = false; return newdw; } @@ -943,8 +1029,11 @@ bool buildpath(char *pathname, /** * Syncs a directory. + * TODO: make better error handling (differ between + * directory gone away, and thus cannot work, or network + * failed) * - * @param watch the index in dir_watches to the directory. + * @param watch the index in dir_watches to the directory. * * @returns true when all targets were successful. */ @@ -975,6 +1064,34 @@ bool rsync_dir(int watch) return status; } +/** + * Puts a directory on the TO-DO list. Waiting for its delay + * to be actually executed. + * + * Directly calls rsync_dir if delay == 0; + * + * @param watch the index in dir_watches to the directory. + * @param alarm times() when the directory handling should be fired. + */ +void tackle_dir(int watch, clock_t alarm) +{ + char pathname[PATH_MAX+1]; + + if (delay == 0) { + rsync_dir(watch); + } + + if (!buildpath(pathname, sizeof(pathname), watch, NULL, NULL)) { + return; + } + + if (append_tackle(watch, alarm)) { + printlogf(NORMAL, "Putted %s on a delay", pathname); + } else { + printlogf(NORMAL, "Not acted on %s already on delay", pathname); + } +} + /** * Adds a dir to watch. * @@ -1140,11 +1257,11 @@ int get_dirwatch_offset(int wd) { * * TODO: make special logic to determine who is a subdirectory of whom, and maybe optimizie calls. */ -bool process_tosync_stack() +bool process_tosync_stack(clock_t alarm) { printlogf(DEBUG, "Processing through tosync stack."); while(tosync_pos > 0) { - rsync_dir(tosync[--tosync_pos]); + tackle_dir(tosync[--tosync_pos], alarm); } printlogf(DEBUG, "being done with tosync stack"); return true; @@ -1155,8 +1272,9 @@ bool process_tosync_stack() * Handles an inotify event. * * @param event The event to handle + * @param alarm times() moment t */ -bool handle_event(struct inotify_event *event) +bool handle_event(struct inotify_event *event, clock_t alarm) { char masktext[255] = {0,}; @@ -1166,6 +1284,8 @@ bool handle_event(struct inotify_event *event) struct inotify_mask_text *p; + // creates a string for logging that shows which flags + // were raised in the event for (p = mask_texts; p->mask; p++) { if (mask & p->mask) { if (strlen(masktext) + strlen(p->text) + 3 >= sizeof(masktext)) { @@ -1186,6 +1306,7 @@ bool handle_event(struct inotify_event *event) return true; } + // TODO, is this needed? for (i = 0; i < exclude_dir_n; i++) { if (!strcmp(event->name, exclude_dirs[i])) { return true; @@ -1194,32 +1315,37 @@ bool handle_event(struct inotify_event *event) watch = get_dirwatch_offset(event->wd); if (watch == -1) { + // this should not happen! printlogf(ERROR, "received an inotify event that doesnt match any watched directory :-(%d,%d)", event->mask, event->wd); return false; } + // in case of a new directory create new watches if (((IN_CREATE | IN_MOVED_TO) & event->mask) && (IN_ISDIR & event->mask)) { subwatch = add_dirwatch(event->name, watch, dir_watches[watch].dir_conf); } + // in case of a removed directory remove watches if (((IN_DELETE | IN_MOVED_FROM) & event->mask) && (IN_ISDIR & event->mask)) { remove_dirwatch(event->name, watch); } + // call the binary if something changed if ((IN_ATTRIB | IN_CREATE | IN_CLOSE_WRITE | IN_DELETE | IN_MOVED_TO | IN_MOVED_FROM) & event->mask ) { printlogf(NORMAL, "event %s:%s triggered.", masktext, event->name); - rsync_dir(watch); // TODO, worry about errors? + + tackle_dir(watch, alarm); if (subwatch >= 0) { // sync through the new created directory as well. - rsync_dir(subwatch); + tackle_dir(subwatch, alarm); } } else { printlogf(DEBUG, "... ignored this event."); } - process_tosync_stack(); + process_tosync_stack(alarm); return true; } @@ -1230,27 +1356,72 @@ bool master_loop() { char buf[INOTIFY_BUF_LEN]; int len, i = 0; + long clocks_per_sec = sysconf(_SC_CLK_TCK); + + struct timeval tv; + fd_set readfds; + clock_t now; + clock_t alarm; + int ready; + + FD_ZERO(&readfds); + FD_SET(inotf, &readfds); + + if (delay > 0) { + if (clocks_per_sec <= 0) { + printlogf(ERROR, "Clocks per seoond invalid! %d", printlogf); + terminate(LSYNCD_INTERNALFAIL); + } + printf("clockspersec: %ld\n", clocks_per_sec); + tackle_size = 2; + tackles = s_calloc(tackle_size, sizeof(int)); + tackle_len = 0; + } while (keep_going) { - len = read (inotf, buf, INOTIFY_BUF_LEN); + if (delay > 0 && tackle_len > 0) { + // use select() to determine what happens first + // a new event or "alarm" of an event to actually + // call its binary. + alarm = dir_watches[tackles[0]].alarm; + now = times(NULL); + tv.tv_sec = (alarm - now) / clocks_per_sec; + tv.tv_usec = (alarm - now) * 1000000 / clocks_per_sec % 1000000; + ready = select(inotf + 1, &readfds, NULL, NULL, &tv); + } else { + // if nothing to wait for, enter a blocking read + // (sorry this variable is named a bit confusing + // in this case) + ready = 1; + } + + if (ready) { + len = read (inotf, buf, INOTIFY_BUF_LEN); + } else { + len = 0; + } if (len < 0) { printlogf(ERROR, "failed to read from inotify (%d:%s)", errno, strerror(errno)); return false; } - if (len == 0) { - printlogf(ERROR, "eof?"); - return false; - } + now = times(NULL); + alarm = now + delay * clocks_per_sec; + // first handle all events that might have happened i = 0; - while (i < len) { struct inotify_event *event = (struct inotify_event *) &buf[i]; - handle_event(event); + handle_event(event, alarm); i += sizeof(struct inotify_event) + event->len; } + + while (tackle_len > 0 && dir_watches[tackles[0]].alarm <= times(NULL)) { + printlogf(DEBUG, "time for %d arrived!", tackles[0]); + rsync_dir(tackles[0]); + remove_first_tackle(); + } } return true; @@ -1316,6 +1487,7 @@ void print_help(char *arg0) printf(" (DEFAULT: %s if called without SOURCE/TARGET)\n", conf_filename); #endif printf(" --debug Log debug messages\n"); + printf(" --delay SECS Delay between event and action\n"); printf(" --dryrun Do not call any actions, run dry only\n"); printf(" --exclude-from FILE Exclude file handled to rsync (DEFAULT: None)\n"); printf(" --help Print this help text and exit.\n"); @@ -1487,6 +1659,22 @@ bool parse_settings(xmlNodePtr node) { } if (!xmlStrcmp(snode->name, BAD_CAST "debug")) { loglevel = 1; + } else if (!xmlStrcmp(snode->name, BAD_CAST "delay")) { + char *p; + xc = xmlGetProp(snode, BAD_CAST "value"); + if (xc == NULL) { + printlogf(ERROR, "error in config file: attribute value missing from \n"); + terminate(LSYNCD_BADCONFIGFILE); + } + delay = strtol((char *) xc, &p, 10); + if (*p) { + printlogf(ERROR, " value %s is not an integer.\n", xc); + terminate(LSYNCD_BADCONFIGFILE); + } + if (delay < 0) { + printlogf(ERROR, " value may not be negative.\n"); + terminate(LSYNCD_BADCONFIGFILE); + } } else if (!xmlStrcmp(snode->name, BAD_CAST "dryrun")) { flag_dryrun = 1; } else if (!xmlStrcmp(snode->name, BAD_CAST "exclude-from")) { @@ -1609,6 +1797,7 @@ void parse_options(int argc, char **argv) {"conf", 1, NULL, 0}, #endif {"debug", 0, &loglevel, 1}, + {"delay", 1, NULL, 0}, {"dryrun", 0, &flag_dryrun, 1}, {"exclude-from", 1, NULL, 0}, {"help", 0, NULL, 0}, @@ -1685,6 +1874,19 @@ void parse_options(int argc, char **argv) default_binary = s_strdup(optarg); } + if (!strcmp("delay", long_options[oi].name)) { + char *p; + delay = strtol(optarg, &p, 10); + if (*p) { + printf("%s is not an integer.\n", optarg); + terminate(LSYNCD_BADPARAMETERS); + } + if (delay < 0) { + printf("delay may not be negative.\n"); + terminate(LSYNCD_BADPARAMETERS); + } + } + if (!strcmp("exclude-from", long_options[oi].name)) { default_exclude_file = s_strdup(optarg); }