Added configuration option ignore_old_files

Ryan Steele · Ryan Steele · commit c612bbd3ec34 · 2015-09-15T14:49:44.000-07:00
Files older then n days are ignored
diff --git a/beaver/config.py b/beaver/config.py
@@ -181,6 +181,9 @@ def __init__(self, args, logger=None):
             'debug': '0',
             'daemonize': '0',
             'pid': '',
+
+            # Ignore files older then n days, use 0 to disable
+            'ignore_old_files': 0
         }
 
         self._configfile = args.config
@@ -326,7 +329,8 @@ def _main_parser(config):
                 'kafka_batch_n',
                 'kafka_batch_t',
                 'kafka_ack_timeout',
-                'number_of_consumer_processes'
+                'number_of_consumer_processes',
+                'ignore_old_files'
             ]
             for key in require_int:
                 if config[key] is not None:
diff --git a/beaver/worker/worker.py b/beaver/worker/worker.py
@@ -478,6 +478,10 @@ def update_files(self):
             else:
                 if not stat.S_ISREG(st.st_mode):
                     continue
+                elif int(self._beaver_config.get('ignore_old_files')) > 0 and \
+                        datetime.datetime.fromtimestamp(st.st_mtime) < (datetime.datetime.today() - datetime.timedelta(days=int(self._beaver_config.get('ignore_old_files')))):
+                    self._logger.debug('[{0}] - file {1} older then {2} day so ignoring it'.format(self.get_file_id(st), absname, self._beaver_config.get('ignore_old_files')))
+                    continue
                 fid = self.get_file_id(st)
                 ls.append((fid, absname))
 
