diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..7d9a31c7 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,24 @@ +version: 2 +updates: +- package-ecosystem: pip + directory: "/" + schedule: + interval: daily + open-pull-requests-limit: 10 + ignore: + - dependency-name: moto + versions: + - 1.3.16 + - 2.0.0 + - 2.0.1 + - 2.0.2 + - 2.0.3 + - 2.0.4 + - dependency-name: pyzmq + versions: + - 21.0.2 + - 22.0.0 + - 22.0.2 + - dependency-name: python-daemon + versions: + - 2.2.4 diff --git a/.travis.yml b/.travis.yml index 2f97b411..045120e6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,9 @@ -sudo: false +dist: trusty language: python branches: except: - gh-pages python: - - "2.6" - "2.7" addons: apt: diff --git a/CHANGES.rst b/CHANGES.rst index a426a7bf..8ff4c31e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,225 +1,215 @@ Changelog ========= -36.2.1 (2016-09-20) +36.3.1 (2019-04-03) ------------------- +------------------------ +- Require pika version below 1.0.0. [Michael Contento] + +36.3.0 (2018-10-14) +------------------- + +Fix +~~~ +- Correct release process. [Jose Diaz-Gonzalez] + +Other +~~~~~ +- Fix typo in RABBITMQ_ARGUMENT system property. [eroberts] +- Add arbitrary RabbitMQ arguments - Updated transport to parse + arguments -Updated config.py to add system property - Updated user + usage documentation - Confirmed no change in functionality if args are + not passed. [eroberts] +- Usage fix. [Scott Brenner] + + Fixed missing/incorrect TCP option. +- Remove truncating of logline after 32766 chars (#422) [Robert + Wunderer] + + * Remove truncating of logline after 32766 chars + + It is not necessary to limit unicode()'s input to 32766 chars, while + doing puts an arbitrary limit on the length of lines beaver can + handle. + + * Pin moto version to fix tests + + * multiline_regex_before won't work when logfile ends with empty line + + * Revert "multiline_regex_before won't work when logfile ends with empty line" + + This reverts commit 5196789916afa184beb71a0ef86344d7580d9136. + + * Fix README link + + * Remove truncating of logline after 32766 chars + + It is not necessary to limit unicode()'s input to 32766 chars, while + doing puts an arbitrary limit on the length of lines beaver can + handle. +- Fix README link. [Jamie Cressey] +- Revert "multiline_regex_before won't work when logfile ends with empty + line" [Jamie Cressey] + + This reverts commit 5196789916afa184beb71a0ef86344d7580d9136. +- Multiline_regex_before won't work when logfile ends with empty line. + [jeroenmaelbrancke] +- Pin moto version to fix tests. [Jamie Cressey] +- Fixing primary key reuse issues. [Javier Villar] +- Limiting number of records in batch to 500 as this is the kinesis + limit. [Greg Sterin] +- Updating docs for number_of_consumer_processes config. [Greg Sterin] - Avoid unreference variable 'st' error if file was removed. Raise exception if unrecognized environment error from os.stat. [Greg Sterin] - - Fix GitHub links. [Robin Baumgartner] The project was renamed to 'python-beaver', but the links do not reflect that change. + 36.2.0 (2016-04-12) ------------------- - - Replaced Mosquitto with Paho-mqtt for mqtt transport. [Justin van Heerde] - - Add log file rotate in limit size. [soarpenguin] - - Fix README.rst docs link. [Andrew Grigorev] + 36.0.1 (2016-01-13) ------------------- - - Fix README.rst formatting for rst-lint. [Jose Diaz-Gonzalez] - - Remove tabs and use "except .. as .." [EYJ] - - Try to fix hanging test by joining leftover thread. [EYJ] - - Fix Queue timeout occurring after successful reconnect. [EYJ] - - Fix rabbitmq reconnection behaviour to use the beaver reconnect mechanism. [EYJ] - - Migrating repo locations. [Jamie Cressey] - - Fixups for CONTRIBUTING. [Jamie Cressey] - - Fixing formatting. [Jamie Cressey] - - Changes to guidelines and adding reference to README. [Jamie Cressey] - - Adding contributing guidelines. [Jamie Cressey] + 36.0.0 (2015-12-15) ------------------- - - Adding max_queue_size to docs. [Jamie Cressey] - - Pinning kafka-python version. [Jamie Cressey] - - Ensure we test against the latest version of kafka-python. [Jamie Cressey] - - Attempt to reconnect to Kafka on failure. [Jamie Cressey] - - Adding SQS tests. [Jamie Cressey] - - Exclude gh-pages from TravisCI runs. [Jamie Cressey] - - Adding coverage results to README. [Jamie Cressey] - - Adding coverage tests. [Jamie Cressey] - - We say Py2.6+ is a requirement, but do the tests actually pass on 2.6? [Jamie Cressey] - - Dont test py3, yet... [Jamie Cressey] - - Testing python 3.x. [Jamie Cressey] - - Using new travis config. [Jamie Cressey] - - Added requests as a dependency. [Jose Diaz-Gonzalez] Closes #304 - - Bump debian version on release. [David Moravek] - - Support both older and newer pika. [Tim Stoop] - - Make reconnecting to a lost RabbitMQ work. [Tim Stoop] - - Remove old worker code in favor of the - now non-experimental - TailManager. [Jose Diaz-Gonzalez] + 35.0.2 (2015-12-03) ------------------- - - Write to the SQS object not the dict when using sqs_bulk_lines flag. [Jamie Cressey] + 35.0.1 (2015-11-26) ------------------- - - Remove autospec attribute. [Jose Diaz-Gonzalez] For some reason, this broke attribute setting on the mock SelectConnection. - - - Fix pika version to version with all named parameters. [Jose Diaz- Gonzalez] - - Peg kafka to a known-good version. [Jose Diaz-Gonzalez] + 35.0.0 (2015-11-26) ------------------- - - Remove gitchangelog.rc. [Jose Diaz-Gonzalez] - - Merging changes. [Jamie Cressey] - - Added configuration option ignore_old_files. [Ryan Steele] Files older then n days are ignored - - - Support writes into multiple redis namespaces. [Andrei Vaduva] - - Adding support for multiple SQS queues. [Jamie Cressey] - - Ensure log lines confirm to utf-8 standard. [Jamie Cressey] We've come across cases when certain characters break Beaver transmitting log lines. This PR ensures all log lines correctly conform to UTF-8 when they're formatted for transmission. - - Set timeout to 1 second. [Tim Stoop] Apparantly, it needs to be an integer, so we cannot use pika's default of .25. - - - Revert "Lower the default to .25, which is pika's default." [Tim Stoop] This reverts commit 17157990a272e458cc9253666f01c6002b84bda8. - - - Lower the default to .25, which is pika's default. [Tim Stoop] As suggested by @kitchen. - - - Pieter's patch for rabbitmq timeout. [Tim Stoop] - - Typo in config variable default value. [Jamie Cressey] - - Fix regressed change. [Jamie Cressey] - - Ability to send multiple log entries per single SQS message. [Jamie Cressey] - - Adding AWS profile authentication to SQS transport. [Jamie Cressey] + 34.1.0 (2015-08-10) ------------------- - - Adding AWS SNS as a transport option. [Jamie Cressey] + 34.0.1 (2015-08-07) ------------------- - - Revert some breakages caused by d159ec579c01b8fab532b3814c64b0ff8b2063ff. [Jose Diaz-Gonzalez] Closes #331 - - Set default for command. [Jose Diaz-Gonzalez] - - #323 - fix tests to run with pika SelectConnection. [Tom Kregenbild] - - #323 - fix RabbitMQ transport _on_open_connection_error function to print connection errors. [Tom Kregenbild] - - #323 1. Add clear debug prints with queue size (one print every 1000 items in order not to hurt performance) 2. If main queue is empty keep running and do nothing 3. In case of a timeout from main queue restart queue. [Tom Kregenbild] - - #323 - Change RabbitMQ pika to use Asynchronous SelectConnection instead of BlockingConnection for better performance. [Tom Kregenbild] - - #323 - add the ability to increase the number of Queue consumers by creating additional processes while running with --experimental flag. [Tom Kregenbild] - - #323 - add the ability to increase the number of Queue consumers by creating additional processes. [Tom Kregenbild] - - #323 - print current queue size and number of total number transports in debug mode in order to find problem in transport rate. [Tom Kregenbild] + 34.0.0 (2015-07-24) ------------------- - - Added ssl-tcp key file support. [babbleshack] - - Rename configuration dir and debian bin to python-beaver. [David Moravek] - - Rename debian package back to python-beaver. [David Moravek] - - Debian packaging code review; thx @mnicky. [David Moravek] - - Improves debian packaging. [David Moravek] - - Fix tests when ZMQ is not installed. [David Moravek] - - Fix tests for python 2.7 (add funcsigs test dependency) [David Moravek] - - Move badge to below header. [Jose Diaz-Gonzalez] - - Add constants for data types, validate in init, use callback map. [Hector Castro] - - Move data type method conditional outside of loop. [Hector Castro] - - Add channel support to Redis transport. [Hector Castro] This changeset adds support for publishing log entries to a Redis @@ -230,125 +220,90 @@ Changelog is `list`. Attempts to resolve #266. - - - Introduced a stomp transport for beaver using stomp.py. [Peter Lenderyou] - - Fix references to ConfigParser error classes. [Jose Diaz-Gonzalez] - - Redis transport: handle multiple connections and use them in round robin style. [musil] - - Fixes GELF format according to specs. [Marvin Frick] GELF formatted messages need to be \0 ended. At least for sending over TCP. - - - Kafka round robin partitioner. [David Moravek] - - Solve error: cannot convert argument to integer. [Theofilis George- Nektarios] See at #312 + 33.3.0 (2015-04-08) ------------------- - - Basic docs for GELF formatter. [Oleg Rekutin] Also fixes formatting issues with the immediately-preceding HTTP transport example section. - - - Adds a GELF formatter. [Oleg Rekutin] short_message is truncated to 250 characters and only the first line is retained. Pair with the HTTP POST output to write directly to graylog2. - - - Issue #305, accept any 2xx code for http_transport. [Oleg Rekutin] + 33.2.0 (2015-03-11) ------------------- - - Improved kafka test. [Marcel Casado] - - Added example of kafka transport usage in the user docs. [Marcel Casado] - - Added placeholder "dist" directory to download kafka binaries. [Marcel Casado] - - Added integration test support for Kafka transport. [Marcel Casado] - - Wrapped kafka client init in a try catch. [Marcel Casado] - - Initial kafka transport impl. [Marcel Casado] - - Updating config examples and docs. [Jonathan Sabo] - - Adding support for sqs queues in different accounts. [Jonathan Sabo] + 33.1.0 (2015-02-04) ------------------- - - Improved error message for missing logstash_version. [Florian Hopf] Added a comment that the version needs to be set in the config - - Specify stricter dependency on python-daemon, fixes #286. [Graham Floyd] - - Add message_batch size checking since SQS can only handle 256KiB in a batch. Flush queue if message_batch is 10 messages or >= 250KiB. [Lance O'Connor] - - Explained valid values and meaning for rabbitmq_delivery_mode. [Fabio Coatti] - - Added documentation for rabbitmq_delivery_mode configuration parameter. [Fabio Coatti] - - A small change in except syntax. This should make happy python3 and work also in 2.6 and later. [Fabio Coatti] - - When sending a message, now we can tell rabbitmq which delivery mode we want, according to main configuration option rabbitmq_delivery_mode. [Fabio Coatti] - - Added configuration option for rabbitmq deliveryMode. Basically it works like a boolean, but having 1 and 2 as allowed values, we consider it integer and validate it as such. [Fabio Coatti] - - Newline removed. [Fabio Coatti] - - Added stanzas specific redis_namespace key to documentation. [Fabio Coatti] - - Added a space after comma, more compliant with python style guide. [Fabio Coatti] - - Revert "ignored eric files" [Fabio Coatti] This reverts commit ea2a6b27437570aeda3ee53b6c6ebd7ebb1f4f2a. as suggested, leave alone .gitignore :) - - - This small commit allows to specify a redis namespace in file section of configuration file (stanzas). Basically, beaver checks if a redis_namespace is defined for the current file. If yes, it is used for the redis payload. If not (or null), beaver uses the redis_namespace value specified in global section. [Fabio Coatti] - - Added a section (stanza) configuration option in order to be able to specify a redis namespace. If set, it will override the namespace set in main section. Default is null. [Fabio Coatti] - - Ignored eric files. [Fabio Coatti] - - Remove `python-daemon` from requirements on win32. [Ryan Davis] If we're installing on windows, don't require `python-daemon`. This @@ -356,62 +311,48 @@ Changelog trying to install `python-daemon`. refs #141 - - - Use new repository name for travis-ci badge. [Jose Diaz-Gonzalez] + 33.0.0 (2014-10-14) ------------------- - - Extend release script to support new, semver-tagged releases. [Jose Diaz-Gonzalez] - - Add gitchangelog.rc to fix changelog generation. [Jose Diaz-Gonzalez] + 32 (2014-10-14) --------------- - - Allow for the config file to override the logfile's setting. [Aaron France] - - Force update of sincedb when beaver stop. [Pierre Fersing] - - Fixed sincedb_write_interval (Bugs #229). [Pierre Fersing] - - Fix config.get('ssh_options') [svengerlach] ssh_options could never be returned due to a wrong type check - - Add debian packaging based on dh-virtualenv. [Jose Diaz-Gonzalez] - - Zmq3 split HWM into SNDHWM/RCVHWM. Closes #246. [Pete Fritchman] - - Fix typo in usage.rst. [Hugo Lopes Tavares] s/logstash_verion/logstash_version/ - - Fixed badge to point to master branch only. [Jose Diaz-Gonzalez] + 31 (2014-01-25) --------------- Fix ~~~ - - Beaver user can't write its pid nor its log. [Mathieu Lecarme] - Using a folder is the tactic used by Redis on Debian. - + Using a folder is the tactic used by Redis on Debian. Other ~~~~~ - - Add required spacing to readme for proper pypi doc support. [Jose Diaz-Gonzalez] - - Change release process to include processing of documentation. [Jose Diaz-Gonzalez] - - Use GlobSafeConfigParser to parse config files. [Clay Pence] In order to support all of the kinds of globs, pass GlobSafeConfigParser @@ -426,16 +367,11 @@ Other This should fix the unit test to run properly when run from the main directory. - - - Fix redis_transport.py redis exception handling. Fixes #238. [Hugo Lopes Tavares] - - Attempt to fix memory leaks. Closes #186. [Jose Diaz-Gonzalez] - - Allow for newer versions of boto to be used. Closes #236. [Jose Diaz- Gonzalez] - - When shipping logs, use millisecond-precision timestamps. [Ryan Park] Logstash 1.3.2 has a problem with microsecond-precision timestamps in the @@ -450,90 +386,58 @@ Other This patch reduces @timestamp to millisecond precision, which should correct the problem with Beaver 1.3.2. - - - Improve compatibility with case-sensitive filesystems. [Jose Diaz- Gonzalez] - - Modify test cases to support logstash_version. [Jose Diaz-Gonzalez] - - Document usage of logstash_version. [Peter Burkholder] - - Add add_field_env option to the config file to allow fields to be added using values from the environment. [Lance O'Connor] Closes #214 - - - Add SSL/TLS support to the RabbitMQ transport. Closes #217. [Jonathan Harker] - - Added http transport option. Closes #218. [Jeff Bryner] - - Adding missing config file option 'rabbitmq_queue_durable'. [Daniel Whelan] - - `StrictRedis.from_url` is better than DIY-ing it. [Kristian Glass] Note currently `fakeredis` doesn't support `from_url` - this is blocking on https://github.com/jamesls/fakeredis/pull/29 being merged in (I've bumped version requirement in `tests.txt` accordingly) - - - Python 2.6 ConfigParser does not handle non-string Fixed typo. [tommyulfsparre] - - Dont add empty object to input list. [tommyulfsparre] - - Import threading library in tail manager since we want to use it. [Chris Roberts] - - Add SSL to the TCP Transport. [Simon McCartney] - - Redirect all docs to readthedocs. Refs #150. [Jose Diaz-Gonzalez] - - Readthedocs support. Closes #150. [Jose Diaz-Gonzalez] - - Convert producer to process. Allow timed producer culling. [Chris Roberts] - - Make consumer check threaded to prevent wedge state. [Chris Roberts] - - Don't crash on a string decoding exception. [Adam Twardowski] - - Set transport as valid on connect (properly resets for reconnect) [Chris Roberts] - - Handle publication failures in the TCP transport correctly. [Kiall Mac Innes] - - Add config option to manipulate ssh_options. [Andreas Lappe] This option allows to pass all ssh options to the tunnel. - - - Fix version lookup. [Jose Diaz-Gonzalez] - - Moved multiline_merge function to utils.py. [Pierre Fersing] - - Support for multi-line and tail_lines options. [Pierre Fersing] - - Support for multi-line events in tail-version. [Pierre Fersing] - - Support for multi-line events. [Pierre Fersing] - - Ignore invalid rawjson log. [Tomoyuki Sakurai] this ensures beaver keeps running even when other application logged logs in invalid json format. - - - Removed duplicate self._current_host from @source field. Fixes #180. [Alexander Papaspyrou] + 30 (2013-08-22) --------------- - - Use os._exit over sys.exit in signal handlers to quit cleanly. [Kristian Glass] @@ -544,8 +448,6 @@ Other (http://docs.python.org/2/library/sys.html#sys.exit) which can be caught by try/except blocks that might have been executing at time of signal handling, resulting in beaver failing to quit - - - Allow string escapes in delimiter. [Michael Mittelstadt] As far as I can tell, there is no way for me to represent a newline as @@ -567,88 +469,57 @@ Other By doing this sort of multiline parsing with beaver, it allows one to run logstash without the multiline filter, which due to its lack of thread-safety, forces you to run logstash with only one worker thread. - - - CONFIG_DIR to CONFD_PATH. [iyingchi] - - Added doc for -C option for config directory. [iyingchi] - - Fixed example in Readme.rst for sqs_aws_secret_key. [Jonathan Quail] - - Allow path to be None. [Lars Hansson] Allow path to be set empty (None) in the configuration filer. This way all files and globs can be configured in files in confd_path. - - - Fix zmq transport tests. [Scott Smith] - - Move zmq address config parsing into _main_parser. [Scott Smith] - - Allow specifying multiple zmq addresses to bind/connect. [Scott Smith] - - Redis 2.4.11 is no longer available on Pypi. [Andrew Gross] Fixes issue #167 - - Add a TCP transport. [Kiall Mac Innes] - - Isolate connection logic. Provide proper reconnect support. [Chris Roberts] - - Corrected documentation for exclude tag. Closes #157. [Jose Diaz- Gonzalez] - - Add missing sqlite3 module to documentation. [Andreas Lappe] - - Tests status. [Denis Orlikhin] - - Travis integration. [Denis Orlikhin] - - Tests fix (conf_d does work without existing file) [Denis Orlikhin] - - Implicit broken zmq error handling. [Denis Orlikhin] + 29 (2013-05-24) --------------- - - Do not harcode path in TailManager. Closes #143. [Jose Diaz-Gonzalez] - - Use /etc/beaver/conf for path and provide conf.d example. Closes #149. [Jose Diaz-Gonzalez] - - Added mqtt as option in argparse configuration for the transport flag. [Jose Diaz-Gonzalez] - - Fixed broken MqttTransport naming. [Jose Diaz-Gonzalez] - - Refactored BeaverSubprocess to maintain the running command as an attribute. [Jose Diaz-Gonzalez] - - Properly parse the beaver conf.d path for new sections. Closes #144. Closes #145. Refs #107. [Jose Diaz-Gonzalez] - - Use a Buffered Tokenizer to read large/fast incoming log input. Refs #135. Refs #105. [Jose Diaz-Gonzalez] - - Close queue after worker has been stopped. Refs #135. [Jose Diaz- Gonzalez] - - Wrap manager.close() call in try/except to mimic the worker dispatcher. [Jose Diaz-Gonzalez] - - Properly parse out the port from the `ssh_tunnel` option. Closes #142. [Jose Diaz-Gonzalez] - - Subclass the BaseLog class in BeaverSubprocess. Refs #142. [Jose Diaz- Gonzalez] - - Move base_log module higher up in hierarchy. Refs #142. [Jose Diaz- Gonzalez] - - Disable daemonization on the windows platform. Closes #141. [Jose Diaz-Gonzalez] - - Move file unwatching in old-style worker out of for-loop. Refs #139. [Jose Diaz-Gonzalez] @@ -659,52 +530,35 @@ Other We should also take care to catch `RuntimeError` which may arise when closing the Worker out of band - such as in the `cleanup` step of the worker dispatcher - but nowhere else. This should fix issues where logrotate suddenly causes files to disappear for a time and beaver tries to tail the file at the exact time it is being recreated. - - - Typo in SQS docs. [Jonathan Quail] - - Remove ujson requirement. [Jose Diaz-Gonzalez] This allows users that do not have a compiler in their deployment area to install beaver. Closes #137 - - Turn on logfile output when running in non-daemon contexts. Closes #131. [Jose Diaz-Gonzalez] - - Expand logging output path. Closes #133. [Jose Diaz-Gonzalez] - - Ensure logging to a file does not destroy regular logging. Closes #132. [Jose Diaz-Gonzalez] - - Properly handle unreadable files by logging a warning instead of crashing. Closes #130. [Jose Diaz-Gonzalez] - - Rename null_formatter to raw_formatter in BaseTransport class. [Jose Diaz-Gonzalez] - - Ensure that the RedisTransport calls the super invalidate method. Refs #93. [Jose Diaz-Gonzalez] - - Fix issue where input type was not being detected properly. [Jose Diaz-Gonzalez] - - Use logfile flag for sending all output to a file in daemon contexts. [Jose Diaz-Gonzalez] - - Expand path for pidfile creation. [Jose Diaz-Gonzalez] - - Properly handle redis reconnects when the datastore becomes unreacheable. Refs #93. [Jose Diaz-Gonzalez] - - 'type' instead of 'exchange_type' in recent pika vers. [Pravir Chandra] - - Adding options to make queues durable and HA. [Pravir Chandra] - - Respect stat_interval file configuration in stable worker. [Jose Diaz- Gonzalez] - - Unified configuration file using conf_d module. [Jose Diaz-Gonzalez] This change adds support for a conf.d directory - configured only via the `--confd-path` flag - which allows beaver to read configuration from multiple files. @@ -716,299 +570,201 @@ Other Please note that this commit BREAKS custom transport classes, as the interface for creating a transport class has changed. If you are referencing a `file_config.get(field, filename)` anywhere, please omit this and refer to `beaver_config.get_field(field, filename)`. Closes #107 - - - Hack to prevent stupid TypeError: 'NoneType' when running tests via setup.py. [Jose Diaz-Gonzalez] - - Properly handle rotated files on Darwin architectures. [Jose Diaz- Gonzalez] - - Log to debug instead of warning for file reloading on Darwin architectures. [Jose Diaz-Gonzalez] - - Speed up experimental worker. [Jose Diaz-Gonzalez] - Removed inline sleep call, which slowed down passes n*0.1 seconds, where n is the number of files being tailed - Inline methods that update data structures which should speed up larger installations - Make self.active() an attribute lookup instead of a method call - - - Use latest version of message pack interface (0.3.0). Closes #128. [Jose Diaz-Gonzalez] - - Alternative for reading python requirements. [Justin Lambert] - - Fix options sent from original worker to queue. Refs #119. [Jose Diaz- Gonzalez] - - Allow users to ignore the results of a copytruncate from logrotate. Refs #105. [Jose Diaz-Gonzalez] - - Fix rpm package building. Closes #123. [Jose Diaz-Gonzalez] - - Added experimental tail-version of beaver. [Jose Diaz-Gonzalez] - - Beginning work to move from an omniscient worker to individual tail objects. [Jose Diaz-Gonzalez] - - Fix kwargs call. [Jose Diaz-Gonzalez] - - Add formatting to mqtt transport. Closes #115. [Jose Diaz-Gonzalez] - - Retrieve more data from callback to minimize dictionary lookups. [Jose Diaz-Gonzalez] - - Prefer single quotes to double quotes where possible. [Jose Diaz- Gonzalez] - - Ensure stat_interval and tail_lines are both integer values. [Jose Diaz-Gonzalez] - - Alphabetize config variables for file_config. [Jose Diaz-Gonzalez] - - Ensure that debug flag is a boolean. [Jose Diaz-Gonzalez] - - Follow logstash covention for 'format' instead of 'message_format' [Jose Diaz-Gonzalez] - - Use passed in 'ignore_empty' field instead of a file_config lookup in queue module. [Jose Diaz-Gonzalez] - - Prefer discover_interval over update_file_mapping_time. [Jose Diaz- Gonzalez] - - Fix TransportException import. Closes #122. [Jose Diaz-Gonzalez] - - Auto-reconnect mechanism for the SSH tunnel. [Michael Franz Aigner] - - Use an alternative method of reading in requirements. Refs #120. [Jose Diaz-Gonzalez] - - Fix import of REOPEN_FILES constant in dispatcher.py. [Jose Diaz- Gonzalez] - - Fix a PEP8 violation. [Jose Diaz-Gonzalez] - - Ensure all files are utf-8 encoded. [Jose Diaz-Gonzalez] - - Namespace transport classes in the transport module. [Jose Diaz- Gonzalez] - - Allow specifying debug mode via argument. [Jose Diaz-Gonzalez] - - Added thread-safety to datetime calls. [Jose Diaz-Gonzalez] - - Added support for message_format. Closes #91. [Jose Diaz-Gonzalez] - - Add msgpack_pure as fallback for C-Based msgpack package. [Jose Diaz- Gonzalez] - - Fix issues in sincedb implementation. Refs #116. [Jose Diaz-Gonzalez] - - Fix casting issue when checking start_position. [Jose Diaz-Gonzalez] - - Properly handle Queue.Full exceptions. [Jose Diaz-Gonzalez] - - More logging. [Jose Diaz-Gonzalez] - - Expand the sincedb path on configuration parse. [Jose Diaz-Gonzalez] - - Ignore since.db files. [Jose Diaz-Gonzalez] - - Simplified sincedb support to handle an edge case. Refs #116. [Jose Diaz-Gonzalez] - - Remove errant print. [Jose Diaz-Gonzalez] - - Added support for file exclusion in config stanzas. Closes #106. [Jose Diaz-Gonzalez] - - Added python regex exclusion support to eglob. Refs #106. [Jose Diaz- Gonzalez] - - PEP8. [Jose Diaz-Gonzalez] - - Added a tests directory with some sample tests from users. [Jose Diaz- Gonzalez] - - Convert the 'sincedb_write_interval' option to an integer. Refs #116. [Jose Diaz-Gonzalez] - - Moved logger call to a more intelligent spot. [Jose Diaz-Gonzalez] - - Ensure that we use the proper encoding when opening a file. Closes #104. [Jose Diaz-Gonzalez] - - Centralize file-reading using classmethod open() [Jose Diaz-Gonzalez] - - Fixed issue where tailed lines were not being properly sent to the callback. [Jose Diaz-Gonzalez] - - Remove unnecessary argument from Worke.__init__() [Jose Diaz-Gonzalez] - - Force-parse non-unicode files using unicode_dammit. [Jose Diaz- Gonzalez] - - Set utf-8 as default encoding on all python files. [Jose Diaz- Gonzalez] - - Fixed pyflakes issues. [rtoma] - - Syntax fix of list. [rtoma] - - Raise an AssertionError when run in daemon without a pid path specified. Closes #112. [Jose Diaz-Gonzalez] - - Add support for ignoring empty lines. [Jose Diaz-Gonzalez] - - Properly cast boolean values from strings. [Jose Diaz-Gonzalez] - - Ensure all sections have the proper values on start. [Jose Diaz- Gonzalez] - - Ensure internal file_config state is updated. [Jose Diaz-Gonzalez] - - Pass in timestamp from worker class for more accurate timestamps at the cost of speed of sending. [Jose Diaz-Gonzalez] - - Centralize timestamp retrieval to base transport class. [Jose Diaz- Gonzalez] - - Added support for gzipped files. refs #39. [Jose Diaz-Gonzalez] - - Added support for sqlite3-based sincedb. Refs #6 and #39. [Jose Diaz- Gonzalez] - - Refactored worker so as to allow further data to be added to the file_map. [Jose Diaz-Gonzalez] - - Refactor seek_to_end to properly support file tailing. [Jose Diaz- Gonzalez] - - Added support for pubsub zmq. [Jose Diaz-Gonzalez] - - Added support for mosquitto transport. [Jose Diaz-Gonzalez] - - Added support for specifying file encoding, using io.open vs os.open. [Jose Diaz-Gonzalez] - - Fix issue where a field may not exist in the data. [Jose Diaz- Gonzalez] - - Added support for rawjson format. [Jose Diaz-Gonzalez] - - Fixed zeromq tests. [Jose Diaz-Gonzalez] - - Added SQS transport. [Jonathan Quail] - - Fixing outdated transport docs. [Morgan Delagrange] + 28 (2013-03-05) --------------- - - BeaverSubprocess is now a new-style class. Fixes ssh_tunneling. [Jose Diaz-Gonzalez] + 27 (2013-03-05) --------------- - - Fix issue where super method was not called in BeaverSshTunnel. [Jose Diaz-Gonzalez] + 26 (2013-03-05) --------------- - - Add optional reconnect support for transports. Refs #93. [Jose Diaz- Gonzalez] - - Add a method for checking the validity of a Transport. Refs #93. [Jose Diaz-Gonzalez] - - Added a configurable subprocess poll sleep. [Jose Diaz-Gonzalez] - - Add a deafult sleep timeout to BeaverSubprocess polling. [Jose Diaz- Gonzalez] - - Use a larger sleep time to get around redis over ssh connection issues. [Jose Diaz-Gonzalez] + 25 (2013-03-05) --------------- - - Use True instead of 1 for while check. [Jose Diaz-Gonzalez] - - Fix orphan child processes. Closes #103. [Jose Diaz-Gonzalez] + 24 (2013-02-26) --------------- - - Ensure new files are added to a transports configuration. Closes #96. Closes #101. [Jose Diaz-Gonzalez] - - Allow float numbers for update_file_mapping_time. [Jose Diaz-Gonzalez] - - Fix invalid casting of boolean values. [Jose Diaz-Gonzalez] - - Perform all conversions in config.py. Closes #99. [Jose Diaz-Gonzalez] + 23 (2013-02-20) --------------- - - Worker: pretty format debug message "Iteration took %.6f" [Sergey Shepelev] - - Zeromq_hwm int() conversion moved to config. [Denis Orlikhin] - - Zeromq_hwm config entry. [Denis Orlikhin] - - Zeromq_hwm support. [Denis Orlikhin] - - Add test requirements to setup. [Paul Garner] - - Allow beaver to accept custom transport classes. [Paul Garner] - - Rabbitmq_exchange_type option fixed in the README. [Xabier de Zuazo] - - Make beaver slightly more amenable to test mocking and sort of fix the broken zmq test. [Paul Garner] + 22 (2013-01-15) --------------- - - Handle sigterm properly. Refs #87. [Jose Diaz-Gonzalez] - - Add --loglevel as alias for --output. Closes #92. [Jose Diaz-Gonzalez] - - Added logging on connection exception. [Thomas Morse] - - Adding exception when redis connection can't be confirmed. [William Jimenez] - - Add '--format raw' to pass through input unchanged. [Stephen Sugden] - - Fix string & null formatters in beaver.transport. [Stephen Sugden] the inline definitions were expecting a self parameter, which is *not* passed when you assign a function to an attribute on an object instance. - - Call file.readlines() with sizehint in a loop to avoid reading in massive files all at once. [Jose Diaz-Gonzalez] + 21 (2013-01-04) --------------- - - Move runner into a dispatcher class to solve installation issues. [Jose Diaz-Gonzalez] - - Added note for Python 2.6+ support. [Jose Diaz-Gonzalez] + 20 (2013-01-03) --------------- - - Copy the readme over to avoid pypi packaging warnings. [Jose Diaz- Gonzalez] - - Implement fully recursive file globing. [Brian L. Troutwine] Python's base glob.iglob does not operate as if globstar were in effect. To @@ -1025,39 +781,31 @@ Other webmachine logs would be ignored by beaver. This is no longer the case, to an arbitrary depth. - Signed-off-by: Brian L. Troutwine - 19 (2013-01-01) --------------- - - Fix issue with supporting command line args. [Jose Diaz-Gonzalez] + 18 (2012-12-31) --------------- - - Add timing debug information to the worker loop. [Jose Diaz-Gonzalez] - - Use redis pipelining when sending events. [Jose Diaz-Gonzalez] - - Formatting. [Jose Diaz-Gonzalez] - - Do not output debug statement for file_config.get call. [Jose Diaz- Gonzalez] - - Pass in logger object to create_ssh_tunnel() [Jose Diaz-Gonzalez] + 17 (2012-12-28) --------------- - - Added missing python-daemon requirement. [Jose Diaz-Gonzalez] + 16 (2012-12-27) --------------- - - Specify a max queue size of 100 to limit overrunning memory. [Jose Diaz-Gonzalez] - - Use multiprocessing for handling larger queue sizes. [Jose Diaz- Gonzalez] @@ -1066,63 +814,41 @@ Other This patch adds the ability to use an internal work queue for log lines. Whenever file.readlines() is called, the lines are placed in the queue, which is shared with a child process. The child process creates its own transport, allowing us to potentially create a Process Pool in the future to handle a larger queue size. Note that the limitation of file.readlines() reading in too many lines is still in existence, and may continue to cause issues for certain log files. - - - Add default redis_password to BeaverConfig class. [Jose Diaz-Gonzalez] - - Fix missing underscore causing transport to break. [Norman Joyner] - - Implement redis auth support. [Norman Joyner] - - Add beaver init script for daemonization mode. [Jose Diaz-Gonzalez] - - Use python logger when using StdoutTransport. [Jose Diaz-Gonzalez] - - Add short arg flags for hostname and format. [Jose Diaz-Gonzalez] - - Add the ability to daemonize. Closes #79. [Jose Diaz-Gonzalez] - - Pass around a logger instance to all transports. [Jose Diaz-Gonzalez] - - Revert "Added a lightweight Event class" [Jose Diaz-Gonzalez] After deliberation, beaver is meant to be "light-weight". Lets leave the heavy-hitting to the big-boys. This reverts commit 1619d33ef4803c3fe910cf4ff197d0dd0039d2eb. - - - Added a lightweight Event class. [Jose Diaz-Gonzalez] This class's sole responsibility will be the processing of a given line as an event. It's future goal will be to act as a lightweight implementation of the filter system within Logstash - - - Remove argparse requirement for python 2.7 and above. [Jose Diaz- Gonzalez] + 15 (2012-12-25) --------------- - - Pull argument parsing out of beaver __init__.py. [Jose Diaz-Gonzalez] - - Move app-running into __init__.py. [Jose Diaz-Gonzalez] - - Standardize on _parse() as method for parsing config. [Jose Diaz- Gonzalez] - - Automatically parse the path config option. [Jose Diaz-Gonzalez] - - Remove extensions argument on Worker class. [Jose Diaz-Gonzalez] This argument was only used when no globs were specified in a config file. Since it is not configurable, there is no sense leaving around the extra logic. - - - Remove extra callback invocation on readlines. [Jose Diaz-Gonzalez] - - Remove extra file_config module. [Jose Diaz-Gonzalez] - - General code reorganization. [Jose Diaz-Gonzalez] Move both BeaverConfig and FileConfig into a single class @@ -1134,27 +860,18 @@ Other Moved extra configuration and setup code to beaver.utils module. In many cases, code was added hastily before. Made many logger calls debug as opposed to info. The info level should be generally reserved for instances where files are watched, unwatched, or some change in the file state has occurred. - - - Remove duplicative and old beaver instructions from binary. [Jose Diaz-Gonzalez] - - Remove unnecessary passing of ssh_tunnel subprocess. [Jose Diaz- Gonzalez] - - Added docstrings to ssh_tunnel module. [Jose Diaz-Gonzalez] - - Follow convention of underscore for object properties. [Jose Diaz- Gonzalez] - - Follow convention of underscore for object properties. [Jose Diaz- Gonzalez] - - Added a NullFormatter. [Jose Diaz-Gonzalez] Useful for cases where we do not want any extra overhead on message formatting - - - Refactored message formatting in base Transport class. [Jose Diaz- Gonzalez] @@ -1163,61 +880,44 @@ Other In the case of string output, we define a custom formatter using an anonymous function and specify that as the formatter. - - - Moved create_transport to transport module. [Jose Diaz-Gonzalez] - - Moved create_ssh_tunnel to ssh_tunnel module. [Jose Diaz-Gonzalez] - - Fixed order of beaver_config and file_config in args. [Jose Diaz- Gonzalez] - - Reduce overhead of parsing configuration for globs and files. [Jose Diaz-Gonzalez] - - Removed ordereddict dependency. [Jose Diaz-Gonzalez] - - Do not output info level when outputing version. [Jose Diaz-Gonzalez] - - Allow usage of ujson >= 1.19. Closes #76. [Jose Diaz-Gonzalez] + 14 (2012-12-18) --------------- - - Removed erroneous redundant code. [Jose Diaz-Gonzalez] - - Workaround for differing iteration implementation in Python 2.6. [Jose Diaz-Gonzalez] - - Properly detect non-linux platforms. [Jose Diaz-Gonzalez] - - Improve Python 2.6 support. [Jose Diaz-Gonzalez] - - Fix broken python readme. [Jose Diaz-Gonzalez] + 13 (2012-12-17) --------------- - - Fixed certain environment variables. [Jose Diaz-Gonzalez] - - SSH Tunnel Support. [Jose Diaz-Gonzalez] This code should allow us to create an ssh tunnel between two distinct servers for the purposes of sending and receiving data. This is useful in certain cases where you would otherwise need to whitelist in your Firewall or iptables setup, such as when running in two different regions on AWS. - - - Allow for initial connection lag. Helpful when waiting for an SSH proxy to connect. [Jose Diaz-Gonzalez] - - Fix issue where certain config defaults were of an improper value. [Jose Diaz-Gonzalez] - - Allow specifying host via flag. Closes #70. [Jose Diaz-Gonzalez] + 12 (2012-12-17) --------------- - - Reload tailed files on non-linux platforms. [Jose Diaz-Gonzalez] Python has an issue on OS X were the underlying C implementation of @@ -1229,8 +929,6 @@ Other Note that this also causes debug mode to be very noisy on OS X. We all have to make sacrifices... - - - Deprecate all environment variables. [Jose Diaz-Gonzalez] This shifts configuration management into the BeaverConfig class. @@ -1239,16 +937,13 @@ Other Refs #72 Closes #60 - - - Warn when using deprecated ENV variables for configuration. Refs #72. [Jose Diaz-Gonzalez] - - Minor changes for PEP8 conformance. [Jose Diaz-Gonzalez] + 11 (2012-12-16) --------------- - - Add optional support for socket.getfqdn. [Jeremy Kitchen] For my setup I need to have the fqdn used at all times since my @@ -1260,8 +955,6 @@ Other FQDN, it's now an option to explicitly always use the fqdn. Fixes #68 - - - Check for log file truncation fixes #55. [Jeremy Kitchen] This adds a simple check for log file truncation and resets the watch @@ -1278,183 +971,126 @@ Other Additionally, the files beaver will most likely be called upon to watch which may be truncated are generally going to be large enough and slow-filling enough that this won't crop up in the wild. - - - Add a version number to beaver. [Jose Diaz-Gonzalez] + 10 (2012-12-15) --------------- - - Fixed package name. [Jose Diaz-Gonzalez] - - Regenerate CHANGES.rst on release. [Jose Diaz-Gonzalez] - - Adding support for /path/{foo,bar}.log. [Josh Braegger] - - Consistency. [Chris Faulkner] - - Stating the obvious. [Chris Faulkner] - - Grist for the mill. [Chris Faulkner] - - Drop redundant README.txt. [Chris Faulkner] - - Ignore file errors in unwatch method -- the file might not exists. [Josh Braegger] - - Unwatch file when encountering a stale NFS handle. When an NFS file handle becomes stale (ie, file was removed), it was crashing beaver. Need to just unwatch file. [Josh Braegger] - - Consistency. [Chris Faulkner] - - Pull install requirements from requirements/base.txt so they don't get out of sync. [Chris Faulkner] - - Include changelog in setup. [Chris Faulkner] - - Convert changelog to RST. [Chris Faulkner] - - Actually show the license. [Chris Faulkner] - - Consistent casing. [Chris Faulkner] - - Don't use empty string for tag when no tags configured in config file. [Stylianos Modes] - - Making 'mode' option work for zmqtransport. Adding setuptools and tests (use ./setup.py nosetests). Adding .gitignore. [Josh Braegger] + 9 (2012-11-28) -------------- - - More release changes. [Jose Diaz-Gonzalez] - - Fixed deprecated warning when declaring exchange type. [Rafael Fonseca] + 8 (2012-11-28) -------------- - - Removed deprecated usage of e.message. [Rafael Fonseca] - - Fixed exception trapping code. [Rafael Fonseca] - - Added some resiliency code to rabbitmq transport. [Rafael Fonseca] + 7 (2012-11-28) -------------- - - Added a helper script for creating releases. [Jose Diaz-Gonzalez] - - Partial fix for crashes caused by globbed files. [Jose Diaz-Gonzalez] -6 (2012-11-26) --------------- +6 (2012-12-15) +-------------- - Fix issue where polling for files was done incorrectly. [Jose Diaz- Gonzalez] - - Added ubuntu init.d example config. [Jose Diaz-Gonzalez] -5 (2012-11-26) --------------- +5 (2012-12-15) +-------------- - Try to poll for files on startup instead of throwing exceptions. Closes #45. [Jose Diaz-Gonzalez] - - Added python 2.6 to classifiers. [Jose Diaz-Gonzalez] -4 (2012-11-26) --------------- +4 (2012-12-15) +-------------- - Remove unused local vars. [Jose Diaz-Gonzalez] - - Allow rabbitmq exchange type and durability to be configured. [Jose Diaz-Gonzalez] - - Remove unused import. [Jose Diaz-Gonzalez] - - Formatted code to fix PEP8 violations. [Jose Diaz-Gonzalez] - - Use alternate dict syntax for Python 2.6 support. Closes #43. [Jose Diaz-Gonzalez] - - Fixed release date for version 3. [Jose Diaz-Gonzalez] + 3 (2012-11-25) -------------- - - Added requirements files to manifest. [Jose Diaz-Gonzalez] - - Include all contrib files in release. [Jose Diaz-Gonzalez] - - Revert "removed redundant README.txt" to follow pypi standards. [Jose Diaz-Gonzalez] This reverts commit e667f63706e0af8bc82c0eac6eac43318144e107. - - - Added bash startup script. Closes #35. [Jose Diaz-Gonzalez] - - Added an example supervisor config for redis. closes #34. [Jose Diaz- Gonzalez] - - Removed redundant README.txt. [Jose Diaz-Gonzalez] - - Added classifiers to package. [Jose Diaz-Gonzalez] - - Re-order workers. [Jose Diaz-Gonzalez] - - Re-require pika. [Jose Diaz-Gonzalez] - - Make zeromq installation optional. [Morgan Delagrange] - - Formatting. [Jose Diaz-Gonzalez] - - Added changes to changelog for version 3. [Jose Diaz-Gonzalez] - - Timestamp in ISO 8601 format with the "Z" sufix to express UTC. [Xabier de Zuazo] - - Adding udp support. [Morgan Delagrange] - - Lpush changed to rpush on redis transport. This is required to always read the events in the correct order on the logstash side. See: https: //github.com/logstash/logstash/blob/6f745110671b5d9d66bf082fbfed99d145 af4620/lib/logstash/outputs/redis.rb#L4. [Xabier de Zuazo] -2 (2012-10-25) --------------- +2 (2012-11-25) +-------------- - Example upstart script. [Michael D'Auria] - - Fixed a few more import statements. [Jose Diaz-Gonzalez] - - Fixed binary call. [Jose Diaz-Gonzalez] - - Refactored logging. [Jose Diaz-Gonzalez] - - Improve logging. [Michael D'Auria] - - Removed unnecessary print statements. [Jose Diaz-Gonzalez] - - Add default stream handler when transport is stdout. Closes #26. [bear (Mike Taylor)] - - Better exception handling for unhandled exceptions. [Michael D'Auria] - - Handle the case where the config file is not present. [Michael D'Auria] - - Fix wrong addfield values. [Alexander Fortin] - - Add add_field to config example. [Alexander Fortin] - - Add support for add_field into config file. [Alexander Fortin] - - Minor readme updates. [Jose Diaz-Gonzalez] - - Add support for type reading from INI config file. [Alexander Fortin] Add support for symlinks in config file @@ -1474,116 +1110,70 @@ Other Conflicts: README.rst - - - Support globs in file paths. [Darren Worrall] - - When sending data over the wire, use UTC timestamps. [Darren Worrall] - - Added msgpack support. [Jose Diaz-Gonzalez] - - Use the python logging framework. [Jose Diaz-Gonzalez] - - Fixed Transport.format() method. [Jose Diaz-Gonzalez] - - Properly parse BEAVER_FILES env var. [Jose Diaz-Gonzalez] - - Refactor transports. [Jose Diaz-Gonzalez] - Fix the json import to use the fastest json module available - Move formatting into Transport class - - - Attempt to fix defaults from env variables. [Jose Diaz-Gonzalez] - - Fix README and beaver CLI help to reference correct RABBITMQ_HOST environment variable. [jdutton] - - Add RabbitMQ support. [Alexander Fortin] - - Added real-world example of beaver usage for tailing a file. [Jose Diaz-Gonzalez] - - Removed unused argument. [Jose Diaz-Gonzalez] - - Ensure that python-compatible readme is included in package. [Jose Diaz-Gonzalez] - - Fix variable naming and timeout for redis transport. [Jose Diaz- Gonzalez] - - Installation instructions. [Jose Diaz-Gonzalez] - - Use restructured text for readme instead of markdown. [Jose Diaz- Gonzalez] - - Removed unnecessary .gitignore. [Jose Diaz-Gonzalez] -1 (2012-08-06) --------------- +1 (2012-11-25) +-------------- - Moved app into python package format. [Jose Diaz-Gonzalez] - - Moved binary beaver.py to bin/beaver, as per python packaging. [Jose Diaz-Gonzalez] - - Moved around transports to be independent of each other. [Jose Diaz- Gonzalez] - - Reorder transports. [Jose Diaz-Gonzalez] - - Rewrote run_worker to throw exception if all transport options have been exhausted. [Jose Diaz-Gonzalez] - - Rename Amqp -> Zmq to avoid confusion with RabbitMQ. [Alexander Fortin] - - Added choices to the --transport argument. [Jose Diaz-Gonzalez] - - Fixed derpy formatting. [Jose Diaz-Gonzalez] - - Added usage to the readme. [Jose Diaz-Gonzalez] - - Support usage of environment variables instead of arguments. [Jose Diaz-Gonzalez] - - Fixed files argument parsing. [Jose Diaz-Gonzalez] - - One does not simply license all the things. [Jose Diaz-Gonzalez] - - Add todo to readme. [Jose Diaz-Gonzalez] - - Added version to pyzmq. [Jose Diaz-Gonzalez] - - Added license. [Jose Diaz-Gonzalez] - - Reordered imports. [Jose Diaz-Gonzalez] - - Moved all transports to beaver/transports.py. [Jose Diaz-Gonzalez] - - Calculate current timestamp at most once per callback fired. [Jose Diaz-Gonzalez] - - Modified transports to include proper information for ingestion in logstash. [Jose Diaz-Gonzalez] - - Fixed package imports. [Jose Diaz-Gonzalez] - - Removed another compiled python file. [Jose Diaz-Gonzalez] - - Use ujson instead of simplejson. [Jose Diaz-Gonzalez] - - Ignore compiled python files. [Jose Diaz-Gonzalez] - - Fixed imports. [Jose Diaz-Gonzalez] - - Fixed up readme instructions. [Jose Diaz-Gonzalez] - - Refactor transports so that connections are no longer global. [Jose Diaz-Gonzalez] - - Readme and License. [Jose Diaz-Gonzalez] - - First commit. [Jose Diaz-Gonzalez] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fd31ff61..a5e4a466 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -118,7 +118,7 @@ included in the project: [interactive rebase](https://help.github.com/articles/about-git-rebase/) feature to tidy up your commits before making them public. - Test cases should accompany any change in functionality to the code-base - - Any change to end-user functionality should include corrosponding documentation + - Any change to end-user functionality should include corresponding documentation 5. Locally merge (or rebase) the upstream development branch into your topic branch: diff --git a/README.rst b/README.rst index 1975d52a..ed89ca02 100644 --- a/README.rst +++ b/README.rst @@ -23,11 +23,11 @@ Using PIP: From Github:: - pip install git+git://github.com/python-beaver/python-beaver.git@36.2.1#egg=beaver + pip install git+git://github.com/python-beaver/python-beaver.git@36.3.1#egg=beaver From PyPI:: - pip install beaver==36.2.1 + pip install beaver==36.3.1 Documentation ============= @@ -51,7 +51,7 @@ HTML docs will be available in `beaver/docs/_build/html`. Contributing ============ -When contributing to Beaver, please review the full guidelines here: https://github.com/python-beaver/python-beaver/blob/master/CONTRIBUTING.rst. +When contributing to Beaver, please review the full guidelines here: https://github.com/python-beaver/python-beaver/blob/master/CONTRIBUTING.md. If you would like, you can open an issue to let others know about your work in progress. Documentation must be included and tests must pass on Python 2.6 and 2.7 for pull requests to be accepted. Credits diff --git a/beaver/__init__.py b/beaver/__init__.py index 3e28d757..f42ae5bf 100644 --- a/beaver/__init__.py +++ b/beaver/__init__.py @@ -1,2 +1,2 @@ # -*- coding: utf-8 -*- -__version__ = '36.2.1' +__version__ = '36.3.1' diff --git a/beaver/config.py b/beaver/config.py index a246dc14..14092c53 100644 --- a/beaver/config.py +++ b/beaver/config.py @@ -82,6 +82,7 @@ def __init__(self, args, logger=None): 'rabbitmq_exchange_durable': os.environ.get('RABBITMQ_EXCHANGE_DURABLE', '0'), 'rabbitmq_queue_durable': os.environ.get('RABBITMQ_QUEUE_DURABLE', '0'), 'rabbitmq_ha_queue': os.environ.get('RABBITMQ_HA_QUEUE', '0'), + 'rabbitmq_arguments': os.environ.get('RABBITMQ_ARGUMENTS', {}), 'rabbitmq_key': os.environ.get('RABBITMQ_KEY', 'logstash-key'), 'rabbitmq_exchange': os.environ.get('RABBITMQ_EXCHANGE', 'logstash-exchange'), 'rabbitmq_timeout': '1', @@ -248,6 +249,7 @@ def use_ssh_tunnel(self): def _check_for_deprecated_usage(self): env_vars = [ + 'RABBITMQ_ARGUMENTS' 'RABBITMQ_HOST', 'RABBITMQ_PORT', 'RABBITMQ_VHOST', diff --git a/beaver/tests/test_kinesis_transport.py b/beaver/tests/test_kinesis_transport.py new file mode 100644 index 00000000..dbea972e --- /dev/null +++ b/beaver/tests/test_kinesis_transport.py @@ -0,0 +1,155 @@ +# -*- coding: utf-8 -*- +import sys +if sys.version_info < (2, 7): + import unittest2 as unittest +else: + import unittest + +import mock +import tempfile +import logging + +import beaver +from beaver.config import BeaverConfig +from beaver.transports import create_transport +from beaver.unicode_dammit import unicode_dammit + +from fixtures import Fixture + +from moto import mock_kinesis +import boto.kinesis + +class KinesisTests(unittest.TestCase): + + @mock_kinesis + def _create_streams(self): + conn = boto.kinesis.connect_to_region("us-east-1") + conn.create_stream("stream1", 1) + conn.create_stream("stream2", 1) + + @classmethod + def setUpClass(cls): + cls.logger = logging.getLogger(__name__) + + empty_conf = tempfile.NamedTemporaryFile(delete=True) + cls.beaver_config = BeaverConfig(mock.Mock(config=empty_conf.name)) + cls.beaver_config.set('transport', 'kinesis') + cls.beaver_config.set('logstash_version', 1) + + output_file = Fixture.download_official_distribution() + Fixture.extract_distribution(output_file) + + @mock_kinesis + def test_kinesis_default_auth_profile(self): + self._create_streams() + self.beaver_config.set('kinesis_aws_profile_name', None) + self.beaver_config.set('kinesis_aws_access_key', None) + self.beaver_config.set('kinesis_aws_secret_key', None) + self.beaver_config.set('kinesis_aws_stream', 'stream1') + + transport = create_transport(self.beaver_config, logger=self.logger) + + self.assertIsInstance(transport, beaver.transports.kinesis_transport.KinesisTransport) + transport.interrupt() + + @mock_kinesis + def test_kinesis_auth_profile(self): + self._create_streams() + self.beaver_config.set('kinesis_aws_profile_name', 'beaver_stream') + self.beaver_config.set('kinesis_aws_access_key', None) + self.beaver_config.set('kinesis_aws_secret_key', None) + self.beaver_config.set('kinesis_aws_stream', 'stream1') + + transport = create_transport(self.beaver_config, logger=self.logger) + + self.assertIsInstance(transport, beaver.transports.kinesis_transport.KinesisTransport) + + @mock_kinesis + def test_kinesis_auth_key(self): + self._create_streams() + self.beaver_config.set('kinesis_aws_profile_name', None) + self.beaver_config.set('kinesis_aws_access_key', 'beaver_test_key') + self.beaver_config.set('kinesis_aws_secret_key', 'beaver_test_secret') + self.beaver_config.set('kinesis_aws_stream', 'stream1') + + transport = create_transport(self.beaver_config, logger=self.logger) + + self.assertIsInstance(transport, beaver.transports.kinesis_transport.KinesisTransport) + transport.interrupt() + + @mock_kinesis + def test_kinesis_auth_account_id(self): + self._create_streams() + self.beaver_config.set('kinesis_aws_stream_owner_acct_id', 'abc123') + self.beaver_config.set('kinesis_aws_profile_name', None) + self.beaver_config.set('kinesis_aws_access_key', 'beaver_test_key') + self.beaver_config.set('kinesis_aws_secret_key', 'beaver_test_secret') + self.beaver_config.set('kinesis_aws_stream', 'stream1') + + transport = create_transport(self.beaver_config, logger=self.logger) + + self.assertIsInstance(transport, beaver.transports.kinesis_transport.KinesisTransport) + transport.interrupt() + + @mock_kinesis + def test_kinesis_send_stream(self): + self._create_streams() + self.beaver_config.set('kinesis_aws_stream', 'stream1') + self.beaver_config.set('kinesis_aws_profile_name', None) + self.beaver_config.set('kinesis_aws_access_key', None) + self.beaver_config.set('kinesis_aws_secret_key', None) + self.beaver_config.set('kinesis_bulk_lines', False) + + transport = create_transport(self.beaver_config, logger=self.logger) + mock_send_batch = mock.Mock() + transport._send_message_batch = mock_send_batch + + self.assertIsInstance(transport, beaver.transports.kinesis_transport.KinesisTransport) + + data = {} + lines = [] + n=500 + for i in range(n): + lines.append('log' + str(i) + '\n') + new_lines = [] + for line in lines: + message = unicode_dammit(line) + if len(message) == 0: + continue + new_lines.append(message) + data['lines'] = new_lines + data['fields'] = [] + self.assertTrue(transport.callback("test.log", **data)) + self.assertEqual(1, mock_send_batch.call_count) + + + @mock_kinesis + def test_kinesis_send_stream_with_record_count_cutoff(self): + self._create_streams() + self.beaver_config.set('kinesis_aws_stream', 'stream1') + self.beaver_config.set('kinesis_aws_profile_name', None) + self.beaver_config.set('kinesis_aws_access_key', None) + self.beaver_config.set('kinesis_aws_secret_key', None) + self.beaver_config.set('kinesis_bulk_lines', False) + + transport = create_transport(self.beaver_config, logger=self.logger) + mock_send_batch = mock.Mock() + transport._send_message_batch = mock_send_batch + + self.assertIsInstance(transport, beaver.transports.kinesis_transport.KinesisTransport) + + data = {} + lines = [] + n = 501 + for i in range(n): + lines.append('log' + str(i) + '\n') + new_lines = [] + for line in lines: + message = unicode_dammit(line) + if len(message) == 0: + continue + new_lines.append(message) + data['lines'] = new_lines + data['fields'] = [] + self.assertTrue(transport.callback("test.log", **data)) + self.assertEqual(2, mock_send_batch.call_count) diff --git a/beaver/transports/base_transport.py b/beaver/transports/base_transport.py index 7f0d54f9..cecba2bd 100644 --- a/beaver/transports/base_transport.py +++ b/beaver/transports/base_transport.py @@ -116,7 +116,7 @@ def callback(self, filename, lines): def format(self, filename, line, timestamp, **kwargs): """Returns a formatted log line""" - line = unicode(line.encode("utf-8")[:32766], "utf-8", errors="ignore") + line = unicode(line.encode("utf-8"), "utf-8", errors="ignore") formatter = self._beaver_config.get_field('format', filename) if formatter not in self._formatters: formatter = self._default_formatter diff --git a/beaver/transports/kinesis_transport.py b/beaver/transports/kinesis_transport.py index 46aff6ba..e790a40c 100644 --- a/beaver/transports/kinesis_transport.py +++ b/beaver/transports/kinesis_transport.py @@ -20,6 +20,9 @@ def __init__(self, beaver_config, logger=None): # self-imposed max batch size to minimize the number of records in a given call to Kinesis self._batch_size_max = beaver_config.get('kinesis_aws_batch_size_max', '512000') + # Kinesis Limit http://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecords.html#API_PutRecords_RequestSyntax + self._max_records_per_batch = 500 + try: if self._access_key is None and self._secret_key is None: self._connection = boto.kinesis.connect_to_region(self._region) @@ -55,7 +58,7 @@ def callback(self, filename, lines, **kwargs): continue # Check the self-enforced/declared batch size and flush before moving forward if we've eclipsed the max - if (len(message_batch) > 0) and ((message_batch_size + message_size) >= self._batch_size_max): + if len(message_batch) > 0 and ((message_batch_size + message_size) >= self._batch_size_max or len(message_batch) == self._max_records_per_batch): self._logger.debug('Flushing {0} messages to Kinesis stream {1} bytes'.format( len(message_batch), message_batch_size)) self._send_message_batch(message_batch) diff --git a/beaver/transports/rabbitmq_transport.py b/beaver/transports/rabbitmq_transport.py index 46d10b84..8992fb81 100644 --- a/beaver/transports/rabbitmq_transport.py +++ b/beaver/transports/rabbitmq_transport.py @@ -18,11 +18,17 @@ def __init__(self, beaver_config, logger=None): config_to_store = [ 'key', 'exchange', 'username', 'password', 'host', 'port', 'vhost', 'queue', 'queue_durable', 'ha_queue', 'exchange_type', 'exchange_durable', - 'ssl', 'ssl_key', 'ssl_cert', 'ssl_cacert', 'timeout', 'delivery_mode' + 'ssl', 'ssl_key', 'ssl_cert', 'ssl_cacert', 'timeout', 'delivery_mode', 'arguments' ] for key in config_to_store: self._rabbitmq_config[key] = beaver_config.get('rabbitmq_' + key) + if self._rabbitmq_config['arguments']: + self._rabbitmq_config['arguments'] = self.get_rabbitmq_args() + if self._rabbitmq_config['ha_queue']: + self._rabbitmq_config['arguments']['x-ha-policy'] = 'all' + + self._connection = None self._channel = None @@ -33,6 +39,20 @@ def __init__(self, beaver_config, logger=None): self._is_valid = True self._connect() + def get_rabbitmq_args(self): + res = {} + args = self._rabbitmq_config['arguments'].split(',') + for x in args: + k, v = x.split(':') + try: + # convert str to int if not a str + v = int(v) + except ValueError: + pass # is a str, not an int + res[k] = v + return res + + def _on_connection_open(self,connection): self._logger.debug("RabbitMQ: Connection Created") self._channel = connection.channel(self._on_channel_open) @@ -49,7 +69,7 @@ def _on_exchange_declareok(self,unused): self._channel.queue_declare(self._on_queue_declareok, queue=self._rabbitmq_config['queue'], durable=self._rabbitmq_config['queue_durable'], - arguments={'x-ha-policy': 'all'} if self._rabbitmq_config['ha_queue'] else {}) + arguments=self._rabbitmq_config['arguments']) def _on_queue_declareok(self,unused): self._logger.debug("RabbitMQ: Queue Declared") diff --git a/beaver/utils.py b/beaver/utils.py index d74a02de..bf7fb20e 100644 --- a/beaver/utils.py +++ b/beaver/utils.py @@ -180,7 +180,7 @@ def _replace_all(path, replacements): def multiline_merge(lines, current_event, re_after, re_before): """ Merge multi-line events based. - Some event (like Python trackback or Java stracktrace) spawn + Some event (like Python traceback or Java stacktrace) spawn on multiple line. This method will merge them using two regular expression: regex_after and regex_before. @@ -188,8 +188,8 @@ def multiline_merge(lines, current_event, re_after, re_before): If a line match re_before, it will be merged with previous line. - This function return a list of complet event. Note that because - we don't know if an event is complet before another new event + This function return a list of complete event. Note that because + we don't know if an event is complete before another new event start, the last event will not be returned but stored in current_event. You should pass the same current_event to successive call to multiline_merge. current_event is a list diff --git a/beaver/worker/tail.py b/beaver/worker/tail.py index 4fa15548..b5f81a9a 100644 --- a/beaver/worker/tail.py +++ b/beaver/worker/tail.py @@ -422,7 +422,7 @@ def _sincedb_update_position(self, lines=0, force_update=False): conn = sqlite3.connect(self._sincedb_path, isolation_level=None) cursor = conn.cursor() - query = 'insert or ignore into sincedb (fid, filename) values (:fid, :filename);' + query = 'insert or replace into sincedb (fid, filename) values (:fid, :filename);' cursor.execute(query, { 'fid': self._fid, 'filename': self._filename diff --git a/debian/changelog b/debian/changelog index f1a43b2a..935126ee 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +python-beaver (36.3.1) stable; urgency=low + + * Release version 36.3.1 + + -- Jose Diaz-Gonzalez Wed, 03 Apr 2019 16:48:10 +0000 + +python-beaver (36.3.0) stable; urgency=low + + * Release version 36.3.0 + + -- Jose Diaz-Gonzalez Sun, 14 Oct 2018 23:42:32 +0000 + python-beaver (36.2.1) stable; urgency=low * Release version 36.2.1 diff --git a/docs/conf.py b/docs/conf.py index 9824317a..9d4bc5e9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -53,9 +53,9 @@ # built documents. # # The short X.Y version. -version = '36.2.1' +version = '36.3.1' # The full version, including alpha/beta/rc tags. -release = '36.2.1' +release = '36.3.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/user/usage.rst b/docs/user/usage.rst index c32b77a2..86792ed6 100644 --- a/docs/user/usage.rst +++ b/docs/user/usage.rst @@ -57,6 +57,8 @@ Beaver can optionally get data from a ``configfile`` using the ``-c`` flag. This * mqtt_clientid: Default ``paho``. Paho client id * mqtt_keepalive: Default ``60``. mqtt keepalive ping * mqtt_topic: Default ``/logstash``. Topic to publish to +* number_of_consumer_processes: Default ``1``. Number of parallel consumer processes that read and process messages from the beaver queue. +* rabbitmq_arguments: Defaults ``{}``. RabbitMQ arguments comma separated, colon separated key value pairs. i.e ``rabbitmq_arguments: x-max-length:750000,x-max-length-bytes:1073741824`` * rabbitmq_host: Defaults ``localhost``. Host for RabbitMQ * rabbitmq_port: Defaults ``5672``. Port for RabbitMQ * rabbitmq_ssl: Defaults ``0``. Connect using SSL/TLS @@ -96,7 +98,7 @@ Beaver can optionally get data from a ``configfile`` using the ``-c`` flag. This * kinesis_aws_batch_size_max: Default ``512000``. Arbitrary flush size to limit size of logs in transit. * tcp_host: Default ``127.0.0.1``. TCP Host * tcp_port: Default ``9999``. TCP Port -* tcp_ssl Defaults ``0``. Connect using SSL/TLS +* tcp_ssl_enabled: Defaults ``0``. Connect using SSL/TLS * tcp_ssl_key Optional. Defaults ``None``. Path to client private key for SSL/TLS * tcp_ssl_cert Optional. Defaults ``None``. Path to client certificate for SSL/TLS * tcp_ssl_cacert Optional. Defaults ``None``. Path to CA certificate for SSL/TLS @@ -222,7 +224,7 @@ Read config from config.ini and put to stdout:: tags: tag1,tag2 add_field: fieldname1,fieldvalue1[,fieldname2,fieldvalue2, ...] - ; follow all logs in /var/log except those with `messages` or `secure` in the name. + ; follow all logs in /var/log except those with `messages` or `secure` in the name *of the file*. Currently it is not possible to exclude certain lines of a file. ; The exclude tag must be a valid python regular expression. [/var/log/*log] type: syslog diff --git a/install-dependencies.sh b/install-dependencies.sh index 325580b3..50e4d652 100755 --- a/install-dependencies.sh +++ b/install-dependencies.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -pip install -r requirements/zeromq.txt -r requirements/tests.txt --use-mirrors +pip install -r requirements/zeromq.txt -r requirements/tests.txt mkdir ~/.aws/ cat > ~/.aws/credentials << EOL diff --git a/release b/release index acb200b5..b48a67ef 100755 --- a/release +++ b/release @@ -37,7 +37,7 @@ fi echo -e "\n${GREEN}STARTING RELEASE PROCESS${COLOR_OFF}\n" set +e; -git status | grep "working directory clean" &> /dev/null +git status | grep -Eo "working (directory|tree) clean" &> /dev/null if [ ! $? -eq 0 ]; then # working directory is NOT clean echo -e "${RED}WARNING: You have uncomitted changes, you may have forgotten something${COLOR_OFF}\n" exit 1 diff --git a/requirements/base.txt b/requirements/base.txt index 6e282a9f..33e02ce6 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,10 +1,10 @@ boto>=2.18.0 conf_d>=0.0.4 -glob2==0.3 +glob2==0.7 kafka-python==0.9.5 paho-mqtt==1.1 msgpack-pure>=0.1.3 -pika>=0.9.14 +pika>=0.9.14,<1.0.0 python-daemon>=1.5.2,<=1.6.1 redis>=2.7.5 requests diff --git a/requirements/tests.txt b/requirements/tests.txt index a7775cc6..e087027f 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -5,4 +5,4 @@ nose six unittest2 coveralls -moto +moto==0.4.31