diff --git a/.gitignore b/.gitignore index b6ee79e..66a4ee4 100644 --- a/.gitignore +++ b/.gitignore @@ -28,5 +28,6 @@ bin/tt pkg/ vendor/gems/ vendor/bundle/ +vendor/bundler/ bin/httparty bin/weather diff --git a/Gemfile b/Gemfile index 3b512e9..3b77fdc 100644 --- a/Gemfile +++ b/Gemfile @@ -1,33 +1,38 @@ -# Version:20130523 +# Version:20150104 source "http://rubygems.org" source "http://gems.github.com" -gem 'sqlite3' -gem 'activesupport', '~>3.2.13' -gem 'activerecord', '~>3.2.13' -gem 'actionmailer', '~>3.2.13' -gem 'hashie' -gem 'gcalapi' -gem 'xml-simple' -gem 'feedbag' -gem 'nokogiri' -gem 'twitter' -gem 'weather_hacker' -gem 'pocket-ruby' -gem 'hipchat' +gem 'json', '~> 1.8.1' +gem 'sqlite3', '~> 1.3.10' +gem 'activesupport', '~> 4.1.8' +gem 'activerecord', '~> 4.1.8' +gem 'hashie', '~> 3.3.2' +gem 'gcalapi', '~> 0.1.2' +gem 'xml-simple', '~> 1.1.4' +gem 'feedbag', '~> 0.9.5' +gem 'nokogiri', '~> 1.6.5' +gem 'sanitize', '~> 3.1.0' +gem 'faraday', '~> 0.8.9' +gem 'twitter', '~> 4.8.0' +gem 'pocket-ruby', '~> 0.0.5' +gem 'weather_hacker', '~> 0.1.7' +gem 'hipchat', '~> 1.4.0' +gem 'fluent-logger', '~> 0.4.7' +gem 'dalli', '~> 2.7.2' +gem 'aws-sdk', '~> 1.60.2' group :test do - gem 'rspec' + gem 'rspec', '~> 2.14.1' gem 'rcov', :platforms => :mri_18 - gem 'simplecov' - gem 'simplecov-rcov' - gem 'koseki-mocksmtpd' + gem 'simplecov', '~> 0.9.1' + gem 'simplecov-rcov', '~> 0.2.3' + gem 'koseki-mocksmtpd', '~> 0.0.3' end group :development do - gem "cucumber" - gem 'bundler', '~>1.3.5' - gem 'builder', '~>3.0.4' + gem 'cucumber' + gem 'bundler' + gem 'builder' gem "jeweler" end diff --git a/README.md b/README.md index 8f3038b..1cfacd5 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Automatic Ruby ============== -**Ruby General Automation Framework** +**Ruby framework for the general-purpose automatic processing** Description @@ -75,7 +75,7 @@ For more info, refer to the document (doc/README). Environment ----------- -Ruby 1.8 - 2.0, see Gemfile. +After Ruby 1.9. See Gemfile. Development @@ -123,13 +123,11 @@ This naming convention is to mimic Ubuntu. Developers ---------- -See doc/AUTHORS. +See doc/AUTHORS or following link. ++ https://github.com/automaticruby?tab=members -Author ------- - -**774** +Project created by + http://id774.net + http://github.com/id774 diff --git a/Rakefile b/Rakefile index 7e6b791..531bd17 100644 --- a/Rakefile +++ b/Rakefile @@ -32,7 +32,7 @@ Jeweler::Tasks.new do |gem| gem.homepage = "http://github.com/automaticruby/automaticruby" gem.license = "GPL" gem.summary = %Q{Automatic Ruby} - gem.description = %Q{Ruby General Automation Framework} + gem.description = %Q{Ruby framework for the general-purpose automatic processing} gem.email = "idnanashi@gmail.com" gem.authors = ["id774"] # dependencies defined in Gemfile diff --git a/VERSION b/VERSION index d224e69..f16c023 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -13.5.0 +14.12.2 diff --git a/automatic.gemspec b/automatic.gemspec index acb0b4f..4757f0f 100644 --- a/automatic.gemspec +++ b/automatic.gemspec @@ -2,15 +2,17 @@ # DO NOT EDIT THIS FILE DIRECTLY # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec' # -*- encoding: utf-8 -*- +# stub: automatic 14.12.2 ruby lib Gem::Specification.new do |s| s.name = "automatic" - s.version = "13.5.0" + s.version = "14.12.2" s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= + s.require_paths = ["lib"] s.authors = ["id774"] - s.date = "2013-05-18" - s.description = "Ruby General Automation Framework" + s.date = "2015-01-04" + s.description = "Ruby framework for the general-purpose automatic processing" s.email = "idnanashi@gmail.com" s.executables = ["automatic"] s.extra_rdoc_files = [ @@ -35,84 +37,121 @@ Gem::Specification.new do |s| "doc/README.ja", "lib/automatic.rb", "lib/automatic/environment.rb", + "lib/automatic/feed_maker.rb", "lib/automatic/feed_parser.rb", "lib/automatic/log.rb", "lib/automatic/opml.rb", "lib/automatic/pipeline.rb", "lib/automatic/recipe.rb", + "lib/automatic/version.rb", "plugins/custom_feed/svn_log.rb", "plugins/filter/absolute_uri.rb", + "plugins/filter/accept.rb", + "plugins/filter/clear.rb", + "plugins/filter/description_link.rb", "plugins/filter/full_feed.rb", + "plugins/filter/github_feed.rb", + "plugins/filter/google_news.rb", "plugins/filter/ignore.rb", "plugins/filter/image.rb", "plugins/filter/image_source.rb", "plugins/filter/one.rb", "plugins/filter/rand.rb", + "plugins/filter/sanitize.rb", "plugins/filter/sort.rb", "plugins/filter/tumblr_resize.rb", "plugins/notify/ikachan.rb", + "plugins/provide/fluentd.rb", + "plugins/publish/amazon_s3.rb", "plugins/publish/console.rb", + "plugins/publish/console_link.rb", + "plugins/publish/eject.rb", + "plugins/publish/fluentd.rb", "plugins/publish/google_calendar.rb", "plugins/publish/hatena_bookmark.rb", + "plugins/publish/hipchat.rb", "plugins/publish/instapaper.rb", - "plugins/publish/mail.rb", + "plugins/publish/memcached.rb", "plugins/publish/pocket.rb", - "plugins/publish/smtp.rb", "plugins/publish/twitter.rb", "plugins/store/database.rb", + "plugins/store/file.rb", "plugins/store/full_text.rb", "plugins/store/permalink.rb", - "plugins/store/target_link.rb", + "plugins/subscription/chan_toru.rb", "plugins/subscription/feed.rb", - "plugins/subscription/google_reader_star.rb", + "plugins/subscription/g_guide.rb", "plugins/subscription/link.rb", + "plugins/subscription/pocket.rb", "plugins/subscription/text.rb", "plugins/subscription/tumblr.rb", "plugins/subscription/twitter.rb", + "plugins/subscription/twitter_search.rb", "plugins/subscription/weather.rb", + "plugins/subscription/xml.rb", "script/build", + "spec/fixtures/sampleFeeds.tsv", + "spec/fixtures/sampleFeeds2.tsv", "spec/fixtures/sampleRecipe.yml", + "spec/lib/automatic/log_spec.rb", "spec/lib/automatic/pipeline_spec.rb", "spec/lib/automatic/recipe_spec.rb", "spec/lib/automatic_spec.rb", "spec/plugins/custom_feed/svn_log_spec.rb", "spec/plugins/filter/absolute_uri_spec.rb", + "spec/plugins/filter/accept_spec.rb", + "spec/plugins/filter/clear_spec.rb", + "spec/plugins/filter/description_link_spec.rb", "spec/plugins/filter/full_feed_spec.rb", + "spec/plugins/filter/github_feed_spec.rb", + "spec/plugins/filter/google_news_spec.rb", "spec/plugins/filter/ignore_spec.rb", "spec/plugins/filter/image_source_spec.rb", "spec/plugins/filter/image_spec.rb", "spec/plugins/filter/one_spec.rb", "spec/plugins/filter/rand_spec.rb", + "spec/plugins/filter/sanitize_spec.rb", "spec/plugins/filter/sort_spec.rb", "spec/plugins/filter/tumblr_resize_spec.rb", "spec/plugins/notify/ikachan_spec.rb", + "spec/plugins/provide/fluentd_spec.rb", + "spec/plugins/publish/amazon_s3_spec.rb", "spec/plugins/publish/console_spec.rb", + "spec/plugins/publish/eject_spec.rb", + "spec/plugins/publish/fluentd_spec.rb", "spec/plugins/publish/google_calendar_spec.rb", "spec/plugins/publish/hatena_bookmark_spec.rb", + "spec/plugins/publish/hipchat_spec.rb", "spec/plugins/publish/instapaper_spec.rb", - "spec/plugins/publish/mail_spec.rb", + "spec/plugins/publish/memcached_spec.rb", "spec/plugins/publish/pocket_spec.rb", - "spec/plugins/publish/smtp_spec.rb", "spec/plugins/publish/twitter_spec.rb", + "spec/plugins/store/file_spec.rb", "spec/plugins/store/full_text_spec.rb", "spec/plugins/store/permalink_spec.rb", - "spec/plugins/store/target_link_spec.rb", + "spec/plugins/subscription/chan_toru_spec.rb", "spec/plugins/subscription/feed_spec.rb", - "spec/plugins/subscription/google_reader_star_spec.rb", + "spec/plugins/subscription/g_guide_spec.rb", "spec/plugins/subscription/link_spec.rb", + "spec/plugins/subscription/pocket_spec.rb", "spec/plugins/subscription/text_spec.rb", "spec/plugins/subscription/tumblr_spec.rb", + "spec/plugins/subscription/twitter_search_spec.rb", "spec/plugins/subscription/twitter_spec.rb", "spec/plugins/subscription/weather_spec.rb", + "spec/plugins/subscription/xml_spec.rb", "spec/spec_helper.rb", "spec/user_dir/plugins/store/mock.rb", "test/fixtures/sampleOPML.xml", "test/integration/test_absoluteurl.yml", "test/integration/test_activerecord.yml", "test/integration/test_add_pocket.yml", + "test/integration/test_chan_toru.yml", + "test/integration/test_descriptionlink.yml", + "test/integration/test_fluentd.yml", "test/integration/test_fulltext.yml", + "test/integration/test_google_news.yml", "test/integration/test_googlealert.yml", - "test/integration/test_googlestar.yml", "test/integration/test_hatenabookmark.yml", "test/integration/test_ignore.yml", "test/integration/test_ignore2.yml", @@ -120,71 +159,95 @@ Gem::Specification.new do |s| "test/integration/test_instapaper.yml", "test/integration/test_link2local.yml", "test/integration/test_one.yml", + "test/integration/test_pocket.yml", "test/integration/test_rand.yml", + "test/integration/test_sanitize.yml", "test/integration/test_sort.yml", "test/integration/test_svnlog.yml", "test/integration/test_text2feed.yml", "test/integration/test_tumblr2local.yml", + "test/integration/test_twitter_search.yml", "test/integration/test_weather.yml", + "test/integration/test_xml2fluentd.yml", "vendor/.gitkeep" ] s.homepage = "http://github.com/automaticruby/automaticruby" s.licenses = ["GPL"] - s.require_paths = ["lib"] - s.rubygems_version = "1.8.24" + s.rubygems_version = "2.4.5" s.summary = "Automatic Ruby" if s.respond_to? :specification_version then - s.specification_version = 3 + s.specification_version = 4 if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then - s.add_runtime_dependency(%q, [">= 0"]) - s.add_runtime_dependency(%q, ["~> 3"]) - s.add_runtime_dependency(%q, [">= 0"]) - s.add_runtime_dependency(%q, ["~> 3"]) - s.add_runtime_dependency(%q, ["~> 3"]) - s.add_runtime_dependency(%q, [">= 0"]) - s.add_runtime_dependency(%q, [">= 0"]) - s.add_runtime_dependency(%q, [">= 0"]) - s.add_runtime_dependency(%q, [">= 0"]) - s.add_runtime_dependency(%q, [">= 0"]) - s.add_runtime_dependency(%q, [">= 0"]) - s.add_runtime_dependency(%q, [">= 0"]) + s.add_runtime_dependency(%q, ["~> 1.8.1"]) + s.add_runtime_dependency(%q, ["~> 1.3.10"]) + s.add_runtime_dependency(%q, ["~> 4.1.8"]) + s.add_runtime_dependency(%q, ["~> 4.1.8"]) + s.add_runtime_dependency(%q, ["~> 3.3.2"]) + s.add_runtime_dependency(%q, ["~> 0.1.2"]) + s.add_runtime_dependency(%q, ["~> 1.1.4"]) + s.add_runtime_dependency(%q, ["~> 0.9.5"]) + s.add_runtime_dependency(%q, ["~> 1.6.5"]) + s.add_runtime_dependency(%q, ["~> 3.1.0"]) + s.add_runtime_dependency(%q, ["~> 0.8.9"]) + s.add_runtime_dependency(%q, ["~> 4.8.0"]) + s.add_runtime_dependency(%q, ["~> 0.0.5"]) + s.add_runtime_dependency(%q, ["~> 0.1.7"]) + s.add_runtime_dependency(%q, ["~> 1.4.0"]) + s.add_runtime_dependency(%q, ["~> 0.4.7"]) + s.add_runtime_dependency(%q, ["~> 2.7.2"]) + s.add_runtime_dependency(%q, ["~> 1.60.2"]) s.add_development_dependency(%q, [">= 0"]) s.add_development_dependency(%q, [">= 0"]) + s.add_development_dependency(%q, [">= 0"]) s.add_development_dependency(%q, [">= 0"]) else - s.add_dependency(%q, [">= 0"]) - s.add_dependency(%q, ["~> 3"]) - s.add_dependency(%q, [">= 0"]) - s.add_dependency(%q, ["~> 3"]) - s.add_dependency(%q, ["~> 3"]) - s.add_dependency(%q, [">= 0"]) - s.add_dependency(%q, [">= 0"]) - s.add_dependency(%q, [">= 0"]) - s.add_dependency(%q, [">= 0"]) - s.add_dependency(%q, [">= 0"]) - s.add_dependency(%q, [">= 0"]) - s.add_dependency(%q, [">= 0"]) + s.add_dependency(%q, ["~> 1.8.1"]) + s.add_dependency(%q, ["~> 1.3.10"]) + s.add_dependency(%q, ["~> 4.1.8"]) + s.add_dependency(%q, ["~> 4.1.8"]) + s.add_dependency(%q, ["~> 3.3.2"]) + s.add_dependency(%q, ["~> 0.1.2"]) + s.add_dependency(%q, ["~> 1.1.4"]) + s.add_dependency(%q, ["~> 0.9.5"]) + s.add_dependency(%q, ["~> 1.6.5"]) + s.add_dependency(%q, ["~> 3.1.0"]) + s.add_dependency(%q, ["~> 0.8.9"]) + s.add_dependency(%q, ["~> 4.8.0"]) + s.add_dependency(%q, ["~> 0.0.5"]) + s.add_dependency(%q, ["~> 0.1.7"]) + s.add_dependency(%q, ["~> 1.4.0"]) + s.add_dependency(%q, ["~> 0.4.7"]) + s.add_dependency(%q, ["~> 2.7.2"]) + s.add_dependency(%q, ["~> 1.60.2"]) s.add_dependency(%q, [">= 0"]) s.add_dependency(%q, [">= 0"]) + s.add_dependency(%q, [">= 0"]) s.add_dependency(%q, [">= 0"]) end else - s.add_dependency(%q, [">= 0"]) - s.add_dependency(%q, ["~> 3"]) - s.add_dependency(%q, [">= 0"]) - s.add_dependency(%q, ["~> 3"]) - s.add_dependency(%q, ["~> 3"]) - s.add_dependency(%q, [">= 0"]) - s.add_dependency(%q, [">= 0"]) - s.add_dependency(%q, [">= 0"]) - s.add_dependency(%q, [">= 0"]) - s.add_dependency(%q, [">= 0"]) - s.add_dependency(%q, [">= 0"]) - s.add_dependency(%q, [">= 0"]) + s.add_dependency(%q, ["~> 1.8.1"]) + s.add_dependency(%q, ["~> 1.3.10"]) + s.add_dependency(%q, ["~> 4.1.8"]) + s.add_dependency(%q, ["~> 4.1.8"]) + s.add_dependency(%q, ["~> 3.3.2"]) + s.add_dependency(%q, ["~> 0.1.2"]) + s.add_dependency(%q, ["~> 1.1.4"]) + s.add_dependency(%q, ["~> 0.9.5"]) + s.add_dependency(%q, ["~> 1.6.5"]) + s.add_dependency(%q, ["~> 3.1.0"]) + s.add_dependency(%q, ["~> 0.8.9"]) + s.add_dependency(%q, ["~> 4.8.0"]) + s.add_dependency(%q, ["~> 0.0.5"]) + s.add_dependency(%q, ["~> 0.1.7"]) + s.add_dependency(%q, ["~> 1.4.0"]) + s.add_dependency(%q, ["~> 0.4.7"]) + s.add_dependency(%q, ["~> 2.7.2"]) + s.add_dependency(%q, ["~> 1.60.2"]) s.add_dependency(%q, [">= 0"]) s.add_dependency(%q, [">= 0"]) + s.add_dependency(%q, [">= 0"]) s.add_dependency(%q, [">= 0"]) end end diff --git a/bin/automatic b/bin/automatic index d10cb49..22b126c 100755 --- a/bin/automatic +++ b/bin/automatic @@ -3,8 +3,8 @@ # Name:: Automatic::Ruby # Author:: 774 # Created:: Feb 18, 2012 -# Updated:: Apr 10, 2013 -# Copyright:: 774 Copyright (c) 2012-2013 +# Updated:: Feb 21, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. root_dir = File.expand_path("..", File.dirname(__FILE__)) @@ -66,7 +66,7 @@ subparsers = { 'feedparser' => lambda {|argv| require 'automatic/feed_parser' url = argv.shift || abort_with_usage("feedparser", "") - rss_results = Automatic::FeedParser.get(url) + rss_results = Automatic::FeedParser.get_url(url) pp rss_results }, 'inspect' => lambda {|argv| @@ -74,7 +74,7 @@ subparsers = { url = argv.shift || abort_with_usage("inspect", "") feeds = Feedbag.find(url) pp feeds - rss_results = Automatic::FeedParser.get(feeds.pop) + rss_results = Automatic::FeedParser.get_url(feeds.pop) pp rss_results }, 'opmlparser' => lambda {|argv| diff --git a/doc/ChangeLog b/doc/ChangeLog index 40388fc..755dd80 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,3 +1,109 @@ +=== 14.12.0 / 2014-12-27 + +* Rewrite obsolete methods. + +* Update Gemfile. + + +=== 14.10.0 / 2014-10-23 + +* Using Logger + +* Improvement File Saving Plugin + + * Store::FullText will ignore duplicated titled entry. + +* Added new plugins. + + * Filter::Clear + * Filter::GoogleNews + * Filter::DescriptionLink + + +=== 14.5.0 / 2014-05-31 + +* Update to activesupport/record 4.1. + + +=== 14.3.0 / 2014-03-23 + +* Downgrade nokogiri version for OS X. + + +=== 14.2.0 / 2014-02-26 + +* Added new plugins. + + * Publish::AmazonS3 + +* Improvement File Saving Plugin + + * Store::TargetLink rename to Store::File. + * Store::File rewrite link to file URI scheme. + * Enable file URI scheme in RSS 2.0 Feed. + * Store::File enable get file from Amazon S3. + +* Improvement Subscription::Text + + * Read text from TSV formatted files. + * Add fields description, author, comments to RSS 2.0 Feed. + +* Into details Log messages. + +* Use 'double' instead of 'mock', 'stub' for rspec-mocks. + +* Improvement CI with network connection. + +* Refactoring to divide Feed::Parser to Feed::Maker. + + +=== 14.1.0 / 2014-01-24 + +* Fix dependency conflict. + +* Refactoring error retry handling. + + +=== 13.12.0 / 2013-12-17 + +* Fix twitter gem version. + + +=== 13.7.0 / 2013-07-28 + +* Added new plugins. + + * Subscription::Xml + * Provide::Fluentd + +* Fix some problems. + + +=== 13.6.0 / 2013-06-28 + +* Added new plugins. + + * Subscription::Pocket + * Subscription::TwitterSearch + * Subscription::GGuide + * Subscription::ChanToru + * Filter::Accept + * Filter::Sanitize + * Filter::GithubFeed + * Publish::ConsoleLink + * Publish::Hipchat + * Publish::Memcached + * Publish::Fluentd + +* And other Improvements, bug fixes. + +* Correspond to activesupport & activerecord 4.0. + + * Mail plugins temporary removed (for actionmailer 4.0). + +* Ruby 1.8 support was outdated. + + === 13.5.0 / 2013-05-18 * Added new plugins. diff --git a/doc/PLUGINS b/doc/PLUGINS index 0d92393..03ae7f7 100644 --- a/doc/PLUGINS +++ b/doc/PLUGINS @@ -38,42 +38,43 @@ SubscriptionLink retry: RETRY_COUNT -SubscriptionTumblr ------------------- +SubscriptionXml +--------------- [Path] - /plugins/subscription/tumblr.rb + /plugins/subscription/xml.rb [Abstract] - Subscribe tumblr and extract the links. - If pages specified, including past pages. + Subscribe XML API with GET method. + This plug-in store encoded XML in content_encoded of RSS feed. [Syntax] - - module: SubscriptionTumblr + - module: SubscriptionXml config: urls: - URL - - URL ... - pages: PAGES + - URL ... (XML base API) interval: INTERVAL_FOR_SCRAPING (in seconds.) retry: RETRY_COUNT -SubscriptionGoogleReaderStar ----------------------------- +SubscriptionTumblr +------------------ [Path] - /plugins/subscription/google_reader_star.rb + /plugins/subscription/tumblr.rb [Abstract] - Subscribe to Google reader star feed. + Subscribe tumblr and extract the links. + If pages specified, including past pages. [Syntax] - - module: SubscriptionGoogleReaderStar + - module: SubscriptionTumblr config: - feeds: - - FEED - - FEED ... + urls: + - URL + - URL ... + pages: PAGES + interval: INTERVAL_FOR_SCRAPING (in seconds.) retry: RETRY_COUNT - interval: INTERVAL_FOR_RETRY (in seconds.) SubscriptionTwitter @@ -94,28 +95,25 @@ SubscriptionTwitter retry: RETRY_COUNT -SubscriptionText ----------------- +SubscriptionTwitterSearch +------------------------- [Path] - /plugins/subscription/text.rb + /plugins/subscription/twitter_search.rb [Abstract] - Create new feeds from urls, titles, feeds on text. + The search results of a twitter are acquired. [Syntax] - - module: SubscriptionText + - module: SubscriptionTwitterSearch config: - titles: - - 'title' - urls: - - 'url' - feeds: - - - title: 'title' - url: 'url' - - - title: 'title' - url: 'url' + search: search word + opt: option + consumer_key: 'your_consumer_key' + consumer_secret: 'your_consumer_secret' + oauth_token: 'your_oauth_token' + oauth_token_secret: 'your_oauth_token_secret' + interval: INTERVAL_FOR_SCRAPING (in seconds.) + retry: RETRY_COUNT SubscriptionWeather @@ -135,8 +133,36 @@ SubscriptionWeather retry: RETRY_COUNT -SubscriptionPocket +SubscriptionText ---------------- +[Path] + /plugins/subscription/text.rb + +[Abstract] + Create new feeds from urls, titles, feeds on text. + Reading data from TSV (sep = '\t') formatted files. + +[Syntax] + - module: SubscriptionText + config: + titles: + - 'title' + urls: + - 'url' + feeds: + - + title: 'title' + url: 'url' + - + title: 'title' + url: 'url' + files: + - 'somefile1.tsv' + - 'somefile2.tsv' + + +SubscriptionPocket +------------------ [Path] /plugins/subscription/pocket.rb @@ -155,6 +181,53 @@ SubscriptionPocket retry: RETRY_COUNT +SubscriptionGGuide +------------------ +[Path] + /plugins/subscription/g_guide.rb + +[Abstract] + search keyword to the G guide (information on a TV program) rss + +[Syntax] + - module: SubscriptionGGuide + config: + keyword: 'anime,shogi' # search word (',' dividing and coming out two or more search) + station: '地上波' # (It means terrestrial broadcasting in Japanese.) + interval: INTERVAL_FOR_SCRAPING (in seconds.) + retry: RETRY_COUNT + + +SubscriptionChanToru +-------------------- +[Path] + /plugins/subscription/chan_toru.rb + +[Abstract] + Search keyword to CHAN-TORU (information on a TV program) rss + (SubscriptionGGuide Plug-in dependence) + +[Syntax] + - module: SubscriptionChanToru + config: + keyword: 'anime' # search word + station: '地上波' # (It means terrestrial broadcasting in Japanese.) + interval: INTERVAL_FOR_SCRAPING (in seconds.) + retry: RETRY_COUNT + + +FilterClear +----------- +[Path] + /plugins/filter/clear.rb + +[Abstract] + Clear Pipeline. + +[Syntax] + - module: FilterClear + + FilterSort ---------- [Path] @@ -245,7 +318,7 @@ FilterOne [Syntax] - module: FilterOne config: - pick: last (If none, pick the first item.) + pick: last (If none, pick the first item.) FilterRand @@ -260,6 +333,22 @@ FilterRand - module: FilterRand +FilterSanitize +-------------- +[Path] + /plugins/filter/sanitize.rb + +[Abstract] + Strip the html tags. + Reference: https://github.com/rgrove/sanitize + +[Syntax] + - module: FilterSanitize + config: + mode: basic or relaxed or restricted + (default: restricted) + + FilterFullFeed -------------- [Path] @@ -289,7 +378,7 @@ FilterAbsoluteURI FilterTumblrResize --------------------- +------------------ [Path] /plugins/filter/tumblr_resize.rb @@ -304,6 +393,34 @@ FilterTumblrResize - module: FilterTumblrResize +FilterDescriptionLink +--------------------- +[Path] + /plugins/filter/description_link.rb + +[Abstract] + Pickup http or https URL from description. + Re-write link to the URL. + +[Syntax] + - module: FilterDescriptionLink + config: + clear_description: 1 # Set empty to description. + get_title: 1 # Get title after re-write link. + + +FilterGoogleNews +---------------- +[Path] + /plugins/filter/google_news.rb + +[Abstract] + Re-write Google News link to a new link of the redirect. + +[Syntax] + - module: FilterGoogleNews + + StorePermalink -------------- [Path] @@ -336,21 +453,26 @@ StoreFullText db: DB_NAME -StoreTargetLink ---------------- +StoreFile +--------- [Path] - /plugins/store/target_link.rb + /plugins/store/file.rb [Abstract] - Store target locally. - This emulate "Right click to save". + Store target to local file system. + This emulate "Right click to save" + If scheme begin with "s3n://", get file from Amazon S3. + Re-write URI scheme to file URI scheme and return feeds. [Syntax] - - Module: StoreTargetLink + - Module: StoreFile config: path: SAVE_TO_PATH retry: RETRY_COUNT interval: INTERVAL_FOR_DOWNLOAD (in seconds.) + access_key: ACCESS_KEY_ID (Only using S3) + secret_key: SECRET_ACCESS_KEY (Only using S3) + bucket_name: AMAZONS3_BUCKET_NAME (Only using S3) PublishConsole @@ -365,6 +487,90 @@ PublishConsole - module: PublishConsole +PublishAmazonS3 +--------------- +[Path] + /plugins/publish/amazon_s3.rb + +[Abstract] + Upload file to Amazon S3. + +[Description] + Read file URI scheme and upload it to aws. + Postulate feed link must be file URI scheme. + +[Syntax] + - module: PublishAmazonS3 + config: + access_key: ACCESS_KEY_ID + secret_key: SECRET_ACCESS_KEY + bucket_name: AMAZONS3_BUCKET_NAME + target_path: PATH_ON_BUCKET + + +PublishFluentd +-------------- +[Path] + /plugins/publish/fluentd.rb + +[Abstract] + Output to fluentd with fluent-logger. + +[Description] + This plugin outputs feed to fluentd. + Following fluentd config example. + + + type forward + port 9999 + + +[Syntax] + - module: PublishFluentd + config: + host: HOSTNAME + port: PORT_NUMBER + tag: TAGS (example: automatic.feed) + + +ProvideFluentd +-------------- +[Path] + /plugins/provide/fluentd.rb + +[Abstract] + This plugin output content_encoded to fluentd. + +[Description] + Provide::Fluentd is different from Publish::Fluentd. + + Publish::Fluentd -> Output feed to flulentd. + Provide::Fluentd -> Output content_encoded to fluentd. + +[Syntax] + - module: ProvideFluentd + config: + host: HOSTNAME + port: PORT_NUMBER + tag: TAGS (example: automatic.feed) + + +PublishMemcached +---------------- +[Path] + /plugins/publish/memcached.rb + +[Abstract] + Output feeds converted hash to memcached with dalli. + +[Syntax] + - module: PublishMemcached + config: + host: HOSTNAME + port: PORT_NUMBER + key: KEY + + PublishHatenaBookmark --------------------- [Path] @@ -378,7 +584,7 @@ PublishHatenaBookmark Hatena Bookmark API. [Syntax] - - Modu - module: PublishHatenaBookmark + - module: PublishHatenaBookmark config: username: HATENA_ID passowrd: PASSWORD @@ -406,7 +612,7 @@ PublishTwitter PublishPocket ------------------ +------------- [Path] /plugins/publish/pocket.rb @@ -464,8 +670,22 @@ PublishGoogleCalendar interval: INTERVAL +PublishEject +------------ +[Path] + /plugins/publish/eject.rb + +[Abstract] + eject by the feed link count. + +[Syntax] + - module: PublishEject + config: + interval: INTERVAL + + PublishHipchat ---------------------- +-------------- [Path] /plugins/publish/hipchat.rb diff --git a/doc/PLUGINS.ja b/doc/PLUGINS.ja index 6390c4e..6826bc8 100644 --- a/doc/PLUGINS.ja +++ b/doc/PLUGINS.ja @@ -38,42 +38,44 @@ SubscriptionLink retry: エラー時のリトライ回数 (回数, 省略時 0) -SubscriptionTumblr ------------------- +SubscriptionXml +--------------- [パス] - /plugins/subscription/tumblr.rb + /plugins/subscription/xml.rb [概要] - Tumblr を購読し、リンクを抽出する - pages が指定された場合、過去のページも遡って購読する + XML で提供された API を GET メソッドで購読する + このプラグインはエンコードされた XML を RSS フィードの + content_encoded に保存する [レシピ記法] - - module: SubscriptionTumblr + - module: SubscriptionXml config: urls: - URL - - URL ... - pages: PAGES + - URL ... (XML ベース API) interval: スクレイピングの間隔 (秒数, 省略可) retry: エラー時のリトライ回数 (回数, 省略時 0) -SubscriptionGoogleReaderStar ----------------------------- +SubscriptionTumblr +------------------ [パス] - /plugins/subscription/google_reader_star.rb + /plugins/subscription/tumblr.rb [概要] - Google リーダーのスターフィードを購読する + Tumblr を購読し、リンクを抽出する + pages が指定された場合、過去のページも遡って購読する [レシピ記法] - - module: SubscriptionGoogleReaderStar + - module: SubscriptionTumblr config: - feeds: - - フィード名 - - フィード名 ... + urls: + - URL + - URL ... + pages: PAGES + interval: スクレイピングの間隔 (秒数, 省略可) retry: エラー時のリトライ回数 (回数, 省略時 0) - interval: エラー時のリトライ間隔 (秒数, 省略可) SubscriptionTwitter @@ -94,6 +96,27 @@ SubscriptionTwitter retry: エラー時のリトライ回数 (回数, 省略時 0) +SubscriptionTwitterSearch +------------------------- +[Path] + /plugins/subscription/twitter_search.rb + +[Abstract] + Twitterの検索結果を取得 + +[Syntax] + - module: SubscriptionTwitterSearch + config: + search: 検索ワード + opt: 検索時に使用するオプション + consumer_key: 'your_consumer_key' + consumer_secret: 'your_consumer_secret' + oauth_token: 'your_oauth_token' + oauth_token_secret: 'your_oauth_token_secret' + interval: スクレイピングの間隔 (秒数, 省略可) + retry: エラー時のリトライ回数 (回数, 省略時 0) + + SubscriptionWeather ------------------- [パス] @@ -118,6 +141,7 @@ SubscriptionText [概要] テキストベースの urls, titles, feeds から新規のフィードを作成する + タブ区切りの TSV 形式のファイルからフィードの情報を読み込む [レシピ記法] - module: SubscriptionText @@ -131,12 +155,14 @@ SubscriptionText - title: TITLE url: URL ... - interval: スクレイピングの間隔 (秒数, 省略可) - retry: エラー時のリトライ回数 (回数, 省略時 0) + files: + - 'somefile1.tsv' + - 'somefile2.tsv' ... SubscriptionPocket ----------------- + 検索キーワードからCHAN-TORUページの取得(Gk) +------------------ [パス] /plugins/subscription/pocket.rb @@ -155,6 +181,53 @@ SubscriptionPocket retry: エラー時のリトライ回数 (回数, 省略時 0) +SubscriptionGGuide +------------------ +[パス] + /plugins/subscription/g_guide.rb + +[概要] + 検索キーワードからGガイド(テレビ番組の情報)rssの取得 + +[レシピ記法] + - module: SubscriptionGGuide + config: + keyword: 'アニメ,将棋' # 検索ワード(','区切りで複数検索) + station: '地上波' # 衛星放送の区域(現在は地上波のみ対応) + interval: スクレイピングの間隔 (秒数, 省略可) + retry: エラー時のリトライ回数 (回数, 省略時 0) + + +SubscriptionChanToru +-------------------- +[パス] + /plugins/subscription/chan_toru.rb + +[概要] + 検索キーワードからCHAN-TORU(テレビ番組の情報)rssの取得 + (SubscriptionGGuideプラグイン依存) + +[レシピ記法] + - module: SubscriptionChanToru + config: + keyword: 'アニメ,将棋' # 検索ワード(','区切りで複数検索) + station: '地上波' # 衛星放送の区域(現在は地上波のみ対応) + interval: スクレイピングの間隔 (秒数, 省略可) + retry: エラー時のリトライ回数 (回数, 省略時 0) + + +FilterClear +----------- +[パス] + /plugins/filter/clear.rb + +[概要] + パイプラインをカラにする + +[レシピ記法] + - module: FilterClear + + FilterSort ---------- [パス] @@ -235,6 +308,48 @@ FilterImageSource - module: FilterImageSource +FilterOne +--------- +[パス] + /plugins/filter/one.rb + +[概要] + アイテムをフィードから 1 件だけ取り出す + +[レシピ記法] + - module: FilterOne + config: + pick: last (省略時は先頭から) + + +FilterRand +---------- +[パス] + /plugins/filter/rand.rb + +[概要] + フィードの順番をランダムにする + +[レシピ記法] + - module: FilterRand + + +FilterSanitize +-------------- +[パス] + /plugins/filter/sanitize.rb + +[概要] + HTML タグを除去する + 参照 https://github.com/rgrove/sanitize + +[レシピ記法] + - module: FilterSanitize + config: + mode: basic or relaxed or restricted + (デフォルト restricted) + + FilterFullFeed -------------- [パス] @@ -264,7 +379,7 @@ FilterAbsoluteURI FilterTumblrResize --------------------- +------------------ [パス] /plugins/filter/tumblr_resize.rb @@ -279,30 +394,34 @@ FilterTumblrResize - module: FilterTumblrResize -FilterOne ---------- +FilterDescriptionLink +--------------------- [パス] - /plugins/filter/one.rb + /plugins/filter/description_link.rb [概要] - アイテムをフィードから 1 件だけ取り出す + 日付の昇順にソートする + もし config で sort に desc が指定されると + 昇順ではなく降順にソートする [レシピ記法] - - module: FilterOne + - module: FilterDescriptionLink config: - pick: last (省略時は先頭から) + clear_description: 1 # Set empty to description. + get_title: 1 # Get title after re-write link. -FilterRand ----------- +FilterGoogleNews +---------------- [パス] - /plugins/filter/rand.rb + /plugins/filter/google_news.rb [概要] - フィードの順番をランダムにする + Google News のリンクを + リダイレクト先のものに書き換える [レシピ記法] - - module: FilterRand + - module: FilterGoogleNews StorePermalink @@ -337,24 +456,29 @@ StoreFullText db: SQLite3 DB 名 -StoreTargetLink --------------- +StoreFile +--------- [パス] - /plugins/store/target_link.rb + /plugins/store/file.rb [概要] - リンク先のコンテンツをローカルに保存する + リンク先のコンテンツをローカルファイルシステムに保存する + s3n:// で始まる場合は Amazon S3 からファイルを取得する + リンクを保存したファイルのパス (file スキーム) に書き換えてフィードを返す [レシピ記法] - - module: StoreTargetLink + - module: StoreFile config: path: 保存先のフォルダ retry: エラー時のリトライ回数 (回数, 省略時 0) interval: 保存する間隔 (秒) + access_key: アクセスキー (S3 使用時のみ) + secret_key: シークレットアクセスキー (S3 使用時のみ) + bucket_name: S3 のバケット名 (S3 使用時のみ) PublishConsole ------------- +-------------- [パス] /plugins/publish/console.rb @@ -365,6 +489,90 @@ PublishConsole - module: PublishConsole +PublishAmazonS3 +--------------- +[パス] + /plugins/publish/amazon_s3.rb + +[概要] + Amazon S3 にファイルをアップロードする + +[説明] + フィードのリンクにあるファイルを Amazon S3 にアップする + リンクが file URI スキームであることが前提である + +[レシピ記法] + - module: PublishAmazonS3 + config: + access_key: アクセスキー + secret_key: シークレットアクセスキー + bucket_name: S3 のバケット名 + target_path: バケット上でのファイルのパス + + +PublishFluentd +-------------- +[パス] + /plugins/publish/fluentd.rb + +[概要] + fluentd にフィードを出力する + +[説明] + このプラグインは fluentd にフィードを出力する + 受け手側では以下のように fluentd を設定し起動しておく + + + type forward + port 9999 + + +[レシピ記法] + - module: PublishFluentd + config: + host: ホスト名 + port: ポート番号 + tag: タグ (例 automatic.feed) + + +ProvideFluentd +-------------- +[パス] + /plugins/provide/fluentd.rb + +[概要] + fluentd に content_encoded の内容を出力する + +[説明] + このプラグインは Publish::Fluentd とは異なる + + Publish::Fluentd -> フィード全体を fluentd に出力 + Provide::Fluentd -> content_encoded の内容を fluentd に出力 + +[レシピ記法] + - module: ProvideFluentd + config: + host: ホスト名 + port: ポート番号 + tag: タグ (例 automatic.feed) + + +PublishMemcached +---------------- +[パス] + /plugins/publish/memcached.rb + +[概要] + フィードをハッシュに変換して memcached に出力する + +[レシピ記法] + - module: PublishMemcached + config: + host: ホスト名 + port: ポート番号 + key: 出力するキーの名称 + + PublishHatenaBookmark --------------------- [パス] @@ -406,7 +614,7 @@ PublishTwitter PublishPocket ------------------ +------------- [パス] /plugins/publish/pocket.rb @@ -464,8 +672,22 @@ PublishGoogleCalendar interval: 複数の予定の投稿間隔 (秒) +PublishEject +------------ +[パス] + /plugins/publish/eject.rb + +[概要] + Feedのリンク個数分だけeject + +[レシピ記法] + - module: PublishEject + config: + interval: 複数の予定の投稿間隔 (秒) + + PublishHipchat ---------------------- +-------------- [パス] /plugins/publish/hipchat.rb diff --git a/doc/README b/doc/README index 9ceb898..f19fa39 100644 --- a/doc/README +++ b/doc/README @@ -1,7 +1,7 @@ automaticruby Name - Automatic Ruby - Ruby General Automation Framework + Automatic Ruby - Ruby framework for the general-purpose automatic processing Syntax $ automatic @@ -112,7 +112,7 @@ plugins: - module: FilterTumblrResize - - module: StoreTargetLink + - module: StoreFile config: path: /Users/yourname/Desktop/ interval: 1 @@ -121,7 +121,7 @@ In the case of sample this recipe. 1. Subscribe to RSS feeds of Tumblr by SubScriptionFeed. 2. Save permalink to database by StorePermalink. 3. Specify the URL of the image by FilterImage and FilterTumblrResize. -4. Downloading image file of Tumblr by StoreTargetLink. +4. Downloading image file of Tumblr by StoreFile. Showing another example as follows. @@ -496,7 +496,7 @@ TODO Environment =========== - Ruby 1.8 - 2.0. + After Ruby 1.9. Depend on the packages that described in Gemfile. diff --git a/doc/README.ja b/doc/README.ja index 0a47933..3e4875f 100644 --- a/doc/README.ja +++ b/doc/README.ja @@ -117,7 +117,7 @@ plugins: - module: FilterTumblrResize - - module: StoreTargetLink + - module: StoreFile config: path: /Users/yourname/Desktop/ interval: 1 @@ -126,7 +126,7 @@ plugins: 1. Subscription Feed で Tumblr のフィードを購読し 2. StorePermalink でパーマリンクをデータベースに保存し 3 FilterImage と FilterTumblrResize で画像の URL を指定して -4. StoreTargetLink で Tumblr の画像をダウンロードする +4. StoreFile で Tumblr の画像をダウンロードする という一連の処理をプラグインの組み合わせで実現している。 @@ -504,7 +504,7 @@ TODO 動作環境 ======== -Ruby 1.8 - 2.0 +Ruby 1.9 以降 Gemfile に記述された gem パッケージに依存 diff --git a/lib/automatic.rb b/lib/automatic.rb index bba9ee9..07408f1 100755 --- a/lib/automatic.rb +++ b/lib/automatic.rb @@ -2,8 +2,8 @@ # Name:: Automatic::Ruby # Author:: 774 # Created:: Feb 18, 2012 -# Updated:: May 19, 2013 -# Copyright:: 774 Copyright (c) 2012-2013 +# Updated:: Feb 21, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. module Automatic @@ -12,8 +12,9 @@ module Automatic require 'automatic/pipeline' require 'automatic/log' require 'automatic/feed_parser' + require 'automatic/feed_maker' + require 'automatic/version' - VERSION = "13.5.0-devel" USER_DIR = "/.automatic" class << self diff --git a/lib/automatic/feed_maker.rb b/lib/automatic/feed_maker.rb new file mode 100644 index 0000000..eacacad --- /dev/null +++ b/lib/automatic/feed_maker.rb @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::FeedMaker +# Author:: 774 +# Created:: Feb 21, 2014 +# Updated:: Feb 26, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. +# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. + +module Automatic + module FeedMaker + require 'rss' + require 'uri' + require 'nokogiri' + + class FeedObject + attr_accessor :title, :link, :description, :author, :comments + def initialize + @link = 'http://dummy' + @title = 'dummy' + @description = '' + @author = '' + @comments = '' + end + end + + def self.generate_feed(feed) + feed_object = FeedObject.new + feed_object.title = feed['title'] unless feed['title'].nil? + feed_object.link = feed['url'] unless feed['url'].nil? + feed_object.description = feed['description'] unless feed['description'].nil? + feed_object.author = feed['author'] unless feed['author'].nil? + feed_object.comments = feed['comments'] unless feed['comments'].nil? + feed_object + end + + def self.create_pipeline(feeds = []) + RSS::Maker.make("2.0") {|maker| + xss = maker.xml_stylesheets.new_xml_stylesheet + maker.channel.title = "Automatic Ruby" + maker.channel.description = "Automatic::FeedMaker" + maker.channel.link = "https://github.com/automaticruby/automaticruby" + maker.items.do_sort = true + + unless feeds.nil? + feeds.each {|feed| + unless feed.link.nil? + Automatic::Log.puts("info", "Create Pipeline: #{feed.link}") + item = maker.items.new_item + item.title = feed.title + item.link = feed.link + begin + item.description = feed.description + item.author = feed.author + item.comments = feed.comments + item.date = feed.pubDate || Time.now + rescue NoMethodError + Automatic::Log.puts("warn", "Undefined field detected in feed.") + end + end + } + end + } + end + + def self.content_provide(url, data) + RSS::Maker.make("2.0") {|maker| + xss = maker.xml_stylesheets.new_xml_stylesheet + maker.channel.title = "Automatic Ruby" + maker.channel.description = "Automatic::FeedMaker" + maker.channel.link = "https://github.com/automaticruby/automaticruby" + maker.items.do_sort = true + item = maker.items.new_item + item.title = "Automatic Ruby" + item.link = url + item.content_encoded = data + item.date = Time.now + } + end + end +end diff --git a/lib/automatic/feed_parser.rb b/lib/automatic/feed_parser.rb index 500d13e..15c749b 100644 --- a/lib/automatic/feed_parser.rb +++ b/lib/automatic/feed_parser.rb @@ -2,8 +2,8 @@ # Name:: Automatic::FeedParser # Author:: 774 # Created:: Feb 19, 2012 -# Updated:: Jan 8, 2013 -# Copyright:: 774 Copyright (c) 2012-2013 +# Updated:: Feb 26, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. module Automatic @@ -12,10 +12,10 @@ module FeedParser require 'uri' require 'nokogiri' - def self.get(url) + def self.get_url(url) begin unless url.nil? - Automatic::Log.puts("info", "Parsing: #{url}") + Automatic::Log.puts("info", "Parsing Feed: #{url}") feed = URI.parse(url).normalize open(feed) {|http| response = http.read @@ -27,40 +27,11 @@ def self.get(url) end end - def self.create(feeds = []) + def self.parse_html(html) RSS::Maker.make("2.0") {|maker| xss = maker.xml_stylesheets.new_xml_stylesheet maker.channel.title = "Automatic Ruby" - maker.channel.description = "Automatic Ruby" - maker.channel.link = "https://github.com/automaticruby/automaticruby" - maker.items.do_sort = true - - unless feeds.nil? - feeds.each {|feed| - unless feed.link.nil? - Automatic::Log.puts("info", "Creating: #{feed.link}") - item = maker.items.new_item - item.title = feed.title - item.link = feed.link - begin - item.description = feed.description - item.author = feed.author - item.comments = feed.comments - item.date = feed.pubDate || Time.now - rescue NoMethodError - Automatic::Log.puts("warn", "Undefined field detected in feed.") - end - end - } - end - } - end - - def self.parse(html) - RSS::Maker.make("2.0") {|maker| - xss = maker.xml_stylesheets.new_xml_stylesheet - maker.channel.title = "Automatic Ruby" - maker.channel.description = "Automatic Ruby" + maker.channel.description = "Automatic::FeedParser" maker.channel.link = "https://github.com/automaticruby/automaticruby" maker.items.do_sort = true @@ -76,6 +47,5 @@ def self.parse(html) } } end - end end diff --git a/lib/automatic/log.rb b/lib/automatic/log.rb index 930072b..248d0be 100644 --- a/lib/automatic/log.rb +++ b/lib/automatic/log.rb @@ -2,10 +2,12 @@ # Name:: Automatic::Log # Author:: 774 # Created:: Feb 20, 2012 -# Updated:: Mar 11, 2012 -# Copyright:: 774 Copyright (c) 2012 +# Updated:: Oct 09, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. +require 'logger' + module Automatic module Log LOG_LEVELS = ['info', 'warn', 'error', 'none'] @@ -14,13 +16,16 @@ def self.level(level) @level = level end - def self.puts(level, message) + def self.logger + @logger ||= Logger.new(STDOUT) + end + + def self.puts(level = :info, message) if LOG_LEVELS.index(@level).to_i > LOG_LEVELS.index(level).to_i return end - t = Time.now.strftime("%Y/%m/%d %X") - print log = "#{t} [#{level}] #{message}\n" - return log + logger.send(level, message) end + end end diff --git a/lib/automatic/pipeline.rb b/lib/automatic/pipeline.rb index c34efff..6f29e4d 100644 --- a/lib/automatic/pipeline.rb +++ b/lib/automatic/pipeline.rb @@ -2,11 +2,13 @@ # Name:: Automatic::Pipeline # Author:: 774 # Created:: Feb 22, 2012 -# Updated:: Mar 10, 2012 -# Copyright:: 774 Copyright (c) 2012 +# Updated:: Mar 16, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. +require 'active_support' require 'active_support/core_ext' +require 'active_support/deprecation' module Automatic module Plugin end diff --git a/lib/automatic/recipe.rb b/lib/automatic/recipe.rb index dd46ac0..bfc4242 100644 --- a/lib/automatic/recipe.rb +++ b/lib/automatic/recipe.rb @@ -4,7 +4,7 @@ # 774 # Created:: Feb 18, 2012 # Updated:: Jun 14, 2012 -# Copyright:: 774 Copyright (c) 2012 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require 'hashie' @@ -24,7 +24,7 @@ def load_recipe(path) @procedure = Hashie::Mash.new(YAML.load(File.read(path))) log_level = @procedure.global && @procedure.global.log && @procedure.global.log.level Automatic::Log.level(log_level) - Automatic::Log.puts("info", "Loading: #{path}") + Automatic::Log.puts("info", "Loading Recipe: #{path}") @procedure end diff --git a/lib/automatic/version.rb b/lib/automatic/version.rb new file mode 100644 index 0000000..791778c --- /dev/null +++ b/lib/automatic/version.rb @@ -0,0 +1,3 @@ +module Automatic + VERSION = "14.12.2-devel" +end diff --git a/plugins/custom_feed/svn_log.rb b/plugins/custom_feed/svn_log.rb index f1a1f94..6210a01 100644 --- a/plugins/custom_feed/svn_log.rb +++ b/plugins/custom_feed/svn_log.rb @@ -3,7 +3,7 @@ # Author:: kzgs # Created:: Feb 29, 2012 # Updated:: Mar 3, 2012 -# Copyright:: kzgs Copyright (c) 2012 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require 'rss/maker' @@ -19,12 +19,12 @@ def initialize(config, pipeline=[]) def run revisions = XmlSimple.xml_in(`svn log #{svn_log_argument}`)["logentry"] - @pipeline << RSS::Maker.make("1.0") { |maker| + @pipeline << RSS::Maker.make("1.0") {|maker| maker.channel.title = @config["title"] || "" maker.channel.about = "" maker.channel.description = "" maker.channel.link = base_url - revisions.each { |rev| + revisions.each {|rev| item = maker.items.new_item item.title = "#{rev["msg"]} by #{rev["author"]}" item.link = base_url+"/!svn/bc/#{rev["revision"]}" diff --git a/plugins/filter/absolute_uri.rb b/plugins/filter/absolute_uri.rb index 85c9c44..12f9a4d 100644 --- a/plugins/filter/absolute_uri.rb +++ b/plugins/filter/absolute_uri.rb @@ -2,8 +2,8 @@ # Name:: Automatic::Plugin::Filter::AbsoluteURI # Author:: 774 # Created:: Jun 20, 2012 -# Updated:: Apr 5, 2013 -# Copyright:: 774 Copyright (c) 2012-2013 +# Updated:: Oct 29, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. module Automatic::Plugin @@ -37,7 +37,7 @@ def rewrite(string) @config['url'] = @config['url'] + '/' end string = @config['url'] + string.sub(/^\./,'').sub(/^\//,'') - string = URI.encode(string) + string = URI::Parser.new.escape(string) return string end end diff --git a/plugins/filter/accept.rb b/plugins/filter/accept.rb index ceeff6b..e2e901d 100644 --- a/plugins/filter/accept.rb +++ b/plugins/filter/accept.rb @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- # Name:: Automatic::Plugin::Filter::Accept # Author:: soramugi +# 774 # Created:: Jun 4, 2013 -# Updated:: Jun 4, 2013 -# Copyright:: soramugi Copyright (c) 2013 +# Updated:: Feb 21, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. module Automatic::Plugin @@ -22,7 +23,7 @@ def run new_feeds << items if contain(items) == true } end - @return_feeds << Automatic::FeedParser.create(new_feeds) if new_feeds.length > 0 + @return_feeds << Automatic::FeedMaker.create_pipeline(new_feeds) if new_feeds.length > 0 } @return_feeds end diff --git a/plugins/filter/clear.rb b/plugins/filter/clear.rb new file mode 100644 index 0000000..6c3951d --- /dev/null +++ b/plugins/filter/clear.rb @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::Plugin::Filter::Clear +# Author:: 774 +# Created:: Oct 20, 2014 +# Updated:: Oct 20, 2014 +# Copyright:: Copyright (c) 2014 Automatic Ruby Developers. +# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. + +module Automatic::Plugin + class FilterClear + def initialize(config, pipeline=[]) + @config = config + @pipeline = pipeline + end + + def run + [] + end + end +end diff --git a/plugins/filter/description_link.rb b/plugins/filter/description_link.rb new file mode 100644 index 0000000..7f10b44 --- /dev/null +++ b/plugins/filter/description_link.rb @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::Plugin::Filter::DescriptionLink +# Author:: 774 +# Created:: Oct 03, 2014 +# Updated:: Oct 16, 2014 +# Copyright:: Copyright (c) 2014 Automatic Ruby Developers. +# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. + +module Automatic::Plugin + class FilterDescriptionLink + require 'uri' + require 'nkf' + + def initialize(config, pipeline=[]) + @config = config + @pipeline = pipeline + end + + def run + @return_feeds = [] + @pipeline.each {|feeds| + new_feeds = [] + unless feeds.nil? + feeds.items.each {|feed| + new_feeds << rewrite_link(feed) + } + end + @return_feeds << Automatic::FeedMaker.create_pipeline(new_feeds) + } + @return_feeds + end + + private + + def get_title(url) + new_title = nil + if url.class == String + url.gsub!(Regexp.new("[^#{URI::PATTERN::ALNUM}\/\:\?\=&~,\.\(\)#]")) {|match| ERB::Util.url_encode(match)} + begin + read_data = NKF.nkf("--utf8", open(url).read) + get_text = Nokogiri::HTML.parse(read_data, nil, 'utf8').xpath('//title').text + new_title = get_text if get_text.class == String + rescue + Automatic::Log.puts("warn", "Failed in get title for: #{url}") + end + end + + new_title + end + + def rewrite_link(feed) + new_link = URI.extract(feed.description, %w{http https}).uniq.last + feed.link = new_link unless new_link.nil? + + if @config.class == Hash + if @config['clear_description'] == 1 + feed.description = "" + end + + if @config['get_title'] == 1 + begin + new_title = get_title(feed.link) + feed.title = new_title unless new_title.nil? + rescue OpenURI::HTTPError + Automatic::Log.puts("warn", "404 Not Found in get title process.") + end + end + end + + feed + end + end +end + diff --git a/plugins/filter/full_feed.rb b/plugins/filter/full_feed.rb index 3b7ad17..c667afb 100644 --- a/plugins/filter/full_feed.rb +++ b/plugins/filter/full_feed.rb @@ -3,9 +3,8 @@ # Author:: progd # 774 # Created:: Apr 29, 2012 -# Updated:: Apr 5, 2013 -# Copyright:: progd -# 774 Copyright (c) 2012-2013 +# Updated:: Jun 23, 2013 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. module Automatic::Plugin @@ -38,7 +37,7 @@ def run def get_siteinfo Automatic::Log.puts(:info, "Loading siteinfo from #{@config['siteinfo']}") - siteinfo = JSON.load(open(File.join(assets_dir, @config['siteinfo'])).read) + siteinfo = JSON.load(open(File.join(assets_dir, @config['siteinfo'])).read.force_encoding("UTF-8")) siteinfo.select! { |info| SITEINFO_TYPES.include? (info['data']['type']) } siteinfo.sort! { |a, b| atype, btype = a['data']['type'], b['data']['type'] @@ -58,7 +57,7 @@ def assets_dir def fulltext(feed) return feed unless feed.link - @siteinfo.each { |info| + @siteinfo.each {|info| begin if feed.link.match(info['data']['url']) Automatic::Log.puts(:info, "Siteinfo matched: #{info['data']['url']}") diff --git a/plugins/filter/github_feed.rb b/plugins/filter/github_feed.rb index d852eb7..a16ef7d 100644 --- a/plugins/filter/github_feed.rb +++ b/plugins/filter/github_feed.rb @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- # Name:: Automatic::Plugin::Filter::GithubFeed # Author:: Kohei Hasegawa -# Created:: Jun 6, 2013 -# Updated:: Jun 6, 2013 -# Copyright:: Kohei Hasegawa Copyright (c) 2013 +# 774 +# Created:: Jun 6, 2013 +# Updated:: Feb 21, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. module Automatic::Plugin @@ -17,20 +18,20 @@ def initialize(config, pipeline=[]) def run @return_feeds = [] @pipeline.each {|feeds| - dummyFeeds = [] + new_feeds = [] unless feeds.nil? feeds.items.each {|feed| Automatic::Log.puts("info", "Invoked: FilterGithubFeed") - dummy = Hashie::Mash.new - dummy.title = feed.title.content - dummy.link = feed.id.content - dummy.description = feed.content.content - dummyFeeds << dummy + hashie = Hashie::Mash.new + hashie.title = feed.title.content + hashie.link = feed.id.content + hashie.description = feed.content.content + new_feeds << hashie } end - @return_feeds << Automatic::FeedParser.create(dummyFeeds) + @return_feeds << Automatic::FeedMaker.create_pipeline(new_feeds) } - @pipeline = @return_feeds + @return_feeds end end end diff --git a/plugins/filter/google_news.rb b/plugins/filter/google_news.rb new file mode 100644 index 0000000..8bd5ec7 --- /dev/null +++ b/plugins/filter/google_news.rb @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::Plugin::Filter::GoogleNews +# Author:: 774 +# Created:: Oct 12, 2014 +# Updated:: Oct 14, 2014 +# Copyright:: Copyright (c) 2014 Automatic Ruby Developers. +# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. + +module Automatic::Plugin + class FilterGoogleNews + require 'uri' + require 'nkf' + + def initialize(config, pipeline=[]) + @config = config + @pipeline = pipeline + end + + def run + @return_feeds = [] + @pipeline.each {|feeds| + new_feeds = [] + unless feeds.nil? + feeds.items.each {|feed| + new_feeds << rewrite_link(feed) unless feed.link.nil? + } + end + @return_feeds << Automatic::FeedMaker.create_pipeline(new_feeds) + } + @return_feeds + end + + private + + def rewrite_link(feed) + if feed.link.class == String + if feed.link.index("http://news.google.com") + matched = feed.link.match(/(&url=)/) + unless matched.nil? + new_link = matched.post_match + feed.link = new_link unless new_link.nil? + end + end + end + + feed + end + end +end + diff --git a/plugins/filter/ignore.rb b/plugins/filter/ignore.rb index 6057d60..a198a8d 100644 --- a/plugins/filter/ignore.rb +++ b/plugins/filter/ignore.rb @@ -2,8 +2,8 @@ # Name:: Automatic::Plugin::Filter::Ignore # Author:: 774 # Created:: Feb 22, 2012 -# Updated:: Apr 5, 2013 -# Copyright:: 774 Copyright (c) 2012-2013 +# Updated:: Feb 21, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. module Automatic::Plugin @@ -22,37 +22,39 @@ def run new_feeds << items if exclude(items) == false } end - @return_feeds << Automatic::FeedParser.create(new_feeds) if new_feeds.length > 0 + @return_feeds << Automatic::FeedMaker.create_pipeline(new_feeds) if new_feeds.length > 0 } @return_feeds end private + def detect_exclude(item, evaluation, reason) + begin + if item.include?(evaluation) + Automatic::Log.puts("info", "Excluded by #{reason}: #{item}") + return true + end + rescue NoMethodError + Automatic::Log.puts("warn", "Invalid feed detected in ignore process with #{item}") + return false + end + end def exclude(items) detection = false unless @config['title'].nil? @config['title'].each {|e| - if items.title.include?(e.chomp) - detection = true - Automatic::Log.puts("info", "Excluded by title: #{items.link}") - end + detection = true if detect_exclude(items.title, e.chomp, 'title') } end unless @config['link'].nil? @config['link'].each {|e| - if items.link.include?(e.chomp) - detection = true - Automatic::Log.puts("info", "Excluded by link: #{items.link}") - end + detection = true if detect_exclude(items.link, e.chomp, 'link') } end unless @config['description'].nil? @config['description'].each {|e| - if items.description.include?(e.chomp) - detection = true - Automatic::Log.puts("info", "Excluded by description: #{items.link}") - end + detection = true if detect_exclude(items.description, e.chomp, 'description') } end detection diff --git a/plugins/filter/image.rb b/plugins/filter/image.rb index 5a299d9..fae0b3f 100644 --- a/plugins/filter/image.rb +++ b/plugins/filter/image.rb @@ -3,7 +3,7 @@ # Author:: 774 # Created:: Sep 18, 2012 # Updated:: Apr 5, 2013 -# Copyright:: 774 Copyright (c) 2012-2013 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. module Automatic::Plugin diff --git a/plugins/filter/image_source.rb b/plugins/filter/image_source.rb index 016ec83..fe22c3a 100644 --- a/plugins/filter/image_source.rb +++ b/plugins/filter/image_source.rb @@ -2,8 +2,8 @@ # Name:: Automatic::Plugin::Filter::ImageSource # Author:: 774 # Created:: Feb 28, 2012 -# Updated:: Apr 5, 2013 -# Copyright:: 774 Copyright (c) 2012-2013 +# Updated:: Feb 26, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. module Automatic::Plugin @@ -19,30 +19,31 @@ def initialize(config, pipeline=[]) def run @return_feeds = [] @pipeline.each {|feeds| - dummyFeeds = Array.new + new_feeds = Array.new unless feeds.nil? feeds.items.each {|feed| arr = rewrite_link(feed) if arr.length > 0 arr.each {|link| - Automatic::Log.puts("info", "Parsing: #{link}") - dummy = Hashie::Mash.new - dummy.title = 'FilterImageSource' - dummy.link = link - dummyFeeds << dummy + Automatic::Log.puts("info", "Extract Image: #{link}") + hashie = Hashie::Mash.new + hashie.title = 'FilterImageSource' + hashie.link = link + new_feeds << hashie } end } end - @return_feeds << Automatic::FeedParser.create(dummyFeeds) + @return_feeds << Automatic::FeedMaker.create_pipeline(new_feeds) } - @pipeline = @return_feeds + @return_feeds end private + def rewrite_link(feed) array = Array.new - feed.description.scan(/ +# 774 # Created:: May 8, 2013 -# Updated:: May 8, 2013 -# Copyright:: soramugi Copyright (c) 2013 +# Updated:: Feb 21, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. module Automatic::Plugin @@ -27,7 +28,7 @@ def run if item.count == 0 item << feed.items.shift end - @return_feeds << Automatic::FeedParser.create(item) + @return_feeds << Automatic::FeedMaker.create_pipeline(item) end } @return_feeds diff --git a/plugins/filter/rand.rb b/plugins/filter/rand.rb index ffd7e48..36c9239 100644 --- a/plugins/filter/rand.rb +++ b/plugins/filter/rand.rb @@ -3,8 +3,8 @@ # Author:: soramugi # 774 # Created:: Mar 6, 2013 -# Updated:: Mar 7, 2013 -# Copyright:: soramugi Copyright (c) 2013 +# Updated:: Feb 21, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. module Automatic::Plugin @@ -19,7 +19,7 @@ def run @return_feeds = [] @pipeline.each {|feed| unless feed.nil? - @return_feeds << Automatic::FeedParser.create(feed.items.shuffle) + @return_feeds << Automatic::FeedMaker.create_pipeline(feed.items.shuffle) end } @return_feeds diff --git a/plugins/filter/sanitize.rb b/plugins/filter/sanitize.rb new file mode 100644 index 0000000..2d80a74 --- /dev/null +++ b/plugins/filter/sanitize.rb @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::Plugin::Filter::Sanitize +# Author:: 774 +# Created:: Jun 20, 2013 +# Updated:: Feb 21, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. +# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. + +module Automatic::Plugin + class FilterSanitize + require 'sanitize' + + def initialize(config, pipeline=[]) + @config = config + @pipeline = pipeline + case @config['mode'] + when "basic" + @mode = Sanitize::Config::BASIC + when "relaxed" + @mode = Sanitize::Config::RELAXED + else + @mode = Sanitize::Config::RESTRICTED + end + end + + def run + @return_feeds = [] + @pipeline.each {|feeds| + unless feeds.nil? + feeds.items.each {|feed| + feed = sanitize(feed) + } + @return_feeds << feeds + end + } + @return_feeds + end + + private + def sanitize(feed) + begin + feed.description = Sanitize.clean(feed.description, @mode) unless feed.description.nil? + rescue + Automatic::Log.puts("warn", "Undefined field detected in feed.") + end + feed + end + end +end diff --git a/plugins/filter/sort.rb b/plugins/filter/sort.rb index 86556c1..ba65454 100644 --- a/plugins/filter/sort.rb +++ b/plugins/filter/sort.rb @@ -3,7 +3,7 @@ # Author:: 774 # Created:: Mar 23, 2012 # Updated:: Jan 23, 2013 -# Copyright:: 774 Copyright (c) 2012 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. module Automatic::Plugin diff --git a/plugins/filter/tumblr_resize.rb b/plugins/filter/tumblr_resize.rb index 278e2ca..0b1d6e4 100644 --- a/plugins/filter/tumblr_resize.rb +++ b/plugins/filter/tumblr_resize.rb @@ -3,7 +3,7 @@ # Author:: 774 # Created:: Feb 28, 2012 # Updated:: Apr 5, 2013 -# Copyright:: 774 Copyright (c) 2012-2013 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. module Automatic::Plugin diff --git a/plugins/notify/ikachan.rb b/plugins/notify/ikachan.rb index 05d4fb0..c33b706 100644 --- a/plugins/notify/ikachan.rb +++ b/plugins/notify/ikachan.rb @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- +require 'active_support' require 'active_support/core_ext' +require 'active_support/deprecation' module Automatic::Plugin class Ikachan @@ -75,7 +77,7 @@ def run feeds.items.each {|feed| Automatic::Log.puts("info", %Q(Ikachan: [#{feed.link}] sending with params #{ikachan.params.to_s}...)) ikachan.post(feed.link, feed.title) - sleep @config['interval'].to_i unless @config['interval'].nil? + sleep ||= @config['interval'].to_i } end } diff --git a/plugins/provide/fluentd.rb b/plugins/provide/fluentd.rb new file mode 100644 index 0000000..b0256c1 --- /dev/null +++ b/plugins/provide/fluentd.rb @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::Plugin::Provide::Fluentd +# Author:: 774 +# Created:: Jul 12, 2013 +# Updated:: May 16, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. +# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. + +module Automatic::Plugin + class ProvideFluentd + require 'fluent-logger' + + def initialize(config, pipeline=[]) + @config = config + @pipeline = pipeline + @mode = @config['mode'] + @fluentd = Fluent::Logger::FluentLogger.open(nil, + host = @config['host'], + port = @config['port']) unless @mode == 'test' + end + + def run + @pipeline.each {|feeds| + unless feeds.nil? + feeds.items.each {|feed| + begin + @fluentd.post(@config['tag'], feed.content_encoded) unless @mode == 'test' + rescue + Automatic::Log.puts("error", "Fluent::Logger.post failed, the content_encoded of item may be not kind of Hash.") + end + } + end + } + @pipeline + end + end +end diff --git a/plugins/publish/amazon_s3.rb b/plugins/publish/amazon_s3.rb new file mode 100644 index 0000000..b36a50e --- /dev/null +++ b/plugins/publish/amazon_s3.rb @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::Plugin::Publish::AmazonS3 +# Author:: 774 +# Created:: Feb 24, 2014 +# Updated:: Feb 24, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. +# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. + +module Automatic::Plugin + class PublishAmazonS3 + require 'uri' + require 'aws-sdk' + + def initialize(config, pipeline=[]) + @config = config + @pipeline = pipeline + s3 = AWS::S3.new( + :access_key_id => @config['access_key'], + :secret_access_key => @config['secret_key'] + ) + @bucket = s3.buckets[@config['bucket_name']] + @mode = @config['mode'] + end + + def run + @pipeline.each {|feeds| + unless feeds.nil? + feeds.items.each {|feed| + unless feed.link.nil? + begin + uri = URI.parse(feed.link) + if uri.scheme == 'file' + upload_amazons3(uri.path, @config['target_path']) + else + Automatic::Log.puts("warn", "Skip feed due to uri scheme is not file.") + end + rescue + Automatic::Log.puts("error", "Error detected with #{feed.link} in uploading AmazonS3.") + end + end + } + end + } + @pipeline + end + + private + + def upload_amazons3(filename, target_path) + target = File.join(target_path, File.basename(filename)) + object = @bucket.objects[target] + source = Pathname.new(filename) + object.write(source) unless @mode == "test" + Automatic::Log.puts("info", "Uploaded: file #{source} to the bucket #{target} on #{@bucket.name}.") + return source, target + end + end +end diff --git a/plugins/publish/console.rb b/plugins/publish/console.rb index 87dad2c..700d664 100644 --- a/plugins/publish/console.rb +++ b/plugins/publish/console.rb @@ -3,7 +3,7 @@ # Author:: 774 # Created:: Feb 23, 2012 # Updated:: Feb 24, 2012 -# Copyright:: 774 Copyright (c) 2012 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. module Automatic::Plugin @@ -17,9 +17,9 @@ def initialize(config, pipeline=[]) end def run - @pipeline.each { |feeds| + @pipeline.each {|feeds| unless feeds.nil? - feeds.items.each { |feed| + feeds.items.each {|feed| @output.puts("info", feed.pretty_inspect) } end diff --git a/plugins/publish/console_link.rb b/plugins/publish/console_link.rb index 60abbf0..c2e643e 100644 --- a/plugins/publish/console_link.rb +++ b/plugins/publish/console_link.rb @@ -3,7 +3,7 @@ # Author:: soramugi # Created:: Jun 02, 2013 # Updated:: Jun 02, 2013 -# Copyright:: soramugi Copyright (c) 2013 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. module Automatic::Plugin @@ -17,9 +17,9 @@ def initialize(config, pipeline=[]) end def run - @pipeline.each { |feeds| + @pipeline.each {|feeds| unless feeds.nil? - feeds.items.each { |feed| + feeds.items.each {|feed| @output.puts("info", feed.link) } end diff --git a/plugins/publish/eject.rb b/plugins/publish/eject.rb new file mode 100644 index 0000000..5bd6cf8 --- /dev/null +++ b/plugins/publish/eject.rb @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::Plugin::Publish::Eject +# Author:: soramugi +# 774 +# Created:: Jun 9, 2013 +# Updated:: May 16, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. +# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. + +module Automatic::Plugin + class PublishEject + + def initialize(config, pipeline=[]) + @config = config + @pipeline = pipeline + end + + def run + @pipeline.each {|feeds| + unless feeds.nil? + feeds.items.each {|feed| + unless feed.link.nil? + `#{eject_cmd}` + Automatic::Log.puts('info', "Eject: #{feed.link}") + + interval = @config['interval'].to_i unless @config['interval'].nil? unless @config.nil? + sleep ||= @config['interval'].to_i + end + } + end + } + @pipeline + end + + def eject_cmd + if `which eject` != '' # linux + 'eject ; eject -t' + elsif `which drutil` != '' # mac + 'drutil tray eject ; drutil tray close' + end + end + end +end diff --git a/plugins/publish/fluentd.rb b/plugins/publish/fluentd.rb new file mode 100644 index 0000000..8fea52a --- /dev/null +++ b/plugins/publish/fluentd.rb @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::Plugin::Publish::Fluentd +# Author:: 774 +# Created:: Jun 21, 2013 +# Updated:: Feb 25, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. +# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. + +module Automatic::Plugin + class PublishFluentd + require 'fluent-logger' + + def initialize(config, pipeline=[]) + @config = config + @pipeline = pipeline + @mode = @config['mode'] + @fluentd = Fluent::Logger::FluentLogger.open(nil, + host = @config['host'], + port = @config['port']) unless @mode == 'test' + end + + def run + @pipeline.each {|feeds| + unless feeds.nil? + feeds.items.each {|feed| + begin + @fluentd.post(@config['tag'], { + :title => feed.title, + :link => feed.link, + :description => feed.description, + :content => feed.content_encoded, + :created_at => Time.now.strftime("%Y/%m/%d %X") + }) unless @mode == 'test' + rescue + Automatic::Log.puts("warn", "Skip feed due to fault in forward.") + end + } + end + } + @pipeline + end + end +end diff --git a/plugins/publish/google_calendar.rb b/plugins/publish/google_calendar.rb index dca31aa..83c6118 100644 --- a/plugins/publish/google_calendar.rb +++ b/plugins/publish/google_calendar.rb @@ -2,8 +2,8 @@ # Name:: Automatic::Plugin::Publish::Googlecalendar # Author:: 774 # Created:: Feb 24, 2012 -# Updated:: Jan 8, 2013 -# Copyright:: 774 Copyright (c) 2012-2013 +# Updated:: Jan 15, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. module Automatic::Plugin @@ -37,8 +37,8 @@ def add(arg) end end - Automatic::Log.puts("info", "日付 ： #{date}") - Automatic::Log.puts("info", "タイトル ： #{text}") + Automatic::Log.puts("info", "Date ： #{date}") + Automatic::Log.puts("info", "Title ： #{text}") # Register to calendar require 'rubygems' @@ -74,7 +74,7 @@ def run unless feeds.nil? feeds.items.each {|feed| @gc.add('今日 ' + feed.title) - sleep @config['interval'].to_i unless @config['interval'].nil? + sleep ||= @config['interval'].to_i } end } diff --git a/plugins/publish/hatena_bookmark.rb b/plugins/publish/hatena_bookmark.rb index ce9f80c..d7f0ca4 100644 --- a/plugins/publish/hatena_bookmark.rb +++ b/plugins/publish/hatena_bookmark.rb @@ -2,8 +2,8 @@ # Name:: Automatic::Plugin::Publish::HatenaBookmark # Author:: 774 # Created:: Feb 22, 2012 -# Updated:: Mar 7, 2013 -# Copyright:: 774 Copyright (c) 2012-2013 +# Updated:: Jan 15, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. module Automatic::Plugin @@ -89,7 +89,7 @@ def run unless feeds.nil? feeds.items.each {|feed| hb.post(rewrite(feed.link), nil) - sleep @config['interval'].to_i unless @config['interval'].nil? + sleep ||= @config['interval'].to_i } end } diff --git a/plugins/publish/hipchat.rb b/plugins/publish/hipchat.rb index c2ee492..419e038 100644 --- a/plugins/publish/hipchat.rb +++ b/plugins/publish/hipchat.rb @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- # Name:: Automatic::Plugin::Publish::Hipchat # Author:: Kohei Hasegawa -# Created:: Jun 6, 2013 -# Updated:: Jun 6, 2013 -# Copyright:: Kohei Hasegawa Copyright (c) 2013 +# Created:: Jun 6, 2013 +# Updated:: Jan 15, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. module Automatic::Plugin @@ -26,16 +26,17 @@ def run unless feeds.nil? feeds.items.each {|feed| retries = 0 + retry_max = @config['retry'].to_i || 0 begin @client.send(@config['username'], feed.description, @options) - Automatic::Log.puts("info", "post: #{feed.description.gsub(/[\r\n]/,'')[0..50]}...") rescue nil + Automatic::Log.puts("info", "Hipchat post: #{feed.description.gsub(/[\r\n]/,'')[0..50]}...") rescue nil rescue => e retries += 1 Automatic::Log.puts("error", "ErrorCount: #{retries}, #{e.message}") - sleep @config['interval'].to_i unless @config['interval'].nil? - retry if retries <= @config['retry'].to_i unless @config['retry'].nil? + sleep ||= @config['interval'].to_i + retry if retries <= retry_max end - sleep @config['interval'].to_i unless @config['interval'].nil? + sleep ||= @config['interval'].to_i } end } diff --git a/plugins/publish/instapaper.rb b/plugins/publish/instapaper.rb index 142cedc..3cc240f 100644 --- a/plugins/publish/instapaper.rb +++ b/plugins/publish/instapaper.rb @@ -3,8 +3,8 @@ # Author:: soramugi # 774 # Created:: Feb 9, 2013 -# Updated:: Mar 23, 2013 -# Copyright:: soramugi Copyright (c) 2013 +# Updated:: Jan 15, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. module Automatic::Plugin @@ -69,15 +69,16 @@ def run feeds.items.each {|feed| Automatic::Log.puts("info", "add: #{feed.link}") retries = 0 + retry_max = @config['retry'].to_i || 0 begin instapaper.add(feed.link, feed.title, feed.description) rescue retries += 1 Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault in publish to instapaper.") - sleep @config['interval'].to_i unless @config['interval'].nil? - retry if retries <= @config['retry'].to_i unless @config['retry'].nil? + sleep ||= @config['interval'].to_i + retry if retries <= retry_max end - sleep @config['interval'].to_i unless @config['interval'].nil? + sleep ||= @config['interval'].to_i } end } diff --git a/plugins/publish/mail.rb b/plugins/publish/mail.rb deleted file mode 100644 index 5e26367..0000000 --- a/plugins/publish/mail.rb +++ /dev/null @@ -1,64 +0,0 @@ -# -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::Publish::Mail -# Author:: 774 -# Created:: Apr 5, 2012 -# Updated:: Apr 5, 2012 -# Copyright:: 774 Copyright (c) 2012 -# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. - -module Automatic::Plugin - class PublishMail - require 'action_mailer' - - def initialize(config, pipeline=[]) - @config = config - @pipeline = pipeline - end - - def run - Mailer.smtp_settings = default_setting.update( - { - :port => @config["port"], - #:address => @config["address"], - #:authentication => @config["auth"], - #:user_name => @config["username"], - #:password => @config["password"], - #:domain => @config["domain"], - }) - - Mailer.raise_delivery_errors = true - @pipeline.each { |feeds| - unless feeds.nil? - feeds.items.each { |feed| - Mailer.notify(@config, feed) - } - end - } - @pipeline - end - - def default_setting - return {} - end - - class Mailer < ActionMailer::Base - def notify(option, feed) - m = mail( - :date => Time.now, - :subject => feed.title, - :to => option["mailto"], - #:cc => option["mailcc"], - #:bcc => option["mailbcc"], - :from => option["mailfrom"]) - - unless feed.content_encoded.nil? - m.body = feed.content_encoded - else - m.body = feed.description - end - - m.deliver - end - end - end -end diff --git a/plugins/publish/memcached.rb b/plugins/publish/memcached.rb new file mode 100644 index 0000000..f3e41e5 --- /dev/null +++ b/plugins/publish/memcached.rb @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::Plugin::Publish::Memcached +# Author:: 774 +# Created:: Jun 25, 2013 +# Updated:: Jun 25, 2013 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. +# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. + +module Automatic::Plugin + class PublishMemcached + require 'dalli' + + def initialize(config, pipeline=[]) + @config = config + @pipeline = pipeline + @cache = Dalli::Client.new( + @config['host'] + ":" + + @config['port']) + end + + def run + hash = {} + @pipeline.each {|feeds| + unless feeds.nil? + feeds.items.each {|feed| + hash[feed.link] = + { + :title => feed.title, + :description => feed.description, + :content => feed.content_encoded, + :created_at => Time.now.strftime("%Y/%m/%d %X") + } + } + end + } + begin + @cache.set(@config['key'], hash) + rescue + Automatic::Log.puts("warn", "Skip feed due to fault in put to memcached.") + end + @pipeline + end + end +end diff --git a/plugins/publish/pocket.rb b/plugins/publish/pocket.rb index ebc980c..2100a0e 100644 --- a/plugins/publish/pocket.rb +++ b/plugins/publish/pocket.rb @@ -2,8 +2,8 @@ # Name:: Automatic::Plugin::Publish::Pocket # Author:: soramugi # Created:: May 15, 2013 -# Updated:: May 15, 2013 -# Copyright:: soramugi Copyright (c) 2013 +# Updated:: Jan 15, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. module Automatic::Plugin @@ -25,16 +25,17 @@ def run unless feeds.nil? feeds.items.each {|feed| retries = 0 + retry_max = @config['retry'].to_i || 0 begin @client.add(:url => feed.link) Automatic::Log.puts("info", "add: #{feed.link}") rescue retries += 1 Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault in publish to pocket.") - sleep @config['interval'].to_i unless @config['interval'].nil? - retry if retries <= @config['retry'].to_i unless @config['retry'].nil? + sleep ||= @config['interval'].to_i + retry if retries <= retry_max end - sleep @config['interval'].to_i unless @config['interval'].nil? + sleep ||= @config['interval'].to_i } end } diff --git a/plugins/publish/smtp.rb b/plugins/publish/smtp.rb deleted file mode 100644 index 300c4d9..0000000 --- a/plugins/publish/smtp.rb +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::Publish::Smtp -# Author:: kzgs -# Created:: Mar 17, 2012 -# Updated:: Mar 17, 2012 -# Copyright:: kzgs Copyright (c) 2012 -# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. - -module Automatic::Plugin - class PublishSmtp - require 'action_mailer' - - def initialize(config, pipeline=[]) - @config = config - @pipeline = pipeline - end - - def run - Mailer.smtp_settings = default_setting.update( - { - :port => @config["port"], - }) - Mailer.raise_delivery_errors = true - @pipeline.each { |feeds| - unless feeds.nil? - feeds.items.each { |feed| - Mailer.notify(@config, feed) - } - end - } - @pipeline - end - - def default_setting - return {} - end - - class Mailer < ActionMailer::Base - def notify(option, feed) - m = mail(:subject => option["subject"], - :to => option["mailto"], - :from => option["mailfrom"]) - m.body = feed.link - m.deliver - end - end - end -end diff --git a/plugins/publish/twitter.rb b/plugins/publish/twitter.rb index d915485..286b2fc 100644 --- a/plugins/publish/twitter.rb +++ b/plugins/publish/twitter.rb @@ -3,7 +3,7 @@ # Author:: soramugi # Created:: May 5, 2013 # Updated:: May 5, 2013 -# Copyright:: soramugi Copyright (c) 2013 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. module Automatic::Plugin @@ -34,8 +34,9 @@ def run @pipeline.each {|feeds| unless feeds.nil? feeds.items.each {|feed| - Automatic::Log.puts("info", "tweet: #{feed.link}") + Automatic::Log.puts("info", "Publish Tweet: #{feed.link}") retries = 0 + retry_max = @config['retry'].to_i || 0 begin tweet = @tweet_tmp.gsub(/\{(.+?)\}/) do |text| feed.__send__($1) @@ -44,10 +45,10 @@ def run rescue retries += 1 Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault in publish to twitter.") - sleep @config['interval'].to_i unless @config['interval'].nil? - retry if retries <= @config['retry'].to_i unless @config['retry'].nil? + sleep ||= @config['interval'].to_i + retry if retries <= retry_max end - sleep @config['interval'].to_i unless @config['interval'].nil? + sleep ||= @config['interval'].to_i } end } diff --git a/plugins/store/database.rb b/plugins/store/database.rb index cedd414..7c6165d 100644 --- a/plugins/store/database.rb +++ b/plugins/store/database.rb @@ -4,8 +4,8 @@ # 774 # soramugi # Created:: Feb 27, 2012 -# Updated:: Dec 20, 2012 -# Copyright:: kzgs Copyright (c) 2012 +# Updated:: Oct 09, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require 'active_record' @@ -14,20 +14,25 @@ module Automatic::Plugin module Database def for_each_new_feed prepare_database - existing_records = model_class.find(:all) + existing_records = model_class.all @return_feeds = [] @pipeline.each {|feeds| unless feeds.nil? new_feeds = [] feeds.items.each {|feed| unless feed.link.nil? - unless existing_records.detect {|b| b.try(unique_key) == feed.link } + if unique_keys.length > 1 + detection = existing_records.detect {|b| b.try(unique_keys[0]) == feed.link || b.try(unique_keys[1]) == feed.title } + else + detection = existing_records.detect {|b| b.try(unique_keys[0]) == feed.link } + end + unless detection yield(feed) new_feeds << feed end end } - @return_feeds << Automatic::FeedParser.create(new_feeds) if new_feeds.length > 0 + @return_feeds << Automatic::FeedMaker.create_pipeline(new_feeds) if new_feeds.length > 0 end } @return_feeds @@ -54,7 +59,7 @@ def db_dir def prepare_database db = File.join(db_dir, @config['db']) - Automatic::Log.puts("info", "Database: #{db}") + Automatic::Log.puts("info", "Using Database: #{db}") ActiveRecord::Base.establish_connection( :adapter => "sqlite3", :database => db) diff --git a/plugins/store/file.rb b/plugins/store/file.rb new file mode 100644 index 0000000..e72509b --- /dev/null +++ b/plugins/store/file.rb @@ -0,0 +1,95 @@ +#!/usr/bin/env ruby +# -*- coding: utf-8 -*- +# Name:: Automatic::Plugin::Store::File +# Author:: 774 +# Created:: Feb 28, 2012 +# Updated:: Feb 25, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. +# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. + +require 'open-uri' +require 'uri' +require 'aws-sdk' + +module Automatic::Plugin + class StoreFile + + def initialize(config, pipeline=[]) + @config = config + @pipeline = pipeline + unless @config['bucket_name'].nil? + s3 = AWS::S3.new( + :access_key_id => @config['access_key'], + :secret_access_key => @config['secret_key'] + ) + @bucket = s3.buckets[@config['bucket_name']] + end + @return_feeds = [] + end + + def run + @pipeline.each {|feeds| + unless feeds.nil? + feeds.items.each {|feed| + unless feed.link.nil? + Automatic::Log.puts("info", "Downloading File: #{feed.link}") + FileUtils.mkdir_p(@config['path']) unless FileTest.exist?(@config['path']) + retries = 0 + retry_max = @config['retry'].to_i || 0 + begin + retries += 1 + feed.link = get_file(feed.link) + sleep ||= @config['interval'].to_i + @return_feeds << feed + rescue + Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault during file download.") + sleep ||= @config['interval'].to_i + retry if retries <= retry_max + end + end + } + end + } + @pipeline = [] + @pipeline << Automatic::FeedMaker.create_pipeline(@return_feeds) if @return_feeds.length > 0 + @pipeline + end + + private + + def get_file(url) + uri = URI.parse(url) + case uri.scheme + when "s3n" + return_path = get_aws(uri) + else + return_path = wget(uri, url) + end + Automatic::Log.puts("info", "Saved File: #{return_path}") + "file://" + return_path + end + + def wget(uri, url) + filename = File.basename(uri.path) + filepath = File.join(@config['path'], filename) + open(url) {|source| + open(filepath, "w+b") { |o| + o.print(source.read) + } + } + filepath + end + + def get_aws(uri) + filename = File.basename(uri.path) + filepath = File.join(@config['path'], filename) + object = @bucket.objects[uri.path] + File.open(filepath, 'wb') do |file| + object.read do |chunk| + file.write(chunk) + end + end + filepath + end + end +end diff --git a/plugins/store/full_text.rb b/plugins/store/full_text.rb index 2b4becb..63e7e28 100644 --- a/plugins/store/full_text.rb +++ b/plugins/store/full_text.rb @@ -2,8 +2,8 @@ # Name:: Automatic::Plugin::Store::FullText # Author:: 774 # Created:: Feb 26, 2012 -# Updated:: Jun 14, 2012 -# Copyright:: 774 Copyright (c) 2012 +# Updated:: Oct 09, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require 'plugins/store/database' @@ -30,8 +30,8 @@ def column_definition } end - def unique_key - :link + def unique_keys + [:link, :title] end def model_class @@ -39,8 +39,8 @@ def model_class end def run - for_each_new_feed { |feed| - Automatic::Log.puts("info", "Saving: #{feed.link}") + for_each_new_feed {|feed| + Automatic::Log.puts("info", "Saving FullText: #{feed.link}") begin Blog.create( :title => feed.title, diff --git a/plugins/store/permalink.rb b/plugins/store/permalink.rb index f714481..acd8c70 100644 --- a/plugins/store/permalink.rb +++ b/plugins/store/permalink.rb @@ -3,8 +3,8 @@ # Name:: Automatic::Plugin::Store::Permalink # Author:: 774 # Created:: Feb 22, 2012 -# Updated:: Sep 18, 2012 -# Copyright:: 774 Copyright (c) 2012 +# Updated:: Oct 09, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require 'plugins/store/database' @@ -28,8 +28,8 @@ def column_definition } end - def unique_key - :url + def unique_keys + [:url] end def model_class @@ -42,7 +42,7 @@ def run Permalink.create( :url => feed.link, :created_at => Time.now.strftime("%Y/%m/%d %X")) - Automatic::Log.puts("info", "Saving: #{feed.link}") + Automatic::Log.puts("info", "Saving Permalink: #{feed.link}") end } end diff --git a/plugins/store/target_link.rb b/plugins/store/target_link.rb deleted file mode 100644 index 610b622..0000000 --- a/plugins/store/target_link.rb +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env ruby -# -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::Store::TargetLink -# Author:: 774 -# Created:: Feb 28, 2012 -# Updated:: Feb 9, 2013 -# Copyright:: 774 Copyright (c) 2012-2013 -# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. - -require 'open-uri' - -module Automatic::Plugin - class StoreTargetLink - - def initialize(config, pipeline=[]) - @config = config - @pipeline = pipeline - end - - def run - @pipeline.each {|feeds| - unless feeds.nil? - feeds.items.each {|feed| - unless feed.link.nil? - Automatic::Log.puts("info", "Downloading: #{feed.link}") - FileUtils.mkdir_p(@config['path']) unless FileTest.exist?(@config['path']) - retries = 0 - begin - retries += 1 - wget(feed.link) - sleep @config['interval'].to_i unless @config['interval'].nil? - rescue - Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault during file download.") - sleep @config['interval'].to_i unless @config['interval'].nil? - retry if retries <= @config['retry'].to_i unless @config['retry'].nil? - end - end - } - end - } - @pipeline - end - - private - def wget(url) - filename = url.split(/\//).last - open(url) { |source| - open(File.join(@config['path'], filename), "w+b") { |o| - o.print source.read - } - } - end - end -end diff --git a/plugins/subscription/chan_toru.rb b/plugins/subscription/chan_toru.rb new file mode 100644 index 0000000..4af0a86 --- /dev/null +++ b/plugins/subscription/chan_toru.rb @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::Plugin::Subscription::ChanToru +# Author:: soramugi +# Created:: Jun 28, 2013 +# Updated:: Jan 15, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. +# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. + +require File.expand_path(File.dirname(__FILE__) + '/g_guide') + +module Automatic::Plugin + class SubscriptionChanToru + + def initialize(config, pipeline=[]) + @config = config + @pipeline = pipeline + end + + def g_guide_pipeline + SubscriptionGGuide.new(@config, @pipeline).run + end + + def run + retries = 0 + retry_max = @config['retry'].to_i || 0 + begin + pipeline = g_guide_pipeline + pipeline.each {|feeds| + feeds.items.each {|feed| + feed = link_change(feed) + } + } + @pipeline = pipeline + rescue + retries += 1 + Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault in parsing: #{retries}") + sleep ||= @config['interval'].to_i + retry if retries <= retry_max + end + + @pipeline + end + + def link_change(feed) + feed.link.gsub(/([0-9]+)/) do |pid| + if pid != '' + feed.link = "https://tv.so-net.ne.jp/chan-toru/intent" + + "?cat=1&area=23&pid=#{pid}&from=tw" + else + feed.link = nil + end + end + feed + end + + end +end diff --git a/plugins/subscription/feed.rb b/plugins/subscription/feed.rb index 752fb2a..a997655 100644 --- a/plugins/subscription/feed.rb +++ b/plugins/subscription/feed.rb @@ -2,8 +2,8 @@ # Name:: Automatic::Plugin::SubscriptionFeed # Author:: 774 # Created:: Feb 22, 2012 -# Updated:: Feb 8, 2013 -# Copyright:: 774 Copyright (c) 2012-2013 +# Updated:: Feb 21, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. module Automatic::Plugin @@ -16,14 +16,15 @@ def initialize(config, pipeline=[]) def run @config['feeds'].each {|feed| retries = 0 + retry_max = @config['retry'].to_i || 0 begin - rss = Automatic::FeedParser.get(feed) + rss = Automatic::FeedParser.get_url(feed) @pipeline << rss rescue retries += 1 Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault in parsing: #{feed}") - sleep @config['interval'].to_i unless @config['interval'].nil? - retry if retries <= @config['retry'].to_i unless @config['retry'].nil? + sleep ||= @config['interval'].to_i + retry if retries <= retry_max end } @pipeline diff --git a/plugins/subscription/g_guide.rb b/plugins/subscription/g_guide.rb new file mode 100644 index 0000000..99a6c0a --- /dev/null +++ b/plugins/subscription/g_guide.rb @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::Plugin::Subscription::GGuide +# Author:: soramugi +# 774 +# Created:: Jun 28, 2013 +# Updated:: Oct 29, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. +# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. + +module Automatic::Plugin + class SubscriptionGGuide + G_GUIDE_RSS = 'http://tv.so-net.ne.jp/rss/schedulesBySearch.action?' + + def initialize(config, pipeline=[]) + @config = config + @pipeline = pipeline + unless @config['keyword'].nil? || @config['keyword'].index(',').nil? + @keywords = @config['keyword'].split(',') + else + @keywords = [@config['keyword']] + end + end + + def run + retries = 0 + retry_max = @config['retry'].to_i || 0 + begin + @keywords.each {|keyword| + @pipeline << Automatic::FeedParser.get_url(feed_url keyword) + } + rescue + retries += 1 + Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault in parsing: #{retries}") + sleep ||= @config['interval'].to_i + retry if retries <= retry_max + end + @pipeline + end + + def feed_url keyword = nil + feed = G_GUIDE_RSS + unless keyword.nil? + feed += "condition.keyword=#{keyword}&" + end + feed += station_param + URI::Parser.new.escape(feed) + end + + def station_param + station = 0 + unless @config['station'].nil? + station = '1' if @config['station'] == '地上波' + end + "stationPlatformId=#{station}&" + end + end +end diff --git a/plugins/subscription/google_reader_star.rb b/plugins/subscription/google_reader_star.rb deleted file mode 100644 index 48d1749..0000000 --- a/plugins/subscription/google_reader_star.rb +++ /dev/null @@ -1,66 +0,0 @@ -# -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::Subscription::GoogleReaderStar -# Author:: soramugi -# 774 -# Created:: Feb 10, 2013 -# Updated:: Feb 17, 2013 -# Copyright:: soramugi Copyright (c) 2013 -# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. - -module Automatic::Plugin - class SubscriptionGoogleReaderStar - require 'open-uri' - require 'nokogiri' - require 'rss' - - def initialize(config, pipeline=[]) - @config = config - @pipeline = pipeline - end - - def run - @return_feeds = [] - @config['feeds'].each {|feed| - retries = 0 - begin - Automatic::Log.puts("info", "Parsing: #{feed}") - @return_feeds << self.parse(Automatic::FeedParser.get(feed).items) - rescue - retries += 1 - Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault in parsing: #{feed}") - sleep @config['interval'].to_i unless @config['interval'].nil? - retry if retries <= @config['retry'].to_i unless @config['retry'].nil? - end - } - @return_feeds - end - - def parse(feeds = []) - RSS::Maker.make("2.0") {|maker| - maker.xml_stylesheets.new_xml_stylesheet - maker.channel.title = "Automatic Ruby" - maker.channel.description = "Automatic Ruby" - maker.channel.link = "https://github.com/automaticruby/automaticruby" - maker.items.do_sort = true - - unless feeds.nil? - feeds.each {|feed| - unless feed.link.href.nil? - Automatic::Log.puts("info", "Creating: #{feed.link.href}") - item = maker.items.new_item - item.title = feed.title.content # google reader feed - item.link = feed.link.href # google reader feed - begin - item.description = feed.content - item.date = feed.dc_date || Time.now - rescue NoMethodError - Automatic::Log.puts("warn", "Undefined field detected in feed.") - end - end - } - end - } - end - - end -end diff --git a/plugins/subscription/link.rb b/plugins/subscription/link.rb index efece8c..f4c859f 100644 --- a/plugins/subscription/link.rb +++ b/plugins/subscription/link.rb @@ -2,14 +2,13 @@ # Name:: Automatic::Plugin::Subscription::Link # Author:: 774 # Created:: Sep 18, 2012 -# Updated:: Apr 5, 2013 -# Copyright:: 774 Copyright (c) 2012-2013 +# Updated:: Oct 29, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. module Automatic::Plugin class SubscriptionLink require 'open-uri' - require 'nokogiri' require 'rss' def initialize(config, pipeline=[]) @@ -21,25 +20,27 @@ def run @return_feeds = [] @config['urls'].each {|url| retries = 0 + retry_max = @config['retry'].to_i || 0 begin - create_rss(URI.encode(url)) + create_rss(URI::Parser.new.escape(url)) rescue retries += 1 Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault in parsing: #{url}") - sleep @config['interval'].to_i unless @config['interval'].nil? - retry if retries <= @config['retry'].to_i unless @config['retry'].nil? + sleep ||= @config['interval'].to_i + retry if retries <= retry_max end } @return_feeds end private + def create_rss(url) - Automatic::Log.puts("info", "Parsing: #{url}") + Automatic::Log.puts("info", "Parsing Link: #{url}") html = open(url).read unless html.nil? - rss = Automatic::FeedParser.parse(html) - sleep @config['interval'].to_i unless @config['interval'].nil? + rss = Automatic::FeedParser.parse_html(html) + sleep ||= @config['interval'].to_i @return_feeds << rss end end diff --git a/plugins/subscription/pocket.rb b/plugins/subscription/pocket.rb index b9c9cbd..dfd59d2 100644 --- a/plugins/subscription/pocket.rb +++ b/plugins/subscription/pocket.rb @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- # Name:: Automatic::Plugin::Subscription::Pocket # Author:: soramugi +# 774 # Created:: May 21, 2013 -# Updated:: May 21, 2013 -# Copyright:: soramugi Copyright (c) 2013 +# Updated:: Feb 21, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. module Automatic::Plugin @@ -22,30 +23,29 @@ def initialize(config, pipeline=[]) def run retries = 0 + retry_max = @config['retry'].to_i || 0 begin - dummyfeeds = cleate_dummy_feed(@client.retrieve(@config['optional'])) - @pipeline << Automatic::FeedParser.create(dummyfeeds) + return_feeds = generate_feed(@client.retrieve(@config['optional'])) + @pipeline << Automatic::FeedMaker.create_pipeline(return_feeds) rescue retries += 1 Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault in parsing: #{retries}") - sleep @config['interval'].to_i unless @config['interval'].nil? - retry if retries <= @config['retry'].to_i unless @config['retry'].nil? + sleep ||= @config['interval'].to_i + retry if retries <= retry_max end - @pipeline end - def cleate_dummy_feed(retrieve) - dummyFeeds = [] + def generate_feed(retrieve) + return_feeds = [] retrieve['list'].each {|key,list| - - dummy = Hashie::Mash.new - dummy.title = list['given_title'] - dummy.link = list['given_url'] - dummy.description = list['excerpt'] - dummyFeeds << dummy + hashie = Hashie::Mash.new + hashie.title = list['given_title'] + hashie.link = list['given_url'] + hashie.description = list['excerpt'] + return_feeds << hashie } - dummyFeeds + return_feeds end end end diff --git a/plugins/subscription/text.rb b/plugins/subscription/text.rb index 9a6292b..f811a5c 100644 --- a/plugins/subscription/text.rb +++ b/plugins/subscription/text.rb @@ -1,77 +1,67 @@ # -*- coding: utf-8 -*- # Name:: Automatic::Plugin::Subscription::Text # Author:: soramugi -# Created:: May 6, 2013 -# Updated:: May 6, 2013 -# Copyright:: soramugi Copyright (c) 2013 +# 774 +# Created:: May 6, 2013 +# Updated:: Feb 21, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. module Automatic::Plugin - class TextFeed - def initialize - @link = 'http://dummy' - @title = 'dummy' - end - - def link - @link - end - - def title - @title - end - - def set_link(link) - @link = link - end - - def set_title(title) - @title = title - end - end - class SubscriptionText - def initialize(config, pipeline=[]) @config = config @pipeline = pipeline + @return_feeds = [] + end + def run + create_feed + @pipeline << Automatic::FeedMaker.create_pipeline(@return_feeds) if @return_feeds.length > 0 + @pipeline + end + + private + + def create_feed unless @config.nil? @dummyfeeds = [] unless @config['titles'].nil? @config['titles'].each {|title| - textFeed = TextFeed.new - textFeed.set_title(title) - @dummyfeeds << textFeed + feed = {} + feed['title'] = title + @return_feeds << Automatic::FeedMaker.generate_feed(feed) } end unless @config['urls'].nil? @config['urls'].each {|url| - textFeed = TextFeed.new - textFeed.set_link(url) - @dummyfeeds << textFeed + feed = {} + feed['url'] = url + @return_feeds << Automatic::FeedMaker.generate_feed(feed) } end unless @config['feeds'].nil? @config['feeds'].each {|feed| - textFeed = TextFeed.new - textFeed.set_title(feed['title']) unless feed['title'].nil? - textFeed.set_link(feed['url']) unless feed['url'].nil? - @dummyfeeds << textFeed + @return_feeds << Automatic::FeedMaker.generate_feed(feed) } end - end - - end - def run - if @dummyfeeds != [] - @pipeline << Automatic::FeedParser.create(@dummyfeeds) + unless @config['files'].nil? + @config['files'].each {|f| + open(File.expand_path(f)) do |file| + file.each_line do |line| + feed = {} + feed['title'], feed['url'], feed['description'], feed['author'], + feed['comments'] = line.force_encoding("utf-8").strip.split("\t") + @return_feeds << Automatic::FeedMaker.generate_feed(feed) + end + end + } + end end - - @pipeline end + end end diff --git a/plugins/subscription/tumblr.rb b/plugins/subscription/tumblr.rb index c1b0832..18a30d5 100644 --- a/plugins/subscription/tumblr.rb +++ b/plugins/subscription/tumblr.rb @@ -2,8 +2,8 @@ # Name:: Automatic::Plugin::Subscription::Tumblr # Author:: 774 # Created:: Oct 16, 2012 -# Updated:: Apr 5, 2013 -# Copyright:: 774 Copyright (c) 2012-2013 +# Updated:: Feb 21, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. module Automatic::Plugin @@ -19,6 +19,7 @@ def run @return_feeds = [] @config['urls'].each {|url| retries = 0 + retry_max = @config['retry'].to_i || 0 begin create_rss(url) unless @config['pages'].nil? @@ -32,8 +33,8 @@ def run rescue retries += 1 Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault in parsing: #{url}") - sleep @config['interval'].to_i unless @config['interval'].nil? - retry if retries <= @config['retry'].to_i unless @config['retry'].nil? + sleep ||= @config['interval'].to_i + retry if retries <= retry_max end } @return_feeds @@ -41,17 +42,17 @@ def run private def create_rss(url) - Automatic::Log.puts("info", "Parsing: #{url}") + Automatic::Log.puts("info", "Parsing Tumblr: #{url}") html = open(url).read unless html.nil? uri = URI.parse(url) - rss = Automatic::FeedParser.parse(html) + rss = Automatic::FeedParser.parse_html(html) rss.items.each {|item| unless item.link =~ Regexp.new(uri.host) item.link = nil end } - sleep @config['interval'].to_i unless @config['interval'].nil? + sleep ||= @config['interval'].to_i @return_feeds << rss end end diff --git a/plugins/subscription/twitter.rb b/plugins/subscription/twitter.rb index a330c00..abc4cdf 100644 --- a/plugins/subscription/twitter.rb +++ b/plugins/subscription/twitter.rb @@ -2,8 +2,8 @@ # Name:: Automatic::Plugin::Subscription::Twitter # Author:: 774 # Created:: Sep 9, 2012 -# Updated:: Apr 5, 2013 -# Copyright:: 774 Copyright (c) 2012-2013 +# Updated:: Jan 15, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. module Automatic::Plugin @@ -21,21 +21,23 @@ def run @return_feeds = [] @config['urls'].each {|url| retries = 0 + retry_max = @config['retry'].to_i || 0 begin create_rss(url) rescue retries += 1 Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault in parsing: #{url}") - sleep @config['interval'].to_i unless @config['interval'].nil? - retry if retries <= @config['retry'].to_i unless @config['retry'].nil? + sleep ||= @config['interval'].to_i + retry if retries <= retry_max end } @return_feeds end private + def create_rss(url) - Automatic::Log.puts("info", "Parsing: #{url}") + Automatic::Log.puts("info", "Parsing Twitter: #{url}") html = open(url).read unless html.nil? rss = RSS::Maker.make("2.0") {|maker| @@ -59,7 +61,7 @@ def create_rss(url) item.description = content.search('[@class="js-tweet-text"]').text.to_s } } - sleep @config['interval'].to_i unless @config['interval'].nil? + sleep ||= @config['interval'].to_i @return_feeds << rss end end diff --git a/plugins/subscription/twitter_search.rb b/plugins/subscription/twitter_search.rb index 6abdc08..8974034 100644 --- a/plugins/subscription/twitter_search.rb +++ b/plugins/subscription/twitter_search.rb @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- # Name:: Automatic::Plugin::Subscription::TwitterSearch # Author:: soramugi +# 774 # Created:: May 30, 2013 -# Updated:: May 30, 2013 -# Copyright:: soramugi Copyright (c) 2013 +# Updated:: Feb 21, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. module Automatic::Plugin @@ -24,25 +25,24 @@ def initialize(config, pipeline=[]) def run @pipeline = [] retries = 0 + retry_max = @config['retry'].to_i || 0 begin - feeds = [] + return_feeds = [] @client.search(@config['search'],@config['opt']).results.each do |status| - - dummy = Hashie::Mash.new - dummy.title = 'Twitter Search' - dummy.link = "https://twitter.com/#{status.user['screen_name']}/status/#{status.id}" - dummy.description = status.text - dummy.author = status.user['screen_name'] - dummy.date = status.created_at - - feeds << dummy + hashie = Hashie::Mash.new + hashie.title = 'Twitter Search' + hashie.link = "https://twitter.com/#{status.user['screen_name']}/status/#{status.id}" + hashie.description = status.text + hashie.author = status.user['screen_name'] + hashie.date = status.created_at + return_feeds << hashie end - @pipeline << Automatic::FeedParser.create(feeds) + @pipeline << Automatic::FeedMaker.create_pipeline(return_feeds) rescue retries += 1 Automatic::Log.puts("error", "ErrorCount: #{retries}") - sleep @config['interval'].to_i unless @config['interval'].nil? - retry if retries <= @config['retry'].to_i unless @config['retry'].nil? + sleep ||= @config['interval'].to_i + retry if retries <= retry_max end @pipeline end diff --git a/plugins/subscription/weather.rb b/plugins/subscription/weather.rb index 4469a0d..844f5ef 100644 --- a/plugins/subscription/weather.rb +++ b/plugins/subscription/weather.rb @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- # Name:: Automatic::Plugin::Subscription::Weather # Author:: soramugi +# 774 # Created:: May 12, 2013 -# Updated:: May 12, 2013 -# Copyright:: soramugi Copyright (c) 2013 +# Updated:: Feb 21, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. module Automatic::Plugin @@ -21,13 +22,11 @@ def initialize(config, pipeline=[]) def run weather = @weather.send(@day)['weather'] unless @weather.send(@day).nil? if weather != nil - - dummy = Hashie::Mash.new - dummy.title = weather - dummy.link = 'http://weather.dummy.' + @day - @pipeline << Automatic::FeedParser.create([dummy]) + hashie = Hashie::Mash.new + hashie.title = weather + hashie.link = 'http://weather.dummy.' + @day + @pipeline << Automatic::FeedMaker.create_pipeline([hashie]) end - @pipeline end end diff --git a/plugins/subscription/xml.rb b/plugins/subscription/xml.rb new file mode 100644 index 0000000..6b74bf0 --- /dev/null +++ b/plugins/subscription/xml.rb @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::Plugin::Subscription::Xml +# Author:: 774 +# Created:: Jul 12, 2013 +# Updated:: Oct 29, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. +# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. + +module Automatic::Plugin + class SubscriptionXml + require 'open-uri' + require 'active_support' + require 'active_support/core_ext' + require 'active_support/deprecation' + require 'rss' + + def initialize(config, pipeline=[]) + @config = config + @pipeline = pipeline + end + + def run + @return_feeds = [] + @config['urls'].each {|url| + retries = 0 + retry_max = @config['retry'].to_i || 0 + begin + create_rss(URI::Parser.new.escape(url)) + rescue + retries += 1 + Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault in parsing: #{url}") + sleep ||= @config['interval'].to_i + retry if retries <= retry_max + end + } + @return_feeds + end + + private + + def create_rss(url) + Automatic::Log.puts("info", "Parsing XML: #{url}") + hash = Hash.from_xml(open(url).read) + json = hash.to_json + data = ActiveSupport::JSON.decode(json) + unless data.nil? + rss = Automatic::FeedMaker.content_provide(url, data) + sleep ||= @config['interval'].to_i + @return_feeds << rss + end + end + end +end diff --git a/script/build b/script/build index 45dd9c8..f26f34c 100755 --- a/script/build +++ b/script/build @@ -6,6 +6,8 @@ # # Maintainer: id774 # +# v1.5 6/28,2013 +# Always using latest gems. # v1.4 5/13,2013 # Fix bundler environment. # v1.3 2/26,2013 @@ -21,14 +23,15 @@ kickstart() { export RACK_ROOT="." export RACK_ENV="test" - export DATABASE_URL="$RACK_ROOT/db" + #export DATABASE_URL="$RACK_ROOT/db" ruby -v test_subcommand log info started. } exec_rspec() { $RACK_ROOT/bin/automatic -v - bundle install --path vendor/gems + #bundle install --path vendor/gems + bundle update bundle exec rake simplecov } diff --git a/spec/fixtures/sampleFeeds.tsv b/spec/fixtures/sampleFeeds.tsv new file mode 100644 index 0000000..e6135d0 --- /dev/null +++ b/spec/fixtures/sampleFeeds.tsv @@ -0,0 +1 @@ +testTitle http://dummy.from.file testDescription testAuthor testComments diff --git a/spec/fixtures/sampleFeeds2.tsv b/spec/fixtures/sampleFeeds2.tsv new file mode 100644 index 0000000..5064327 --- /dev/null +++ b/spec/fixtures/sampleFeeds2.tsv @@ -0,0 +1,2 @@ +testTitle1 http://dummy.from.file1 testDescription1 testComments1 +testTitle2 http://dummy.from.file2 testDescription2 testAuthor2 diff --git a/spec/lib/automatic/log_spec.rb b/spec/lib/automatic/log_spec.rb index ab34b8f..5bd38d5 100644 --- a/spec/lib/automatic/log_spec.rb +++ b/spec/lib/automatic/log_spec.rb @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- # Name:: Auaotmatic::Log # Author:: soramugi +# 774 # Created:: May 19, 2013 -# Updated:: May 19, 2013 -# Copyright:: soramugi Copyright (c) 2013 +# Updated:: Oct 09, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.join(File.dirname(__FILE__) ,'../../spec_helper')) diff --git a/spec/lib/automatic/pipeline_spec.rb b/spec/lib/automatic/pipeline_spec.rb index bfd8957..ee2764f 100644 --- a/spec/lib/automatic/pipeline_spec.rb +++ b/spec/lib/automatic/pipeline_spec.rb @@ -3,8 +3,8 @@ # Author:: ainame # 774 # Created:: Mar 10, 2012 -# Updated:: Jun 16, 2012 -# Copyright:: ainame Copyright (c) 2012 +# Updated:: Feb 25, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.join(File.dirname(__FILE__) ,'../../spec_helper')) @@ -38,10 +38,10 @@ describe "#run" do it "run a recipe with FilterIgnore module" do - plugin = mock("plugin") + plugin = double("plugin") plugin.should_receive(:module).and_return("FilterIgnore") plugin.should_receive(:config) - recipe = mock("recipe") + recipe = double("recipe") recipe.should_receive(:each_plugin).and_yield(plugin) Automatic::Pipeline.run(recipe).should == [] end diff --git a/spec/lib/automatic/recipe_spec.rb b/spec/lib/automatic/recipe_spec.rb index 1923da9..589a517 100644 --- a/spec/lib/automatic/recipe_spec.rb +++ b/spec/lib/automatic/recipe_spec.rb @@ -3,7 +3,7 @@ # Author:: 774 # Created:: Jun 14, 2012 # Updated:: Jun 14, 2012 -# Copyright:: ainame Copyright (c) 2012 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.join(File.dirname(__FILE__) ,'../../spec_helper')) @@ -20,14 +20,19 @@ Automatic.user_dir = nil end - describe "with a normal recipe" do - it "correctly load recipe" do + context "with a normal recipe" do + subject { recipe = Automatic::Recipe.new(TEST_RECIPE) - expect =[{"module"=>"SubscriptionFeed", - "config"=>{"feeds"=>["http://blog.id774.net/post/feed/"]}}, - {"module"=>"FilterIgnore", "config"=>{"link"=>["hoge"]}}, - {"module"=>"StorePermalink", "config"=>{"db"=>"test_permalink.db"}}] - recipe.each_plugin{recipe}.should == expect + recipe.each_plugin{recipe} + } + let(:expected) { [{"module"=>"SubscriptionFeed", + "config"=>{"feeds"=>["http://blog.id774.net/post/feed/"]}}, + {"module"=>"FilterIgnore", "config"=>{"link"=>["hoge"]}}, + {"module"=>"StorePermalink", "config"=>{"db"=>"test_permalink.db"}}] + } + + it "correctly load recipe" do + expect(subject).to eq expected end end diff --git a/spec/lib/automatic_spec.rb b/spec/lib/automatic_spec.rb index 0260709..3c033f4 100644 --- a/spec/lib/automatic_spec.rb +++ b/spec/lib/automatic_spec.rb @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- -# Name:: Automatic +# Name:: Automatic::Ruby # Author:: kzgs # 774 # Created:: Mar 9, 2012 -# Updated:: May 19, 2013 -# Copyright:: kzgs Copyright (c) 2012-2013 +# Updated:: Jan 24, 2015 +# Copyright:: Copyright (c) 2012-2015 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.join(File.dirname(__FILE__), '../spec_helper')) @@ -14,27 +14,33 @@ describe Automatic do describe "#run" do describe "with a root dir which has default recipe" do - specify { - lambda{ + subject { Automatic.run(:recipe => "", :root_dir => APP_ROOT, :user_dir => APP_ROOT + "/spec/user_dir") - }.should_not raise_exception(Errno::ENOENT) } + + it { expect { subject }.to raise_error } end end describe "#version" do - specify { - Automatic.const_get(:VERSION).should == "13.5.0-devel" - } + subject { Automatic.const_get(:VERSION) } + + it { expect(subject).to eq "14.12.2-devel" } end - describe "#(root|config)_dir" do - specify { - Automatic.root_dir.should == APP_ROOT - Automatic.config_dir.should == APP_ROOT+"/config" - } + describe "#(root)_dir" do + subject { Automatic.root_dir } + + it { expect(subject).to eq APP_ROOT } + + end + + describe "#(config)_dir" do + subject { Automatic.config_dir } + + it { expect(subject).to eq APP_ROOT+"/config" } end describe "#user_dir= in test env" do @@ -43,14 +49,18 @@ end describe "#user_dir" do + subject { Automatic.user_dir } + it "return valid value" do - Automatic.user_dir.should == File.join(APP_ROOT, "spec/user_dir") + expect(subject).to eq File.join(APP_ROOT, "spec/user_dir") end end describe "#user_plugins_dir" do + subject { Automatic.user_plugins_dir } + it "return valid value" do - Automatic.user_plugins_dir.should == File.join(APP_ROOT, "spec/user_dir/plugins") + expect(subject).to eq File.join(APP_ROOT, "spec/user_dir/plugins") end end @@ -66,14 +76,18 @@ end describe "#user_dir" do + subject { Automatic.user_dir } + it "return valid value" do - Automatic.user_dir.should == File.expand_path("~/") + "/.automatic" + expect(subject).to eq File.expand_path("~/") + "/.automatic" end end describe "#user_plugins_dir" do + subject { Automatic.user_plugins_dir } + it "return valid value" do - Automatic.user_plugins_dir.should == File.expand_path("~/") + "/.automatic/plugins" + expect(subject).to eq File.expand_path("~/") + "/.automatic/plugins" end end diff --git a/spec/plugins/custom_feed/svn_log_spec.rb b/spec/plugins/custom_feed/svn_log_spec.rb index 381a70c..7d921f8 100644 --- a/spec/plugins/custom_feed/svn_log_spec.rb +++ b/spec/plugins/custom_feed/svn_log_spec.rb @@ -3,7 +3,7 @@ # Author:: kzgs # Created:: Feb 29, 2012 # Updated:: Nov 3, 2012 -# Copyright:: kzgs Copyright (c) 2012 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') diff --git a/spec/plugins/filter/absolute_uri_spec.rb b/spec/plugins/filter/absolute_uri_spec.rb index 5b06f1c..effdc53 100644 --- a/spec/plugins/filter/absolute_uri_spec.rb +++ b/spec/plugins/filter/absolute_uri_spec.rb @@ -3,7 +3,7 @@ # Author:: 774 # Created:: Jun 20, 2012 # Updated:: Sep 18, 2012 -# Copyright:: 774 Copyright (c) 2012 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') diff --git a/spec/plugins/filter/accept_spec.rb b/spec/plugins/filter/accept_spec.rb index bfa64dd..f767ad4 100644 --- a/spec/plugins/filter/accept_spec.rb +++ b/spec/plugins/filter/accept_spec.rb @@ -3,7 +3,7 @@ # Author:: soramugi # Created:: Jun 4, 2013 # Updated:: Jun 4, 2013 -# Copyright:: soramugi Copyright (c) 2013 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') diff --git a/spec/plugins/filter/clear_spec.rb b/spec/plugins/filter/clear_spec.rb new file mode 100644 index 0000000..fbbe207 --- /dev/null +++ b/spec/plugins/filter/clear_spec.rb @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::Plugin::Filter::Clear +# Author:: 774 +# Created:: Oct 20, 2014 +# Updated:: Oct 20, 2014 +# Copyright:: Copyright (c) 2014 Automatic Ruby Developers. +# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. + +require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') + +require 'filter/clear' + +describe Automatic::Plugin::FilterClear do + + context "should be cleared" do + subject { + Automatic::Plugin::FilterClear.new({ + }, + AutomaticSpec.generate_pipeline { + feed { + item "http://hogefuga.com", "", + "aaaabbbccc" + } + feed { + item "http://aaaabbbccc.com", "", + "hogefugahoge" + } + feed { + item "http://aaabbbccc.com", "", + "aaaaaaaaaacccdd" + } + feed { + item "http://aaccc.com", "", + "aaaabbbccc" + item "http://aabbccc.com", "", + "aabbbccc" + } + feed { + item "http://cccddd.com", "", + "aabbbcccdd" + } + } + )} + + describe "#run" do + its(:run) { should have(0).feeds } + end + end +end diff --git a/spec/plugins/filter/description_link_spec.rb b/spec/plugins/filter/description_link_spec.rb new file mode 100644 index 0000000..3afe13e --- /dev/null +++ b/spec/plugins/filter/description_link_spec.rb @@ -0,0 +1,138 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::Plugin::Filter::DescriptionLink +# Author:: 774 +# Created:: Oct 03, 2014 +# Updated:: Oct 03, 2014 +# Copyright:: Copyright (c) 2014 Automatic Ruby Developers. +# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. + +require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') + +require 'filter/description_link' + +describe Automatic::Plugin::FilterDescriptionLink do + + context "It should be rewrite link based on the description" do + + subject { + Automatic::Plugin::FilterDescriptionLink.new( + {}, + AutomaticSpec.generate_pipeline { + feed { + item "http://test1.id774.net", + "dummy title", + "aaa bbb ccc http://test2.id774.net ddd eee", + "Mon, 07 Mar 2011 15:54:11 +0900" + } + } + ) + } + + describe "#run" do + its(:run) { should have(1).feeds } + + specify { + subject.run + subject.instance_variable_get(:@pipeline)[0].items[0].link. + should == "http://test2.id774.net" + subject.instance_variable_get(:@pipeline)[0].items[0].description. + should == "aaa bbb ccc http://test2.id774.net ddd eee" + } + end + end + + context "It should be empty description if clear_description specified" do + + subject { + Automatic::Plugin::FilterDescriptionLink.new({ + 'clear_description' => 1, + }, + AutomaticSpec.generate_pipeline { + feed { + item "http://test1.id774.net", + "dummy title", + "aaa bbb ccc http://test2.id774.net ddd eee", + "Mon, 07 Mar 2011 15:54:11 +0900" + } + } + ) + } + + describe "#run" do + its(:run) { should have(1).feeds } + + specify { + subject.run + subject.instance_variable_get(:@pipeline)[0].items[0].link. + should == "http://test2.id774.net" + subject.instance_variable_get(:@pipeline)[0].items[0].description. + should == "" + } + end + end + + context "It should be got title if get_title specified" do + + subject { + Automatic::Plugin::FilterDescriptionLink.new({ + 'get_title' => 1, + }, + AutomaticSpec.generate_pipeline { + feed { + item "http://test1.id774.net", + "dummy title", + "aaa bbb ccc http://blog.id774.net/post/2014/10/01/531/ ddd eee", + "Mon, 07 Mar 2011 15:54:11 +0900" + } + } + ) + } + + describe "#run" do + its(:run) { should have(1).feeds } + + specify { + subject.run + subject.instance_variable_get(:@pipeline)[0].items[0].link. + should == "http://blog.id774.net/post/2014/10/01/531/" + subject.instance_variable_get(:@pipeline)[0].items[0].title. + should == "二穂様は俺の嫁 | 774::Blog" + subject.instance_variable_get(:@pipeline)[0].items[0].description. + should == "aaa bbb ccc http://blog.id774.net/post/2014/10/01/531/ ddd eee" + } + end + end + + context "It should be handling error if 404 Not Found" do + + subject { + Automatic::Plugin::FilterDescriptionLink.new({ + 'get_title' => 1, + }, + AutomaticSpec.generate_pipeline { + feed { + item "http://test1.id774.net", + "dummy title", + "aaa bbb ccc http://blog.id774.net/post/2014/10/01/532/ ddd eee", + "Mon, 07 Mar 2011 15:54:11 +0900" + } + } + ) + } + + describe "#run" do + its(:run) { should have(1).feeds } + + specify { + subject.run + subject.instance_variable_get(:@pipeline)[0].items[0].link. + should == "http://blog.id774.net/post/2014/10/01/532/" + subject.instance_variable_get(:@pipeline)[0].items[0].title. + should == "dummy title" + subject.instance_variable_get(:@pipeline)[0].items[0].description. + should == "aaa bbb ccc http://blog.id774.net/post/2014/10/01/532/ ddd eee" + } + end + end + +end diff --git a/spec/plugins/filter/full_feed_spec.rb b/spec/plugins/filter/full_feed_spec.rb index 9b106b3..dc1f66e 100644 --- a/spec/plugins/filter/full_feed_spec.rb +++ b/spec/plugins/filter/full_feed_spec.rb @@ -3,7 +3,7 @@ # Author:: 774 # Created:: Jan 24, 2013 # Updated:: May 23, 2013 -# Copyright:: 774 Copyright (c) 2013 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') diff --git a/spec/plugins/filter/github_feed_spec.rb b/spec/plugins/filter/github_feed_spec.rb index 517a818..2b76476 100644 --- a/spec/plugins/filter/github_feed_spec.rb +++ b/spec/plugins/filter/github_feed_spec.rb @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- # Name:: Automatic::Plugin::Filter::GithubFeed # Author:: Kohei Hasegawa +# 774 # Created:: Jun 6, 2013 -# Updated:: Jun 6, 2013 -# Copyright:: Kohei Hasegawa Copyright (c) 2013 +# Updated:: Feb 21, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') @@ -35,18 +36,18 @@ specify { subject.run - subject.instance_variable_get(:@pipeline)[0].items[0].link + subject.instance_variable_get(:@return_feeds)[0].items[0].link .should == '1' - subject.instance_variable_get(:@pipeline)[0].items[0].title + subject.instance_variable_get(:@return_feeds)[0].items[0].title .should == 'title1' - subject.instance_variable_get(:@pipeline)[0].items[0].description + subject.instance_variable_get(:@return_feeds)[0].items[0].description .should == 'description1' - subject.instance_variable_get(:@pipeline)[0].items[1].link + subject.instance_variable_get(:@return_feeds)[0].items[1].link .should == '0' - subject.instance_variable_get(:@pipeline)[0].items[1].title + subject.instance_variable_get(:@return_feeds)[0].items[1].title .should == 'title0' - subject.instance_variable_get(:@pipeline)[0].items[1].description + subject.instance_variable_get(:@return_feeds)[0].items[1].description .should == 'description0' } end diff --git a/spec/plugins/filter/google_news_spec.rb b/spec/plugins/filter/google_news_spec.rb new file mode 100644 index 0000000..6ed249a --- /dev/null +++ b/spec/plugins/filter/google_news_spec.rb @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::Plugin::Filter::GoogleNews +# Author:: 774 +# Created:: Oct 12, 2014 +# Updated:: Oct 12, 2014 +# Copyright:: Copyright (c) 2014 Automatic Ruby Developers. +# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. + +require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') + +require 'filter/google_news' + +describe Automatic::Plugin::FilterGoogleNews do + + context "It should be not rewrite link other urls" do + + subject { + Automatic::Plugin::FilterGoogleNews.new( + {}, + AutomaticSpec.generate_pipeline { + feed { + item "http://test1.id774.net", + "dummy title", + "aaa bbb ccc http://test2.id774.net ddd eee", + "Mon, 07 Mar 2011 15:54:11 +0900" + } + } + ) + } + + describe "#run" do + its(:run) { should have(1).feeds } + + specify { + subject.run + subject.instance_variable_get(:@pipeline)[0].items[0].link. + should == "http://test1.id774.net" + } + end + end + + context "It should be rewrite link for google news urls" do + + subject { + Automatic::Plugin::FilterGoogleNews.new( + {}, + AutomaticSpec.generate_pipeline { + feed { + item "http://news.google.com/news/url?sa=t&fd=R&ct2=us&usg=AFQjCNGhIFo1illQ6jFyVGPZtfkttFaJYQ&clid=c3a7d30bb8a4878e06b80cf16b898331&cid=52779138313507&ei=Pts3VLDAD4X7kgWO9oGIBg&url=http://www.yomiuri.co.jp/world/20141010-OYT1T50090.html", + "dummy title", + "aaa bbb ccc http://test2.id774.net ddd eee", + "Mon, 07 Mar 2011 15:54:11 +0900" + } + } + ) + } + + describe "#run" do + its(:run) { should have(1).feeds } + + specify { + subject.run + subject.instance_variable_get(:@pipeline)[0].items[0].link. + should == "http://www.yomiuri.co.jp/world/20141010-OYT1T50090.html" + } + end + end + +end diff --git a/spec/plugins/filter/ignore_spec.rb b/spec/plugins/filter/ignore_spec.rb index fbce869..5d6aacf 100644 --- a/spec/plugins/filter/ignore_spec.rb +++ b/spec/plugins/filter/ignore_spec.rb @@ -2,7 +2,7 @@ # Name:: Automatic::Plugin::Filter::Ignore # Author:: 774 # Updated:: Jan 19, 2013 -# Copyright:: 774 Copyright (c) 2012-2013 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') diff --git a/spec/plugins/filter/image_source_spec.rb b/spec/plugins/filter/image_source_spec.rb index 35a3439..0fc3881 100644 --- a/spec/plugins/filter/image_source_spec.rb +++ b/spec/plugins/filter/image_source_spec.rb @@ -3,8 +3,8 @@ # Author:: kzgs # 774 # Created:: Mar 1, 2012 -# Updated:: Jun 14, 2012 -# Copyright:: kzgs Copyright (c) 2012 +# Updated:: Feb 21, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') @@ -25,7 +25,7 @@ its(:run) { should have(1).feeds } specify { subject.run - subject.instance_variable_get(:@pipeline)[0].items[0].link. + subject.instance_variable_get(:@return_feeds)[0].items[0].link. should == "http://27.media.tumblr.com/tumblr_lzrubkfPlt1qb8vzto1_500.png" } end @@ -46,7 +46,7 @@ its(:run) { should have(1).feeds } specify { subject.run - subject.instance_variable_get(:@pipeline)[0].items[0].link. + subject.instance_variable_get(:@return_feeds)[0].items[0].link. should == "http://24.media.tumblr.com/tumblr_m07wttnIdy1qzoj1jo1_400.jpg" } end @@ -65,13 +65,13 @@ describe "#run" do before do - subject.stub!(:rewrite_link).and_return(['http://huge.png']) + subject.stub(:rewrite_link).and_return(['http://huge.png']) end its(:run) { should have(1).feeds } specify { subject.run - subject.instance_variable_get(:@pipeline)[0].items[0].link. + subject.instance_variable_get(:@return_feeds)[0].items[0].link. should == 'http://huge.png' } end @@ -80,7 +80,7 @@ before do open = Hashie::Mash.new open.read = '
' - subject.stub!(:open).and_return(open) + subject.stub(:open).and_return(open) end its(:run) { subject.run[0].items.length.should == 2 } diff --git a/spec/plugins/filter/image_spec.rb b/spec/plugins/filter/image_spec.rb index 50c4f82..8c847e8 100644 --- a/spec/plugins/filter/image_spec.rb +++ b/spec/plugins/filter/image_spec.rb @@ -3,7 +3,7 @@ # Author:: 774 # Created:: Sep 18, 2012 # Updated:: Sep 18, 2012 -# Copyright:: 774 Copyright (c) 2012 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') diff --git a/spec/plugins/filter/one_spec.rb b/spec/plugins/filter/one_spec.rb index 62b7585..72bb9d5 100644 --- a/spec/plugins/filter/one_spec.rb +++ b/spec/plugins/filter/one_spec.rb @@ -3,7 +3,7 @@ # Author:: soramugi # Created:: May 8, 2013 # Updated:: May 8, 2013 -# Copyright:: soramugi Copyright (c) 2013 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') diff --git a/spec/plugins/filter/rand_spec.rb b/spec/plugins/filter/rand_spec.rb index 24273d0..1c07101 100644 --- a/spec/plugins/filter/rand_spec.rb +++ b/spec/plugins/filter/rand_spec.rb @@ -4,7 +4,7 @@ # 774 # Created:: May 6, 2013 # Updated:: Mar 7, 2013 -# Copyright:: soramugi Copyright (c) 2013 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') diff --git a/spec/plugins/filter/sanitize_spec.rb b/spec/plugins/filter/sanitize_spec.rb new file mode 100644 index 0000000..59d4766 --- /dev/null +++ b/spec/plugins/filter/sanitize_spec.rb @@ -0,0 +1,153 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::Plugin::Filter::Sanitize +# Author:: 774 +# Created:: Jun 20, 2013 +# Updated:: Jun 20, 2013 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. +# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. + +require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') + +require 'filter/sanitize' + +describe Automatic::Plugin::FilterSanitize do + context "It should be sanitized" do + subject { + Automatic::Plugin::FilterSanitize.new( + {}, + AutomaticSpec.generate_pipeline { + feed { + item "http://testsite.org", "hoge", + "fuga", + "Mon, 07 Mar 2011 15:54:11 +0900" + } + } + ) + } + + describe "#run" do + its(:run) { should have(1).feeds } + + specify { + subject.run + subject.instance_variable_get(:@return_feeds)[0].items. + count.should == 1 + subject.instance_variable_get(:@return_feeds)[0].items[0].description. + should == 'fuga' + } + end + end + + context "It should not be sanitized in basic mode" do + subject { + Automatic::Plugin::FilterSanitize.new( + { + 'mode' => "basic" + }, + AutomaticSpec.generate_pipeline { + feed { + item "http://testsite.org", "hoge", + "fuga", + "Mon, 07 Mar 2011 15:54:11 +0900" + } + } + ) + } + + describe "#run" do + its(:run) { should have(1).feeds } + + specify { + subject.run + subject.instance_variable_get(:@return_feeds)[0].items. + count.should == 1 + subject.instance_variable_get(:@return_feeds)[0].items[0].description. + should == 'fuga' + } + end + end + + context "It should not be sanitized in restricted mode" do + subject { + Automatic::Plugin::FilterSanitize.new( + { + 'mode' => "restricted" + }, + AutomaticSpec.generate_pipeline { + feed { + item "http://testsite.org", "hoge", + "fuga", + "Mon, 07 Mar 2011 15:54:11 +0900" + } + } + ) + } + + describe "#run" do + its(:run) { should have(1).feeds } + + specify { + subject.run + subject.instance_variable_get(:@return_feeds)[0].items. + count.should == 1 + subject.instance_variable_get(:@return_feeds)[0].items[0].description. + should == 'fuga' + } + end + end + + context "It should not be sanitized in relaxed mode" do + subject { + Automatic::Plugin::FilterSanitize.new( + { + 'mode' => "relaxed" + }, + AutomaticSpec.generate_pipeline { + feed { + item "http://testsite.org", "hoge", + "fuga", + "Mon, 07 Mar 2011 15:54:11 +0900" + } + } + ) + } + + describe "#run" do + its(:run) { should have(1).feeds } + + specify { + subject.run + subject.instance_variable_get(:@return_feeds)[0].items. + count.should == 1 + subject.instance_variable_get(:@return_feeds)[0].items[0].description. + should == 'fuga' + } + end + end + + context "It should be sanitized" do + subject { + Automatic::Plugin::FilterSanitize.new( + {}, + AutomaticSpec.generate_pipeline { + feed { + item "http://testsite.org", "hoge" + } + } + ) + } + + describe "#run" do + its(:run) { should have(1).feeds } + + specify { + subject.run + subject.instance_variable_get(:@return_feeds)[0].items. + count.should == 1 + subject.instance_variable_get(:@return_feeds)[0].items[0].description. + should == '' + } + end + end + +end diff --git a/spec/plugins/filter/sort_spec.rb b/spec/plugins/filter/sort_spec.rb index dfdbb99..9781b35 100644 --- a/spec/plugins/filter/sort_spec.rb +++ b/spec/plugins/filter/sort_spec.rb @@ -3,7 +3,7 @@ # Author:: 774 # Created:: Mar 23, 2012 # Updated:: Jan 23, 2013 -# Copyright:: 774 Copyright (c) 2012-2013 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') diff --git a/spec/plugins/filter/tumblr_resize_spec.rb b/spec/plugins/filter/tumblr_resize_spec.rb index 1abb95d..d393908 100644 --- a/spec/plugins/filter/tumblr_resize_spec.rb +++ b/spec/plugins/filter/tumblr_resize_spec.rb @@ -2,7 +2,7 @@ # Name:: Automatic::Plugin::Filter::TumblrResize # Author:: 774 # Updated:: Jun 14, 2012 -# Copyright:: 774 Copyright (c) 2012 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') diff --git a/spec/plugins/notify/ikachan_spec.rb b/spec/plugins/notify/ikachan_spec.rb index c5de16a..bb03f55 100644 --- a/spec/plugins/notify/ikachan_spec.rb +++ b/spec/plugins/notify/ikachan_spec.rb @@ -19,7 +19,7 @@ } it "should post title and link in the feed" do - ikachan = mock("ikachan") + ikachan = double("ikachan") ikachan.should_receive(:post).with("http://github.com", "GitHub") ikachan.should_receive(:params) subject.instance_variable_set(:@ikachan, ikachan) @@ -30,7 +30,7 @@ describe Automatic::Plugin::Ikachan do describe "#post" do subject { - Automatic::Plugin::Ikachan.new.tap { |ikachan| + Automatic::Plugin::Ikachan.new.tap {|ikachan| ikachan.params = { "channels" => "#room", "url" => "http://sample.com", @@ -44,9 +44,9 @@ link = "http://www.google.com" require 'net/http' - res = stub("res") + res = double("res") res.should_receive(:code).and_return("200") - http = mock("http") + http = double("http") http.should_receive(:post).with("/join", "channel=#room") http.should_receive(:post).with( "/notice", "channel=#room&message=#{link}").and_return(res) diff --git a/spec/plugins/provide/fluentd_spec.rb b/spec/plugins/provide/fluentd_spec.rb new file mode 100644 index 0000000..1b07a26 --- /dev/null +++ b/spec/plugins/provide/fluentd_spec.rb @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::Plugin::Provide::Fluentd +# Author:: 774 +# Created:: Jul 12, 2013 +# Updated:: Feb 25, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. +# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. + +require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') + +require 'provide/fluentd' + +describe Automatic::Plugin::ProvideFluentd do + context 'when feed' do + describe 'should forward the feeds' do + hash = {} + hash['test1'] = "test2" + hash['test3'] = "test4" + expect = hash + + feeds = [] + json = hash.to_json + data = ActiveSupport::JSON.decode(json) + url = "http://id774.net/test/xml/data" + rss = Automatic::FeedMaker.content_provide(url, data) + feeds << rss + + subject { + Automatic::Plugin::ProvideFluentd.new( + { + 'host' => "localhost", + 'port' => "10000", + 'tag' => "automatic_spec.provide_fluentd", + 'mode' => "test" + }, + feeds + ) + } + + its (:run) { + fluentd = double("fluentd") + subject.run.should have(1).feed + subject.instance_variable_get(:@pipeline)[0].items[0].content_encoded.class == Hash + subject.instance_variable_get(:@pipeline)[0].items[0].content_encoded.should == expect + } + end + + end +end diff --git a/spec/plugins/publish/amazon_s3_spec.rb b/spec/plugins/publish/amazon_s3_spec.rb new file mode 100644 index 0000000..93bca1e --- /dev/null +++ b/spec/plugins/publish/amazon_s3_spec.rb @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::Plugin::Publish::AmazonS3 +# Author:: 774 +# Created:: Feb 25, 2014 +# Updated:: Feb 25, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. +# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. + +require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') + +require 'publish/amazon_s3' + +describe Automatic::Plugin::PublishAmazonS3 do + context 'when feed' do + describe 'should forward the feeds' do + subject { + Automatic::Plugin::PublishAmazonS3.new( + { + 'access_key' => "aabbcc", + 'secret_key' => "ddeeff", + 'bucket_name' => "test_bucket", + 'target_path' => "test/tmp", + 'mode' => "test" + }, + AutomaticSpec.generate_pipeline{ + feed { + item "http://github.com", "hoge", + "fuga" + } + } + ) + } + + its (:run) { + subject.run.should have(1).feed + } + end + + end +end diff --git a/spec/plugins/publish/console_spec.rb b/spec/plugins/publish/console_spec.rb index 2e30e1b..24e4647 100644 --- a/spec/plugins/publish/console_spec.rb +++ b/spec/plugins/publish/console_spec.rb @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- # Name:: Automatic::Plugin::Publish::Console # Author:: 774 -# Updated:: Jun 14, 2012 -# Copyright:: 774 Copyright (c) 2012 +# Updated:: Feb 25, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') @@ -21,7 +21,7 @@ } it "should output pretty inspect of feeds" do - output = mock("output") + output = double("output") output.should_receive(:puts). with("info", @pipeline[0].items[0].pretty_inspect) subject.instance_variable_set(:@output, output) diff --git a/spec/plugins/publish/eject_spec.rb b/spec/plugins/publish/eject_spec.rb new file mode 100644 index 0000000..5b34301 --- /dev/null +++ b/spec/plugins/publish/eject_spec.rb @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::Plugin::Publish::Eject +# Author:: soramugi +# Created:: Jun 9, 2013 +# Updated:: Jun 9, 2013 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. +# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. + +require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') + +require 'publish/eject' + +describe Automatic::Plugin::PublishEject do + before do + @pipeline = AutomaticSpec.generate_pipeline { + feed { item "http://github.com" }} + end + + subject { + Automatic::Plugin::PublishEject.new({}, @pipeline) + } + + it "should eject of feeds" do + subject.stub(:eject_cmd).and_return('echo') + subject.run.should have(1).items + end + + subject { + Automatic::Plugin::PublishEject.new({'interval' => 0}, @pipeline) + } + + it "should eject of feeds" do + subject.stub(:eject_cmd).and_return('echo') + subject.run.should have(1).items + end + + it "should eject_cmd" do + subject.eject_cmd.should_not == '' + end +end diff --git a/spec/plugins/publish/fluentd_spec.rb b/spec/plugins/publish/fluentd_spec.rb new file mode 100644 index 0000000..ac36dcb --- /dev/null +++ b/spec/plugins/publish/fluentd_spec.rb @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::Plugin::Publish::Fluentd +# Author:: 774 +# Created:: Jun 21, 2013 +# Updated:: Feb 25, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. +# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. + +require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') + +require 'publish/fluentd' + +describe Automatic::Plugin::PublishFluentd do + context 'when feed' do + describe 'should forward the feeds' do + subject { + Automatic::Plugin::PublishFluentd.new( + { + 'host' => "localhost", + 'port' => "10000", + 'tag' => "automatic_spec.publish_fluent", + 'mode' => "test" + }, + AutomaticSpec.generate_pipeline{ + feed { + item "http://github.com", "hoge", + "fuga" + } + } + ) + } + + its (:run) { + fluentd = double("fluentd") + subject.run.should have(1).feed + } + end + + end +end diff --git a/spec/plugins/publish/google_calendar_spec.rb b/spec/plugins/publish/google_calendar_spec.rb index b61718c..f1ae1c5 100644 --- a/spec/plugins/publish/google_calendar_spec.rb +++ b/spec/plugins/publish/google_calendar_spec.rb @@ -2,8 +2,8 @@ # Name:: Automatic::Plugin::CustomFeed::SVNFLog # Author:: kzgs # Created:: Feb 26, 2012 -# Updated:: Mar 3, 2012 -# Copyright:: kzgs Copyright (c) 2012 +# Updated:: Feb 25, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') @@ -24,7 +24,7 @@ } it "should post the link in the feed" do - gc = mock("gc") + gc = double("gc") gc.should_receive(:add).with("今日 GitHub") subject.instance_variable_set(:@gc, gc) subject.run.should have(1).feed @@ -50,7 +50,7 @@ end def all_day_event_mock(title, where, date=nil) - event = mock("event") + event = double("event") { :title => title, :st => date.nil? ? nil : Time.mktime(date.year, date.month, date.day), @@ -69,7 +69,7 @@ def all_day_event_mock(title, where, date=nil) end def cal_mock(event_mock) - cal = mock("cal") + cal = double("cal") cal.should_receive(:create_event).and_return { event_mock } diff --git a/spec/plugins/publish/hatena_bookmark_spec.rb b/spec/plugins/publish/hatena_bookmark_spec.rb index a4adf8d..5bf062f 100644 --- a/spec/plugins/publish/hatena_bookmark_spec.rb +++ b/spec/plugins/publish/hatena_bookmark_spec.rb @@ -2,8 +2,8 @@ # Name:: Automatic::Plugin::Publish::HatenaBookmark # Author:: 774 # Created:: Feb 22, 2012 -# Updated:: Mar 7, 2013 -# Copyright:: 774 Copyright (c) 2012-2013 +# Updated:: Feb 25, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') @@ -21,7 +21,7 @@ } it "should post the link with prefix 'http' in the feed" do - hb = mock("hb") + hb = double("hb") hb.should_receive(:post).with("http://github.com", nil) subject.instance_variable_set(:@hb, hb) subject.run.should have(1).feed @@ -39,7 +39,7 @@ } it "should post the link with prefix '//...' in the feed" do - hb = mock("hb") + hb = double("hb") hb.should_receive(:post).with("http://github.com", nil) subject.instance_variable_set(:@hb, hb) subject.run.should have(1).feed @@ -57,7 +57,7 @@ } it "should post the link with prefix 'https' in the feed" do - hb = mock("hb") + hb = double("hb") hb.should_receive(:post).with("https://github.com", nil) subject.instance_variable_set(:@hb, hb) subject.run.should have(1).feed @@ -75,7 +75,7 @@ } it "should post the link with others in the feed" do - hb = mock("hb") + hb = double("hb") hb.should_receive(:post).with("http://github.com", nil) subject.instance_variable_set(:@hb, hb) subject.run.should have(1).feed @@ -106,9 +106,9 @@ comment = "Can we trust them ?" require 'net/http' - res = stub("res") + res = double("res") res.should_receive(:code).and_return("201") - http = mock("http") + http = double("http") http.should_receive(:post).with("/atom/post", subject.toXml(url, comment), subject.wsse("", "")).and_return(res) http.should_receive(:start).and_yield(http) @@ -121,9 +121,9 @@ comment = "Can we trust them ?" require 'net/http' - res = stub("res") + res = double("res") res.should_receive(:code).twice.and_return("400") - http = mock("http") + http = double("http") http.should_receive(:post).with("/atom/post", subject.toXml(url, comment), subject.wsse("", "")).and_return(res) http.should_receive(:start).and_yield(http) diff --git a/spec/plugins/publish/hipchat_spec.rb b/spec/plugins/publish/hipchat_spec.rb index cd98585..af35743 100644 --- a/spec/plugins/publish/hipchat_spec.rb +++ b/spec/plugins/publish/hipchat_spec.rb @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- # Name:: Automatic::Plugin::Publish::Hipchat # Author:: Kohei Hasegawa -# Created:: Jun 5, 2013 -# Updated:: Jun 5, 2013 -# Copyright:: Kohei Hasegawa Copyright (c) 2013 +# Created:: Jun 5, 2013 +# Updated:: Feb 25, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') @@ -33,13 +33,13 @@ context 'when successfully' do it "should passed proper argument to HipChat::Client" do - client = mock('client').as_null_object + client = double('client').as_null_object HipChat::Client.should_receive(:new).with("bogus_api_token").and_return(client) subject.run end it "should post the link in the feed" do - client = mock("client") + client = double("client") client.should_receive(:send).with('bogus_bot', 'description', {"color"=>"yellow", "notify"=>false}) subject.instance_variable_set(:@client, client) subject.run.should have(1).feed @@ -48,7 +48,7 @@ context 'when raise an error during post' do it do - client = mock("client") + client = double("client") client.stub(:send).and_raise subject.instance_variable_set(:@client, client) Automatic::Log.should_receive(:puts).twice diff --git a/spec/plugins/publish/instapaper_spec.rb b/spec/plugins/publish/instapaper_spec.rb index 1cfa374..fd2cb20 100644 --- a/spec/plugins/publish/instapaper_spec.rb +++ b/spec/plugins/publish/instapaper_spec.rb @@ -2,9 +2,9 @@ # Name:: Automatic::Plugin::Publish::Instapaper # Author:: soramugi # 774 -# Created:: Feb 9, 2013 -# Updated:: Mar 22, 2013 -# Copyright:: soramugi Copyright (c) 2013 +# Created:: Feb 9, 2013 +# Updated:: Feb 25, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') @@ -20,14 +20,12 @@ 'interval' => 5, 'retry' => 5 }, - AutomaticSpec.generate_pipeline{ + AutomaticSpec.generate_pipeline { feed { item "http://github.com" } - } - ) - } + })} it "should post the link in the feed" do - instapaper = mock("instapaper") + instapaper = double("instapaper") instapaper.should_receive(:add).with("http://github.com", nil, '') subject.instance_variable_set(:@instapaper, instapaper) subject.run.should have(1).feed @@ -42,11 +40,9 @@ 'interval' => 1, 'retry' => 1 }, - AutomaticSpec.generate_pipeline{ + AutomaticSpec.generate_pipeline { feed { item "http://github.com" } - } - ) - } + })} its (:run) { subject.run.should have(1).feed } end @@ -60,16 +56,14 @@ 'password' => "pswd", 'interval' => 5, 'retry' => 5 - } - ) - } + })} url = "http://www.google.com" title = "automatic test" description = "automatic test" specify { - res = stub("res") + res = double("res") res.should_receive(:code).and_return("201") subject.should_receive(:request).and_return(res) subject.add(url, title, description) @@ -78,7 +72,7 @@ it 'raise error' do lambda{ - res = mock("res") + res = double("res") res.should_receive(:code).twice.and_return("403") subject.should_receive(:request).and_return(res) subject.add(url, title, description) diff --git a/spec/plugins/publish/mail_spec.rb b/spec/plugins/publish/mail_spec.rb deleted file mode 100644 index cbcb577..0000000 --- a/spec/plugins/publish/mail_spec.rb +++ /dev/null @@ -1,53 +0,0 @@ -# -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::Publish::Mail -# Author:: 774 -# Updated:: Jun 14, 2012 -# Copyright:: 774 Copyright (c) 2012 -# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. - -require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') - -require 'publish/mail' -require 'mocksmtpd' - -describe Automatic::Plugin::PublishMail do - before do - @pipeline = AutomaticSpec.generate_pipeline { - feed { item "http://github.com", - 'aaa', - 'bbb\n', - 'Mon, 30 Jan 2012 13:22:29 +0900' - } - } - end - - subject { - Automatic::Plugin::PublishMail.new({ - "port" => 25, - "mailto" => "", - "mailfrom" => "", - "subject" => "test" - }, @pipeline) - } - - it "should mail to smtp server" do - smtpd = SMTPServer.new({ - :Port => 10025, - :MailHook => lambda { |sender| sender.should == "" }, - :DataHook => lambda { |tmpf, sender, recipients| - tmpf.should include("github.com") - sender.should == "" - recipients.should == [""] - } - } - ) - Thread.start { - smtpd.start - } - loop do - break if smtpd.status == :Running - end - subject.run - smtpd.shutdown - end -end diff --git a/spec/plugins/publish/memcached_spec.rb b/spec/plugins/publish/memcached_spec.rb new file mode 100644 index 0000000..3e12775 --- /dev/null +++ b/spec/plugins/publish/memcached_spec.rb @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::Plugin::Publish::Memcached +# Author:: 774 +# Created:: Jun 25, 2013 +# Updated:: Feb 25, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. +# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. + +require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') + +require 'publish/memcached' + +describe Automatic::Plugin::PublishMemcached do + context 'when feed' do + describe 'should put the feeds to memcached' do + subject { + Automatic::Plugin::PublishMemcached.new( + { + 'host' => "localhost", + 'port' => "11211", + 'key' => "rspec" + }, + AutomaticSpec.generate_pipeline{ + feed { + item "http://blog.id774.net/post/2012/01/30/18/", "ブログをはじめた", + "なぜいまブログなのか + + いままでインターネット全体に公開するブログとして、はてなダイアリーを利用してきた。それ以外のある程度まとまった文章は Facebook に書いてきた。それはそれで良かったのだけど、いろいろと思うところもあり、このたび新しくブログをはじめることにした。 + + はてなダイアリーはシンタックスハイライト (プログラミング言語の色付けのこと) が充実していたので利用していた。しかし最近登場した CoffeeScript や Haml のような新しい言語には対応していない。新しくはてなブログというのも始まったが、ダイアリー以上にシンタックスハイライトが使えないようだ。 + + そこで、今後インターネット上で文章を書いていくにあたりどうするか考えた。" + item "http://blog.id774.net/post/2012/01/30/38/", "Twitter Viewer つくった", + "Twitter を閲覧するための Web アプリをつくった。 + + Twitter Viewer + + やっていることは至ってシンプルで RDB にためた発言をブラウザに表示させているだけである。内容は Rails の Scaffold ほとんどそのまま。 CSS はサイトローカルな Bootstrap を読み込んでいる。簡単なアプリだが、ブラウザにいちど表示させてしまえば電波が入らない地下鉄などでもゆっくり読めるので、モバイル環境で大量の発言をざっとチェックしたいときなどに使えて意外と実用的である。発言のクロールは別途おこなう必要がある。この例では Termtter の ActiveRecord プラグインを利用している。" + item "http://blog.id774.net/post/2012/01/30/48/", "PC-98 とエミュレータ", + "Facebook には少し書いたのだが、今年に入ってから 90 年代に使っていた PC-98 と呼ばれる PC を発掘したので起動した。もう 15 年前後も経っているというのに正常に利用することができて感動してしまった。あの ThinkPad ですら数年ほど電源を入れないで放置しておくと起動しないことが多いのに、さすが発売当初 40 〜 50 万円程もした高級マシンである。そんなわけで今回は PC-98 の話。 + + PC-98 のソフトを使う + 当時のソフトウェアを利用するためには以下のものが必要だ。 + 1. PC-98 エミュレータ + 2. MS-DOS (オペレーティングシステム) + 3. 動作させる対象のソフトウェア" + } + feed { + item "http://d.hatena.ne.jp/Naruhodius/20120130/1327862031", "ブログを移転しました", + "いままでこの「はてなダイアリー」にブログを書いてきましたが、以下のアドレスにブログを移転することにしました。このブログはもう更新されません。以下の新しいブログを購読してください。" + } + } + ) + } + + its (:run) { + fluentd = double("memcached") + subject.run.should have(2).feed + } + end + + end +end diff --git a/spec/plugins/publish/pocket_spec.rb b/spec/plugins/publish/pocket_spec.rb index 916e979..cd57a08 100644 --- a/spec/plugins/publish/pocket_spec.rb +++ b/spec/plugins/publish/pocket_spec.rb @@ -2,8 +2,8 @@ # Name:: Automatic::Plugin::Publish::Pocket # Author:: soramugi # Created:: May 15, 2013 -# Updated:: May 15, 2013 -# Copyright:: soramugi Copyright (c) 2013 +# Updated:: Feb 25, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') @@ -19,12 +19,12 @@ 'interval' => 1, 'retry' => 1 }, - AutomaticSpec.generate_pipeline{ + AutomaticSpec.generate_pipeline { feed { item "http://github.com" } - }) } + })} it "should post the link in the feed" do - client = mock("client") + client = double("client") client.should_receive(:add).with(:url => 'http://github.com') subject.instance_variable_set(:@client, client) subject.run.should have(1).feed diff --git a/spec/plugins/publish/smtp_spec.rb b/spec/plugins/publish/smtp_spec.rb deleted file mode 100644 index 64390f6..0000000 --- a/spec/plugins/publish/smtp_spec.rb +++ /dev/null @@ -1,49 +0,0 @@ -# -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::Publish::Mail -# Author:: 774 -# Updated:: Jun 14, 2012 -# Copyright:: 774 Copyright (c) 2012 -# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. - -require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') - -require 'publish/smtp' -require 'mocksmtpd' - -describe Automatic::Plugin::PublishSmtp do - before do - @pipeline = AutomaticSpec.generate_pipeline { - feed { item "http://github.com" } - } - end - - subject { - Automatic::Plugin::PublishSmtp.new({ - "port" => 10025, - "mailto" => "to@example.com", - "mailfrom" => "from@example.com", - "subject" => "test" - }, @pipeline) - } - - it "should mail to smtp server" do - smtpd = SMTPServer.new({ - :Port => 10025, - :MailHook => lambda { |sender| sender.should == "from@example.com" }, - :DataHook => lambda { |tmpf, sender, recipients| - tmpf.should include("github.com") - sender.should == "from@example.com" - recipients.should == ["to@example.com"] - } - } - ) - Thread.start { - smtpd.start - } - loop do - break if smtpd.status == :Running - end - subject.run - smtpd.shutdown - end -end diff --git a/spec/plugins/publish/twitter_spec.rb b/spec/plugins/publish/twitter_spec.rb index aae6c89..d34c351 100644 --- a/spec/plugins/publish/twitter_spec.rb +++ b/spec/plugins/publish/twitter_spec.rb @@ -2,8 +2,8 @@ # Name:: Automatic::Plugin::Publish::Twitter # Author:: soramugi # Created:: May 5, 2013 -# Updated:: May 5, 2013 -# Copyright:: soramugi Copyright (c) 2013 +# Updated:: Feb 25, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') @@ -21,7 +21,7 @@ })} its (:run) { - twitter = mock("twitter") + twitter = double("twitter") twitter.should_receive(:update).with(" http://github.com") subject.instance_variable_set(:@twitter, twitter) subject.run.should have(1).feed @@ -37,7 +37,7 @@ })} its (:run) { - twitter = mock("twitter") + twitter = double("twitter") twitter.should_receive(:update).with("publish-twitter") subject.instance_variable_set(:@twitter, twitter) subject.run.should have(1).feed diff --git a/spec/plugins/store/target_link_spec.rb b/spec/plugins/store/file_spec.rb similarity index 70% rename from spec/plugins/store/target_link_spec.rb rename to spec/plugins/store/file_spec.rb index 84811bb..87de791 100644 --- a/spec/plugins/store/target_link_spec.rb +++ b/spec/plugins/store/file_spec.rb @@ -1,22 +1,22 @@ # -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::CustomFeed::SVNFLog +# Name:: Automatic::Plugin::Store::File # Author:: kzgs # 774 # Created:: Mar 4, 2012 -# Updated:: May 1, 2013 -# Copyright:: kzgs Copyright (c) 2012-2013 +# Updated:: Feb 25, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') -require 'store/target_link' +require 'store/file' require 'tmpdir' require 'pathname' -describe Automatic::Plugin::StoreTargetLink do +describe Automatic::Plugin::StoreFile do it "should store the target link" do Dir.mktmpdir do |dir| - instance = Automatic::Plugin::StoreTargetLink.new( + instance = Automatic::Plugin::StoreFile.new( { "path" => dir }, AutomaticSpec.generate_pipeline { feed { item "http://id774.net/test/store/rss" } @@ -29,19 +29,19 @@ it "should error during file download" do Dir.mktmpdir do |dir| - instance = Automatic::Plugin::StoreTargetLink.new( + instance = Automatic::Plugin::StoreFile.new( { "path" => dir }, AutomaticSpec.generate_pipeline { feed { item "aaa" } } ) - instance.run.should have(1).feed + instance.run.should have(0).feed end end it "should error and retry during file download" do Dir.mktmpdir do |dir| - instance = Automatic::Plugin::StoreTargetLink.new( + instance = Automatic::Plugin::StoreFile.new( { "path" => dir, 'retry' => 1, @@ -51,7 +51,7 @@ feed { item "aaa" } } ) - instance.run.should have(1).feed + instance.run.should have(0).feed end end diff --git a/spec/plugins/store/full_text_spec.rb b/spec/plugins/store/full_text_spec.rb index 55c0093..f2e5f2b 100644 --- a/spec/plugins/store/full_text_spec.rb +++ b/spec/plugins/store/full_text_spec.rb @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- # Name:: Automatic::Plugin::Store::FullText # Author:: 774 -# Updated:: Jun 14, 2012 -# Copyright:: 774 Copyright (c) 2012 +# Updated:: Oct 16, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') @@ -26,13 +26,20 @@ it "should store 1 record for the new blog entry" do instance = Automatic::Plugin::StoreFullText.new({"db" => @db_filename}, AutomaticSpec.generate_pipeline { - feed { item "http://blog.id774.net/blogs/feed/" } + feed { + item "http://blog.id774.net/blogs/feed/", + "dummy title", + "aaa bbb ccc http://test2.id774.net ddd eee", + "Mon, 07 Mar 2011 15:54:11 +0900" + } } ) + Automatic::Plugin::Blog.count.should eq 0 lambda { instance.run.should have(1).feed }.should change(Automatic::Plugin::Blog, :count).by(1) + Automatic::Plugin::Blog.count.should eq 1 end it "should not store record for the existent blog entry" do @@ -42,10 +49,104 @@ } ) + Automatic::Plugin::Blog.count.should eq 0 instance.run.should have(1).feed lambda { instance.run.should have(0).feed }.should change(Automatic::Plugin::Blog, :count).by(0) + instance.run.should have(0).feed + Automatic::Plugin::Blog.count.should eq 1 + end + + it "should not store record for the existent blog link" do + instance = Automatic::Plugin::StoreFullText.new({"db" => @db_filename}, + AutomaticSpec.generate_pipeline { + feed { + item "http://blog.id774.net/post/100", + "dummy title 1", + "aaa bbb ccc http://test2.id774.net ddd eee", + "Mon, 07 Mar 2011 15:54:11 +0900" + } + } + ) + instance2 = Automatic::Plugin::StoreFullText.new({"db" => @db_filename}, + AutomaticSpec.generate_pipeline { + feed { + item "http://blog.id774.net/post/100", + "dummy title 2", + "aaa bbb ccc http://test2.id774.net ddd eee", + "Mon, 07 Mar 2011 15:54:11 +0900" + } + } + ) + + Automatic::Plugin::Blog.count.should eq 0 + instance.run.should have(1).feed + lambda { + instance2.run.should have(0).feed + }.should change(Automatic::Plugin::Blog, :count).by(0) + instance2.run.should have(0).feed + Automatic::Plugin::Blog.count.should eq 1 end + it "should not store record for the existent blog title" do + instance = Automatic::Plugin::StoreFullText.new({"db" => @db_filename}, + AutomaticSpec.generate_pipeline { + feed { + item "http://blog.id774.net/post/100", + "dummy title 1", + "aaa bbb ccc http://test2.id774.net ddd eee", + "Mon, 07 Mar 2011 15:54:11 +0900" + } + } + ) + instance2 = Automatic::Plugin::StoreFullText.new({"db" => @db_filename}, + AutomaticSpec.generate_pipeline { + feed { + item "http://blog.id774.net/post/200", + "dummy title 1", + "aaa bbb ccc http://test2.id774.net ddd eee", + "Mon, 07 Mar 2011 15:54:11 +0900" + } + } + ) + + Automatic::Plugin::Blog.count.should eq 0 + instance.run.should have(1).feed + lambda { + instance2.run.should have(0).feed + }.should change(Automatic::Plugin::Blog, :count).by(0) + instance2.run.should have(0).feed + Automatic::Plugin::Blog.count.should eq 1 + end + + it "should store 2 records for the independent entries" do + instance = Automatic::Plugin::StoreFullText.new({"db" => @db_filename}, + AutomaticSpec.generate_pipeline { + feed { + item "http://blog.id774.net/post/100", + "dummy title 1", + "aaa bbb ccc http://test2.id774.net ddd eee", + "Mon, 07 Mar 2011 15:54:11 +0900" + } + } + ) + instance2 = Automatic::Plugin::StoreFullText.new({"db" => @db_filename}, + AutomaticSpec.generate_pipeline { + feed { + item "http://blog.id774.net/post/200", + "dummy title 2", + "aaa bbb ccc http://test2.id774.net ddd eee", + "Mon, 07 Mar 2011 15:54:11 +0900" + } + } + ) + + Automatic::Plugin::Blog.count.should eq 0 + instance.run.should have(1).feed + lambda { + instance2.run.should have(1).feed + }.should change(Automatic::Plugin::Blog, :count).by(1) + Automatic::Plugin::Blog.count.should eq 2 + end end diff --git a/spec/plugins/store/permalink_spec.rb b/spec/plugins/store/permalink_spec.rb index a6dc6b2..73e8308 100644 --- a/spec/plugins/store/permalink_spec.rb +++ b/spec/plugins/store/permalink_spec.rb @@ -2,7 +2,7 @@ # Name:: Automatic::Plugin::Store::Permalink # Author:: 774 # Updated:: Dec 17, 2012 -# Copyright:: 774 Copyright (c) 2012 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') diff --git a/spec/plugins/subscription/chan_toru_spec.rb b/spec/plugins/subscription/chan_toru_spec.rb new file mode 100644 index 0000000..b2d204d --- /dev/null +++ b/spec/plugins/subscription/chan_toru_spec.rb @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::Plugin::Subscription::ChanToru +# Author:: soramugi +# Created:: Jun 28, 2013 +# Updated:: Jun 28, 2013 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. +# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. + +require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') + +require 'subscription/chan_toru' + +def chan_toru(config = {}, pipeline = []) + Automatic::Plugin::SubscriptionChanToru.new(config,pipeline) +end + +describe 'Automatic::Plugin::SubscriptionChanToru' do + context 'when feed is empty' do + describe 'attestation error' do + subject { chan_toru } + before do + subject.should_receive(:g_guide_pipeline).times.and_return([]) + end + its(:run) { should be_empty } + end + + describe 'interval & retry was used error' do + config = {'interval' => 1, 'retry' => 1} + subject { chan_toru(config) } + before do + subject. + should_receive(:g_guide_pipeline). + exactly(2). + times.and_return('') + end + its(:run) { should be_empty } + end + end + + context 'when feed' do + describe 'config keyword' do + subject { chan_toru } + before do + pipeline = AutomaticSpec.generate_pipeline { + feed { + item "http://soramugi.net/images/hugehuge" + item "http://tv.so-net.ne.jp/schedule/500333201307052330.action?from=rss" + }} + subject. + should_receive(:g_guide_pipeline). + times.and_return(pipeline) + end + its(:run) { should have(1).feed } + end + end +end diff --git a/spec/plugins/subscription/feed_spec.rb b/spec/plugins/subscription/feed_spec.rb index d18e2da..a0720e0 100644 --- a/spec/plugins/subscription/feed_spec.rb +++ b/spec/plugins/subscription/feed_spec.rb @@ -2,7 +2,7 @@ # Name:: Automatic::Plugin::Subscription::Feed # Author:: 774 # Updated:: Feb 8, 2013 -# Copyright:: 774 Copyright (c) 2012-2013 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') diff --git a/spec/plugins/subscription/g_guide_spec.rb b/spec/plugins/subscription/g_guide_spec.rb new file mode 100644 index 0000000..46c7ef3 --- /dev/null +++ b/spec/plugins/subscription/g_guide_spec.rb @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::Plugin::Subscription::GGuide +# Author:: soramugi +# Created:: Jun 28, 2013 +# Updated:: Jun 28, 2013 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. +# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. + +require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') + +require 'subscription/g_guide' + +def g_guide(config = {}, pipeline = []) + Automatic::Plugin::SubscriptionGGuide.new(config,pipeline) +end + +describe 'Automatic::Plugin::SubscriptionGGuide' do + context 'when config' do + describe 'is empty' do + subject { g_guide } + + its(:feed_url) { + should == Automatic::Plugin::SubscriptionGGuide::G_GUIDE_RSS + + 'stationPlatformId=0&' + } + end + describe 'keyword is anime' do + config = { 'keyword' => 'anime' } + subject { g_guide(config) } + + it 'feed_url' do + subject.feed_url(config['keyword']).should == URI::Parser.new.escape( + Automatic::Plugin::SubscriptionGGuide::G_GUIDE_RSS + + "condition.keyword=#{config['keyword']}&" + + 'stationPlatformId=0&') + end + end + describe 'station is 地上波' do + config = { 'station' => '地上波' } + subject { g_guide(config) } + + its(:feed_url) { + should == URI::Parser.new.escape( + Automatic::Plugin::SubscriptionGGuide::G_GUIDE_RSS + + 'stationPlatformId=1&') + } + end + end + + context 'when feed is empty' do + describe 'attestation error' do + subject { g_guide } + before do + subject.should_receive(:feed_url).and_return('') + end + its(:run) { should be_empty } + end + + describe 'interval & retry was used error' do + config = {'interval' => 1, 'retry' => 1} + subject { g_guide(config) } + before do + subject.should_receive(:feed_url).exactly(2).times.and_return('') + end + its(:run) { should be_empty } + end + end + + context 'when feed' do + describe 'config keyword' do + config = { 'keyword' => 'アニメ', 'station' => '地上波' } + subject { g_guide(config) } + its(:run) { should have(1).feed } + end + + describe 'config keyword ","' do + config = { 'keyword' => 'おじゃる丸,忍たま', 'station' => '地上波' } + subject { g_guide(config) } + its(:run) { should have(2).feed } + end + end +end diff --git a/spec/plugins/subscription/google_reader_star_spec.rb b/spec/plugins/subscription/google_reader_star_spec.rb deleted file mode 100644 index 7f833b0..0000000 --- a/spec/plugins/subscription/google_reader_star_spec.rb +++ /dev/null @@ -1,74 +0,0 @@ -# -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::Subscription::GoogleReaderStar -# Author:: soramugi -# 774 -# Created:: Feb 10, 2013 -# Updated:: Feb 17, 2013 -# Copyright:: soramugi Copyright (c) 2013 -# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. - -require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') - -require 'subscription/google_reader_star' - -describe Automatic::Plugin::SubscriptionGoogleReaderStar do - context "with empty feeds" do - subject { - Automatic::Plugin::SubscriptionGoogleReaderStar.new( - { 'feeds' => [] } - ) - } - - its(:run) { should be_empty } - end - - context "with feeds whose invalid URL" do - subject { - Automatic::Plugin::SubscriptionGoogleReaderStar.new( - { 'feeds' => ["invalid_url"] } - ) - } - - its(:run) { should be_empty } - end - - context "with feeds whose valid URL" do - subject { - Automatic::Plugin::SubscriptionGoogleReaderStar.new( - { 'feeds' => [ - "http://www.google.com/reader/public/atom/user%2F00482198897189159802%2Fstate%2Fcom.google%2Fstarred"] - } - ) - } - - its(:run) { should have(1).feed } - end - - context "with retry to feeds whose valid URL" do - subject { - Automatic::Plugin::SubscriptionGoogleReaderStar.new( - { 'feeds' => [ - "http://www.google.com/reader/public/atom/user%2F00482198897189159802%2Fstate%2Fcom.google%2Fstarred"], - 'retry' => 3, - 'interval' => 5 - } - ) - } - - its(:run) { should have(1).feed } - end - - context "with retry to feeds whose invalid URL" do - subject { - Automatic::Plugin::SubscriptionGoogleReaderStar.new( - { 'feeds' => ["invalid_url"], - 'retry' => 1, - 'interval' => 1 - } - ) - } - - its(:run) { should be_empty } - end - -end diff --git a/spec/plugins/subscription/link_spec.rb b/spec/plugins/subscription/link_spec.rb index 202225d..bcb4568 100644 --- a/spec/plugins/subscription/link_spec.rb +++ b/spec/plugins/subscription/link_spec.rb @@ -3,7 +3,7 @@ # Author:: 774 # Created:: Sep 18, 2012 # Updated:: Feb 8, 2013 -# Copyright:: 774 Copyright (c) 2012-2013 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') diff --git a/spec/plugins/subscription/pocket_spec.rb b/spec/plugins/subscription/pocket_spec.rb index 39cdfd6..39307dd 100644 --- a/spec/plugins/subscription/pocket_spec.rb +++ b/spec/plugins/subscription/pocket_spec.rb @@ -2,8 +2,8 @@ # Name:: Automatic::Plugin::Subscription::Pocket # Author:: soramugi # Created:: May 21, 2013 -# Updated:: May 21, 2013 -# Copyright:: soramugi Copyright (c) 2013 +# Updated:: Feb 25, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') @@ -16,7 +16,6 @@ def pocket(config = {}, pipeline = []) describe 'Automatic::Plugin::SubscriptionPocket' do context 'when feed is empty' do - describe 'attestation error' do subject { pocket } @@ -29,11 +28,9 @@ def pocket(config = {}, pipeline = []) its(:run) { should be_empty } end - end context 'when feed' do - describe 'config optional' do config = { 'optional' => { 'count' => 1, @@ -47,7 +44,7 @@ def pocket(config = {}, pipeline = []) 'given_title' => 'GitHub', 'excerpt' => 'github' }}} - client = mock("client") + client = double("client") client.should_receive(:retrieve). with(config['optional']). and_return(retrieve) @@ -56,6 +53,5 @@ def pocket(config = {}, pipeline = []) its(:run) { should have(1).item } end - end end diff --git a/spec/plugins/subscription/text_spec.rb b/spec/plugins/subscription/text_spec.rb index e2820d9..b7bad74 100644 --- a/spec/plugins/subscription/text_spec.rb +++ b/spec/plugins/subscription/text_spec.rb @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- # Name:: Automatic::Plugin::Subscription::Text # Author:: soramugi +# 774 # Created:: May 6, 2013 -# Updated:: May 6, 2013 -# Copyright:: soramugi Copyright (c) 2013 +# Updated:: Feb 19, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') @@ -51,4 +52,33 @@ its(:run) { should have(1).feed } end + context "with feeds including full fields whose return feed" do + subject { + Automatic::Plugin::SubscriptionText.new( + { 'feeds' => [ {'title' => 'huge', 'url' => "http://hugehuge", 'description' => "aaa", 'comments' => "bbb", 'author' => "ccc" } ] } + ) + } + + its(:run) { should have(1).feed } + end + + context "with file whose return feed" do + subject { + Automatic::Plugin::SubscriptionText.new( + { 'files' => ["spec/fixtures/sampleFeeds.tsv"] } + ) + } + + its(:run) { should have(1).feed } + end + + context "with files whose return feed" do + subject { + Automatic::Plugin::SubscriptionText.new( + { 'files' => ["spec/fixtures/sampleFeeds.tsv", "spec/fixtures/sampleFeeds2.tsv"] } + ) + } + + its(:run) { should have(1).feed } + end end diff --git a/spec/plugins/subscription/tumblr_spec.rb b/spec/plugins/subscription/tumblr_spec.rb index ce379d4..7b17c3f 100644 --- a/spec/plugins/subscription/tumblr_spec.rb +++ b/spec/plugins/subscription/tumblr_spec.rb @@ -3,7 +3,7 @@ # Author:: 774 # Created:: Oct 16, 2012 # Updated:: Feb 8, 2013 -# Copyright:: 774 Copyright (c) 2012-2013 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') @@ -14,7 +14,8 @@ context "with empty URLs" do subject { Automatic::Plugin::SubscriptionTumblr.new( - { 'urls' => [] }) + { 'urls' => [] } + ) } its(:run) { should be_empty } diff --git a/spec/plugins/subscription/twitter_search_spec.rb b/spec/plugins/subscription/twitter_search_spec.rb index dbbca8a..88773a5 100644 --- a/spec/plugins/subscription/twitter_search_spec.rb +++ b/spec/plugins/subscription/twitter_search_spec.rb @@ -2,8 +2,8 @@ # Name:: Automatic::Plugin::Subscription::TwitterSearch # Author:: soramugi # Created:: May 30, 2013 -# Updated:: May 30, 2013 -# Copyright:: soramugi Copyright (c) 2013 +# Updated:: Feb 25, 2014 +# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') @@ -44,7 +44,7 @@ def twitter_search(config = {}, pipeline = []) status.created_at = Time.now search = Hashie::Mash.new search.results = [status] - client = mock("client") + client = double("client") client.should_receive(:search) .with(config['search'],config['opt']) .and_return(search) diff --git a/spec/plugins/subscription/twitter_spec.rb b/spec/plugins/subscription/twitter_spec.rb index 3edb135..f43c982 100644 --- a/spec/plugins/subscription/twitter_spec.rb +++ b/spec/plugins/subscription/twitter_spec.rb @@ -3,7 +3,7 @@ # Author:: 774 # Created:: Sep 10, 2012 # Updated:: Feb 8, 2013 -# Copyright:: 774 Copyright (c) 2012-2013 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') @@ -14,7 +14,8 @@ context "with empty URLs" do subject { Automatic::Plugin::SubscriptionTwitter.new( - { 'urls' => [] }) + { 'urls' => [] } + ) } its(:run) { should be_empty } diff --git a/spec/plugins/subscription/weather_spec.rb b/spec/plugins/subscription/weather_spec.rb index c872eaa..b6f7f9a 100644 --- a/spec/plugins/subscription/weather_spec.rb +++ b/spec/plugins/subscription/weather_spec.rb @@ -3,7 +3,7 @@ # Author:: soramugi # Created:: May 12, 2013 # Updated:: May 12, 2013 -# Copyright:: soramugi Copyright (c) 2013 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') @@ -14,7 +14,7 @@ context "with empty zipcode" do subject { Automatic::Plugin::SubscriptionWeather.new( - { } + {} ) } diff --git a/spec/plugins/subscription/xml_spec.rb b/spec/plugins/subscription/xml_spec.rb new file mode 100644 index 0000000..c5905f8 --- /dev/null +++ b/spec/plugins/subscription/xml_spec.rb @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::Plugin::Subscription::Xml +# Author:: 774 +# Created:: Jul 12, 2013 +# Updated:: Jul 12, 2013 +# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers. +# License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0. + +require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') + +require 'subscription/xml' + +describe Automatic::Plugin::SubscriptionXml do + context "with empty URLs" do + subject { + Automatic::Plugin::SubscriptionXml.new( + { 'urls' => [] }) + } + + its(:run) { should be_empty } + end + + context "with URLs whose invalid URL" do + subject { + Automatic::Plugin::SubscriptionXml.new( + { 'urls' => ["invalid_url"] } + ) + } + + its(:run) { should be_empty } + end + + context "with URLs whose invalid Xml" do + subject { + Automatic::Plugin::SubscriptionXml.new( + { 'urls' => [ + "http://id774.net"] + } + ) + } + + its(:run) { should be_empty } + end + + context "with URLs whose valid XML" do + subject { + Automatic::Plugin::SubscriptionXml.new( + { 'urls' => [ + "http://id774.net/test/xml/data"], + 'interval' => 1 + } + ) + } + + its(:run) { should have(1).item } + end + + context "with retry to URLs whose valid XML" do + subject { + Automatic::Plugin::SubscriptionXml.new( + { 'urls' => [ + "http://id774.net/test/xml/data"], + 'interval' => 2, + 'retry' => 3 + } + ) + } + + its(:run) { should have(1).item } + end + + context "with retry to URLs whose invalid XML" do + subject { + Automatic::Plugin::SubscriptionXml.new( + { 'urls' => ["invalid_url"], + 'interval' => 1, + 'retry' => 1 + } + ) + } + + its(:run) { should be_empty } + end +end diff --git a/test/integration/test_add_pocket.yml b/test/integration/test_add_pocket.yml index a17a395..6f1c48d 100644 --- a/test/integration/test_add_pocket.yml +++ b/test/integration/test_add_pocket.yml @@ -19,5 +19,8 @@ plugins: config: consumer_key: consumer_key access_token: access_token + interval: 2 + retry: 2 #- module: PublishConsole + diff --git a/test/integration/test_chan_toru.yml b/test/integration/test_chan_toru.yml new file mode 100644 index 0000000..5a9f5c2 --- /dev/null +++ b/test/integration/test_chan_toru.yml @@ -0,0 +1,21 @@ +global: + timezone: Asia/Tokyo + cache: + base: /tmp + log: + level: none + +plugins: + - module: SubscriptionChanToru + config: + keyword: 'アニメ' + station: '地上波' + interval: 2 + retry: 2 + + - module: StorePermalink + config: + db: test_chan_toru.db + + #- module: PublishConsoleLink + diff --git a/test/integration/test_descriptionlink.yml b/test/integration/test_descriptionlink.yml new file mode 100644 index 0000000..6a50d64 --- /dev/null +++ b/test/integration/test_descriptionlink.yml @@ -0,0 +1,21 @@ +global: + timezone: Asia/Tokyo + cache: + base: /tmp + log: + level: info + +plugins: + - module: SubscriptionFeed + config: + feeds: + - http://reblog.id774.net/rss + + - module: FilterDescriptionLink + + - module: StorePermalink + config: + db: test_description_link.db + + #- module: PublishConsole + diff --git a/test/integration/test_fluentd.yml b/test/integration/test_fluentd.yml new file mode 100644 index 0000000..4ab7bb3 --- /dev/null +++ b/test/integration/test_fluentd.yml @@ -0,0 +1,22 @@ +global: + timezone: Asia/Tokyo + cache: + base: /tmp + log: + level: info + +plugins: + - module: SubscriptionFeed + config: + feeds: + - http://blog.id774.net/post/feed/ + + - module: PublishFluentd + config: + host: localhost + port: 10000 + tag: automatic_test.publish_fluentd + mode: test + + # - module: PublishConsole + diff --git a/test/integration/test_google_news.yml b/test/integration/test_google_news.yml new file mode 100644 index 0000000..bcd8b43 --- /dev/null +++ b/test/integration/test_google_news.yml @@ -0,0 +1,21 @@ +global: + timezone: Asia/Tokyo + cache: + base: /tmp + log: + level: info + +plugins: + - module: SubscriptionFeed + config: + feeds: + - http://news.google.com/news?hl=ja&ned=us&ie=UTF-8&oe=UTF-8&output=rss + + - module: FilterGoogleNews + + - module: StorePermalink + config: + db: test_google_news.db + + - module: PublishConsole + diff --git a/test/integration/test_googlestar.yml b/test/integration/test_googlestar.yml deleted file mode 100644 index abcbf06..0000000 --- a/test/integration/test_googlestar.yml +++ /dev/null @@ -1,32 +0,0 @@ -global: - timezone: Asia/Tokyo - cache: - base: /tmp - log: - level: info - -plugins: - - module: SubscriptionGoogleReaderStar - config: - feeds: - - http://www.google.com/reader/public/atom/user%2F00482198897189159802%2Fstate%2Fcom.google%2Fstarred - - - module: SubscriptionGoogleReaderStar - config: - feeds: - - http://www.google.com/reader/public/atom/user%2F00482198897189159802%2Fstate%2Fcom.google%2Fstarred - internal: 1 - - - module: SubscriptionGoogleReaderStar - config: - feeds: - - http://www.google.com/reader/public/atom/user%2F00482198897189159802%2Fstate%2Fcom.google%2Fstarred - internal: 5 - retry: 5 - - - module: StorePermalink - config: - db: test_googlestar.db - - #- module: PublishConsole - diff --git a/test/integration/test_image2local.yml b/test/integration/test_image2local.yml index f927390..1352e28 100644 --- a/test/integration/test_image2local.yml +++ b/test/integration/test_image2local.yml @@ -19,11 +19,15 @@ plugins: config: db: test_image.db - - module: StoreTargetLink + - module: StoreFile config: path: /tmp retry: 2 - interval: 4 + interval: 2 - #- module: PublishConsole + - module: StorePermalink + config: + db: test_image2.db + + # - module: PublishConsole diff --git a/test/integration/test_instapaper.yml b/test/integration/test_instapaper.yml index 75fa80a..034b70c 100644 --- a/test/integration/test_instapaper.yml +++ b/test/integration/test_instapaper.yml @@ -19,6 +19,8 @@ plugins: config: email: your_email password: your_pass - interval: 5 + interval: 2 + retry: 2 #- module: PublishConsole + diff --git a/test/integration/test_link2local.yml b/test/integration/test_link2local.yml index cdcfa63..a8e3113 100644 --- a/test/integration/test_link2local.yml +++ b/test/integration/test_link2local.yml @@ -21,8 +21,8 @@ plugins: config: urls: - http://reblog.id774.net - interval: 5 - retry: 5 + interval: 2 + retry: 2 - module: FilterImage diff --git a/test/integration/test_one.yml b/test/integration/test_one.yml index f26eb3a..0418aaa 100644 --- a/test/integration/test_one.yml +++ b/test/integration/test_one.yml @@ -20,3 +20,4 @@ plugins: db: test_one.db # - module: PublishConsole + diff --git a/test/integration/test_pdf2local.yml b/test/integration/test_pdf2local.yml new file mode 100644 index 0000000..ea1a3f1 --- /dev/null +++ b/test/integration/test_pdf2local.yml @@ -0,0 +1,26 @@ +global: + timezone: Asia/Tokyo + cache: + base: /tmp + log: + level: info + +plugins: + - module: SubscriptionFeed + config: + feeds: + - http://webapi.yanoshin.jp/webapi/tdnet/list/recent.rss + + - module: FilterSort + config: + sort: desc + + - module: StorePermalink + config: + db: test_pdf2local.db + + - module: StoreFile + config: + path: /tmp + retry: 3 + interval: 3 diff --git a/test/integration/test_pocket.yml b/test/integration/test_pocket.yml index 8e83b16..1b0ce77 100644 --- a/test/integration/test_pocket.yml +++ b/test/integration/test_pocket.yml @@ -11,7 +11,7 @@ plugins: consumer_key: 'consumer_key' access_token: 'access_token' optional: - favorite: 1 + favorite: 3 count: 2 - module: StorePermalink @@ -19,3 +19,4 @@ plugins: db: test_pocket.db #- module: PublishConsole + diff --git a/test/integration/test_rand.yml b/test/integration/test_rand.yml index 78dea4a..5f3898d 100644 --- a/test/integration/test_rand.yml +++ b/test/integration/test_rand.yml @@ -18,3 +18,4 @@ plugins: db: test_rand.db # - module: PublishConsole + diff --git a/test/integration/test_sanitize.yml b/test/integration/test_sanitize.yml new file mode 100644 index 0000000..c2d734a --- /dev/null +++ b/test/integration/test_sanitize.yml @@ -0,0 +1,23 @@ +global: + timezone: Asia/Tokyo + cache: + base: /tmp + log: + level: info + +plugins: + - module: SubscriptionFeed + config: + feeds: + - http://blog.id774.net/post/feed/ + + - module: FilterSanitize + config: + mode: restricted + + - module: StoreFullText + config: + db: test_sanitize.db + + # - module: PublishConsole + diff --git a/test/integration/test_sort.yml b/test/integration/test_sort.yml index 3332d72..faf5cd3 100644 --- a/test/integration/test_sort.yml +++ b/test/integration/test_sort.yml @@ -21,8 +21,8 @@ plugins: config: feeds: - http://blog.id774.net/post/feed/ - interval: 5 - retry: 5 + interval: 2 + retry: 2 - module: FilterSort config: diff --git a/test/integration/test_text2feed.yml b/test/integration/test_text2feed.yml index d333dfe..819097e 100644 --- a/test/integration/test_text2feed.yml +++ b/test/integration/test_text2feed.yml @@ -19,9 +19,18 @@ plugins: - title: 'Tumblr' url: 'http://www.tumblr.com/dashboard' + description: 'aaa' + comments: 'bbb' + author: 'ccc' + files: + - 'spec/fixtures/sampleFeeds.tsv' + files: + - 'spec/fixtures/sampleFeeds.tsv' + - 'spec/fixtures/sampleFeeds2.tsv' - module: StorePermalink config: db: test_text2feed.db # - module: PublishConsole + diff --git a/test/integration/test_tumblr2local.yml b/test/integration/test_tumblr2local.yml index 9a3dc1d..a81323f 100644 --- a/test/integration/test_tumblr2local.yml +++ b/test/integration/test_tumblr2local.yml @@ -24,8 +24,8 @@ plugins: urls: - http://reblog.id774.net pages: 1 - interval: 5 - retry: 5 + interval: 2 + retry: 2 - module: FilterImage @@ -33,5 +33,11 @@ plugins: config: db: test_tumblr.db + - module: StoreFile + config: + path: /tmp + retry: 2 + interval: 2 + #- module: PublishConsole diff --git a/test/integration/test_twitter_search.yml b/test/integration/test_twitter_search.yml new file mode 100644 index 0000000..679e481 --- /dev/null +++ b/test/integration/test_twitter_search.yml @@ -0,0 +1,22 @@ +global: + timezone: Asia/Tokyo + cache: + base: /tmp + log: + level: none + +plugins: + - module: SubscriptionTwitterSearch + config: + consumer_key: 'consumer_key' + consumer_secret: 'consumer_secret' + oauth_token: 'oauth_token' + oauth_token_secret: 'oauth_token_secret' + search: 'ruby' + + - module: StorePermalink + config: + db: test_twitter_search.db + + #- module: PublishConsole + diff --git a/test/integration/test_weather.yml b/test/integration/test_weather.yml index abfee61..204e874 100644 --- a/test/integration/test_weather.yml +++ b/test/integration/test_weather.yml @@ -16,3 +16,4 @@ plugins: db: test_weather.db #- module: PublishConsole + diff --git a/test/integration/test_xml2fluentd.yml b/test/integration/test_xml2fluentd.yml new file mode 100644 index 0000000..3f5fb60 --- /dev/null +++ b/test/integration/test_xml2fluentd.yml @@ -0,0 +1,21 @@ +global: + timezone: Asia/Tokyo + cache: + base: /tmp + log: + level: info + +plugins: + - module: SubscriptionXml + config: + urls: + - http://id774.net/test/xml/data + + - module: ProvideFluentd + config: + host: localhost + port: 10000 + tag: automatic_test.provide_fluentd + + # - module: PublishConsole +