From 24c9b5b76df1f85aea015d52a06de9a5af51956c Mon Sep 17 00:00:00 2001 From: David Personette Date: Wed, 24 Feb 2021 10:08:30 -0500 Subject: [PATCH] update to feedjira 3 --- Gemfile | 3 ++- Gemfile.lock | 19 ++++++++--------- app/tasks/fetch_feed.rb | 7 +++++-- app/utils/feed_discovery.rb | 12 ++++++----- spec/tasks/fetch_feed_spec.rb | 35 +++++++++++++++++++++---------- spec/utils/feed_discovery_spec.rb | 33 +++++++++++++++++++---------- 6 files changed, 69 insertions(+), 40 deletions(-) diff --git a/Gemfile b/Gemfile index 02907e99b..70cf9fc9b 100644 --- a/Gemfile +++ b/Gemfile @@ -30,7 +30,8 @@ gem "bcrypt", "~> 3.1" gem "delayed_job", "~> 4.1" gem "delayed_job_active_record", "~> 4.1" gem "feedbag", "~> 0.9.5" -gem "feedjira", "~> 2.1.3" +gem "feedjira", "~> 3.0" +gem "httparty" gem "i18n" gem "loofah", "~> 2.3" gem "nokogiri", "~> 1.11" diff --git a/Gemfile.lock b/Gemfile.lock index a27c1c98b..d2c5ea6a7 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -49,17 +49,14 @@ GEM execjs (2.7.0) faker (1.6.3) i18n (~> 0.5) - faraday (0.13.1) - multipart-post (>= 1.2, < 3) - faraday_middleware (0.12.2) - faraday (>= 0.7.4, < 1.0) feedbag (0.9.5) nokogiri (~> 1.0) - feedjira (2.1.3) - faraday (>= 0.9) - faraday_middleware (>= 0.9) - loofah (>= 2.0) + feedjira (3.1.2) + loofah (>= 2.3.1) sax-machine (>= 1.0) + httparty (0.18.1) + mime-types (~> 3.0) + multi_xml (>= 0.5.2) i18n (0.9.5) concurrent-ruby (~> 1.0) jaro_winkler (1.5.4) @@ -69,12 +66,13 @@ GEM crass (~> 1.0.2) nokogiri (>= 1.5.9) method_source (0.8.2) - mime-types (3.0) + mime-types (3.3.1) mime-types-data (~> 3.2015) mime-types-data (3.2016.0221) mini_portile2 (2.5.0) minitest (5.12.2) multi_json (1.14.1) + multi_xml (0.6.0) multipart-post (2.0.0) mustermann (1.0.3) nokogiri (1.11.0) @@ -194,7 +192,8 @@ DEPENDENCIES delayed_job_active_record (~> 4.1) faker (~> 1.2) feedbag (~> 0.9.5) - feedjira (~> 2.1.3) + feedjira (~> 3.0) + httparty i18n loofah (~> 2.3) nokogiri (~> 1.11) diff --git a/app/tasks/fetch_feed.rb b/app/tasks/fetch_feed.rb index 941a9005c..c0af4efa0 100644 --- a/app/tasks/fetch_feed.rb +++ b/app/tasks/fetch_feed.rb @@ -1,13 +1,15 @@ require "feedjira" +require "httparty" require_relative "../repositories/story_repository" require_relative "../repositories/feed_repository" require_relative "../commands/feeds/find_new_stories" class FetchFeed - def initialize(feed, parser: Feedjira::Feed, logger: nil) + def initialize(feed, parser: Feedjira, client: HTTParty, logger: nil) @feed = feed @parser = parser + @client = client @logger = logger end @@ -30,7 +32,8 @@ def fetch private def fetch_raw_feed - @parser.fetch_and_parse(@feed.url) + response = @client.get(@feed.url).to_s + @parser.parse(response) end def feed_not_modified diff --git a/app/utils/feed_discovery.rb b/app/utils/feed_discovery.rb index f0fb64697..dd65d4740 100644 --- a/app/utils/feed_discovery.rb +++ b/app/utils/feed_discovery.rb @@ -1,20 +1,22 @@ require "feedbag" require "feedjira" +require "httparty" class FeedDiscovery - def discover(url, finder = Feedbag, parser = Feedjira::Feed) - get_feed_for_url(url, parser) do + def discover(url, finder = Feedbag, parser = Feedjira, client = HTTParty) + get_feed_for_url(url, parser, client) do urls = finder.find(url) return false if urls.empty? - get_feed_for_url(urls.first, parser) do + get_feed_for_url(urls.first, parser, client) do return false end end end - def get_feed_for_url(url, parser) - feed = parser.fetch_and_parse(url) + def get_feed_for_url(url, parser, client) + response = client.get(url).to_s + feed = parser.parse(response) feed.feed_url ||= url feed rescue StandardError diff --git a/spec/tasks/fetch_feed_spec.rb b/spec/tasks/fetch_feed_spec.rb index 7408e854e..8448bbd12 100644 --- a/spec/tasks/fetch_feed_spec.rb +++ b/spec/tasks/fetch_feed_spec.rb @@ -18,24 +18,26 @@ context "when feed has not been modified" do it "should not try to fetch posts" do - parser = double(fetch_and_parse: 304) + client = class_spy(HTTParty) + parser = class_double(Feedjira, parse: 304) expect(StoryRepository).not_to receive(:add) - FetchFeed.new(daring_fireball, parser: parser) + FetchFeed.new(daring_fireball, parser: parser, client: client) end end context "when no new posts have been added" do it "should not add any new posts" do fake_feed = double(last_modified: Time.new(2012, 12, 31)) - parser = double(fetch_and_parse: fake_feed) + client = class_spy(HTTParty) + parser = class_double(Feedjira, parse: fake_feed) allow_any_instance_of(FindNewStories).to receive(:new_stories).and_return([]) expect(StoryRepository).not_to receive(:add) - FetchFeed.new(daring_fireball, parser: parser).fetch + FetchFeed.new(daring_fireball, parser: parser, client: client).fetch end end @@ -45,7 +47,8 @@ let(:old_story) { double } let(:fake_feed) { double(last_modified: now, entries: [new_story, old_story]) } - let(:fake_parser) { double(fetch_and_parse: fake_feed) } + let(:fake_client) { class_spy(HTTParty) } + let(:fake_parser) { class_double(Feedjira, parse: fake_feed) } before { allow_any_instance_of(FindNewStories).to receive(:new_stories).and_return([new_story]) } @@ -53,35 +56,45 @@ expect(StoryRepository).to receive(:add).with(new_story, daring_fireball) expect(StoryRepository).not_to receive(:add).with(old_story, daring_fireball) - FetchFeed.new(daring_fireball, parser: fake_parser).fetch + FetchFeed.new( + daring_fireball, + parser: fake_parser, + client: fake_client + ).fetch end it "should update the last fetched time for the feed" do expect(FeedRepository).to receive(:update_last_fetched) .with(daring_fireball, now) - FetchFeed.new(daring_fireball, parser: fake_parser).fetch + FetchFeed.new( + daring_fireball, + parser: fake_parser, + client: fake_client + ).fetch end end context "feed status" do it "sets the status to green if things are all good" do fake_feed = double(last_modified: Time.new(2012, 12, 31), entries: []) - parser = double(fetch_and_parse: fake_feed) + client = class_spy(HTTParty) + parser = class_double(Feedjira, parse: fake_feed) expect(FeedRepository).to receive(:set_status) .with(:green, daring_fireball) - FetchFeed.new(daring_fireball, parser: parser).fetch + FetchFeed.new(daring_fireball, parser: parser, client: client).fetch end it "sets the status to red if things go wrong" do - parser = double(fetch_and_parse: 404) + client = class_spy(HTTParty) + parser = class_double(Feedjira, parse: 404) expect(FeedRepository).to receive(:set_status) .with(:red, daring_fireball) - FetchFeed.new(daring_fireball, parser: parser).fetch + FetchFeed.new(daring_fireball, parser: parser, client: client).fetch end end end diff --git a/spec/utils/feed_discovery_spec.rb b/spec/utils/feed_discovery_spec.rb index 1c10286d9..19be6b4eb 100644 --- a/spec/utils/feed_discovery_spec.rb +++ b/spec/utils/feed_discovery_spec.rb @@ -4,7 +4,8 @@ describe FeedDiscovery do let(:finder) { double } - let(:parser) { double } + let(:client) { class_double(HTTParty) } + let(:parser) { class_double(Feedjira) } let(:feed) { double(feed_url: url) } let(:url) { "http://example.com" } @@ -13,38 +14,48 @@ describe "#discover" do it "returns false if url is not a feed and feed url cannot be discovered" do - expect(parser).to receive(:fetch_and_parse).with(url).and_raise(StandardError) + expect(client).to receive(:get).with(url) + expect(parser).to receive(:parse).and_raise(StandardError) expect(finder).to receive(:find).and_return([]) - result = FeedDiscovery.new.discover(url, finder, parser) + result = FeedDiscovery.new.discover(url, finder, parser, client) expect(result).to eq(false) end it "returns a feed if the url provided is parsable" do - expect(parser).to receive(:fetch_and_parse).with(url).and_return(feed) + expect(client).to receive(:get).with(url) + expect(parser).to receive(:parse).and_return(feed) - result = FeedDiscovery.new.discover(url, finder, parser) + result = FeedDiscovery.new.discover(url, finder, parser, client) expect(result).to eq feed end it "returns false if the discovered feed is not parsable" do - expect(parser).to receive(:fetch_and_parse).with(url).and_raise(StandardError) + expect(client).to receive(:get).with(url) + expect(parser).to receive(:parse).and_raise(StandardError) + expect(finder).to receive(:find).and_return([invalid_discovered_url]) - expect(parser).to receive(:fetch_and_parse).with(invalid_discovered_url).and_raise(StandardError) - result = FeedDiscovery.new.discover(url, finder, parser) + expect(client).to receive(:get).with(invalid_discovered_url) + expect(parser).to receive(:parse).and_raise(StandardError) + + result = FeedDiscovery.new.discover(url, finder, parser, client) expect(result).to eq(false) end it "returns the feed if the discovered feed is parsable" do - expect(parser).to receive(:fetch_and_parse).with(url).and_raise(StandardError) + expect(client).to receive(:get).with(url) + expect(parser).to receive(:parse).and_raise(StandardError) + expect(finder).to receive(:find).and_return([valid_discovered_url]) - expect(parser).to receive(:fetch_and_parse).with(valid_discovered_url).and_return(feed) - result = FeedDiscovery.new.discover(url, finder, parser) + expect(client).to receive(:get).with(valid_discovered_url) + expect(parser).to receive(:parse).and_return(feed) + + result = FeedDiscovery.new.discover(url, finder, parser, client) expect(result).to eq feed end