Skip to content

Commit c31350b

Browse files
hpatel41hpatel41
authored
TDL-13117: Added support for KeyError on fetching data from dictionary (singer-io#115)
* TDL-13117: Added support for KeyError on fetching data from dictionary * Added code for handling KeyError
1 parent 1909ca9 commit c31350b

2 files changed

Lines changed: 160 additions & 3 deletions

File tree

‎tap_github/__init__.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,7 @@ def get_all_teams(schemas, repo_path, state, mdata):
246246
extraction_time = singer.utils.now()
247247

248248
for r in teams:
249+
team_slug = r.get('slug')
249250
r['_sdc_repository'] = repo_path
250251

251252
# transform and write release record
@@ -256,13 +257,11 @@ def get_all_teams(schemas, repo_path, state, mdata):
256257
counter.increment()
257258

258259
if schemas.get('team_members'):
259-
team_slug = r['slug']
260260
for team_members_rec in get_all_team_members(team_slug, schemas['team_members'], repo_path, state, mdata):
261261
singer.write_record('team_members', team_members_rec, time_extracted=extraction_time)
262262
singer.write_bookmark(state, repo_path, 'team_members', {'since': singer.utils.strftime(extraction_time)})
263263

264264
if schemas.get('team_memberships'):
265-
team_slug = r['slug']
266265
for team_memberships_rec in get_all_team_memberships(team_slug, schemas['team_memberships'], repo_path, state, mdata):
267266
singer.write_record('team_memberships', team_memberships_rec, time_extracted=extraction_time)
268267

@@ -878,10 +877,11 @@ def get_all_stargazers(schema, repo_path, state, mdata):
878877
stargazers = response.json()
879878
extraction_time = singer.utils.now()
880879
for stargazer in stargazers:
880+
user_id = stargazer['user']['id']
881881
stargazer['_sdc_repository'] = repo_path
882882
with singer.Transformer() as transformer:
883883
rec = transformer.transform(stargazer, schema, metadata=metadata.to_map(mdata))
884-
rec['user_id'] = rec['user']['id']
884+
rec['user_id'] = user_id
885885
singer.write_record('stargazers', rec, time_extracted=extraction_time)
886886
singer.write_bookmark(state, repo_path, 'stargazers', {'since': singer.utils.strftime(extraction_time)})
887887
counter.increment()

‎tests/unittests/test_key_error.py‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
import unittest
2+
from unittest import mock
3+
import tap_github.__init__ as tap_github
4+
5+
class Mockresponse:
6+
def __init__(self, resp):
7+
self.json_data = resp
8+
9+
def json(self):
10+
return [(self.json_data)]
11+
12+
def get_response(json):
13+
yield Mockresponse(resp=json)
14+
15+
@mock.patch("tap_github.__init__.authed_get_all_pages")
16+
class TestKeyErrorSlug(unittest.TestCase):
17+
18+
@mock.patch("tap_github.__init__.get_all_team_members")
19+
def test_slug_sub_stream_selected_slug_selected(self, mocked_team_members, mocked_request):
20+
json = {"key": "value", "slug": "team-slug"}
21+
22+
mocked_request.return_value = get_response(json)
23+
24+
schemas = {"teams": "None", "team_members": "None"}
25+
mdata =[
26+
{
27+
'breadcrumb': [],
28+
'metadata': {'selected': True, 'table-key-properties': ['id']}
29+
},
30+
{
31+
'breadcrumb': ['properties', 'slug'],
32+
'metadata': {'inclusion': 'available'}
33+
},
34+
{
35+
"breadcrumb": [ "properties", "name"],
36+
"metadata": {"inclusion": "available"}
37+
}]
38+
tap_github.get_all_teams(schemas, "tap-github", {}, mdata)
39+
self.assertEquals(mocked_team_members.call_count, 1)
40+
41+
@mock.patch("tap_github.__init__.get_all_team_members")
42+
def test_slug_sub_stream_not_selected_slug_selected(self, mocked_team_members, mocked_request):
43+
json = {"key": "value", "slug": "team-slug"}
44+
45+
mocked_request.return_value = get_response(json)
46+
47+
schemas = {"teams": "None"}
48+
mdata =[
49+
{
50+
'breadcrumb': [],
51+
'metadata': {'selected': True, 'table-key-properties': ['id']}
52+
},
53+
{
54+
'breadcrumb': ['properties', 'slug'],
55+
'metadata': {'inclusion': 'available'}
56+
},
57+
{
58+
"breadcrumb": [ "properties", "name"],
59+
"metadata": {"inclusion": "available"}
60+
}]
61+
tap_github.get_all_teams(schemas, "tap-github", {}, mdata)
62+
self.assertEquals(mocked_team_members.call_count, 0)
63+
64+
@mock.patch("tap_github.__init__.get_all_team_members")
65+
def test_slug_sub_stream_selected_slug_not_selected(self, mocked_team_members, mocked_request):
66+
json = {"key": "value", "slug": "team-slug"}
67+
68+
mocked_request.return_value = get_response(json)
69+
70+
schemas = {"teams": "None", "team_members": "None"}
71+
mdata =[
72+
{
73+
'breadcrumb': [],
74+
'metadata': {'selected': True, 'table-key-properties': ['id']}
75+
},
76+
{
77+
'breadcrumb': ['properties', 'slug'],
78+
'metadata': {'inclusion': 'available', 'selected': False}
79+
},
80+
{
81+
"breadcrumb": [ "properties", "name"],
82+
"metadata": {"inclusion": "available"}
83+
}]
84+
tap_github.get_all_teams(schemas, "tap-github", {}, mdata)
85+
self.assertEquals(mocked_team_members.call_count, 1)
86+
87+
@mock.patch("tap_github.__init__.get_all_team_members")
88+
def test_slug_sub_stream_not_selected_slug_not_selected(self, mocked_team_members, mocked_request):
89+
json = {"key": "value", "slug": "team-slug"}
90+
91+
mocked_request.return_value = get_response(json)
92+
93+
schemas = {"teams": "None"}
94+
mdata =[
95+
{
96+
'breadcrumb': [],
97+
'metadata': {'selected': True, 'table-key-properties': ['id']}
98+
},
99+
{
100+
'breadcrumb': ['properties', 'slug'],
101+
'metadata': {'inclusion': 'available', 'selected': False}
102+
},
103+
{
104+
"breadcrumb": [ "properties", "name"],
105+
"metadata": {"inclusion": "available"}
106+
}]
107+
tap_github.get_all_teams(schemas, "tap-github", {}, mdata)
108+
self.assertEquals(mocked_team_members.call_count, 0)
109+
110+
@mock.patch("tap_github.__init__.authed_get_all_pages")
111+
class TestKeyErrorUser(unittest.TestCase):
112+
113+
@mock.patch("singer.write_record")
114+
def test_user_not_selected_in_stargazers(self, mocked_write_records, mocked_request):
115+
json = {"key": "value", "user": {"id": 1}}
116+
117+
mocked_request.return_value = get_response(json)
118+
119+
schemas = {"teams": "None"}
120+
mdata =[
121+
{
122+
'breadcrumb': [],
123+
'metadata': {'selected': True, 'table-key-properties': ['user_id']}
124+
},
125+
{
126+
"breadcrumb": ["properties", "user"],
127+
"metadata": {"inclusion": "available", "selected": False}
128+
},
129+
{
130+
"breadcrumb": ["properties", "starred_at"],
131+
"metadata": {"inclusion": "available"}
132+
}]
133+
tap_github.get_all_stargazers(schemas, "tap-github", {}, mdata)
134+
self.assertEquals(mocked_write_records.call_count, 1)
135+
136+
@mock.patch("singer.write_record")
137+
def test_user_selected_in_stargazers(self, mocked_write_records, mocked_request):
138+
json = {"key": "value", "user": {"id": 1}}
139+
140+
mocked_request.return_value = get_response(json)
141+
142+
schemas = {"stargazers": "None"}
143+
mdata =[
144+
{
145+
'breadcrumb': [],
146+
'metadata': {'selected': True, 'table-key-properties': ['user_id']}
147+
},
148+
{
149+
"breadcrumb": ["properties", "user"],
150+
"metadata": {"inclusion": "available"}
151+
},
152+
{
153+
"breadcrumb": ["properties", "starred_at"],
154+
"metadata": {"inclusion": "available"}
155+
}]
156+
tap_github.get_all_stargazers(schemas, "tap-github", {}, mdata)
157+
self.assertEquals(mocked_write_records.call_count, 1)

0 commit comments

Comments
 (0)