Skip to content
This repository was archived by the owner on Sep 3, 2022. It is now read-only.

Commit ab05ea2

Browse files
supriyagargsupriyagarg
committed
Monitoring API
This initial version of the monitoring client for Datalab allows users to query timeseries data. It introduces a new module under gcp that users can import using: from gcp.stackdriver import monitoring Querying timeseries data: There is a `Query` class that allows users to query timeseries data for their monitored resources. They can initialize the query by specifying a metric type and time interval, and refine it by adding filters to it. The timeseries data is returned as a pandas DataFrame that allows users to further manipulate the data and visualize it within Datalab. IPython magics: There are a couple of IPython magic commands to allow users to list details of the available metrics and resource types. E.g.: %%monitoring list metrics The base library: The _impl directory is a snapshot of an earlier version of a monitoring client library that we have submitted to gcloud-python: googleapis/google-cloud-python#1691 It works as the base library on top of which the Datalab library adds interactive features. The authors of this code are @rimey and myself, both at Google.
1 parent 4a86c50 commit ab05ea2

17 files changed

Lines changed: 2550 additions & 0 deletions

File tree

‎content/datalab/tutorials/Monitoring/Getting started.ipynb‎

Lines changed: 750 additions & 0 deletions
Large diffs are not rendered by default.

‎sources/lib/api/gcp/stackdriver/__init__.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# Copyright 2016 Google Inc. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""Google Cloud Platform library - Stackdriver Functionality."""

‎sources/lib/api/gcp/stackdriver/monitoring/__init__.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# Copyright 2016 Google Inc. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""Google Cloud Platform library - Monitoring Functionality."""
16+
17+
from ._timeseries import Query

‎sources/lib/api/gcp/stackdriver/monitoring/_impl/__init__.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Copyright 2016 Google Inc. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""Google Monitoring API V3."""
16+
17+
from .client import Client
18+
from .group import Group
19+
from .label import LabelDescriptor
20+
from .metric import Metric, MetricDescriptor
21+
from .resource import Resource, ResourceDescriptor
22+
from .timeseries import Query

‎sources/lib/api/gcp/stackdriver/monitoring/_impl/api.py‎

Lines changed: 279 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,279 @@
1+
# Copyright 2016 Google Inc. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""HTTP API wrapper for the Google Monitoring API."""
16+
17+
from gcp._util import Http
18+
19+
20+
class Api(object):
21+
"""A helper class to issue Monitoring API HTTP requests."""
22+
23+
_ENDPOINT = 'https://monitoring.googleapis.com/v3/'
24+
_GROUPS_PATH = 'projects/%s/groups/%s'
25+
_GROUP_MEMBERS_PATH = '{}/members'.format(_GROUPS_PATH)
26+
_METRIC_PATH = 'projects/%s/metricDescriptors/%s'
27+
_RESOURCE_PATH = 'projects/%s/monitoredResourceDescriptors/%s'
28+
_TIMESERIES_PATH = 'projects/%s/timeSeries'
29+
30+
def __init__(self, credentials):
31+
"""Initializes the Monitoring helper with credentials.
32+
33+
Args:
34+
credentials: The credentials to use to authorize requests.
35+
"""
36+
self._credentials = credentials
37+
38+
def groups_get(self, group_id, project_id):
39+
"""Issues a request to retrieve information about a group.
40+
41+
Args:
42+
group_id: The ID of the group.
43+
project_id: The project ID to use to fetch the results.
44+
45+
Returns:
46+
A parsed result object.
47+
"""
48+
url = self._ENDPOINT + self._GROUPS_PATH % (project_id, group_id)
49+
return Http.request(url, credentials=self._credentials)
50+
51+
def groups_list(self, project_id, children_of_group=None,
52+
ancestors_of_group=None, descendants_of_group=None,
53+
page_size=None, page_token=None):
54+
"""Issues a request to list the groups in the project.
55+
56+
At most one of the parameters children_of_group, ancestors_of_group, and
57+
descendants_of_group may be specified to narrow the query.
58+
59+
Args:
60+
project_id: The project ID to use to fetch the results.
61+
children_of_group: The ID of the group whose children are to be listed.
62+
ancestors_of_group: The ID of the group whose ancestors are to be listed.
63+
descendants_of_group: The ID of the group whose descendants are to be
64+
listed.
65+
page_size: The maximum number of results to return per page.
66+
page_token: An optional token to continue the retrieval.
67+
68+
Returns:
69+
A parsed result object.
70+
"""
71+
url = self._ENDPOINT + self._GROUPS_PATH % (project_id, '')
72+
args = {}
73+
74+
if children_of_group is not None:
75+
args['childrenOfGroup'] = self._GROUPS_PATH % (project_id,
76+
children_of_group)
77+
if ancestors_of_group is not None:
78+
args['ancestorsOfGroup'] = self._GROUPS_PATH % (project_id,
79+
ancestors_of_group)
80+
if descendants_of_group is not None:
81+
args['descendantsOfGroup'] = self._GROUPS_PATH % (project_id,
82+
descendants_of_group)
83+
if page_size is not None:
84+
args['pageSize'] = page_size
85+
if page_token is not None:
86+
args['pageToken'] = page_token
87+
88+
return Http.request(url, args=args, credentials=self._credentials)
89+
90+
def groups_members_list(
91+
self, group_id, project_id, end_time=None, start_time=None, filter=None,
92+
page_size=None, page_token=None):
93+
"""Issues a request to retrieve the members of a group.
94+
95+
Args:
96+
group_id: The ID of the group.
97+
project_id: The project ID to use to fetch the results.
98+
end_time: The end time (inclusive) of the group membership.
99+
start_time: The start time (exclusive) of the group membership.
100+
filter: A filter to restrict the resources returned. E.g.:
101+
'resource.type = "gce_instance"'
102+
page_size: The maximum number of results to return per page.
103+
page_token: An optional token to continue the retrieval.
104+
105+
Returns:
106+
A parsed result object.
107+
"""
108+
# Allow "filter" as a parameter name: pylint: disable=redefined-builtin
109+
url = self._ENDPOINT + self._GROUP_MEMBERS_PATH % (project_id, group_id)
110+
args = {}
111+
if end_time is not None:
112+
args['interval.endTime'] = _format_timestamp_as_string(end_time)
113+
if start_time is not None:
114+
args['interval.startTime'] = _format_timestamp_as_string(start_time)
115+
if filter is not None:
116+
args['filter'] = filter
117+
if page_size is not None:
118+
args['pageSize'] = page_size
119+
if page_token is not None:
120+
args['pageToken'] = page_token
121+
return Http.request(url, args=args, credentials=self._credentials)
122+
123+
def metric_descriptors_get(self, metric_id, project_id):
124+
"""Issues a request to retrieve information about a metric.
125+
126+
Args:
127+
metric_id: The ID of the metric.
128+
project_id: The project ID to use to fetch the results.
129+
130+
Returns:
131+
A parsed result object.
132+
"""
133+
url = self._ENDPOINT + self._METRIC_PATH % (project_id, metric_id)
134+
return Http.request(url, credentials=self._credentials)
135+
136+
def metric_descriptors_list(
137+
self, project_id, filter=None, page_size=None, page_token=None):
138+
"""Issues a request to list the descriptors of all metrics in the project.
139+
140+
Args:
141+
project_id: The project ID to use to fetch the results.
142+
filter: A filter to restrict the metrics returned. E.g.:
143+
'metric.type = starts_with("compute")'
144+
page_size: The maximum number of results to return per page.
145+
page_token: An optional token to continue the retrieval.
146+
147+
Returns:
148+
A parsed result object.
149+
"""
150+
# Allow "filter" as a parameter name: pylint: disable=redefined-builtin
151+
url = self._ENDPOINT + self._METRIC_PATH % (project_id, '')
152+
args = {}
153+
if filter is not None:
154+
args['filter'] = filter
155+
if page_size is not None:
156+
args['pageSize'] = page_size
157+
if page_token is not None:
158+
args['pageToken'] = page_token
159+
return Http.request(url, args=args, credentials=self._credentials)
160+
161+
def monitored_resource_descriptors_get(self, resource_type, project_id):
162+
"""Issues a request to retrieve information about a resource type.
163+
164+
Args:
165+
resource_type: The type of resource to get the descriptor for.
166+
project_id: The project ID to use to fetch the results.
167+
168+
Returns:
169+
A parsed result object.
170+
"""
171+
url = self._ENDPOINT + self._RESOURCE_PATH % (project_id, resource_type)
172+
return Http.request(url, credentials=self._credentials)
173+
174+
def monitored_resource_descriptors_list(
175+
self, project_id, filter=None, page_size=None, page_token=None):
176+
"""Issues a request to list the descriptors of available resource types.
177+
178+
Args:
179+
project_id: The project ID to use to fetch the results.
180+
filter: A filter for the resource descriptors. E.g.:
181+
'resource.type = starts_with("gce_")'
182+
page_size: The maximum number of results to return per page.
183+
page_token: An optional token to continue the retrieval.
184+
185+
Returns:
186+
A parsed result object.
187+
"""
188+
# Allow "filter" as a parameter name: pylint: disable=redefined-builtin
189+
url = self._ENDPOINT + self._RESOURCE_PATH % (project_id, '')
190+
args = {}
191+
if filter is not None:
192+
args['filter'] = filter
193+
if page_size is not None:
194+
args['pageSize'] = page_size
195+
if page_token is not None:
196+
args['pageToken'] = page_token
197+
return Http.request(url, args=args, credentials=self._credentials)
198+
199+
def time_series_list(
200+
self, project_id, filter, end_time, start_time=None,
201+
per_series_aligner=None, alignment_period_seconds=None,
202+
cross_series_reducer=None, group_by_fields=(),
203+
view=None,
204+
page_size=None,
205+
page_token=None):
206+
"""Issues a request to retrieve metric data.
207+
208+
Args:
209+
project_id: The project ID to use to fetch the results.
210+
filter: The filter for the resource and metrics to fetch.
211+
end_time: The end time (inclusive) of the timeseries data.
212+
start_time: The start time (exclusive) of the timeseries data.
213+
per_series_aligner: An alignment period for the timeseries data.
214+
alignment_period_seconds: An int specifying the alignment period.
215+
cross_series_reducer: The reduce method for aggregating multiple
216+
timeseries.
217+
group_by_fields: An iterable of fields to preserve when
218+
cross_series_reducer is specified. E.g.: ["resource.zone"]
219+
view: Specifies which information is returned about the time series.
220+
Must be one of "FULL" or "HEADERS".
221+
page_size: The maximum number of results to return per page.
222+
page_token: An optional token to continue the retrieval.
223+
224+
Returns:
225+
A parsed result object.
226+
"""
227+
# Allow "filter" as a parameter name: pylint: disable=redefined-builtin
228+
229+
url = self._ENDPOINT + self._TIMESERIES_PATH % project_id
230+
231+
# Assemble the arguments for the RPC.
232+
args = {
233+
'filter': filter,
234+
'interval.endTime': _format_timestamp_as_string(end_time),
235+
}
236+
if start_time is not None:
237+
args['interval.startTime'] = _format_timestamp_as_string(start_time)
238+
if per_series_aligner is not None:
239+
args['aggregation.perSeriesAligner'] = per_series_aligner
240+
if alignment_period_seconds is not None:
241+
args['aggregation.alignmentPeriod'] = '{:d}s'.format(
242+
alignment_period_seconds)
243+
if cross_series_reducer is not None:
244+
args['aggregation.crossSeriesReducer'] = cross_series_reducer
245+
if view is not None:
246+
args['view'] = view
247+
if page_size is not None:
248+
args['pageSize'] = page_size
249+
if page_token is not None:
250+
args['pageToken'] = page_token
251+
252+
# Convert to a list before adding repeated fields.
253+
args = args.items()
254+
255+
args.extend(('aggregation.groupByFields', field)
256+
for field in group_by_fields)
257+
258+
return Http.request(url, args=args, credentials=self._credentials)
259+
260+
261+
def _format_timestamp_as_string(timestamp):
262+
"""Converts a datetime object to a string as required by the API.
263+
264+
Args:
265+
timestamp: A Python datetime object or a timestamp string in RFC3339
266+
UTC "Zulu" format.
267+
268+
Returns:
269+
The string version of the timestamp. For example:
270+
"2016-02-17T19:18:01.763000Z".
271+
"""
272+
if isinstance(timestamp, basestring):
273+
return timestamp
274+
275+
if timestamp.tzinfo is not None:
276+
# Convert to UTC and remove the time zone info.
277+
timestamp = timestamp.replace(tzinfo=None) - timestamp.utcoffset()
278+
279+
return timestamp.isoformat() + 'Z'

0 commit comments

Comments
 (0)