Skip to content

Commit 61490fc

Browse files
liyanhui1228liyanhui1228
committed
Stop writing to '/var/log/app_engine/' and write logs to Stackdriver logging API
1 parent ecd4da4 commit 61490fc

10 files changed

Lines changed: 150 additions & 86 deletions

File tree

‎logging/google/cloud/logging/client.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -303,7 +303,7 @@ def get_default_handler(self):
303303
"""
304304
if (_APPENGINE_FLEXIBLE_ENV_VM in os.environ or
305305
_APPENGINE_FLEXIBLE_ENV_FLEX in os.environ):
306-
return AppEngineHandler()
306+
return AppEngineHandler(self)
307307
elif _CONTAINER_ENGINE_ENV in os.environ:
308308
return ContainerEngineHandler()
309309
else:

‎logging/google/cloud/logging/handlers/app_engine.py‎

Lines changed: 48 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -27,47 +27,63 @@
2727
# /appengine-vmruntime/vmruntime/cloud_logging.py
2828

2929
import logging.handlers
30-
import os
3130

32-
from google.cloud.logging.handlers._helpers import format_stackdriver_json
31+
from google.cloud.logging.handlers.handlers import CloudLoggingHandler
32+
from google.cloud.logging.handlers.transports import BackgroundThreadTransport
33+
from google.cloud.logging.resource import Resource
3334

34-
_LOG_PATH_TEMPLATE = '/var/log/app_engine/app.{pid}.json'
35-
_MAX_LOG_BYTES = 128 * 1024 * 1024
36-
_LOG_FILE_COUNT = 3
35+
DEFAULT_LOGGER_NAME = 'python'
3736

37+
EXCLUDED_LOGGER_DEFAULTS = ('google.cloud', 'oauth2client')
3838

39-
class AppEngineHandler(logging.handlers.RotatingFileHandler):
40-
"""A handler that writes to the App Engine fluentd Stackdriver log file.
39+
_GLOBAL_RESOURCE = Resource(type='global', labels={})
4140

42-
Writes to the file that the fluentd agent on App Engine Flexible is
43-
configured to discover logs and send them to Stackdriver Logging.
44-
Log entries are wrapped in JSON and with appropriate metadata. The
45-
process of converting the user's formatted logs into a JSON payload for
46-
Stackdriver Logging consumption is implemented as part of the handler
47-
itself, and not as a formatting step, so as not to interfere with
48-
user-defined logging formats.
41+
42+
class AppEngineHandler(CloudLoggingHandler):
43+
"""A handler that directly makes Stackdriver logging API calls.
44+
45+
This handler can be used to route Python standard logging messages directly
46+
to the Stackdriver Logging API.
47+
48+
This handler supports both an asynchronous and synchronous transport.
49+
50+
:type client: :class:`google.cloud.logging.client`
51+
:param client: the authenticated Google Cloud Logging client for this
52+
handler to use
53+
54+
:type name: str
55+
:param name: the name of the custom log in Stackdriver Logging. Defaults
56+
to 'python'. The name of the Python logger will be represented
57+
in the ``python_logger`` field.
58+
59+
:type transport: type
60+
:param transport: Class for creating new transport objects. It should
61+
extend from the base :class:`.Transport` type and
62+
implement :meth`.Transport.send`. Defaults to
63+
:class:`.BackgroundThreadTransport`. The other
64+
option is :class:`.SyncTransport`.
65+
66+
:type resource: :class:`~google.cloud.logging.resource.Resource`
67+
:param resource: Monitored resource of the entry, defaults
68+
to the global resource type.
4969
"""
5070

51-
def __init__(self):
52-
"""Construct the handler
71+
def __init__(self, client,
72+
name=DEFAULT_LOGGER_NAME,
73+
transport=BackgroundThreadTransport,
74+
resource=_GLOBAL_RESOURCE):
75+
super(AppEngineHandler, self).__init__(client, name, transport)
76+
self.resource=resource
5377

54-
Large log entries will get mangled if multiple workers write to the
55-
same file simultaneously, so we'll use the worker's PID to pick a log
56-
filename.
57-
"""
58-
self.filename = _LOG_PATH_TEMPLATE.format(pid=os.getpid())
59-
super(AppEngineHandler, self).__init__(self.filename,
60-
maxBytes=_MAX_LOG_BYTES,
61-
backupCount=_LOG_FILE_COUNT)
78+
def emit(self, record):
79+
"""Actually log the specified logging record.
6280
63-
def format(self, record):
64-
"""Format the specified record into the expected JSON structure.
81+
Overrides the default emit behavior of ``StreamHandler``.
6582
66-
:type record: :class:`~logging.LogRecord`
67-
:param record: the log record
83+
See: https://docs.python.org/2/library/logging.html#handler-objects
6884
69-
:rtype: str
70-
:returns: JSON str to be written to the log file
85+
:type record: :class:`logging.LogRecord`
86+
:param record: The record to be logged.
7187
"""
72-
message = super(AppEngineHandler, self).format(record)
73-
return format_stackdriver_json(record, message)
88+
message = super(CloudLoggingHandler, self).format(record)
89+
self.transport.send(record, message, self.resource)

‎logging/google/cloud/logging/handlers/transports/background_thread.py‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,15 @@
2828
from six.moves import queue
2929

3030
from google.cloud.logging.handlers.transports.base import Transport
31+
from google.cloud.logging.resource import Resource
32+
3133

3234
_DEFAULT_GRACE_PERIOD = 5.0 # Seconds
3335
_DEFAULT_MAX_BATCH_SIZE = 10
3436
_WORKER_THREAD_NAME = 'google.cloud.logging.Worker'
3537
_WORKER_TERMINATOR = object()
3638
_LOGGER = logging.getLogger(__name__)
39+
_GLOBAL_RESOURCE = Resource(type='global', labels={})
3740

3841

3942
def _get_many(queue_, max_items=None):
@@ -203,7 +206,7 @@ def _main_thread_terminated(self):
203206
else:
204207
print('Failed to send %d pending logs.' % (self._queue.qsize(),))
205208

206-
def enqueue(self, record, message):
209+
def enqueue(self, record, message, resource=None):
207210
"""Queues a log entry to be written by the background thread.
208211
209212
:type record: :class:`logging.LogRecord`
@@ -212,13 +215,17 @@ def enqueue(self, record, message):
212215
:type message: str
213216
:param message: The message from the ``LogRecord`` after being
214217
formatted by the associated log formatters.
218+
219+
:type resource: :class:`~google.cloud.logging.resource.Resource`
220+
:param resource: (Optional) Monitored resource of the entry
215221
"""
216222
self._queue.put_nowait({
217223
'info': {
218224
'message': message,
219225
'python_logger': record.name,
220226
},
221227
'severity': record.levelname,
228+
'resource': resource,
222229
})
223230

224231

@@ -248,7 +255,7 @@ def __init__(self, client, name, grace_period=_DEFAULT_GRACE_PERIOD,
248255
logger = self.client.logger(name)
249256
self.worker = _Worker(logger)
250257

251-
def send(self, record, message):
258+
def send(self, record, message, resource=_GLOBAL_RESOURCE):
252259
"""Overrides Transport.send().
253260
254261
:type record: :class:`logging.LogRecord`
@@ -257,5 +264,9 @@ def send(self, record, message):
257264
:type message: str
258265
:param message: The message from the ``LogRecord`` after being
259266
formatted by the associated log formatters.
267+
268+
:type resource: :class:`~google.cloud.logging.resource.Resource`
269+
:param resource: Monitored resource of the entry, defaults
270+
to the global resource type.
260271
"""
261-
self.worker.enqueue(record, message)
272+
self.worker.enqueue(record, message, resource)

‎logging/google/cloud/logging/handlers/transports/base.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ class Transport(object):
2222
client and name object, and must override :meth:`send`.
2323
"""
2424

25-
def send(self, record, message):
25+
def send(self, record, message, resource):
2626
"""Transport send to be implemented by subclasses.
2727
2828
:type record: :class:`logging.LogRecord`
@@ -31,5 +31,9 @@ def send(self, record, message):
3131
:type message: str
3232
:param message: The message from the ``LogRecord`` after being
3333
formatted by the associated log formatters.
34+
35+
:type resource: :class:`~google.cloud.logging.resource.Resource`
36+
:param resource: Monitored resource of the entry, defaults
37+
to the global resource type.
3438
"""
3539
raise NotImplementedError

‎logging/google/cloud/logging/handlers/transports/sync.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,10 @@
1818
"""
1919

2020
from google.cloud.logging.handlers.transports.base import Transport
21+
from google.cloud.logging.resource import Resource
22+
23+
24+
_GLOBAL_RESOURCE = Resource(type='global', labels={})
2125

2226

2327
class SyncTransport(Transport):
@@ -29,7 +33,7 @@ class SyncTransport(Transport):
2933
def __init__(self, client, name):
3034
self.logger = client.logger(name)
3135

32-
def send(self, record, message):
36+
def send(self, record, message, resource=_GLOBAL_RESOURCE):
3337
"""Overrides transport.send().
3438
3539
:type record: :class:`logging.LogRecord`
@@ -40,4 +44,4 @@ def send(self, record, message):
4044
formatted by the associated log formatters.
4145
"""
4246
info = {'message': message, 'python_logger': record.name}
43-
self.logger.log_struct(info, severity=record.levelname)
47+
self.logger.log_struct(info, severity=record.levelname, resource=resource)

‎logging/tests/unit/handlers/test_app_engine.py‎

Lines changed: 36 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15+
import logging
1516
import unittest
17+
from google.cloud.logging.resource import Resource
1618

1719

1820
class TestAppEngineHandlerHandler(unittest.TestCase):
@@ -24,34 +26,42 @@ def _get_target_class(self):
2426
return AppEngineHandler
2527

2628
def _make_one(self, *args, **kw):
27-
import tempfile
29+
return self._get_target_class()(*args, **kw)
2830

29-
from google.cloud._testing import _Monkey
30-
from google.cloud.logging.handlers import app_engine as _MUT
31+
def test_ctor(self):
32+
client = _Client(self.PROJECT)
33+
handler = self._make_one(client, transport=_Transport)
34+
self.assertEqual(handler.client, client)
3135

32-
tmpdir = tempfile.mktemp()
33-
with _Monkey(_MUT, _LOG_PATH_TEMPLATE=tmpdir):
34-
return self._get_target_class()(*args, **kw)
36+
def test_emit(self):
37+
RESOURCE = Resource(
38+
type='gae_app',
39+
labels={
40+
'module_id': 'default',
41+
'version_id': 'test',
42+
})
3543

36-
def test_format(self):
37-
import json
38-
import logging
39-
40-
handler = self._make_one()
44+
client = _Client(self.PROJECT)
45+
handler = self._make_one(client, transport=_Transport, resource=RESOURCE)
4146
logname = 'loggername'
4247
message = 'hello world'
43-
record = logging.LogRecord(logname, logging.INFO, None,
44-
None, message, None, None)
45-
record.created = 5.03
46-
expected_payload = {
47-
'message': message,
48-
'timestamp': {
49-
'seconds': 5,
50-
'nanos': int(.03 * 1e9),
51-
},
52-
'thread': record.thread,
53-
'severity': record.levelname,
54-
}
55-
payload = handler.format(record)
56-
57-
self.assertEqual(payload, json.dumps(expected_payload))
48+
record = logging.LogRecord(logname, logging, None, None, message,
49+
None, None)
50+
handler.emit(record)
51+
52+
self.assertEqual(handler.transport.send_called_with, (record, message, RESOURCE))
53+
54+
55+
class _Client(object):
56+
57+
def __init__(self, project):
58+
self.project = project
59+
60+
61+
class _Transport(object):
62+
63+
def __init__(self, client, name):
64+
pass
65+
66+
def send(self, record, message, resource):
67+
self.send_called_with = (record, message, resource)

‎logging/tests/unit/handlers/transports/test_background_thread.py‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
import logging
1616
import unittest
17+
from google.cloud.logging.resource import Resource
1718

1819
import mock
1920

@@ -46,20 +47,28 @@ def test_constructor(self):
4647
self.assertEqual(logger.name, name)
4748

4849
def test_send(self):
50+
RESOURCE = Resource(
51+
type='gae_app',
52+
labels={
53+
'module_id': 'default',
54+
'version_id': 'test',
55+
})
56+
4957
client = _Client(self.PROJECT)
5058
name = 'python_logger'
5159

5260
transport, _ = self._make_one(client, name)
5361

5462
python_logger_name = 'mylogger'
5563
message = 'hello world'
64+
5665
record = logging.LogRecord(
5766
python_logger_name, logging.INFO,
5867
None, None, message, None, None)
5968

60-
transport.send(record, message)
69+
transport.send(record, message, RESOURCE)
6170

62-
transport.worker.enqueue.assert_called_once_with(record, message)
71+
transport.worker.enqueue.assert_called_once_with(record, message, RESOURCE)
6372

6473

6574
class Test_Worker(unittest.TestCase):
@@ -259,15 +268,16 @@ def join(self, timeout=None):
259268

260269

261270
class _Batch(object):
271+
from google.cloud.logging.logger import _GLOBAL_RESOURCE
262272

263273
def __init__(self):
264274
self.entries = []
265275
self.commit_called = False
266276
self.commit_count = None
267277

268-
def log_struct(self, info, severity=logging.INFO):
269-
self.log_struct_called_with = (info, severity)
270-
self.entries.append(info)
278+
def log_struct(self, record, severity=logging.INFO, resource=_GLOBAL_RESOURCE):
279+
self.log_struct_called_with = (record, severity, resource)
280+
self.entries.append(record)
271281

272282
def commit(self):
273283
self.commit_called = True

‎logging/tests/unit/handlers/transports/test_base.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,4 +31,4 @@ def _make_one(self, *args, **kw):
3131
def test_send_is_abstract(self):
3232
target = self._make_one()
3333
with self.assertRaises(NotImplementedError):
34-
target.send(None, None)
34+
target.send(None, None, None)

0 commit comments

Comments
 (0)