Merge branch 'develop'

avara1986 · avara1986 · commit 4a8532ebfa6f · 2018-05-05T13:46:33.000+02:00
diff --git a/.gitignore b/.gitignore
@@ -86,7 +86,7 @@ celerybeat-schedule
 .venv
 venv/
 ENV/
-
+venv2
 # Spyder project settings
 .spyderproject
 .spyproject
@@ -104,4 +104,4 @@ ENV/
 
 pylintReport.txt
 db.sqlite3
-_build
+_build
diff --git a/project/__init__.py b/project/__init__.py
@@ -1,18 +1,28 @@
 # encoding: utf-8
 
+import logging
 import os
 
 from flasgger import Swagger
 from flask import Flask
+from flask_injector import FlaskInjector
+from injector import Injector
 
 from project.config import CONFIG
+from pyms.healthcheck import healthcheck_blueprint
+from pyms.logger import CustomJsonFormatter
+from pyms.models import db
+from pyms.tracer.main import TracerModule
 
 __author__ = "Alberto Vara"
 __email__ = "a.vara.1986@gmail.com"
 __version__ = "0.0.1"
 
 ENVIRONMENT = os.environ.get("ENVIRONMENT", "default")
 
+logger = logging.getLogger('jaeger_tracing')
+logger.setLevel(logging.DEBUG)
+
 SWAGGER_CONFIG = {
     "headers": [
     ],
@@ -46,24 +56,33 @@
 
 
 class PrefixMiddleware(object):
+    """Set a prefix path to all routes. This action is needed if you have a stack of microservices and each of them
+    exist in the same domain but different path. Por example:
+    * mydomain.com/ms1/
+    * mydomain.com/ms2/
+    """
 
     def __init__(self, app, prefix=''):
         self.app = app
         self.prefix = prefix
 
     def __call__(self, environ, start_response):
-
         if environ['PATH_INFO'].startswith(self.prefix):
             environ['PATH_INFO'] = environ['PATH_INFO'][len(self.prefix):]
             environ['SCRIPT_NAME'] = self.prefix
             return self.app(environ, start_response)
+        elif environ['PATH_INFO'].startswith("/healthcheck"):
+            return self.app(environ, start_response)
         else:
             start_response('404', [('Content-Type', 'text/plain')])
             return ["This url does not belong to the app.".encode()]
 
 
 def create_app():
-    from project.models import db
+    """Initialize the Flask app, register blueprints and intialize all libraries like Swagger, database, the trace system...
+    return the app and the database objects.
+    :return:
+    """
     from project.views import views_bp as views_blueprint
     from project.views.oauth import jwt, bcrypt
     environment = os.environ.get("ENVIRONMENT", "default")
@@ -90,7 +109,24 @@ def create_app():
     )
     Swagger(app, config=SWAGGER_CONFIG)
 
+    # Initialize Blueprints
     app.register_blueprint(views_blueprint)
+    app.register_blueprint(healthcheck_blueprint)
+
+    # Inject Modules
+    # Inject Modules
+    if not app.config["TESTING"] and not app.config["DEBUG"]:
+        log_handler = logging.StreamHandler()
+        formatter = CustomJsonFormatter('(timestamp) (level) (name) (module) (funcName) (lineno) (message)')
+        formatter.add_service_name(app.config["APP_NAME"])
+        tracer = TracerModule(app)
+        injector = Injector([tracer])
+        FlaskInjector(app=app, injector=injector)
+        formatter.add_trace_span(tracer.tracer)
+        log_handler.setFormatter(formatter)
+        app.logger.addHandler(log_handler)
+        app.logger.setLevel(logging.INFO)
+
     with app.test_request_context():
         db.create_all()
     return app, db
diff --git a/project/config.py b/project/config.py
@@ -28,7 +28,6 @@ class DevConfig(Config):
     """Configuration to run in local environments"""
 
     DEBUG = True
-    TESTING = True
     SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.join(BASE_DIR, "db.sqlite3")
 
 
diff --git a/project/models/__init__.py b/project/models/__init__.py
@@ -1,6 +0,0 @@
-# encoding: utf-8
-from __future__ import absolute_import, print_function, unicode_literals
-
-from flask_sqlalchemy import SQLAlchemy
-
-db = SQLAlchemy()
diff --git a/project/models/models.py b/project/models/models.py
@@ -6,7 +6,7 @@
 from flask_login import UserMixin
 from sqlalchemy import Column, Integer, String, Boolean
 
-from project.models import db
+from pyms.models import db
 
 
 class User(db.Model, UserMixin):
diff --git a/project/views/views.py b/project/views/views.py
@@ -4,7 +4,6 @@
 from flask import request, jsonify
 from flask_jwt import jwt_required, current_identity
 
-from project.models.models import User, db
 from project.views import views_bp
 from project.views.oauth import jwt, authenticate
 
@@ -34,6 +33,7 @@ def login():
       400:
         description: User login failed.
     """
+    # current_app.logger.info("{}".format(request.headers))
     try:
         username = request.form.get("username")
         password = request.form.get("password")
@@ -53,6 +53,7 @@ def login():
     except UserNotFoundException:
         resp = jsonify({"message": "Bad username and/or password"})
         resp.status_code = 401
+    print(resp.headers)
     return resp
 
 
@@ -73,4 +74,4 @@ def protected():
     resp = jsonify({"protected": "{}".format(current_identity)})
     resp.status_code = 200
 
-    return resp
+    return resp
diff --git a/pyms/__init__.py b/pyms/__init__.py
diff --git a/pyms/healthcheck/__init__.py b/pyms/healthcheck/__init__.py
@@ -0,0 +1,8 @@
+# coding=utf-8
+from __future__ import unicode_literals, print_function, absolute_import, division
+
+from flask import Blueprint
+
+healthcheck_blueprint = Blueprint('healthcheck', __name__, static_url_path='/static')
+
+from pyms.healthcheck import healthcheck
diff --git a/pyms/healthcheck/healthcheck.py b/pyms/healthcheck/healthcheck.py
@@ -0,0 +1,6 @@
+from pyms.healthcheck import healthcheck_blueprint
+
+
+@healthcheck_blueprint.route('/healthcheck', methods=['GET'])
+def healthcheck():
+    return "OK"
diff --git a/pyms/logger/__init__.py b/pyms/logger/__init__.py
@@ -0,0 +1 @@
+from pyms.logger.logger import CustomJsonFormatter
diff --git a/pyms/logger/logger.py b/pyms/logger/logger.py
@@ -0,0 +1,37 @@
+import datetime
+
+import opentracing
+from flask_opentracing import FlaskTracer
+from pythonjsonlogger import jsonlogger
+
+
+class CustomJsonFormatter(jsonlogger.JsonFormatter):
+    service_name = ""
+    tracer = ""
+
+    def add_fields(self, log_record, record, message_dict):
+        super(CustomJsonFormatter, self).add_fields(log_record, record, message_dict)
+        if not log_record.get('timestamp'):
+            # this doesn't use record.created, so it is slightly off
+            now = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%fZ')
+            log_record['timestamp'] = now
+        if log_record.get('level'):
+            log_record['severity'] = log_record['level'].upper()
+        else:
+            log_record['severity'] = record.levelname
+        log_record["service"] = self.service_name
+
+        # Add traces
+        if self.tracer:
+            span = self.tracer.get_span()
+            headers = {}
+            self.tracer._tracer.inject(span, opentracing.Format.HTTP_HEADERS, headers)
+            log_record["trace"] = headers.get('X-B3-TraceId', '')
+            log_record["span"] = headers.get('X-B3-SpanId', '')
+            log_record["parent"] = headers.get('X-B3-ParentSpanId', '')
+
+    def add_service_name(self, project_name):
+        self.service_name = project_name.lower()
+
+    def add_trace_span(self, tracer: FlaskTracer):
+        self.tracer = tracer
diff --git a/pyms/models/__init__.py b/pyms/models/__init__.py
@@ -0,0 +1,6 @@
+# encoding: utf-8
+from __future__ import absolute_import, print_function, unicode_literals
+
+from flask_sqlalchemy import SQLAlchemy
+
+db = SQLAlchemy()
diff --git a/pyms/tracer/__init__.py b/pyms/tracer/__init__.py
diff --git a/pyms/tracer/main.py b/pyms/tracer/main.py
@@ -0,0 +1,27 @@
+from flask_opentracing import FlaskTracer
+from injector import Module
+from jaeger_client import Config
+
+
+def init_jaeger_tracer(service_name='your-app-name'):
+    """This scaffold is configured whith `Jeager <https://github.com/jaegertracing/jaeger>`_ but you can use
+    one of the `opentracing tracers <http://opentracing.io/documentation/pages/supported-tracers.html>`_
+    :param service_name: the name of your application to register in the tracer
+    :return: opentracing.Tracer
+    """
+    config = Config(config={
+        # 'propagation': 'b3',
+        'sampler': {'type': 'const', 'param': 1}, 'logging': True,
+    }, service_name=service_name)
+    return config.initialize_tracer()
+
+
+class TracerModule(Module):
+    tracer = ""
+
+    def __init__(self, app):
+        self.app = app
+
+    def configure(self, binder):
+        self.tracer = FlaskTracer(init_jaeger_tracer(self.app.config["APP_NAME"]), True, self.app)
+        binder.bind(FlaskTracer, to=self.tracer)
diff --git a/requirements-tests.txt b/requirements-tests.txt
@@ -1,4 +1,6 @@
 -r requirements.txt
 coverage==4.5.1
 coveralls==1.3.0
-pylint==1.8.3
+nose==1.3.7
+pylint==1.8.3
+tox==2.7.0
diff --git a/requirements.txt b/requirements.txt
@@ -1,10 +1,13 @@
+-e git+https://github.com/jaegertracing/jaeger-client-python.git#egg=jaeger_client
 flasgger==0.8.1
 Flask==0.12.2
 Flask-Bcrypt==0.7.1
+Flask-Injector==0.10.1
 Flask-JWT==0.3.2
 Flask-Login==0.4.1
+Flask-OpenTracing==0.1.8
 Flask-Script==2.0.6
 Flask-SQLAlchemy==2.3.2
-Flask-Testing==0.7.1
 SQLAlchemy==1.2.5
+python-json-logger==0.1.8
 requests==2.18.4

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+from pyms.logger.logger import CustomJsonFormatter`