Tag: django

Configuring django for centralised log monitoring with ELK stack with custom logging option (eg. client ip, username, request & response data)

When you are lucky enough to have enough users that you decide to roll another cloud instance for your django app, logging becomes a little bit tough because in your architecture now you would be needing a load balancer which will be proxying request from one instance to another instance based on requirement. Previously we had log in one machine to log monitoring was easier, when someone reported a error we went to that instance and looked for errors, but now as we have multiple instance we have to go to all the instance, regardless of security risks, i would say it is a lot of work. So I think it would be wise to have a centralized log aggregating service.

For log management and monitoring we are using Elastic Logstash and Kibana popularly known as ELK stack. For this blog we will be logging pretty much all the request and its corresponding responses so that debugging process gets handy for us. To serve this purpose we will leverage django middlewares and python-logstash.

First of all let’s configure our settings.py for logging:

LOGGING = {
    'version': 1,
    'disable_existing_loggers': True,
    'formatters': {
        
        'standard': {
            'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
        },
        'logstash': {
            '()': 'proj_name.formatter.SadafNoorLogstashFormatter',
        },
    },
    'handlers': {
        'default': {
            'level':'DEBUG',
            'class':'logging.handlers.RotatingFileHandler',
            'filename': '/var/log/proj_name/django.log',
            'maxBytes': 1024*1024*5, # 5 MB
            'backupCount': 5,
            'formatter':'standard',
        },  
        'logstash': {
          'level': 'DEBUG',
          'class': 'logstash.TCPLogstashHandler',
          'host': 'ec2*****.compute.amazonaws.com',
          'port': 5959, # Default value: 5959
          'version': 1, # Version of logstash event schema. Default value: 0 (for backward compatibility of the library)
          'message_type': 'logstash',  # 'type' field in logstash message. Default value: 'logstash'.
          'fqdn': False, # Fully qualified domain name. Default value: false.
          #'tags': ['tag1', 'tag2'], # list of tags. Default: None.
          'formatter': 'logstash',
      },

        'request_handler': {
            'level':'DEBUG',
            'class':'logging.handlers.RotatingFileHandler',
            'filename': '/var/log/proj_name/django.log',
            'maxBytes': 1024*1024*5, # 5 MB
            'backupCount': 5,
            'formatter': 'standard',
        },
    },
    'loggers': {
        'sadaf_logger': {
            'handlers': ['default', 'logstash'],
            'level': 'DEBUG',
            'propagate': True
        },
    }
}

As you can see we are using a custom logging format. We can leave this configuration and by default LogstashFormatterVersion1 is the logging format that will work just fine. But I chose to define my own logging format because my requirement is different, I am running behind a proxy server, I want to log who has done that and from which IP. So roughly my Log Formatter looks like following:

from logstash.formatter import LogstashFormatterVersion1

from django.utils.deprecation import MiddlewareMixin
class SadafNoorLogstashFormatter(LogstashFormatterVersion1):
    def __init__(self,*kargs, **kwargs):
        print(*kargs, **kwargs)
        super().__init__(*kargs, **kwargs)


    def format(self, record,sent_request=None):
        print(record)
        print(sent_request, "old req")
        caddr = "unknown"
        #print(record.request.META)

        if 'HTTP_X_FORWARDED_FOR' in record.request.META:
            caddr = record.request.META['HTTP_X_FORWARDED_FOR'] #.split(",")[0].strip()
        
#        print(record.request.POST,record.request.GET, record.request.user)
        message = {
            '@timestamp': self.format_timestamp(record.created),
            '@version': '1',
            'message': record.getMessage(),
            'host': self.host,
            
            'client': caddr,
            'username': str(record.request.user),

            'path': record.pathname,
            'tags': self.tags,
            'type': self.message_type,
            #'request': self.record

            # Extra Fields
            'level': record.levelname,
            'logger_name': record.name,
        }

        # Add extra fields
#        print(type(self.get_extra_fields(record)['request']))
        message.update(self.get_extra_fields(record))

        # If exception, add debug info
        if record.exc_info:
            message.update(self.get_debug_fields(record))

        return self.serialize(message)

As our requirement is to log every request our middleware may look like following:

import logging

request_logger = logging.getLogger('sadaf_logger')
from datetime import datetime
from django.utils.deprecation import MiddlewareMixin
class LoggingMiddleware(MiddlewareMixin):
    """
    Provides full logging of requests and responses
    """
    _initial_http_body = None
    def __init__(self, get_response):
        self.get_response = get_response

    def process_request(self, request):
        self._initial_http_body = request.body # this requires because for some reasons there is no way to access request.body in the 'process_response' method.


    def process_response(self, request, response):
        """
        Adding request and response logging
        """
#        print(response.content, "xxxx")
        if request.path.startswith('/') and \
                (request.method == "POST" and
                         request.META.get('CONTENT_TYPE') == 'application/json'
                 or request.method == "GET"):
            status_code = getattr(response, 'status_code', None)
            print(status_code)

            if status_code:
                if status_code >= 400:
                    log_lvl = logging.ERROR
                else:
                    log_lvl = logging.INFO

            #request_logger.log(logging.DEBUG,)
            request_logger.log(log_lvl,
                               "GET: {}"
                               ""
                               .format(
                                   request.GET,
                                   ), 
                                   extra ={
                                       'request': request,
                                       'request_method': request.method,
                                       'request_url': request.build_absolute_uri(),
                                       'request_body': self._initial_http_body.decode("utf-8"),
                                       'response_body':response.content,
                                       'status': response.status_code
                                   }
                                       #extra={
                    #'tags': {
                    #    'url': request.build_absolute_uri()
                    #}
                #}
                )
#            print(request.POST,"fff")
        print("hot")
        return response

So pretty much you are done. Go login to your Kibana dashboard, make index pattern that you are interest and see your log:

Dealing with mandatory ForiegnkeyField for fields that is not in django rest framework serializers

Although I am big fan of django rest framework but sometime i feel it is gruesome to deal with nested serializers (Maybe I am doing something wrong, feel free to suggest me your favourite trick.)

Suppose we have two models, ASerializer is based on A model, BSerializer is based on `B` model. A and B models are related, say B has a foreign key to A. So while creating B it is mandatory to define A but A serializer is full of so much data that I don’t want to have that unnecessary overhead at my BSerializer, but when creating B I must have it. Here how I solved it:

For the sake of brevity let’s say A is our Category, and B is Product. Every Product has a Category, so Product has a foreign key of Category, but I am not making it visible at ProductSerializer given that category has a lot of unnecessary information that is not necessary.

from django.shortcuts import get_object_or_404
class ProductSerializer(serializers.ModelSerializer):
    def to_internal_value(self, data):
        if data.get('category'):
            self.fields['category'] = serializers.PrimaryKeyRelatedField(
                queryset=Category.objects.all())

            cat_slug = data['category']['slug']
            cat = get_object_or_404(Category, slug=cat_slug)
            
            data['category']= cat.id



        return super().to_internal_value(data)

A Django Rest Framework Jwt middleware to support request.user

I am using following Django (2.0.1), djangorestframework (3.7.7), djangorestframework-jwt (1.11.0) on top of python 3.6.3. By default djangorestframework-jwt does not include users in django’s usual requst.user. If you are using code>djangorestframework, chances are you have a huge code base at API which is leveraging this, not to mention at your permission_classes at viewsets. Since you are convinced about the fact that jwts are the best tools for your project, no wonder that you would love to migrate from your old token to new jwt tokens. To make the migration steps easier, we will write a middleware that will set request.user for us.

from django.utils.functional import SimpleLazyObject
from rest_framework_jwt.serializers import VerifyJSONWebTokenSerializer
from rest_framework.exceptions import ValidationError

#from rest_framework.request from Request
class AuthenticationMiddlewareJWT(object):
    def __init__(self, get_response):
        self.get_response = get_response


    def __call__(self, request):
        request.user = SimpleLazyObject(lambda: self.__class__.get_jwt_user(request))
        if not request.user.is_authenticated:
            token = request.META.get('HTTP_AUTHORIZATION', " ").split(' ')[1]
            print(token)
            data = {'token': token}
            try:
                valid_data = VerifyJSONWebTokenSerializer().validate(data)
                user = valid_data['user']
                request.user = user
            except ValidationError as v:
                print("validation error", v)
            

        return self.get_response(request)

And you need to register your middleware in settings:



MIDDLEWARE = [
    #...
    'path.to.AuthenticationMiddlewareJWT',
]

Integrating amazon s3 with django using django-storage and boto3

If we are lucky enough to get high amount of traffic at our website, next thing we start to think about is performance. The throughput of a website loading depends on the speed we are being able to deliver the contents of the website, to our users from our storages. In vanilla django, all assets including css, js, files and images are being stored locally in a predefined or preconfigured folder. To enhance performance we may have to decide to use a third party storage service that alleviate the headache of caching, zoning, replicating and to build the infrastructure of a Content Delivery Network. Ideally we would like to have a pluggable solution, something that allows us to switch storages from this to that, based on configuration. django-storages is one of the cool libraries from django community that helps to maintain 3rd party storage services like aws s3, google cloud, ftp, dropbox and so on. Amazon Webservice is one of the trusted service that offers a large range of services, s3 is one of the cool services from AWS that helps us to store static assets. boto3 is a python library being distributed by amazon to interact with amazon s3.

First thing first, to be able to store files on s3 we would need permission. In AWS world, all sorts of permissions are being managed using Identity Access Management (IAM).
i) In amazon console, you will be able to find IAM under Security, Identity & Compliance. Go there.
ii) We would need to add user with programmatic access.
iii) We would need to add new group.
iv) We would need to set policy for the group. Amazon provides bunch of predefined policies. For our use case, we can choose AmazonS3FullAccess
v) We have to store User, Access key ID and the Secret access key.

In s3 we can organize our contents into multiple buckets. We can use several buckets for a single Django project, sometime it is more efficient to use more but for now we will use only one. We will need to create bucket.

Now we need to install:

pip install boto3
pip install django-storages

We will need to add storages inside our INSTALLED_APPS of settings.py along with other configuration files of

django-storage
INSTALLED_APPS = [
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    'storages',
]


AWS_ACCESS_KEY_ID = '#######'
AWS_SECRET_ACCESS_KEY = '#####'
AWS_STORAGE_BUCKET_NAME = '####bucket-name'
AWS_S3_CUSTOM_DOMAIN = '%s.s3.amazonaws.com' % AWS_STORAGE_BUCKET_NAME
#AWS_S3_OBJECT_PARAMETERS = {
#    'CacheControl': 'max-age=86400',
#}
AWS_LOCATION = 'static'

STATICFILES_DIRS = [
    os.path.join(BASE_DIR, 'mysite/static'),
]
STATIC_URL = 'https://%s/%s/' % (AWS_S3_CUSTOM_DOMAIN, AWS_LOCATION)
STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'

When we are using django even when we don’t write any html, css or js files for our projects, it already has few because many of classes that we will be using at our views, its parent class may have static template files, base html files, css, js files. These static assets are being stored in our python library folder. To move then from library folder to s3 we will need to use following command:

python manage.py collectstatic

Thing to notice here is that, previously static referred to localhost:port but now it is being referred to s3 link.

{% static 'img/logo.png' %}

We may like to have some custom configuration for file storage, say we may like to put media files in a separate directory, we may not like it to be overwritten by another user. In that case we can define a child class of S3Boto3Storage and change the value of DEFAULT_FILE_STORAGE.

#storage_backends.py

from storages.backends.s3boto3 import S3Boto3Storage

class MyStorage(S3Boto3Storage):
    location = 'media'
    file_overwrite = False

DEFAULT_FILE_STORAGE = 'mysite.storage_backends.MediaStorage'

Now all our file related fields like models.FileField(), models.ImageField() will be uploading file in our s3 bucket inside the directory ‘media’.

Now we may have different types of storages, some of them will be storing documents, some of them will be publicly accessible, some of them will be classified. Their directory could be different and so on so forth.

class MyPrivateFileStorage(S3Boto3Storage):
    location = 'classified'
    default_acl = 'private'
    file_overwrite = False
    custom_domain = False

If we want to use any other storages that is not defined in DEFAULT_FILE_STORAGE in settings.py. We would need to define it at the field of our model models.FileField(storage=PrivateMediaStorage()).

[Code snippet] Django rest framework social oauth2 user sign up and sign in

At my serializers.py I have got the following:

from django.contrib.auth.models import User
from rest_framework import serializers
 
 
class UserSerializer(serializers.ModelSerializer):
    class Meta:
        model = User
        fields = ('username', 'email', 'password')

    def create(self, validated_data):
        return User.objects.create_user(**validated_data)

at my views.py I have got the following:

import serializers
from rest_framework.decorators import permission_classes, api_view
from rest_framework.permissions import AllowAny
from django.views.decorators.csrf import csrf_exempt
from rest_framework.parsers import JSONParser
from rest_framework.response import Response
from rest_framework import status
import json
from django.http import JsonResponse

from oauth2_provider.models import Application, AccessToken

@permission_classes((AllowAny,))
@csrf_exempt
@api_view(['POST'])
def create_auth(request, format=None):
    if request.user.is_authenticated():
        return Response({"already_registered": "User with that username has already registered"}, status=701)
    
    data = request.data
    print data

    
    serializer = UserSerializer(data=data, partial=True)
    if serializer.is_valid():
        u=serializer.save(username= data.get(u'username') )
        application=Application.objects.create(user=u, client_type="public", authorization_grant_type="password",name="general")
        client_id = application.client_id #call the url to get your tokens, use urllib or something similar
        client_secret = application.client_secret
        return JsonResponse({'client_id': client_id, 'client_password' : client_secret}, status=201)
    else:
        return JsonResponse({'errors': serializer.errors}, status=400)

I have added following at my urls.py

urlpatterns = patterns(
    '',
    url(r'^register/', 'social_app.views.create_auth'),
    url(r'^auth/', include('rest_framework_social_oauth2.urls')),
)

Testing:

 
sadaf2605@pavilion-g4:~$ curl -X POST -H "Content-Type: application/json" -d '{"email":"boba@athingy09876.com", "password":"p4ssword", "username": "user100"}' http://localhost:8000/register/

returns:

{"client_password": "EjQKMCAGmsUEm3L26uO7XSKnrZZVSVBQJUuvqfwi63pRB7d5y3ndlbZV0cBgQU7t3lCy078DS0FLqhaYoe9JZF0cQCIAgFKo7lfYU3npP7Eyv1PLk2eLPRnD3lF3OUUP", "client_id": "JhbwqqvE34vVjWiuMPnkV1eE636QQ3SzyQXLjmgs"}

sadaf2605@pavilion-g4:~$ curl -X POST -d "client_id=JhbwqqvE34vVjWiuMPnkV1eE636QQ3SzyQXLjmgs&client;_secret=EjQKMCAGmsUEm3L26uO7XSKnrZZVSVBQJUuvqfwi63pRB7d5y3ndlbZV0cBgQU7t3lCy078DS0FLqhaYoe9JZF0cQCIAgFKo7lfYU3npP7Eyv1PLk2eLPRnD3lF3OUUP&grant;_type=password&username;=user100&password;=p4ssword" http://localhost:8000/auth/token
{"access_token": "bssEYlDNaXefq8TPNRuu8oLolqYJp2", "token_type": "Bearer", "expires_in": 36000, "refresh_token": "fankCVPC3P84pQWI5oWOIhtWLCky4w", "scope": "read write"}

Customize django admin page from the scratch

In my last project with django, I had to modify and extend django admin page massively, there were cases when I had to wipe out curren admin page and create on my own. Usually people don’t go through this kind of trouble I guess, they just write their own model and view rather than admin page. That could possibly be a work around. But I decided not to do that because django admin provides a lot more thing and actually it felt to be the right way to do that. There were no straight forward tutorial that covers all that so here I go, I can guide you a little bit on your way.

 

Usually when we don’t define our own admin page, but instead we use admin of django.contrib.

Example:

from django.contrib import admin
admin.site.register(Question)

Now for our own admin page we will extend AdminSite.

class MySiteAdminSite(admin.AdminSite):
    site_header = 'This is my Custom Admin Site'

It will not come into action unless we add this to url.py:

import admin

urlpatterns = patterns('',

    # (r'^admin/', include(django.contrib.admin.site.urls)),
     url(r'^admin/', include(admin.my_admin_site.urls)),

)+ static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)

 

Our admin page is working but there is no model registered to our admin page, so there is nothing to show.

class ArticleAdmin(admin.ModelAdmin):

    fields =('author','title' ,'slug','body', 'categories','cover')
    def change_view(self, request,object_id, form_url='', extra_context=None):
        extra_context = extra_context or {}
        extra_context["show_save_as_draft"] = True

        return super(ArticleAdmin, self).change_view(request,object_id, form_url, extra_context)

    prepopulated_fields = {'slug': ('title',) }

    def save_model(self, request, obj, *kwwargs):
        if not obj.id:
            obj.slug = slugify(obj.title)
            
        if request.user.is_superuser or request.user==obj.author:
            super(ArticleAdmin,self).save_model(request, obj, *kwwargs)
        else:
            obj.author=None
            super(ArticleAdmin,self).save_model(request, obj, *kwwargs)


    def queryset(self, request):
        qs = super(ArticleAdmin, self).queryset(request)

        if request.user.is_superuser:
            return qs
        else:
            return qs.filter(author=request.user)

my_admin_site.register(Article, ArticleAdmin)

I actually wrote quite a lot code at my ArticleAdmin, I am going to explain it now. Basically we override  change_view, save_model and queryset method of modelAdmin class. change_view is responsible for embeding edit or create new form. This is what change_form.html look like:  https://github.com/django/django/blob/master/django/contrib/admin/templates/admin/change_form.html

You can pass your variable via extra_context, for my case I passed show_save_as_draft. and I edited change_form.html and put my edited version of it at template/admin/app/change_form.html

At save model, I actually checked some permission and slugify the link. pretty straight forward.

Now queryset is pretty interesting, it usually shows all the articles, does not it? what if we want that when a certain user logs in you want him to show only his articles? This is exactly what I did with filter.

 

Now we have a working custom admin that shows stuff at its dashboard, we are yet to modify our admin page.

class StripeAdminSite(admin.AdminSite):

    def index(self, request, extra_context=None):
        extra_context = extra_context or {}
        extra_context["site_visited_by"] = 10000


        return super(StripeAdminSite, self).index(request,extra_context)


    def censor_article(self,request,id):
        try:
            article=Article.objects.get(id=id)
            if request.user.is_superuser or article.author==request.user:
                article.censored=True
                article.save()
        except Article.DoesNotExist:
            pass

        from django.http import HttpResponseRedirect
        return HttpResponseRedirect(request.META.get('HTTP_REFERER'))


    def get_urls(self):
        #self.admin_site.admin_view(self.approve_staff)
        urls = super(MyAdminSite, self).get_urls()
        from django.conf.urls import url
        my_urls = [
            url(r'^article/censor/(?P<id>w+)/$', self.censor_article),
        ]

        return my_urls + urls

 

Here, index is what is being displayed when you hit /admin.  Your current index looks like this: https://github.com/sehmaschine/django-grappelli/blob/master/grappelli/templates/admin/index.html

You can override it by putting your own index.html at /template/admin/index.html you can pass variables via extra_context parameter of index function.

I wrote a view fuction censor_article at admin class and added corresponding url at get_urls function. Thats basically it.

One workaroud to pass variables (context) to django admin submit_line.html template

Recently I had to modify django admin page massively, while trying to add a new button at add new model item at admin page I got into trouble, trouble was not to show the button, or get that button working but it was at variable passing. So in this blog I am going to describe, how did I solve it.

I am overriding this template submit_line.html:

{% load i18n admin_urls %}





{% if show_save %}{% endif %}

{% if show_save_as_draft %}
    
    
{% endif %}



{% if show_save_and_add_another %}{% endif %}
{% if show_save_and_continue %}{% endif %}
{% if show_delete_link %}
    {% url opts|admin_urlname:'delete' original.pk|admin_urlquote as delete_url %}
      {% trans "Delete" %}
{% endif %}

Here,

{{show_save_as_draft}}

is at out extra context of our ModelAdmin while it is showing:

#/home/sadaf2605/PycharmProjects/stripe/stripe/news/admin.py
class ArticleAdmin(admin.ModelAdmin):
    change_form_template = 'admin/news/change_form.html'
    def change_view(self, request,object_id, form_url='', extra_context=None):
        extra_context = extra_context or {}
        extra_context["show_save_as_draft"] = True
        return super(ArticleAdmin, self).change_view(request,object_id, form_url, extra_context)

Still

{{show_save_as_draft}}

is not showing up. This is the problem

To solve this problem I actually override a template tag that was responsible for showing buttons, basically that template tag was only keeping few selected context field. In this new tag I am keeping tags which are necessary for my app.

#stripe/stripe/news/templatetags/stripe_admin_tag.py
__author__ = 'sadaf2605'
from django import template
register = template.Library()
from django.contrib.admin.templatetags import admin_modify

@register.inclusion_tag('admin/submit_line.html', takes_context=True)
def submit_line_row(context):
    context = context or {}
    ctx= admin_modify.submit_row(context)
    if "show_save_as_draft" in context.keys():
        ctx["show_save_as_draft"] = context["show_save_as_draft"]
    return  ctx

and then finally I need to override change_form.html as well, I need to replace:

{% block submit_buttons_bottom %}{% submit_row %}{% endblock %}

with:

{% load stripe_admin_tag %}
{% block submit_buttons_bottom %}{% submit_ine_row %}{% endblock %}

/stripe/stripe/stripe/templates/admin/news/change_form.html

{% extends "admin/base_site.html" %}
{% load i18n admin_urls admin_static admin_modify %}

{% block extrahead %}{{ block.super }}

{{ media }}
{% endblock %}

{% block extrastyle %}{{ block.super }}{% endblock %}

{% block coltype %}colM{% endblock %}

{% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} change-form{% endblock %}

{% if not is_popup %}
{% block breadcrumbs %}


{% trans 'Home' %}
› {{ opts.app_config.verbose_name }}
› {% if has_change_permission %}{{ opts.verbose_name_plural|capfirst }}{% else %}{{ opts.verbose_name_plural|capfirst }}{% endif %}
› {% if add %}{% trans 'Add' %} {{ opts.verbose_name }}{% else %}{{ original|truncatewords:"18" }}{% endif %}


{% endblock %}
{% endif %}

{% block content %}

{% block object-tools %}
{% if change %}{% if not is_popup %}
  
{% endif %}{% endif %}
{% endblock %}

{% csrf_token %}{% block form_top %}{% endblock %}


{% if is_popup %}{% endif %}
{% if to_field %}{% endif %}
{# WP Admin start #}
{% if 0 %}{% block submit_buttons_top %}{% submit_row %}{% endblock %}{% endif %}
{# WP Admin end #}
{% if errors %}
    

    {% if errors|length == 1 %}{% trans "Please correct the error below." %}{% else %}{% trans "Please correct the errors below." %}{% endif %}
    

    {{ adminform.form.non_field_errors }}
{% endif %}

{% block field_sets %}
{% for fieldset in adminform %}
  {% include "admin/includes/fieldset.html" %}
{% endfor %}
{% endblock %}

{% block after_field_sets %}{% endblock %}

{% block inline_field_sets %}
{% for inline_admin_formset in inline_admin_formsets %}
    {% include inline_admin_formset.opts.template %}
{% endfor %}
{% endblock %}

{% block after_related_objects %}{% endblock %}
{% load stripe_admin_tag %}
{% block submit_buttons_bottom %}{% submit_line_row %}{% endblock %}

{% if adminform and add %}
    
{% endif %}

{# JavaScript for prepopulated fields #}
{% prepopulated_fields_js %}





{% endblock %}