Category: Uncategorized

docker-compose.yml

version: '3'

services:

  oelk-node1:
    image: amazon/opendistro-for-elasticsearch:0.9.0
    container_name: oelk-node1
    environment:
      - cluster.name=oelk-cluster
      - bootstrap.memory_lock=true # along with the memlock settings below, disables swapping
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
      - opendistro_security.ssl.http.enabled=false
      - path.repo=/usr/share/elasticsearch/backup
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - oelk-data1:/usr/share/elasticsearch/data
      - /var/log/elasticsearchbkup:/usr/share/elasticsearch/backup
    ports:
      - 9200:9200
      - 9600:9600 # required for Performance Analyzer
    networks:
      - oelk-net

  oelk-node2:
    image: amazon/opendistro-for-elasticsearch:0.9.0
    container_name: oelk-node2
    environment:
      - cluster.name=oelk-cluster
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - discovery.zen.ping.unicast.hosts=oelk-node1
      - opendistro_security.ssl.http.enabled=false

    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - oelk-data2:/usr/share/elasticsearch/data
    networks:
      - oelk-net

  kibana:
    image: amazon/opendistro-for-elasticsearch-kibana:0.9.0
    container_name: oelk-kibana
    ports:
      - 5601:5601
    expose:
      - "5601"
    environment:
      ELASTICSEARCH_URL: http://oelk-node1:9200
      ELASTICSEARCH_HOSTS: https://oelk-node1:9200
    networks:
      - oelk-net

  logstash:
    image: docker.elastic.co/logstash/logstash:6.7.1
    volumes:
      - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro
      - ./logstash/pipeline:/usr/share/logstash/pipeline:ro
      - "./certs:/etc/certs"
    ports:
      - "5044:5044"
    environment:
      LS_JAVA_OPTS: "-Xmx256m -Xms256m"
    networks:
      - oelk-net
    depends_on:
      - oelk-node1
      - oelk-node2

  filebeat:
    hostname: filebeat
    build:
      context: filebeat
      dockerfile: Dockerfile
    volumes:
      - "/var/lib/docker/containers:/usr/share/dockerlogs/data:ro"
      - "/var/logs:/usr/share/syslogs:ro"
      - "/var/log/syslog:/var/log/syslog.log:ro"
      - "/var/run/docker.sock:/var/run/docker.sock"
      - "./certs:/etc/certs"
    networks:
      - oelk-net
    depends_on:
      - logstash

volumes:
  oelk-data1:
  oelk-data2:

networks:
  oelk-net:

pipeline/logstash.conf

input{
	 	beats { 
 			port => 5044
			ssl => true
			ssl_certificate_authorities => ["/etc/certs/ca.crt"]
			ssl_certificate => "/etc/certs/logstash.crt"
			ssl_key => "/etc/certs/logstash.key"
			ssl_verify_mode => "force_peer"
		}
        # http{
        #     port => 5044
        # }
}
filter {
#   if [docker][image] =~ /^logstash/ {
#     drop { }
#   }
mutate {
    rename => ["host", "server"]
	convert => {"server" => "string"} #this may be be not necessary but just in case added it
}

}
## Add your filters / logstash plugins configuration here

output {
        elasticsearch {
                hosts => "oelk-node1:9200"
				user => admin
				password => admin
		}
}

filebeat/Dockerfile

FROM docker.elastic.co/beats/filebeat:6.7.1
#FROM docker-logs-elk/filebeat:1.0.0
# Copy our custom configuration file
COPY config/filebeat.yml /usr/share/filebeat/filebeat.yml

USER root
# Create a directory to map volume with all docker log files
#RUN mkdir /usr/share/filebeat/dockerlogs
RUN chown -R root /usr/share/filebeat/filebeat.yml
RUN chmod -R go-w /usr/share/filebeat/filebeat.yml

filebeat.yml

filebeat.inputs:
- type: docker
  combine_partial: true
  containers:
    path: "/usr/share/dockerlogs/data"
    stream: "stdout"
    ids:
      - "*"
# - type: log

#   # Change to true to enable this input configuration.
#   enabled: true

#   # Paths that should be crawled and fetched. Glob based paths.
#   paths:
#     - /var/log/syslog.log

# filebeat.prospectors:
# - type: log
#   enabled: true
#   paths:
#    - '/usr/share/dockerlogs/data/*/*-json.log'
#   json.message_key: log
#   json.keys_under_root: true
#   processors:
#   - add_docker_metadata: ~

output:
  logstash:
    hosts: ["logstash:5044"]
    ssl.certificate_authorities: ["/etc/certs/ca.crt"]
    ssl.certificate: "/etc/certs/beat.crt"
    ssl.key: "/etc/certs/beat.key"

Welcome to WordPress. This is your first post. Edit or delete it, then start writing!

For those who works with closed source tools you won’t understand the freedom that we have on opensource tools because in real life our requirements changed over time, the tool that we are using they also start to grow and start to cover things they did not had in mind when they started the project. As a developer, we want our tools to do different things and as a person to person and project to project, we may have different sense of beauty and different meaning of code organization phylosophy. In computer science in general we always try to map our problem with a known solution that we have already solved before. So when you have access to source code of your tool, you can easily dig up that source code extend or alter the functionality and map your solution that matches your situation.

For example while working on python flask framework after couple years I realized how big they have grown over time, they do pretty much everything django is capable to do and it is even better because of their sense of modularity and flexibility. So for this project I am working on I started using flask, flask-admin for administrative panel and I am using flasks blueprint to separate different components of my project. Flask admin is actually not very comfortable or easy to attach with blueprints, that actually makes sense because because it adds admin panel and admin panel should be attached with the main app rather than a sub app like blueprint. But I actually had different use case and with admin panel I had to add my custom views which I don’t want to put at my app.py rather I want it to be in my controller. Other class architecture I had in mind will cause a circular dependency which I always get me in panic. I may not be very much neat and clean, pretty, tidy, person in personal life, I know I have limitations but I try to keep my code pretty and tidy and a thing of beauty that made me dig up the source code of those libraries at my office hours to rewrite this. Enough talk, if Linus Torvalds visits my blog ever he is going to get real mad at me for talking too much. So here you go, my code that I am using that satisfies my need:

# admin_blueprint.py

from flask import Blueprint
from flask_admin.contrib.sqla import ModelView
from flask_admin import Admin

class AdminBlueprint(Blueprint):
    views=None


    def __init__(self,*args, **kargs):
        self.views = []
        return super(AdminBlueprint, self).__init__('admin2', __name__,url_prefix='/admin2',static_folder='static', static_url_path='/static/admin')


    def add_view(self, view):
        self.views.append(view)

    def register(self,app, options, first_registration=False):
        print app
        admin = Admin(app, name='microblog', template_mode='adminlte')

        for v in self.views:
            admin.add_view(v)

        return super(AdminBlueprint, self).register(app, options, first_registration)

#app/admin/controller.py
from admin_blueprint import AdminBlueprint
from common.models import MyModel, db

app = AdminBlueprint('admin2', __name__,url_prefix='/admin2',static_folder='static', static_url_path='/static/admin')
app.add_view(ModelView(MyModel, db.session))

#app/__init__.py

from flask_sqlalchemy import SQLAlchemy

# Define the WSGI application object
app = Flask(__name__,template_folder="../templates",static_folder="../templates")

from app.api.controllers import app as api
from app.frontend.controllers import app as frontend
from app.admin.controllers import app as admin



# Register blueprint(s)
app.register_blueprint(api)
app.register_blueprint(frontend)
app.register_blueprint(admin)

# replacing the following code that I had
#from flask_admin import Admin
#from flask_admin.contrib.sqla import ModelView
#from common.models import *

#admin = Admin(app, name='microblog', template_mode='adminlte')
#admin.add_view(ModelView(MyModel, db.session))