docker-compose.yml
version: '3' services: oelk-node1: image: amazon/opendistro-for-elasticsearch:0.9.0 container_name: oelk-node1 environment: - cluster.name=oelk-cluster - bootstrap.memory_lock=true # along with the memlock settings below, disables swapping - "ES_JAVA_OPTS=-Xms512m -Xmx512m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM - opendistro_security.ssl.http.enabled=false - path.repo=/usr/share/elasticsearch/backup ulimits: memlock: soft: -1 hard: -1 volumes: - oelk-data1:/usr/share/elasticsearch/data - /var/log/elasticsearchbkup:/usr/share/elasticsearch/backup ports: - 9200:9200 - 9600:9600 # required for Performance Analyzer networks: - oelk-net oelk-node2: image: amazon/opendistro-for-elasticsearch:0.9.0 container_name: oelk-node2 environment: - cluster.name=oelk-cluster - bootstrap.memory_lock=true - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - discovery.zen.ping.unicast.hosts=oelk-node1 - opendistro_security.ssl.http.enabled=false ulimits: memlock: soft: -1 hard: -1 volumes: - oelk-data2:/usr/share/elasticsearch/data networks: - oelk-net kibana: image: amazon/opendistro-for-elasticsearch-kibana:0.9.0 container_name: oelk-kibana ports: - 5601:5601 expose: - "5601" environment: ELASTICSEARCH_URL: http://oelk-node1:9200 ELASTICSEARCH_HOSTS: https://oelk-node1:9200 networks: - oelk-net logstash: image: docker.elastic.co/logstash/logstash:6.7.1 volumes: - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro - ./logstash/pipeline:/usr/share/logstash/pipeline:ro - "./certs:/etc/certs" ports: - "5044:5044" environment: LS_JAVA_OPTS: "-Xmx256m -Xms256m" networks: - oelk-net depends_on: - oelk-node1 - oelk-node2 filebeat: hostname: filebeat build: context: filebeat dockerfile: Dockerfile volumes: - "/var/lib/docker/containers:/usr/share/dockerlogs/data:ro" - "/var/logs:/usr/share/syslogs:ro" - "/var/log/syslog:/var/log/syslog.log:ro" - "/var/run/docker.sock:/var/run/docker.sock" - "./certs:/etc/certs" networks: - oelk-net depends_on: - logstash volumes: oelk-data1: oelk-data2: networks: oelk-net:
pipeline/logstash.conf
input{ beats { port => 5044 ssl => true ssl_certificate_authorities => ["/etc/certs/ca.crt"] ssl_certificate => "/etc/certs/logstash.crt" ssl_key => "/etc/certs/logstash.key" ssl_verify_mode => "force_peer" } # http{ # port => 5044 # } } filter { # if [docker][image] =~ /^logstash/ { # drop { } # } mutate { rename => ["host", "server"] convert => {"server" => "string"} #this may be be not necessary but just in case added it } } ## Add your filters / logstash plugins configuration here output { elasticsearch { hosts => "oelk-node1:9200" user => admin password => admin } }
filebeat/Dockerfile
FROM docker.elastic.co/beats/filebeat:6.7.1 #FROM docker-logs-elk/filebeat:1.0.0 # Copy our custom configuration file COPY config/filebeat.yml /usr/share/filebeat/filebeat.yml USER root # Create a directory to map volume with all docker log files #RUN mkdir /usr/share/filebeat/dockerlogs RUN chown -R root /usr/share/filebeat/filebeat.yml RUN chmod -R go-w /usr/share/filebeat/filebeat.yml
filebeat.yml
filebeat.inputs: - type: docker combine_partial: true containers: path: "/usr/share/dockerlogs/data" stream: "stdout" ids: - "*" # - type: log # # Change to true to enable this input configuration. # enabled: true # # Paths that should be crawled and fetched. Glob based paths. # paths: # - /var/log/syslog.log # filebeat.prospectors: # - type: log # enabled: true # paths: # - '/usr/share/dockerlogs/data/*/*-json.log' # json.message_key: log # json.keys_under_root: true # processors: # - add_docker_metadata: ~ output: logstash: hosts: ["logstash:5044"] ssl.certificate_authorities: ["/etc/certs/ca.crt"] ssl.certificate: "/etc/certs/beat.crt" ssl.key: "/etc/certs/beat.key"