docker安装ELK详细步骤开发者社区

docker安装ELK详细步骤

mkdir -p /data/elk/es/{config,data,logs}

vi /data/elk/es/config/elasticsearch.yml
-----------------------配置内容----------------------------------
cluster.name: "my-es"
network.host: 0.0.0.0
http.port: 9200

chmod 777 /data/elk
chmod 777 /data/elk/es
chmod 777 /data/elk/es/config
chmod 777 /data/elk/es/data
chmod 777 /data/elk/es/logs
#报错挂载⽬录没权限
"Caused by: java.nio.file.AccessDeniedException: /usr/share/elasticsearch/data/nodes",•

#指定好版本
docker run -it -d -p 9200:9200 -p 9300:9300 --name es -e ES_JAVA_OPTS="-Xms1g -Xmx1g" -e "discovery.type=single-node" --restart=always -v /data/elk/es/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml -v /data/elk/es/data:/usr/share/elasticsearch/data -v /data/elk/es/logs:/usr/share/elasticsearch/logs elasticsearch:7.17.5

#获取elasticsearch容器ip
docker inspect --format '{{ .NetworkSettings.IPAddress }}' es

#创建kibana.yml
vi /data/elk/kibana/kibana.yml
-----------------------------------------------------------
#Default Kibana configuration for docker target
server.name: kibana
server.host: "0"
elasticsearch.hosts: ["http://172.17.0.2:9200"]
xpack.monitoring.ui.container.elasticsearch.enabled: true

docker run -d --restart=always --log-driver json-file --log-opt max-size=100m --log-opt max-file=2 --name kibana -p 5601:5601 -v /data/elk/kibana/kibana.yml:/usr/share/kibana/config/kibana.yml kibana:7.17.5

#IP为es-docker内⽹ip地址
vi /data/elk/logstash/logstash.yml
-----------------------------------------------------------------
http.host: "0.0.0.0"
xpack.monitoring.elasticsearch.hosts: [ "http://172.17.0.2:9200" ]
xpack.monitoring.elasticsearch.username: elastic
xpack.monitoring.elasticsearch.password: changeme
#path.config: /data/elk/logstash/conf.d/*.conf
path.config: /data/docker/logstash/conf.d/*.conf
path.logs: /var/log/logstash

#IP该服务器的ip
vi /data/elk/logstash/conf.d/syslog.conf
--------------------------------------------------------------------
input {
 syslog {
 type => "system-syslog"
 port => 5044
output {
 elasticsearch {
 hosts => ["IP:9200"] # 定义es服务器的ip
 index => "system-syslog-%{+YYYY.MM}" # 定义索引
}

#IP该服务器的ip
vi /etc/rsyslog.conf
---------------------------------------------------------------------
*.* @@IP:5044

systemctl restart rsyslog

docker run -d --restart=always --log-driver json-file --log-opt max-size=100m --log-opt max-file=2 -p 5044:5044 --name logstash -v /data/elk/logstash/logstash.yml:/usr/share/logstash/config/logstash.yml -v /data/elk/logstash/conf.d/:/data/docker/logstash/conf.d/ --privileged=true logstash:7.17.5

[root@Server-148a8793-5c2b-466f-9c0e-8da870ed6105 ~]# curl http://localhost:9200/_cat/indices?v
health status index                           uuid                   pri rep docs.count docs.deleted store.size pri.store.size
green  open   .geoip_databases                VF7a6tzVSYOaEBXKeblVbg   1   0         40            0     37.6mb         37.6mb
green  open   .kibana_7.17.5_001              DRcdoBqdRQmkoSajuGGiXg   1   0         19           30      2.3mb          2.3mb
green  open   .apm-custom-link                zxQpJVrXQmmSQrzZ_gP2_g   1   0          0            0       226b           226b
green  open   .apm-agent-configuration        iia52tRnS-aDLFOjWLj97A   1   0          0            0       226b           226b
green  open   .kibana_task_manager_7.17.5_001 6DY-ncFMTr-QweAjaFpSGA   1   0         17         1314    240.8kb        240.8kb

docker run -d -p 2181:2181 -p 2888:2888 -p 3888:3888 --privileged=true \
--restart=always --name=zkNode-1 \
-v /data/elk/zookeeper/conf:/conf \
-v /data/elk/zookeeper/data:/data \
-v /data/elk/zookeeper/datalog:/datalog wurstmeister/zookeeper

docker run -d --restart=always --name kafka \
-p 9092:9092 \
-v /data/elk/kafka/logs:/opt/kafka/logs \
-v /data/elk/kafka/data:/kafka/kafka-logs \
-v /data/elk/kafka/conf:/opt/kafka/config \
-e KAFKA_BROKER_ID=1 \
-e KAFKA_LOG_DIRS="/kafka/kafka-logs" \
-e KAFKA_ZOOKEEPER_CONNECT=10.84.77.10:2181 \
-e KAFKA_DEFAULT_REPLICATION_FACTOR=1 \
-e KAFKA_LOG_RETENTION_HOURS=72 \
-e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://10.84.77.10:9092 \
-e ALLOW_PLAINTEXT_LISTENER=yes \
-e KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092 -t wurstmeister/zookeeper

#进⼊kafka容器的命令⾏
docker exec -it kafka /bin/bash
#进⼊kafka所在⽬录
cd /opt/kafka_2.13-2.8.1/
#没有找到就⽤这个命令搜索
find / -name kafka-topics.sh
#创建topic
./bin/kafka-topics.sh --create --zookeeper 10.175.127.155:2181 --partitions 1 --replication-factor 1 --topic cloud-log
#发送消息
./bin/kafka-console-producer.sh --broker-list localhost:9092 --topic cloud-log
#接收消息
./bin/kafka-console-consumer.sh --zookeeper localhost:9092 --topic cloud-log --from-beginning

input {
 #获取kafka信息
 kafka {
 #⾃定义话题
 topics_pattern => "cloud-log"
 #kafka ip+端⼝
 bootstrap_servers => "10.175.127.155:9092"
 auto_offset_reset => "earliest"
 consumer_threads => 5
 decorate_events => "true"
}#⽂本格式过滤
filter {
 date {
 timezone => "Asia/Shanghai"
 match => ["logtime", "yyyy-MM-dd HH:mm:ss,SSS"]
 target => "@timestamp"
 remove_field => [ "logtime" ]
output {
 #输出⾄es
 elasticsearch {
 hosts => ["http://10.175.127.155:9200"]
 index => "%{[@metadata][kafka][topic]}-%{+YYYY-MM-dd}"

docker安装ELK详细步骤

docker安装ELK详细步骤

简介

安装ELK

安前须知之直接部署

安前须知之docker方式部署

ElasticSearch

kibana

logstash

安装 kafka