elasticsearch : 9200
couchdb : 5984
1. elasticsearch install
sudo apt-get install openjdk-7-jre-headless -y
sudo wget -O - http://packages.elasticsearch.org/GPG-KEY-elasticsearch | apt-key add -
Add the following line to /etc/apt/sources.list
deb http://packages.elasticsearch.org/elasticsearch/1.5/debian stable main
sudo apt-get update
sudo apt-get install elasticsearch
sudo update-rc.d elasticsearch defaults 95 10
sudo /etc/init.d/elasticsearch start
2. fluentd install
curl -L http://toolbelt.treasuredata.com/sh/install-ubuntu-lucid.sh | sh
sudo /usr/lib/fluent/ruby/bin/fluent-gem install fluent-plugin-elasticsearch
3. fluentd 성능 개선
멀티 프로세스
http://docs.fluentd.org/articles/in_multiprocess
루비 메모리 문제
https://github.com/kzk/jemalloc-rb
4. elastic search couchdb 연결
- log 생성
curl -XPUT 'http://localhost:9200/news/'
cd /usr/share/elasticsearch/
./bin/plugin -install elasticsearch/elasticsearch-river-couchdb/1.2.0
curl -XPUT localhost:9200/news/news/_mapping -d '{
"news" : {
"numeric_detection" : false,
"properties" : {
"creation" : {"format":"date_time_no_millis","type" : "date"}
}
}
}'
curl -XPUT localhost:9200/news/news/_mapping -d '{
"news" : {
"numeric_detection" : false,
"properties" : {
"creation" : {"type" : "date"}
}
}
}'
curl -XPUT 'localhost:9200/_river/news/_meta' -d '{
"type" : "couchdb",
"couchdb" : {
"host" : "122.36.208.82",
"port" : 8888,
"db" : "news",
"filter" : null
},
"index" : {
"index" : "news",
"type" : "news",
"bulk_size" : "100",
"bulk_timeout" : "10ms"
}
}'
- river 삭제
curl -XDELETE 'http://localhost:9200/_river/news/'
- log 삭제
curl -XDELETE 'http://localhost:9200/news/'
- mapping 삭제
curl -XDELETE 'http://localhost:9200/news/news/_mapping'
5. fluentd + geoip
sudo apt-get install libgeoip-dev
sudo /usr/lib/fluent/ruby/bin/fluent-gem install fluent-plugin-geoip
<match low.lighttpd.log>
type copy
<store>
type geoip
geoip_lookup_key host
<record>
city ${city['host']}
lat ${latitude['host']}
lon ${longitude['host']}
</record>
remove_tag_prefix low.
tag couch.${tag}
</store>
</match>
------------------------------------------------------------------------------------------------------------------------
vi /etc/td-agent/td-agent.conf
<match td.*.*>
type tdlog
apikey YOUR_API_KEY
auto_create_table
buffer_type file
buffer_path /var/log/td-agent/buffer/td
</match>
<match couch.**>
type couch
database log
# following attibutes are optional
host localhost #default:localhost
port 5984 #default:5984
protocol http #default:http
update_docs false #default:false
doc_key_field nil #default:nil
doc_key_jsonpath nil #default:nil
refresh_view_index list #default:nil
# for Basic Authentication (optional)
user admin
password manson
# Other buffer configurations here
buffer_type file
buffer_path /var/fluentd/buffer
</match>
## match tag=debug.** and dump to console
#<match debug.**>
# type stdout
#</match>
####
## Source descriptions:
##
## built-in TCP input
## @see http://docs.fluentd.org/articles/in_forward
#<source>
# type forward
#</source>
<match low.lighttpd.log>
type copy
<store>
type geoip
geoip_lookup_key host
<record>
city ${city['host']}
lat ${latitude['host']}
lon ${longitude['host']}
</record>
remove_tag_prefix low.
tag nosql.${tag}
</store>
</match>
<match nosql.**>
type copy
<store>
type arango
collection log
buffer_type memory
buffer_chunk_limit 256m
buffer_queue_limit 128
"/etc/td-agent/td-agent.conf" 230L, 4801C 1,1 Top
