= Graylog2 =
Graylog2 is an open source log management solution that stores your logs in ElasticSearch.

== Clean up DB ==
[[http://wiki.hackspherelabs.com/index.php?title=Graylog2#Clean_Out_Graylog2_DB]]

"Cure" for high CPU usage:
{{{#!highlight sh
service graylog2 stop
cd /opt/elasticsearch-0.19.9/data/graylog2
rm * -rf
/opt/mongo/bin/mongo
# use graylog2
# db.message_counts.remove()
# db.hosts.remove()
# exit
service graylog2 start
}}}

== Send log from python to graylog2 through GELF ==
See details in https://pypi.python.org/pypi/graypy

Install with easy_install graypy .
*  pip install graypy .

{{{#!highlight python
#file name testGelf.py
import logging
import graypy
import datetime

my_logger = logging.getLogger('test_logger')
my_logger.setLevel(logging.DEBUG)

handler = graypy.GELFHandler('192.168.1.123', 12201)
my_logger.addHandler(handler)

my_logger.debug('Hello Graylog2.')
my_logger.debug('Hello Graylog2, %s.'%(datetime.datetime.now() ))
my_logger.info('Inf hello Graylog2, %s.'%(datetime.datetime.now() ))
}}}

On graylog2 the following columns are used: 
 * From: hostx
 * Date: Tue Dec 10 13:14:50 +0000 2013
 * Severity: Debug
 * Facility: test_logger
 * File: testGelf.py:10
 * thread_name: MainThread
 * function: <module>
 * process_name: MainProcess
 * pid: 27663

== Drools example .drl ==
{{{
rule "IMEI"
        when
            m : GELFMessage( shortMessage matches ".*\\s\\d{15}\\s.*" )
        then
          Matcher matcher = Pattern.compile("\\s(\\d{15})\\s").matcher(m.getShortMessage());

          if (matcher.find()) {
            m.addAdditionalData("_imei", matcher.group(1) );
          }
end

rule "IP Port"
        when
            m : GELFMessage( shortMessage matches "^.*\\s\\d+.\\d+.\\d+.\\d+:\\d+\\s.*" )
        then
          Matcher matcher = Pattern.compile("\\s(\\d+.\\d+.\\d+.\\d+):(\\d+)\\s").matcher(m.getShortMessage());

          if (matcher.find()) {
            m.addAdditionalData("_ipaddr", matcher.group(1) );
            m.addAdditionalData("_port", matcher.group(2) );
          }
end



rule "Timestamp"
        when
            m : GELFMessage( fullMessage matches "^.*\\s\\d+.\\d+.\\d+-\\d+.\\d+.\\d+.*" )
        then
          Matcher matcher = Pattern.compile("\\s(\\d+).(\\d+).(\\d+)-(\\d+).(\\d+).(\\d+)").matcher(m.getFullMessage());

          if (matcher.find()) {
            m.addAdditionalData("_year", matcher.group(1) );
            m.addAdditionalData("_month", matcher.group(2) );
            m.addAdditionalData("_day", matcher.group(3) );
            m.addAdditionalData("_hour", matcher.group(4) );
            m.addAdditionalData("_minute", matcher.group(5) );
            m.addAdditionalData("_second", matcher.group(6) );
          }
end
}}}

== ElasticSearch Queries ==

http://www.elasticsearchtutorial.com/elasticsearch-in-5-minutes.html

http://joelabrahamsson.com/elasticsearch-101/

{{{
curl 'http://localhost:9200/blog/post/_search?q=user:dilbert&pretty=true'

curl -XGET 'http://localhost:9200/blog/_search?pretty=true' -d '
{ 
    "query" : { 
        "range" : { 
            "postDate" : { "from" : "2011-12-10", "to" : "2011-12-12" } 
        } 
    } 
}'

curl -XPOST "http://localhost:9200/_search" -d'
{
    "query": {
        "filtered": {
            "query": {
                "match_all": {
                }
            },
            "filter": {
                "term": { "FieldX": "value1234" }
            }
        }
    }
}'

curl -XPOST "http://localhost:9200/_search" -d'
{
    "query": {
        "filtered": {
            "query": {
                    "query_string": { "query": "other text" }
            },
            "filter": {
                "term": { "field": "asdf" }
            }
        }
    }
}'

curl -XPOST "http://localhost:9200/_search" -d'
{
  "query": 
  {
    "filtered": 
    {
      "query": { "match_all": { } } ,
      "filter":
      { "and":  
        [  
          {"range":{"_DateTimeX": {"from":"2010-01-01 00:00:00","to":"2010-06-07 23:59:59"} }} , 
          {"term":{"fieldx":"valuex"}} 
        ]  
      }
    } 
  }
}
'

}}}

== 2016 test graylog ==

Elasticsearch
{{{#!highlight bash 
cd /tmp
wget https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.4.2/elasticsearch-2.4.2.tar.gz
tar xvzf elasticsearch-2.4.2.tar.gz 
cd elasticsearch-2.4.2
nano config/elasticsearch.yml # cluster.name: graylog
bin/elasticsearch
}}}

{{{#!highlight bash 
cd /tmp
wget https://packages.graylog2.org/releases/graylog/graylog-2.1.2.tgz
tar xvzf graylog-2.1.2.tgz
cd graylog-2.1.2
mkdir -p /etc/graylog/server
cp /tmp/graylog-2.1.2/graylog.conf.example /etc/graylog/server/server.conf
uuidgen
90f7????-2c8b-????-9c2e-????b3282589
# set password_secret in /etc/graylog/server/server.conf
root_username = admin
# echo -n 12345678 | shasum -a 256 --> root_password_sha2
elasticsearch_cluster_name = graylog
nano /etc/graylog/server/node-id # nodex
mongod &
bin/graylogctl start
bin/graylogctl status
tail -f log/graylog-server.log 
http://localhost:9000/gettingstarted
}}}

{{{

http://localhost:9000/system/inputs # create input for GELF HTTP, launch new input
# Title: GelfHttpTest Node: nodex/localhost bind addr: 0.0.0.0 port 12201
curl -XPOST http://localhost:12201/gelf -p0 -d '{"short_message":"Hello there", "host":"example.org", "facility":"test", "_foo":"bar"}'
}}}

{{{
# added input GelfUdp port 12202
import logging
import graypy
import datetime 

my_logger = logging.getLogger('test_logger')
my_logger.setLevel(logging.DEBUG)

handler = graypy.GELFHandler('127.0.0.1', 12202)
my_logger.addHandler(handler)

my_logger.debug('Hello Graylog2.')
my_logger.debug('Hello Graylog2, %s.'%(datetime.datetime.now() ))
my_logger.info('Inf hello Graylog2, %s.'%(datetime.datetime.now() ))
}}}