#! /bin/bash

. /usr/src/k_framework/main.sh

# ---- VARS -----

VERSION="1.8.6"
SCRIPT_NAME="Elasticsearch Check"

CFG=$SYSTEM_CONFIG
ES_CFG="/etc/elasticsearch/elasticsearch.yml"
SYSCONFIG="/etc/sysconfig/elasticsearch"

TEST=0   # 1 - do not execute changes

# ---- FUNCTIONS -----

# ---- MAIN -----

k_start

if [ "$TEST" == "1" ]; then
    report "***** TEST MODE ON. Changes will not be applied *****" 8
fi

k_config_details

k_default_interface_ip

if [ "$SYSTEM_CONFIG_DIR" == "m2" ]; then
    ES_SRC_DIR=/usr/src/m2/elasticsearch
else
    ES_SRC_DIR=/usr/src/mor/sh_scripts/elasticsearch
fi

#report "Server's IP: $DEFAULT_IP" 3

if  [[ $ES_PRESENT == "" ]]; then
    report "system.conf ES_PRESENT is empty" 1
    report "  set ES_PRESENT in system.conf" fix
    EXIT_CODE=1
else
    report "system.conf ES_PRESENT: $ES_PRESENT" 0
fi


k_db_connection_check
if [[ $EXIT_CODE == 1 ]]; then k_exit 1; fi

if k_db_field_check "servers" "es"; then

    rm -fr /tmp/mysql_error
    es_server_count=`MYSQL_PWD=$DB_PASSWORD /usr/bin/mysql -h $DB_HOST -u $DB_USERNAME $P_OPT $DB_NAME --silent -e "SELECT COUNT(id) FROM servers WHERE es = 1 AND (active = 1 OR server_type='sip_proxy') ;" | grep -v value` 2> /tmp/mysql_error

    FILESIZE=$(stat -c%s "/tmp/mysql_error")
    if [[ $FILESIZE != 0 ]]; then
        report "MySQL error. Old DB?" 1
        report "  ./db_update.sh LATEST"
        k_exit 1
    fi

    if [ $es_server_count != 1 ]; then
        if [ $es_server_count == 0 ]; then
            report "ES is not present in the system. No server is marked to have ES, which is wrong" 1
            report "  Mark properly in GUI-Servers some server where ES is present" fix
        else
            report "ES count [$es_server_count] is wrong in the system (Mark properly in GUI-Servers)" 1
        fi
        k_exit 1
    else
        report "ES present in the GUI-Servers" 0
    fi

fi



conf_ES_IP_count=`MYSQL_PWD=$DB_PASSWORD /usr/bin/mysql -h $DB_HOST -u $DB_USERNAME $P_OPT $DB_NAME --silent -e "SELECT COUNT(id) FROM conflines WHERE name = 'ES_IP';" | grep -v value`
if [[ $conf_ES_IP_count != 1 ]]; then
    report "ES_IP count in DB: $conf_ES_IP_count" 1
    EXIT_CODE=1
else
    report "ES_IP count in DB: 1" 0
fi


conf_ES_IP=`MYSQL_PWD=$DB_PASSWORD /usr/bin/mysql -h $DB_HOST -u $DB_USERNAME $P_OPT $DB_NAME --silent -e "SELECT value FROM conflines WHERE name = 'ES_IP' LIMIT 1;" | grep -v value`

if [[ $conf_ES_IP == "localhost" ]]; then
    conf_ES_IP="127.0.0.1"
    report "ES_IP 'localhost' changed to '127.0.0.1' for internal use" 0
fi

if [[ $conf_ES_IP != "" ]]; then
    report "ES_IP: $conf_ES_IP" 0
else
    report "ES_IP in DB is empty" 1
    EXIT_CODE=1
fi

if [[ $SYSTEM_CONFIG_DIR == "m2" ]]; then
    if [[ $conf_ES_IP == "127.0.0.1" ]]; then
        report "ES_IP == 127.0.0.1. Not allowed. Fix in GUI-Servers." 1
        EXIT_CODE=1
    fi

    es_server_ip=`MYSQL_PWD=$DB_PASSWORD /usr/bin/mysql -h $DB_HOST -u $DB_USERNAME $P_OPT $DB_NAME --silent -e "SELECT server_ip FROM servers WHERE es = 1 AND active = 1 LIMIT 1;" | grep -v value`

    # checking match between DB and m2.conf
    if [[ $EXIT_CODE == 0 ]]; then
        if [[ $ES_PRESENT == 1 ]]; then
            if [[ $es_server_ip == $DEFAULT_IP ]]; then
                report "ES server IP in DB matches this server's IP: $DEFAULT_IP" 0
            else

                es_local_ip=`MYSQL_PWD=$DB_PASSWORD /usr/bin/mysql -h $DB_HOST -u $DB_USERNAME $P_OPT $DB_NAME --silent -e "SELECT local_ip FROM servers WHERE es = 1 AND active = 1 LIMIT 1;" | grep -v value`
                report "ES local IP: $es_local_ip" 3

                if [[ $es_local_ip == $DEFAULT_IP ]]; then
                    report "ES server local IP  [$es_local_ip] matches this server's IP [$DEFAULT_IP]" 0
                    es_server_ip=$es_local_ip
                else
                    report "ES server IP in DB [$es_server_ip] does not match this server's IP [$DEFAULT_IP]" 1
                    EXIT_CODE=1
                fi
            fi
        else
            if [[ $es_server_ip == $DEFAULT_IP ]]; then
                report "ES should be on this server by GUI-Servers, but ES_PRESENT is 0" 1
                EXIT_CODE=1
            else
                report "This server with IP [$DEFAULT_IP] does not have ES. Same as GUI-Servers setting. ES should be on [$es_server_ip]" 0
            fi
        fi
    fi

    if [[ $GUI_PRESENT == 1 ]]; then

        if [[ $es_server_ip != $DEFAULT_IP ]]; then

            report "Checking SSH connection under user apache to server [$es_server_ip]" 3
            es_server_port=`MYSQL_PWD=$DB_PASSWORD /usr/bin/mysql -h $DB_HOST -u $DB_USERNAME $P_OPT $DB_NAME --silent -e "SELECT ssh_port FROM servers WHERE es = 1 AND active = 1 LIMIT 1;" | grep -v value`
            RESPONSE=`sudo -u apache ssh -o BatchMode=yes root@$es_server_ip -p$es_server_port uname -a`
            if echo $RESPONSE | grep -q "Linux"; then
                report "Connection established: $RESPONSE" 0
            else
                report "Connection Failed: $RESPONSE" 1
                EXIT_CODE=1
            fi

        fi
    fi

fi


# ES should not be present/active on the server
if [[ $ES_PRESENT != 1 ]]; then


    if k_service_is_enabled "elasticsearch"; then
        report "ES is enabled, should be disabled" 1
        report "  chkconfig elasticsearch off" fix
    else
        report "ES is disabled" 0
    fi

    if k_service_is_running "elasticsearch"; then
        report "ES is running, should not be" 1
        report "  service elasticsearch stop" fix
    else
        report "ES is not running" 0
    fi


    if grep -REiq '/es_|elastic' /etc/cron.d; then
        report "There are ES cronjobs active, should not be" 1
        report "  Delete all these cronjobs" fix
        grep -REi '/es_|elastic' /etc/cron.d | awk -F ":" '{print $1}'
    else
        report "There are no active ES cronjobs" 0
    fi

    k_exit $EXIT_CODE
fi


# --------- following tests only if ES is on this server -------------



if [ -f $ES_CFG ]; then
    report "$ES_CFG is present" 0
else
    report "$ES_CFG is not present" 1
    k_exit 1
fi


file1=/etc/elasticsearch/logging.yml
file2=$ES_SRC_DIR/logging.yml
if ! cmp -s "$file1" "$file2"; then
    report "$file2 does not match $file1" 2
    report "  cp -fr $file2 $file1"
    EXIT_CODE=2
fi


if k_service_is_running elasticsearch; then
    report "ES service is running" 0
else
    report "ES service is not running" 1
    report "  service elasticsearch start" 1
    #EXIT_CODE=1
    k_exit 1
fi

if [[ $SYSTEM_CONFIG_DIR == "m2" ]]; then
    es_version=`curl -m 2 -s -XGET "http://$es_server_ip:9200" | grep number | awk -F ":" '{print $2}' | tr -d "\"" | tr -d , | xargs`
    if [[ $es_version == "" ]]; then
        report "ES is not accessible over $es_server_ip:9200 " 1
        EXIT_CODE=1
    else
        report "ES is accessible over $es_server_ip:9200 " 0
    fi
fi

es_version=`curl -s -XGET 'http://localhost:9200' | grep number | awk -F ":" '{print $2}' | tr -d "\"" | tr -d , | xargs`
if [[ $es_version != "1.5.2" ]]; then
    report "ES process version: $es_version. Should be 1.5.2" 1
    EXIT_CODE=1
else
    report "ES process version: $es_version" 0
fi


if test -h /usr/bin/elasticsearch; then
    report "Symlink /usr/bin/elasticsearch ok" 0
else
    report "Symlink /usr/bin/elasticsearch does not exist" 1
fi


# check ES script (elasticsearch) version
installed_es_script_version=$(grep -Po 'VERSION=.*' /usr/local/$SYSTEM_CONFIG_DIR/${SYSTEM_CONFIG_DIR}_elasticsearch.sh 2>/dev/null | awk -F'=' '{print $2}')
src_es_script_version=$(grep -Po 'VERSION=.*' $ES_SRC_DIR/${SYSTEM_CONFIG_DIR}_elasticsearch.sh | awk -F'=' '{print $2}')
if [ "$installed_es_script_version" == "$src_es_script_version" ] && [ "$installed_es_script_version" != "" ]; then
    report "ES script installed version: $installed_es_script_version (latest)" 0
else
    report "ES script outdated! Installed version [$installed_es_script_version], source version [$src_es_script_version]" 1
    report "  cp -f $ES_SRC_DIR/${SYSTEM_CONFIG_DIR}_elasticsearch.sh /usr/local/$SYSTEM_CONFIG_DIR/" fix
    report "  ln -fs /usr/local/$SYSTEM_CONFIG_DIR/${SYSTEM_CONFIG_DIR}_elasticsearch.sh /usr/bin/elasticsearch" fix
    EXIT_CODE=1
fi

ES_STATUS=`elasticsearch status | grep status | awk -F ":" '{print $2}' | sed 's/"//g' | sed 's/,//g' | xargs`
if [[ $ES_STATUS != "green" && $ES_STATUS != "yellow" ]]; then
    report "elasticsearch status: $ES_STATUS. Something wrong. Solve it manually." 1
    report "  More info: http://chrissimpson.co.uk/elasticsearch-yellow-cluster-status-explained.html" 1
    EXIT_CODE=1
else
    report "elasticsearch status: $ES_STATUS" 0
fi

ES_NET_9200=`netstat -vatupn | grep ":9200"`
if [[ $ES_NET_9200 != "" ]]; then
    report "ES listens on port 9200" 0
else
    report "ES does not listen on port 9200" 1
    EXIT_CODE=1
fi

ES_NET_9300=`netstat -vatupn | grep ":9300"`
if [[ $ES_NET_9300 != "" ]]; then
    report "ES listens on port 9300" 0
else
    report "ES does not listen on port 9300" 1
    EXIT_CODE=1
fi


if [[ $SYSTEM_CONFIG_DIR == "m2" ]]; then

    ES_NET_LOCAL=`netstat -vatupn | grep "0 127.0.0.1:9200"`
    if [[ $ES_NET_LOCAL != "" ]]; then

        if [[ $conf_ES_IP == "127.0.0.1" ]]; then
            report "ES listens on local IP: 127.0.0.1 and ES_IP == 127.0.0.1" 0
        else
            report "ES listens on local IP: 127.0.0.1 but ES_IP == $conf_ES_IP. Fix manually." 1
            report "  Make sure all network.host instances are commented OUT in $ES_CFG and service elasticsearch restart" 1
            EXIT_CODE=1
        fi

    else

        # ES not on local IP
        if [[ $conf_ES_IP == "127.0.0.1" ]]; then
            report "ES does not listen on local IP: 127.0.0.1 but ES_IP == 127.0.0.1. Fix manually." 1
            report "  ES_IP == 127.0.0.1. Not allowed. Fix in GUI-Servers." 1
            EXIT_CODE=1
        else
            report "ES does not listen on local IP and ES_IP is not local" 0
        fi

    fi

fi

k_iptables_locking_option

# checking firewall
if [[ $conf_ES_IP != "127.0.0.1" ]]; then

    # get all active gui/db server IPs
    server_ips=$(MYSQL_PWD="$DB_PASSWORD" /usr/bin/mysql -h "$DB_HOST" -u $DB_USERNAME $P_OPT "$DB_NAME" --silent -e "SELECT server_ip FROM servers WHERE (gui = 1 OR db = 1) AND active = 1 AND server_ip != '127.0.0.1' AND server_ip != 'localhost';" | grep -v value)

    # clean from \n "
    server_ips=$(echo $server_ips|tr -d '\n')

    if [ ${#server_ips} -gt 0 ]; then
        report "Found active GUI/DB servers with IPs [$server_ips]" 3
    fi

    # make array out of string
    IFS=' ' read -r -a array <<< "$server_ips"

    FILE="/tmp/m2_iptables_es_check"
    rm -fr $FILE

    if iptables -LINPUT -n $l_opt | grep -E '^M2-ES|^MOR-ES|^ES-RULES' | grep -Fq '9200,9300'; then
        iptables -LM2-ES -n $l_opt 2>/dev/null | grep ACCEPT >> $FILE
        iptables -LMOR-ES -n $l_opt 2>/dev/null | grep ACCEPT >> $FILE
        iptables -LES-RULES -n $l_opt 2>/dev/null | grep ACCEPT >> $FILE
    else
        iptables -L -n $l_opt | grep "9200,9300" | grep ACCEPT > $FILE
    fi

    # loop for all server IPs
    for index in "${!array[@]}"
    do
        server_ip=${array[index]}
        #report "Processing server with IP [$server_ip] in $FILE" 3

        if cat $FILE | grep -q $server_ip; then
            report "$server_ip has access to ES in iptables" 0
        else
            report "$server_ip has no access to ES in iptables" 1
            report "  /usr/src/k_framework/helpers/elasticsearch/es_iptables_config.sh" fix
            EXIT_CODE=1
        fi

    done

    if (( $(iptables $l_opt -LMOR-ES -n 2>/dev/null | grep -c DROP) > 1 || $(iptables $l_opt -LM2-ES -n 2>/dev/null | grep -c DROP) > 1 || $(iptables $l_opt -LES-RULES -n 2>/dev/null | grep -c DROP) > 1 )); then
        report "There shoud be only one DROP rule in ES chain" 1
        EXIT_CODE=1
    fi

#    rm -fr $FILE

fi

if iptables $l_opt -L -n | grep DROP | grep -q "9200,9300"; then
    report "Access to ES closed from outside for other IPs" 0
else
    report "Access to ES not closed from outside!" 1
    report "  /usr/src/k_framework/helpers/elasticsearch/es_iptables_config.sh" 1
    EXIT_CODE=1
fi


if /usr/share/elasticsearch/bin/plugin -l | grep -q jdbc; then
    report "jdbc plugin installed" 0
else
    report "jdbc plugin not installed" 1
    EXIT_CODE=1
fi


#NETWORK_=`cat $ES_CFG | grep "network.host: \"127.0.0.1\""`

es_sync_active=0

if /usr/bin/curl -s "http://localhost:9200/_river/_search?q=_type:${SYSTEM_CONFIG_DIR}_jdbc_river&pretty" | grep -q _jdbc_river; then
    report "ES Realtime sync active" 0
    es_sync_active=1
else
    # ES sync not active? Try again in 3sec
    k_delay 3 "ES Sync not active, retesting"
    if /usr/bin/curl -s "http://localhost:9200/_river/_search?q=_type:${SYSTEM_CONFIG_DIR}_jdbc_river&pretty" | grep -q _jdbc_river; then
        report "ES Realtime sync active" 0
        es_sync_active=1
    else
        # Maybe resync is running? In this case realtime sync is created after a delay (once resync finishes syncing first day)
        if [ "$(ps -elf | grep es_resync_by_day.sh | grep -v 'grep' | wc -l)" != "0" ]; then
            report "ES resync is running, can't check ES realtime sync now, try again in few minutes" 2
            EXIT_CODE=2
        else
            report "ES Realtime sync is not active" 1
            report "  elasticsearch sync start" fix
            EXIT_CODE=1
        fi
    fi
fi

if [ "$es_sync_active" == "1" ]; then
    # check if latest version of sync is active
    if /usr/bin/curl -s "http://localhost:9200/_river/_search?q=_type:${SYSTEM_CONFIG_DIR}_jdbc_river&pretty" | grep -qFo 'SELECT calls.id AS _id,calls.id AS id'; then
        report "ES Realtime sync latest version" 0
    else
        report "ES Realtime sync is outdated" 1
        report "  svn update $ES_SRC_DIR" fix
        report "  cp -f $ES_SRC_DIR/${SYSTEM_CONFIG_DIR}_elasticsearch.sh /usr/local/$SYSTEM_CONFIG_DIR/" fix
        report "  elasticsearch resync full" fix
        report "  GET CONFIRMATION FROM THE CLIENT FIRST" fix
        EXIT_CODE=1
    fi
fi

#report "Checking data integrity..." 3

FILE="/tmp/es_data_integrity_check_results"
echo "{ \"size\": 0, \"aggregations\": { \"disp_groups\": { \"terms\": { \"field\": \"disposition\" }}}}" > /tmp/es_check_cmd
curl -s -XGET "localhost:9200/$SYSTEM_CONFIG_DIR/calls/_search?pretty" --data-binary @/tmp/es_check_cmd > $FILE

ES_SHARDS=`cat $FILE | grep 'successful' | awk -F ":" '{print $2}' | sed 's/,//g' | xargs`
if [[ $ES_SHARDS != 5 ]]; then
    report "ES shards looks broken. Successful count: $ES_SHARDS (should be 5)" 1
    EXIT_CODE=1
else
    report "ES successful shards: 5 " 0
fi

ES_RECORDS=`cat $FILE | grep -A1 'hits' | grep 'total' | awk -F ":" '{print $2}' | sed 's/,//g' | xargs`
if [[ $ES_RECORDS != 0 ]]; then

    report "ES records: $ES_RECORDS" 0

    if [ "$(grep "\"key\" : \"no\"," $FILE)" == "" ]; then
        report "ES disposition data integrity valid" 0
        rm -fr $FILE
    else
        report "ES disposition data integrity broken" 1
        report "  Check results in $FILE" 1
        report "  ES delete/reinstall/resync could be a solution :(" 1
        report "  Full horror story: https://support.kolmisoft.com/tickets/ticket_show/51915#comment_473079" 1
        EXIT_CODE=1
    fi
    rm -fr /tmp/es_check_cmd

else
    report "ES has no data, records: 0" 2
fi


# configuration checks

# ram/heap check
# https://ugosan.org/Locking-Memory-for-production/

ES_MEM_CFG_HELP="  Auto-fix: /usr/src/k_framework/helpers/elasticsearch/es_memory_config.sh && service elasticsearch restart"


SYSTEM_RAM=`awk '/MemTotal/ { printf "%.0f \n", $2/1024/1024 }' /proc/meminfo | xargs`
ES_HEAP_SIZE=`cat $SYSCONFIG | grep "ES_HEAP_SIZE" | awk -F "=" '{print $2}' | tr -d "g" | xargs`

# count optimal ES_HEAP_SIZE based on the system RAM
# < 8 gb    -> 1 gb
# 8 - 16 gb -> 2 gb
# > 16 gb   -> 4 gb

if [ $SYSTEM_RAM -lt 8 ]; then
    ES_HEAP_SIZE_OPTIMAL=1;
elif [ $SYSTEM_RAM -lt 16 ]; then
    ES_HEAP_SIZE_OPTIMAL=2;
else
    ES_HEAP_SIZE_OPTIMAL=4;
fi


if [[ $ES_HEAP_SIZE != $ES_HEAP_SIZE_OPTIMAL ]]; then
    report "System RAM: $SYSTEM_RAM Gb, ES_HEAP_SIZE: $ES_HEAP_SIZE Gb != ES_HEAP_SIZE_OPTIMAL: $ES_HEAP_SIZE_OPTIMAL Gb" 1
    report "  Set ES_HEAP_SIZE=${ES_HEAP_SIZE_OPTIMAL}g in $SYSCONFIG && service elasticsearch restart" 1
    report "$ES_MEM_CFG_HELP" 1
    EXIT_CODE=1
else
    report "System RAM: $SYSTEM_RAM Gb, ES_HEAP_SIZE: $ES_HEAP_SIZE Gb == ES_HEAP_SIZE_OPTIMAL: $ES_HEAP_SIZE_OPTIMAL Gb" 0
fi



# ES_JAVA_OPTS

if cat $SYSCONFIG | grep "ES_JAVA_OPTS" | grep -q "#"; then
    report "ES_JAVA_OPTS is commented out in $SYSCONFIG" 1
    report "  Uncomment and set it to ES_JAVA_OPTS=\"-Xms${ES_HEAP_SIZE_OPTIMAL}g -Xmx${ES_HEAP_SIZE_OPTIMAL}g\" && service elasticsearch restart" 1
    report "$ES_MEM_CFG_HELP" 1
    EXIT_CODE=1
else

    ES_JAVA_OPTS=`cat $SYSCONFIG | grep "ES_JAVA_OPTS" | awk -F "=" '{print $2}' | xargs`
    if [[ $ES_JAVA_OPTS != "-Xms${ES_HEAP_SIZE_OPTIMAL}g -Xmx${ES_HEAP_SIZE_OPTIMAL}g" ]]; then
        report "ES_JAVA_OPTS values are wrong: $ES_JAVA_OPTS in $SYSCONFIG" 1
        report "  Set it to ES_JAVA_OPTS=\"-Xms${ES_HEAP_SIZE_OPTIMAL}g -Xmx${ES_HEAP_SIZE_OPTIMAL}g\" && service elasticsearch restart" 1
        report "$ES_MEM_CFG_HELP" 1
    else
        report "ES_JAVA_OPTS: $ES_JAVA_OPTS" 0
    fi

fi


# MAX_LOCKED_MEMORY

MAX_LOCKED_MEMORY=`cat $SYSCONFIG | grep "MAX_LOCKED_MEMORY" | xargs`
if [[ $MAX_LOCKED_MEMORY != "MAX_LOCKED_MEMORY=unlimited" ]]; then
    report "Bad $MAX_LOCKED_MEMORY in $SYSCONFIG" 1
    report "  Set it to MAX_LOCKED_MEMORY=unlimited && service elasticsearch restart" 1
    report "$ES_MEM_CFG_HELP" 1
else
    report "$MAX_LOCKED_MEMORY" 0
fi


# security-limits

FILE="/etc/security/limits.d/elasticsearch.conf"
if [ ! -f $FILE ]; then
    report "$FILE is missing" 1
    report "$ES_MEM_CFG_HELP" 1
    EXIT_CODE=1
else
    if cat $FILE | grep -q "elasticsearch soft memlock unlimited"; then
        report "elasticsearch soft memlock unlimited present in $FILE" 0
    else
        report "elasticsearch soft memlock unlimited missing in $FILE" 1
        report "$ES_MEM_CFG_HELP" 1
        EXIT_CODE=1
    fi

    if cat $FILE | grep -q "elasticsearch hard memlock unlimited"; then
        report "elasticsearch hard memlock unlimited present in $FILE" 0
    else
        report "elasticsearch hard memlock unlimited missing in $FILE" 1
        report "$ES_MEM_CFG_HELP" 1
        EXIT_CODE=1
    fi
fi


# LimitMEMLOCK=infinity

FILE="/usr/lib/systemd/system/elasticsearch.service"
LIMIT_MEMLOCK=`cat $FILE | grep "LimitMEMLOCK" | xargs`
if [[ $LIMIT_MEMLOCK != "LimitMEMLOCK=infinity" ]]; then
    report "Bad $LIMIT_MEMLOCK in $FILE" 1
    report "  Set it to LimitMEMLOCK=infinity && systemctl daemon-reload && service elasticsearch restart" 1
    report "$ES_MEM_CFG_HELP" 1
else
    report "LimitMEMLOCK=infinity in $FILE" 0
fi


# bootstrap.memory_lock: true
BOOTSTRAP=`cat $ES_CFG | grep "bootstrap.memory_lock" | xargs`
if [[ $BOOTSTRAP != "bootstrap.memory_lock: true" ]]; then
    report "Bad bootstrap.memory_lock: $BOOTSTRAP in $ES_CFG" 1
    report "  Set it to or add at the file end: bootstrap.memory_lock: true && service elasticsearch restart" 1
    report "$ES_MEM_CFG_HELP" 1
else
    report "$BOOTSTRAP in $ES_CFG" 0
fi



# accountcode field check
if [[ $SYSTEM_CONFIG_DIR == "m2" ]]; then

    # does river looks for accountcode?
    if curl -s -XGET http://localhost:9200/_river/_search?q=* | grep -q accountcode; then
        report "accountcode found is SQL, not supported anymore" 1
        report "  svn update /usr/src/m2/elasticsearch && elasticsearch sync stop && sleep 10 && elasticsearch sync start" 1
        EXIT_CODE=1
    fi

    # does ES internal structure has accountcode?
    if curl -s -XGET 'http://localhost:9200/m2/_mapping/calls' | grep -q accountcode; then
        report "accountcode found is ES structure, not supported anymore" 2
        report "  svn update $ES_SRC_DIR" fix
        report "  cp -f $ES_SRC_DIR/${SYSTEM_CONFIG_DIR}_elasticsearch.sh /usr/local/$SYSTEM_CONFIG_DIR/" fix
        report "  elasticsearch resync full" fix
        report "  !!!GET CONFIRMATION FROM THE CLIENT FIRST!!!" 2
        EXIT_CODE=2
    fi
fi


# --------- ES control script check ----------

/usr/src/k_framework/helpers/elasticsearch/es_sync_control_check.sh


# ------------- log checks ----------------

k_logs_check "failure" "/var/log/elasticsearch/cluster*"
k_logs_check "Unknown column" "/var/log/elasticsearch/cluster*"

k_logs_check "OutOfMemoryError" "/var/log/elasticsearch/cluster*"
if [[ $LOG_CHECK_FOUND == 1 ]]; then
    report "  Consider increasing ES_HEAP_SIZE in $SYSCONFIG && service elasticsearch restart" 2
fi


k_logs_check "Unable to lock JVM memory" "/var/log/elasticsearch/cluster*"
if [[ $LOG_CHECK_FOUND == 1 ]]; then
    report "  https://stackoverflow.com/questions/51382869/unable-to-lock-jvm-memory-error-12-reason-cannot-allocate-memory-elasticsear" 2
    report "  This could help: /usr/src/k_framework/helpers/elasticsearch/es_memory_config.sh && service elasticsearch restart" 2
fi



k_exit $EXIT_CODE