check_salt_json 7.86 KB
Newer Older
remy's avatar
remy committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
#!/bin/bash

# Following path must be accessible to www user
JSON_EXPORT_PATH="/var/www/html/exports"
TARGET="minion_id.domain.tld"

CURMONTH=`date '+%Y%m'`
CURDAY=`date '+%d'`
#DATE=`date '+%Y%m%d'`
DATE=`date '+%Y%m%d_%H%M'`

SUBDIR=${CURMONTH}"/"${CURDAY}

mkdir -p ${JSON_EXPORT_PATH}/${SUBDIR}

/usr/bin/salt '*' saltutil.sync_modules 1>&2 >/dev/null
remy's avatar
remy committed
17
18
19
20
####################### CHECKING SERVICES FROM PILLAR hosts.sls #####################
# UNCOMMENT NEXT LINE TO USE IT
#/usr/bin/salt '*' state.sls check_services -t 10 --out=json --static |grep -v " did not respond. No job will be sent." > ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_services.json
#####################################################################################
remy's avatar
remy committed
21

remy's avatar
remy committed
22
23
### DEBUG
## to debug previous state:
remy's avatar
remy committed
24
25
26
27
28
29
30
31
#salt '*' state.sls check_services -t 10 --async -v
## then, with the jid from previous command : 
## salt-run jobs.lookup_jid <job id>
## eg :
#salt-run jobs.lookup_jid 20191120154225242785

sleep 3

remy's avatar
remy committed
32
33
34
35
############################ CHECKING D STATES PROCESSES ############################
# UNCOMMENT NEXT LINE TO USE IT
#salt '*' cmd.run "/usr/local/sbin/get_d_states" --static --out=json |grep -v " did not respond. No job will be sent." > ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_dstates.json
#####################################################################################
remy's avatar
remy committed
36

remy's avatar
remy committed
37
################################# PING CHECK ########################################
remy's avatar
remy committed
38
39
40
41
42
43
44
45
46
47
/usr/bin/salt '*' test.ping -t 10 --out=json |sort | grep -Ev "[\{\}],?" |awk '{              
    if (NR == 1) {
      total="\{\n"$0;
    } else {
      total=total",\n"$0;
    }
  }     
  END {               
    print total"\n\}";
  }' 2>/dev/null > ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_hosts_status.json
remy's avatar
remy committed
48
#####################################################################################
remy's avatar
remy committed
49
50
51

sleep 3

remy's avatar
remy committed
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
############################### CHECKING STORAGE ####################################
################################## DISK USAGE #######################################
/usr/bin/salt '*' disk.percent  --out=json --static  |grep -v " did not respond. No job will be sent." > ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_disks_usage.json
#####################################################################################

############################## DISK SMART STATUS ####################################
# using saltstack mine b/c of timeout issues; TARGET is a SaltStack minion knowing the mine. It must be defined above.
# UNCOMMENT NEXT LINES TO USE IT
#nb_lines=`/usr/bin/salt ${TARGET} mine.get '*' 'mine_disks' |grep -v " did not respond. No job will be sent."|wc -l`
#/usr/bin/salt ${TARGET} mine.get "*" "mine_disks" |grep -v " did not respond. No job will be sent."| awk -v last=$nb_lines '{
#    if ( NR == 1 ) {
#        print "{\n\"disks_status\":";
#    } else if (NR == 2) {
#        print "\t{"
#    } else if ($NF ~ ".+:$") {
#        gsub(/[ :]/, "", $0);print "\t\""$0"\":";
#    } else if ($0 ~ "No such file") {
#        print "\t{\n\t\t\"/var\": \"ERROR - UNREADABLE\"\n\t},";
remy's avatar
remy committed
70
71
#    } else if (NR==last) {
#        print "\t\t}\n\t}\n}";
remy's avatar
remy committed
72
73
74
75
76
#    } else {print;} 
#}' > ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_disks_status.json
#####################################################################################

################################# ZPOOL STATUS ######################################
remy's avatar
remy committed
77
# TODO: using salt zpool module
remy's avatar
remy committed
78
79
80
# UNCOMMENT NEXT LINE TO USE IT
#salt '*' cmd.run "/bin/bash -c \"if \[ -f /sbin/zfs \] || \[ -f /usr/local/sbin/zfs \]; then if \\\[ \\\`/usr/local/sbin/get_d_states count\\\` == 0 \\\]; then zpool status -x; else echo \\\"D states processes have been found\\\"; fi; fi\"" --static --out=json | grep -v " did not respond. No job will be sent." > $JSON_EXPORT_PATH/$SUBDIR/"$DATE"_zpool.json
#####################################################################################
remy's avatar
remy committed
81
82


remy's avatar
remy committed
83
84
## old way, without D state process checking (it may be overloaded with additionnal D state processes):
#salt '*' cmd.run "if [ -f /sbin/zfs ] || [ -f /usr/local/sbin/zfs ]; then zpool status -x; fi" --static --out=json | grep -v " did not respond. No job will be sent." > $JSON_EXPORT_PATH/$SUBDIR/"$DATE"_zpool.json
remy's avatar
remy committed
85
86

################################### VERSIONS ########################################
remy's avatar
remy committed
87
# BASIC CHECKING FROM MINIONS GRAINS
remy's avatar
remy committed
88
89
90
91
salt '*' grains.get osfinger --static --out=json |grep -v " did not respond. No job will be sent." > ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_osversion.json
salt '*' grains.get biosreleasedate --static --out=json |grep -v " did not respond. No job will be sent." > ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_biosdate.json
salt '*' grains.get saltversion --static --out=json |grep -v " did not respond. No job will be sent." > ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_saltversion.json
salt '*' grains.get kernelrelease --static --out=json |grep -v " did not respond. No job will be sent." > ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_kernelversion.json
remy's avatar
remy committed
92
#####################################################################################
remy's avatar
remy committed
93
94
95

################################### WEBSITES ########################################
# From https://gitlab.mbb.univ-montp2.fr/remy/website_checks
remy's avatar
remy committed
96
97
98
99
100
101
102
103
# UNCOMMENT NEXT LINES TO USE IT
#bash /usr/local/website_checks/check_urls.sh check 2>/dev/null 1>&2
#cp /usr/local/website_checks/workdir/checksums.json ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_websites_checksums.json
#cp /usr/local/website_checks/workdir/status.json ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_websites_status.json
#bash /usr/local/website_checks/check_certs.sh > ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_websites_certs.json
#####################################################################################


remy's avatar
remy committed
104
105
# to generate host_https_list.txt :
# salt -G 'roles:https' cmd.run 'hostname -f' --out=yaml |awk '{print $2}'
remy's avatar
remy committed
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
# check /usr/local/un-peu-de-sel/salt_pillar/machines/roles_grains.sls content


############################## BORGBACKUP STATUS #####################################
# UNCOMMENT NEXT LINES TO USE IT
#nb_lines=`/usr/bin/salt '*' cmd.run 'if [ -f /var/log/borg/$(date +"%Y%m%d".json) ] ;then if [ -s /var/log/borg/$(date +"%Y%m%d".json) ]; then echo "/var/log/borg/$(date +"%Y%m%d".json):"; sed -e "s|^}|}\n},|" /var/log/borg/$(date +"%Y%m%d".json); else echo "\"/var/log/borg/$(date +"%Y%m%d".json)\": \"borgbackup report file found but is empty!\" \n},"; fi; else echo "\"/var/log/borg/$(date +"%Y%m%d".json)\": \"No borgbackup report file found!\" \n},"; fi' |grep -v " did not respond. No job will be sent."|wc -l`
#/usr/bin/salt "*" cmd.run 'if [ -f /var/log/borg/`date +"%Y%m%d".json` ] ;then if [ -s /var/log/borg/$(date +"%Y%m%d".json) ]; then echo "/var/log/borg/$(date +"%Y%m%d".json):"; sed -e "s|^}|}\n},|" /var/log/borg/`date +"%Y%m%d".json`; else echo "\"/var/log/borg/$(date +"%Y%m%d".json)\": \"borgbackup report file found but is empty!\" \n},"; fi; else echo "\"/var/log/borg/$(date +"%Y%m%d".json)\": \"No borgbackup report file found!\" \n},"; fi' |grep -v " did not respond. No job will be sent." | \
#awk -v last=$nb_lines '{
#    if ( NR == 1 ) {
#        gsub(/[ :]/, "", $0);print "{\n\"borg_status\": {\n\t\""$0"\": {";
#    } else if (NR==last) {
#        print "\t\t}\n\t}\n}";
#    } else if ($NF ~ ".+:$") {
#        if ($0 ~ "/var/log/borg") {
#            gsub(/[ :]/, "", $0);print "\t\t\""$0"\":";
#        } else {
#            gsub(/[ :]/, "", $0);print "\t\""$0"\": {";
#        }
#    } else if ($0 ~ "No such file") {
#        print "\t{\n\t\t\"/var\": \"ERROR - UNREADABLE\"\n\t},";
#    } else {print "\t\t" $0;}
#}' > ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_borg_status.json
#####################################################################################


## Some fix; do not comment.
sed -i 's#\\#\\\\#g' ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_{services,hosts_status,disks_usage,disks_status,zpool}.json
sed -i  's#\\\\"##g' ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_services.json
sed -i  's#\\n#\n#g' ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_borg_status.json