Commit 6010ca0f authored by remy's avatar remy
Browse files

first public commit of Jsonreader2

parent 0ccde580
# jsonreader2
# Jsonreader2
Public copy from internal jsonreader2 project
Jsonreader2 is a PHP Web parser for JSON files mainly used for SaltStack reporting.
\ No newline at end of file
Jsonreader v2 is a PHP Web application parser for JSON files.
I am using it for reporting with a cron and SaltStack outputs in json format, but it can also parse other JSON files (check `include/toolReader.php` on how to do this).
This version 2 is a complete redesign from the MBB `portail_admin` project, that is to say with OOP, a model / database and a more modern UI.
It was a part of MBB `portail_admin` application (wrote mainly by my former wo-worker [_Jimmy Lopez_](https://github.com/Falindir)), but is now a plugin of MBB `portail_admin` and can be used in a standalone version (default).
The first version was a single file; you can see it [here](https://github.com/remyd1/salt_states/tree/master/monitor_salt_json). In this first version, cron files were written into `/var/www/html/exports/YYYYMM/YYYYMMDD_type_of_export.json`. In this version 2, these files are located, by default, in `/var/www/html/exports/YYYYMM/DD/YYYYMMDD_type_of_export.json`.
## Install
Clone this repository in your document root and create the exports directory for json files `/var/www/html/exports`.
Change TARGET by a minion ID in `check_salt_json` and copy `check_salt_json` to `/usr/local/sbin` and set it to be executable by root.
Add a crontab, eg:
```cron
*/30 * * * * /usr/local/sbin/check_salt_json 2>/dev/null
```
```bash
service cron reload
```
Install `php-fpm` and reload your web server:
```bash
# for ubuntu/debian with php7.3 and apache:
apt install -y php-fpm
a2enmod proxy_fcgi setenvif
a2enconf php7.3-fpm
systemctl reload apache2
```
Of course, you also need a mysql/mariadb server:
```bash
apt install -y mariadb-server php-mysql
a2enmod proxy_fcgi setenvif
```
Install the database using `*.sql` in `sql` directory, create mysql users and set it correctly in `config/Conf.php` and `sql/jsonreader.sql`.
```bash
mysql -uroot -p < sql/jsonreader.sql
mysql -uroot -p jsonreader2 < sql/CReader.sql
mysql -uroot -p jsonreader2 < sql/ReaderTable.sql
```
For a plugin version of `portail_admin`, please check `config/Plugin.php`.
## Using SaltStack Formula
`Jsonreader2` is using some specific SaltStack formula + basic formula (`test.ping`, `service.status`, `osfinger` ...).
### Specific formula
- To check services, take a look at the [`check_services`](https://gitlab.mbb.univ-montp2.fr/saltstack-formulas/check_services) formula.
- To check disks, take a look at [`check_disks`](https://gitlab.mbb.univ-montp2.fr/saltstack-formulas/check_disks).
- To check D state processes, take a look at [`get_d_states`](https://gitlab.mbb.univ-montp2.fr/saltstack-formulas) formula.
## Other Json reports
It can also use web reports from [`website_checks`](https://gitlab.mbb.univ-montp2.fr/remy/website_checks).
<?php
require_once "../config/Plugin.php";
define("PAGE","actionJsonReader");
if(!JSONREADER2_STANDALONE) {
session_start ();
if(!isset($_SESSION['username'])) {
header("Location: ".JSR_PATH."/index.php");
}
}
require_once JSR_PATH.'/dao/DBquery.php';
require_once JSR_PATH.'/dao/CReaderQuery.php';
require_once "../model/CReader.php";
$db = new DBquery();
$today = date("Y-m-d G:i:s");
$valueReader = [];
$colorReader = [];
foreach ($_POST as $ckey => $cvalue){
if(strpos($ckey, "colorpicker-regularfont-") !== false){
$colorReader[str_replace("colorpicker-regularfont-", "", $ckey)] = $cvalue;
}
else {
$valueReader[$ckey] = $cvalue;
}
}
foreach ($valueReader as $ckey => $cvalue){
$color = $colorReader[$ckey];
$value = str_replace("\n", ";", $cvalue);
$value = str_replace("\r", "", $value);
$creader = new CReader(0, $ckey, $value, $color);
CReaderQuery::updateCReader($db, $creader);
}
header("Location: ".JSR_PATH."/services/jsonreaderConfig.php");
?>
\ No newline at end of file
<?php
require_once "../config/Plugin.php";
define("PAGE","actionReaderTable");
if(!JSONREADER2_STANDALONE) {
session_start ();
if(!isset($_SESSION['username'])) {
header("Location: ".JSR_PATH."/index.php");
}
}
require_once JSR_PATH.'/dao/DBquery.php';
require_once JSR_PATH.'/dao/ReaderTableDBQuery.php';
require_once "../model/ReaderTable.php";
$db = new DBquery();
$action = NULL;
$message = NULL;
$today = date("Y-m-d G:i:s");
$ID = -1;
$name = NULL;
$title = NULL;
$head = NULL;
$pos = 9999;
$jssort = NULL;
if(isset($_POST['action'])) {
$action = $_POST['action'];
} else {
if(isset($_GET['action'])) {
$action = $_GET['action'];
} else {
$action = "";
}
}
if(isset($_POST['ReaderTableID'])) {
$ID = $_POST['ReaderTableID'];
} else {
if(isset($_GET['ReaderTableID'])) {
$ID = $_GET['ReaderTableID'];
} else {
$ID = -1;
}
}
if(isset($_POST['name'])) {
$name = $_POST['name'];
}
if(isset($_POST['title'])) {
$title = $_POST['title'];
}
if(isset($_POST['pos'])) {
$pos = int($_POST['pos']);
} else {
$pos = ReaderTableDBQuery::getNextPosReaderTable($db);
}
if(isset($_POST['head'])) {
$head = $_POST['head'];
}
if(isset($_POST['jssort'])) {
$jssort = $_POST['jssort'];
}
$ReaderTable = new ReaderTable($ID, $name, $title, $head, $pos, $jssort);
if($action == "create") {
$ReaderTable->escape($db);
$message = ReaderTableDBQuery::createReaderTable($db, $ReaderTable);
if(!JSONREADER2_STANDALONE) {
$log = new Log(-1, "admin", $ReaderTable->name, "insert", $message->value, $today, -1);
LogDBQuery::createLog($db, $log);
}
} else if ($action == "update") {
$ReaderTable->escape($db);
$message = ReaderTableDBQuery::updateReaderTable($db, $ReaderTable);
if(!JSONREADER2_STANDALONE) {
$log = new Log(-1, "admin", $ReaderTable->name, "update", $message->value, $today, -1);
LogDBQuery::createLog($db, $log);
}
} else if ($action == "delete") {
$message = ReaderTableDBQuery::deleteReaderTable($db, $ReaderTable);
if(!JSONREADER2_STANDALONE) {
$log = new Log(-1, "admin", $ReaderTable->ID, "delete", $message->value, $today, -1);
LogDBQuery::createLog($db, $log);
}
}
header("Location: ".JSR_PATH."/services/manageTableReader.php");
\ No newline at end of file
#!/bin/bash
# Following path must be accessible to www user
JSON_EXPORT_PATH="/var/www/html/exports"
TARGET="minion_id.domain.tld"
CURMONTH=`date '+%Y%m'`
CURDAY=`date '+%d'`
#DATE=`date '+%Y%m%d'`
DATE=`date '+%Y%m%d_%H%M'`
SUBDIR=${CURMONTH}"/"${CURDAY}
mkdir -p ${JSON_EXPORT_PATH}/${SUBDIR}
/usr/bin/salt '*' saltutil.sync_modules 1>&2 >/dev/null
####################### CHECKING SERVICES FROM PILLAR hosts.sls ######################
/usr/bin/salt '*' state.sls check_services -t 10 --out=json --static |grep -v " did not respond. No job will be sent." > ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_services.json
## to debug
#salt '*' state.sls check_services -t 10 --async -v
## then, with the jid from previous command :
## salt-run jobs.lookup_jid <job id>
## eg :
#salt-run jobs.lookup_jid 20191120154225242785
sleep 3
############################ CHECKING D STATES PROCESSES #############################
salt '*' cmd.run "/usr/local/sbin/get_d_states" --static --out=json |grep -v " did not respond. No job will be sent." > ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_dstates.json
################################# PING CHECK #########################################
/usr/bin/salt '*' test.ping -t 10 --out=json |sort | grep -Ev "[\{\}],?" |awk '{
if (NR == 1) {
total="\{\n"$0;
} else {
total=total",\n"$0;
}
}
END {
print total"\n\}";
}' 2>/dev/null > ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_hosts_status.json
sleep 3
############################### CHECKING STORAGE #####################################
################################## DISK USAGE ########################################
/usr/bin/salt '*' disk.percent --out=json --static |grep -v " did not respond. No job will be sent." > ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_disks.json
############################## DISK SMART STATUS #####################################
# using saltstack mine b/c of timeout issues
nb_lines=`/usr/bin/salt ${TARGET} mine.get '*' 'mine_disks' |grep -v " did not respond. No job will be sent."|wc -l`
/usr/bin/salt ${TARGET} mine.get "*" "mine_disks" |grep -v " did not respond. No job will be sent."| awk -v last=$nb_lines '{
if ( NR == 1 ) {
print "{\n\"disks_status\":";
} else if (NR == 2) {
print "\t{"
} else if (NR==last) {
print "\t\t}\n\t}\n}";
} else if ($NF ~ ".+:$") {
gsub(/[ :]/, "", $0);print "\t\""$0"\":";
} else if ($0 ~ "No such file") {
print "\t{\n\t\t\"/var\": \"ERROR - UNREADABLE\"\n\t},";
} else {print;}
}' > ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_disks_status.json
################################# ZPOOL STATUS #######################################
#salt '*' cmd.run "if [ -f /sbin/zfs ] || [ -f /usr/local/sbin/zfs ]; then zpool status -x; fi" --static --out=json | grep -v " did not respond. No job will be sent." > $JSON_EXPORT_PATH/$SUBDIR/"$DATE"_zpool.json
# Adding d state process check. Otherwise, it will be overloaded with additionnal D state processes
# TODO: using salt zpool module
salt '*' cmd.run "/bin/bash -c \"if \[ -f /sbin/zfs \] || \[ -f /usr/local/sbin/zfs \]; then if \\\[ \\\`/usr/local/sbin/get_d_states count\\\` == 0 \\\]; then zpool status -x; else echo \\\"D states processes have been found\\\"; fi; fi\"" --static --out=json | grep -v " did not respond. No job will be sent." > $JSON_EXPORT_PATH/$SUBDIR/"$DATE"_zpool.json
sed -i 's#\\#\\\\#g' ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_{services,hosts_status,disks,disks_status,zpool}.json
sed -i 's#\\\\"##g' ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_services.json
################################### VERSIONS ########################################
salt '*' grains.get osfinger --static --out=json |grep -v " did not respond. No job will be sent." > ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_osversion.json
salt '*' grains.get biosreleasedate --static --out=json |grep -v " did not respond. No job will be sent." > ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_biosdate.json
salt '*' grains.get saltversion --static --out=json |grep -v " did not respond. No job will be sent." > ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_saltversion.json
salt '*' grains.get kernelrelease --static --out=json |grep -v " did not respond. No job will be sent." > ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_kernelversion.json
################################### WEBSITES ########################################
# From https://gitlab.mbb.univ-montp2.fr/remy/website_checks
bash /root/website_checks/check_urls.sh check 2>/dev/null 1>&2
cp /root/website_checks/workdir/checksums.json ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_websites_checksums.json
cp /root/website_checks/workdir/status.json ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_websites_status.json
# to generate host_https_list.txt :
# salt -G 'roles:https' cmd.run 'hostname -f' --out=yaml |awk '{print $2}'
# check /root/un-peu-de-sel/salt_pillar/machines/roles_grains.sls content
bash /root/website_checks/check_certs.sh > ${JSON_EXPORT_PATH}/${SUBDIR}/"${DATE}"_websites_certs.json
<?php
/**
* Class used to do static access to specific constants
**/
class Conf {
public static $DB_NAME = 'jsonreader2';
public static $DB_HOSTNAME = 'localhost';
public static $DB_USERNAME = 'jsonreader2';
public static $DB_PP = 'jsonreader_s3cret2';
public static $PA_AUTHOR = 'Rémy Dernat';
public static $PA_COMMITERS = 'Jimmy Lopez, Rémy Dernat';
public static $PA_VERSION = '0.0.0.2';
public static $url = '162.38.197.150';
public static $root_dir = 'html/jsonreader2';
}
?>
<?php
/**
* Class used to do static access to specific constants
**/
class Conf {
public static $DB_NAME = 'jsonreader2';
public static $DB_HOSTNAME = 'localhost';
public static $DB_USERNAME = 'jsonreader2';
public static $DB_PP = 'your_s3cret';
public static $PA_AUTHOR = 'Rémy Dernat';
public static $PA_COMMITERS = 'Jimmy Lopez, Rémy Dernat';
public static $PA_VERSION = '0.0.0.2';
public static $url = '<IP.IP.IP.IP>';
public static $root_dir = 'jsonreader2';
}
?>
<?php
/* Standalone version */
define("JSONREADER2_STANDALONE", TRUE);
define("JSR_PATH", "..");
/* Portail_admin Plugin version */
/*
define("JSONREADER2_STANDALONE", FALSE);
define("JSR_PATH", "../../..");
*/
/* Adapt following to the sections you monitor (case sensitive; see the database table "ReaderTable" for title fields (or directly on the web page)) */
//$SECTIONS=['Hosts Status', 'Services', 'Dstates', 'Disks', 'Disks Status', 'Zpool', 'Bios date', 'Salt version', 'Kernel version', 'OS version', 'Website certs', 'Website checksums', 'Website status'];
$SECTIONS=['Hosts Status', 'Services', 'Dstates', 'Disks', 'Disks Status', 'Zpool', 'Bios date', 'Salt version', 'Kernel version', 'OS version']
?>
<?php
class Reader {
// default value
var $DEFAULT_SUCCESS = "1";
// _hosts_status
// nothing
// _services
// nothing
// _disks
var $DISKS_USAGE_DANGER = 90;
var $DISKS_USAGE_WARNING = 75;
// _disks_status
var $DISKS_STATUS_OK = "OK";
// _zpool
var $ZPOOL_STATUS_SUCCESS = "all pools are healthy";
var $ZPOOL_STATUS_INFO = "no pools available";
var $ZPOOL_STATUS_PRIMARY = "ZFS modules"; // contain this expression
var $ZPOOL_STATUS_WARNING = "ONLINE"; // contain this expression
// _dstates
// nothing
// _biosdate
// nothing
// _saltversion
var $SALTVERSION = ["2019", "3000", "3001", "3002", "3003"];
var $SALTVERSION_SUCCESS = ["3003", "2019"];
var $SALTVERSION_WARNING = ["2000", "3001"];
// _kernelversion
// nothing
// _osversion
var $OSVERSION_SUCCESS = ["Ubuntu-18.04", "Debian-10", "CentOS Linux-7", "Raspbian-9", "Ubuntu-20.04"];
// _websites_checksums & _websites_status
var $WEBSITE_CHECKSUMS_AND_STATUS_SUCCESS = "OK";
var $WEBSITE_CHECKSUMS_AND_STATUS_WARNING = ["Found", "Permanently"];
// _websites_certs
var $WEBSITES_CERTS_SUCCESS = 5;
var $WEBSITES_CERTS_WARNING = 1;
var $WEBSITES_CERTS_STATUS_OK = "Ok";
//=============================================
// COLOR
//=============================================
// default value
var $COLOR_DEFAULT_SUCCESS = "#FFFFFF";
// _hosts_status
// nothing
// _services
// nothing
// _disks
var $COLOR_DISKS_USAGE_DANGER = "#FFFFFF";
var $COLOR_DISKS_USAGE_WARNING = "#FFFFFF";
// _disks_status
var $COLOR_DISKS_STATUS_OK = "#FFFFFF";
// _zpool
var $COLOR_ZPOOL_STATUS_SUCCESS = "#FFFFFF";
var $COLOR_ZPOOL_STATUS_INFO = "#FFFFFF";
var $COLOR_ZPOOL_STATUS_PRIMARY = "#FFFFFF";
var $COLOR_ZPOOL_STATUS_WARNING = "#FFFFFF";
// _dstates
// nothing
// _biosdate
// nothing
// _saltversion
var $COLOR_SALTVERSION = "#FFFFFF";
var $COLOR_SALTVERSION_SUCCESS = "#FFFFFF";
var $COLOR_SALTVERSION_WARNING = "#FFFFFF";
// _kernelversion
// nothing
// _osversion
var $COLOR_OSVERSION_SUCCESS = "#FFFFFF";
// _websites_checksums & _websites_status
var $COLOR_WEBSITE_CHECKSUMS_AND_STATUS_SUCCESS = "#FFFFFF";
var $COLOR_WEBSITE_CHECKSUMS_AND_STATUS_WARNING = "#FFFFFF";
// _websites_certs
var $COLOR_WEBSITES_CERTS_SUCCESS = "#FFFFFF";
var $COLOR_WEBSITES_CERTS_WARNING = "#FFFFFF";
var $COLOR_WEBSITES_CERTS_STATUS_OK = "#FFFFFF";
//=========================================================//
public function __construct($creader, $creadercolor) {
$this->DEFAULT_SUCCESS = $creader["DEFAULT_SUCCESS"][0];
// _hosts_status
// nothing
// _services
// nothing
// _disks
$this->DISKS_USAGE_DANGER = intval($creader["DISKS_USAGE_DANGER"][0]);
$this->DISKS_USAGE_WARNING = intval($creader["DISKS_USAGE_WARNING"][0]);
// _disks_status
$this->DISKS_STATUS_OK = $creader["DISKS_STATUS_OK"][0];
// _zpool
$this->ZPOOL_STATUS_SUCCESS = $creader["ZPOOL_STATUS_SUCCESS"][0];
$this->ZPOOL_STATUS_INFO = $creader["ZPOOL_STATUS_INFO"][0];
$this->ZPOOL_STATUS_PRIMARY = $creader["ZPOOL_STATUS_PRIMARY"][0];
$this->ZPOOL_STATUS_WARNING = $creader["ZPOOL_STATUS_WARNING"][0];
// _dstates
// nothing
// _biosdate
// nothing
// _saltversion
$this->SALTVERSION = $creader["SALTVERSION"];
$this->SALTVERSION_SUCCESS = $creader["SALTVERSION_SUCCESS"];
$this->SALTVERSION_WARNING = $creader["SALTVERSION_WARNING"];
// _kernelversion
// nothing
// _osversion
$this->OSVERSION_SUCCESS = $creader["OSVERSION_SUCCESS"];
// _websites_checksums & _websites_status
$this->WEBSITE_CHECKSUMS_AND_STATUS_SUCCESS = $creader["WEBSITE_CHECKSUMS_AND_STATUS_SUCCESS"][0];
$this->WEBSITE_CHECKSUMS_AND_STATUS_WARNING = $creader["WEBSITE_CHECKSUMS_AND_STATUS_WARNING"];
// _websites_certs
$this->WEBSITES_CERTS_SUCCESS = intval($creader["WEBSITES_CERTS_SUCCESS"][0]);
$this->WEBSITES_CERTS_WARNING = intval($creader["WEBSITES_CERTS_WARNING"][0]);
$this->WEBSITES_CERTS_STATUS_OK = $creader["WEBSITES_CERTS_STATUS_OK"][0];
// =====================================
// COLOR
// =====================================
$this->COLOR_DEFAULT_SUCCESS = $creadercolor["DEFAULT_SUCCESS"];
// _hosts_status
// nothing
// _services
// nothing
// _disks
$this->COLOR_DISKS_USAGE_DANGER = $creadercolor["DISKS_USAGE_DANGER"];
$this->COLOR_DISKS_USAGE_WARNING = $creadercolor["DISKS_USAGE_WARNING"];
// _disks_status
$this->COLOR_DISKS_STATUS_OK = $creadercolor["DISKS_STATUS_OK"];
// _zpool
$this->COLOR_ZPOOL_STATUS_SUCCESS = $creadercolor["ZPOOL_STATUS_SUCCESS"];
$this->COLOR_ZPOOL_STATUS_INFO = $creadercolor["ZPOOL_STATUS_INFO"];
$this->COLOR_ZPOOL_STATUS_PRIMARY = $creadercolor["ZPOOL_STATUS_PRIMARY"];
$this->COLOR_ZPOOL_STATUS_WARNING = $creadercolor["ZPOOL_STATUS_WARNING"];
// _dstates
// nothing
// _biosdate
// nothing
// _saltversion
$this->COLOR_SALTVERSION = $creadercolor["SALTVERSION"];
$this->COLOR_SALTVERSION_SUCCESS = $creadercolor["SALTVERSION_SUCCESS"];
$this->COLOR_SALTVERSION_WARNING = $creadercolor["SALTVERSION_WARNING"];
// _kernelversion
// nothing
// _osversion
$this->COLOR_OSVERSION_SUCCESS = $creadercolor["OSVERSION_SUCCESS"];
// _websites_checksums & _websites_status
$this->COLOR_WEBSITE_CHECKSUMS_AND_STATUS_SUCCESS = $creadercolor["WEBSITE_CHECKSUMS_AND_STATUS_SUCCESS"];
$this->COLOR_WEBSITE_CHECKSUMS_AND_STATUS_WARNING = $creadercolor["WEBSITE_CHECKSUMS_AND_STATUS_WARNING"];
// _websites_certs
$this->COLOR_WEBSITES_CERTS_SUCCESS = $creadercolor["WEBSITES_CERTS_SUCCESS"];
$this->COLOR_WEBSITES_CERTS_WARNING = $creadercolor["WEBSITES_CERTS_WARNING"];
$this->COLOR_WEBSITES_CERTS_STATUS_OK = $creadercolor["WEBSITES_CERTS_STATUS_OK"];
}
}
?>
\ No newline at end of file
table.perso {
border:2px solid black;
}
.true {
background-color:green;
}
.false {
background-color:red;
}
.warning {
background-color:orange;
}
\ No newline at end of file
body { padding-top: 70px; }
.clickable{
cursor: pointer;
}
.panel-heading div {
margin-top: -18px;
font-size: 15px;