Commit a558879a authored by remy's avatar remy
Browse files

compare and compare2init are not permfrming a scan anymore; just a comparison of checksums / diff

parent fc0a8fb1
...@@ -19,6 +19,18 @@ Then, edit `host_https_list.txt` and `url_list.txt` to fit your needs. ...@@ -19,6 +19,18 @@ Then, edit `host_https_list.txt` and `url_list.txt` to fit your needs.
## How to install ## How to install
### Requirements
Requirements are really basics linux softwares:
- gnu make,
- awk & sed,
- sudo (only needed for a system wide install),
- curl & wget,
- sha256sum to perform checksums
___
After editing the 3 configuration files (`profile.conf` and listings `host_https_list.txt`, `url_list.txt`), just run: After editing the 3 configuration files (`profile.conf` and listings `host_https_list.txt`, `url_list.txt`), just run:
```bash ```bash
......
...@@ -4,7 +4,7 @@ ...@@ -4,7 +4,7 @@
###### Ce script verifie l'etat des sites web listés dans url_list.txt ###### Ce script verifie l'etat des sites web listés dans url_list.txt
###### en recuperant la page d'index et leur statut par curl ###### en recuperant la page d'index et leur statut par curl
###### les pages d'index permettent de produire des checksums ###### les pages d'index permettent de produire des checksums
###### qui sont comparees a une version historique sha256sum.txt.orig ###### qui sont comparees a une version historique sha256sum.txt.prev
###### ######
###### ######
###### ######
...@@ -48,7 +48,7 @@ DIFF=`command -v diff` ...@@ -48,7 +48,7 @@ DIFF=`command -v diff`
usage="$0 [compare|init|check|clean|--help]\n\n usage="$0 [compare|init|check|clean|--help]\n\n
\tcompare: display more informations on differences since previous run\n \tcompare: display more informations on differences since previous run\n
\tcompare2init: display more informations on differences since first run\n \tcompare2init: display more informations on differences since first run\n
\tinit: build the sha256sum.txt.orig file and a status.log file (see bellow)\n \tinit: perform a check an compute the sha256sum.txt.prev file and a status.log file (see bellow)\n
\tcheck: run a basic check - get http answer and compare index checksums to the first run\n \tcheck: run a basic check - get http answer and compare index checksums to the first run\n
\tclean: remove the workdir\n \tclean: remove the workdir\n
\t--help: print this help and exit.\n \t--help: print this help and exit.\n
...@@ -205,10 +205,7 @@ OUTPUT_STATUS="${OUTPUT_DIR}/${DATE}_status.json" ...@@ -205,10 +205,7 @@ OUTPUT_STATUS="${OUTPUT_DIR}/${DATE}_status.json"
if [[ "$1" == "compare" ]]; then if [[ "$1" == "compare" ]]; then
cd ${WORKDIR} cd ${WORKDIR}
for file in `ls *.html`; do ext2compare="prev"
mv $file $file.orig
done
ext2compare="orig"
COMPARE=true COMPARE=true
elif [[ "$1" == "compare2init" ]]; then elif [[ "$1" == "compare2init" ]]; then
cd ${WORKDIR} cd ${WORKDIR}
...@@ -216,11 +213,13 @@ elif [[ "$1" == "compare2init" ]]; then ...@@ -216,11 +213,13 @@ elif [[ "$1" == "compare2init" ]]; then
ext2compare="init" ext2compare="init"
elif [[ "$1" == "init" ]]; then elif [[ "$1" == "init" ]]; then
rm -rf ${WORKDIR} rm -rf ${WORKDIR}
CHECK=true
FIRST=true FIRST=true
elif [[ "$1" == "clean" ]];then elif [[ "$1" == "clean" ]];then
rm -rf ${WORKDIR} rm -rf ${WORKDIR}
exit 0 exit 0
else else
CHECK=true
if [[ "$1" != "check" ]]; then if [[ "$1" != "check" ]]; then
echo -e $usage echo -e $usage
exit 0 exit 0
...@@ -231,23 +230,22 @@ fi ...@@ -231,23 +230,22 @@ fi
if [ ! -d ${WORKDIR} ]; then if [ ! -d ${WORKDIR} ]; then
mkdir ${WORKDIR} mkdir ${WORKDIR}
fi fi
cd ${WORKDIR} cd ${WORKDIR}
rm status.log *.html sha256sum.txt 2>/dev/null
while read name website type product plugins activity owner if [ ${CHECK} ]; then
do rm status.log *.html sha256sum.txt 2>/dev/null
while read name website type product plugins activity owner
do
if [[ "$name" =~ [[:space:]]*# ]] || [[ "$name" == "" ]] || [[ "$activity" == "inactive" ]]; then if [[ "$name" =~ [[:space:]]*# ]] || [[ "$name" == "" ]] || [[ "$activity" == "inactive" ]]; then
continue continue
else else
echo -n $website" " >> status.log && $CURL $CURL_OPTS $website|head -1 >> status.log echo -n $website" " >> status.log && $CURL $CURL_OPTS $website|head -1 >> status.log
$WGET -O "${name}.index.html" $website 2>/dev/null $WGET -O "${name}.index.html" $website 2>/dev/null
fi fi
done < ${URL_LIST} done < ${URL_LIST}
/usr/bin/dos2unix status.log 2>/dev/null
/usr/bin/dos2unix status.log 2>/dev/null remove_dyn_elems ${URL_LIST}
remove_dyn_elems ${URL_LIST} fi
if [ -z ${FIRST} ]; then if [ -z ${FIRST} ]; then
$SHA256SUM *.html > sha256sum.txt $SHA256SUM *.html > sha256sum.txt
...@@ -255,30 +253,30 @@ if [ -z ${FIRST} ]; then ...@@ -255,30 +253,30 @@ if [ -z ${FIRST} ]; then
# to check from the file # to check from the file
# sha256sum -c sha256sum.txt # sha256sum -c sha256sum.txt
# or the previous one # or the previous one
# sha256sum -c sha256sum.txt.orig # sha256sum -c sha256sum.txt.prev
if [ ${COMPARE} ]; then if [ ${COMPARE} ]; then
${DIFF} sha256sum.txt sha256sum.txt.orig |tee modified_websites.txt ${DIFF} sha256sum.txt sha256sum.txt.prev |tee modified_websites.txt
websites_modified=`awk '/index.html/ {print $3}' modified_websites.txt|uniq` websites_modified=`awk '/index.html/ {print $3}' modified_websites.txt|uniq`
for website_modified in $websites_modified for website_modified in $websites_modified
do do
orig_file=`echo "${website_modified}.${ext2compare}"` prev_file=`echo "${website_modified}.${ext2compare}"`
echo "" echo ""
echo "#################################################################" echo "#################################################################"
echo "## Checking if there is any file to compare with new result... ##" echo "## Checking if there is any file to compare with new result... ##"
echo "#################################################################" echo "#################################################################"
echo "" echo ""
if [ -f $website_modified ] && [ -f $orig_file ]; then if [ -f $website_modified ] && [ -f $prev_file ]; then
echo $DIFF -Ebw workdir/$orig_file workdir/$website_modified echo $DIFF -Ebw $prev_file $website_modified
echo "" echo ""
$DIFF $orig_file $website_modified $DIFF $prev_file $website_modified
fi fi
done done
#rm modified_websites.txt #rm modified_websites.txt
else else
$DIFF -q sha256sum.txt sha256sum.txt.orig 1>/dev/null 2>&1 $DIFF -q sha256sum.txt sha256sum.txt.prev 1>/dev/null 2>&1
if [ $? -eq 1 ]; then if [ $? -eq 1 ]; then
$DIFF sha256sum.txt sha256sum.txt.orig|$AWK '/[az]/ {print $3}' |uniq $DIFF sha256sum.txt sha256sum.txt.prev|$AWK '/[az]/ {print $3}' |uniq
fi fi
fi fi
...@@ -295,23 +293,29 @@ if [ -z ${FIRST} ]; then ...@@ -295,23 +293,29 @@ if [ -z ${FIRST} ]; then
done < ${URL_LIST} |sort >> checksums.json done < ${URL_LIST} |sort >> checksums.json
echo "}" >> checksums.json echo "}" >> checksums.json
else else
$SHA256SUM *.html > sha256sum.txt.orig $SHA256SUM *.html > sha256sum.txt.prev
for file in `ls *.html`; do for file in `ls *.html`; do
cp $file $file.init cp $file $file.init
done done
fi fi
total=`cat status.log|wc -l` if [ ${CHECK} ]; then
rm status.json 2>/dev/null total=`cat status.log|wc -l`
echo "{" > status.json rm status.json 2>/dev/null
$AWK -v total=$total '{ if ($NF=="200") {$NF="OK";} if (NR==total) {print "\t\""$1"\": \""$NF"\"";} else {print "\t\""$1"\": \""$NF"\",";} }' status.log >> status.json echo "{" > status.json
echo "}" >> status.json $AWK -v total=$total '{ if ($NF=="200") {$NF="OK";} if (NR==total) {print "\t\""$1"\": \""$NF"\"";} else {print "\t\""$1"\": \""$NF"\",";} }' status.log >> status.json
echo "}" >> status.json
for file in `ls *.html`; do
cp $file $file.prev
done
# change current directory to $DIR b/c of issues with relative paths in profile.conf # change current directory to $DIR b/c of issues with relative paths in profile.conf
cd ${DIR} cd ${DIR}
cp ${WORKDIR}/status.json ${OUTPUT_STATUS} cp ${WORKDIR}/status.json ${OUTPUT_STATUS}
if [ -f ${WORKDIR}/checksums.json ]; then if [ -f ${WORKDIR}/checksums.json ]; then
cp ${WORKDIR}/checksums.json ${OUTPUT_CHECKSUMS} cp ${WORKDIR}/checksums.json ${OUTPUT_CHECKSUMS}
fi
fi fi
cd ${CURDIR} cd ${CURDIR}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment