Commit a558879a authored by remy's avatar remy
Browse files

compare and compare2init are not permfrming a scan anymore; just a comparison of checksums / diff

parent fc0a8fb1
......@@ -19,6 +19,18 @@ Then, edit `host_https_list.txt` and `url_list.txt` to fit your needs.
## How to install
### Requirements
Requirements are really basics linux softwares:
- gnu make,
- awk & sed,
- sudo (only needed for a system wide install),
- curl & wget,
- sha256sum to perform checksums
___
After editing the 3 configuration files (`profile.conf` and listings `host_https_list.txt`, `url_list.txt`), just run:
```bash
......
......@@ -4,7 +4,7 @@
###### Ce script verifie l'etat des sites web listés dans url_list.txt
###### en recuperant la page d'index et leur statut par curl
###### les pages d'index permettent de produire des checksums
###### qui sont comparees a une version historique sha256sum.txt.orig
###### qui sont comparees a une version historique sha256sum.txt.prev
######
######
######
......@@ -48,7 +48,7 @@ DIFF=`command -v diff`
usage="$0 [compare|init|check|clean|--help]\n\n
\tcompare: display more informations on differences since previous run\n
\tcompare2init: display more informations on differences since first run\n
\tinit: build the sha256sum.txt.orig file and a status.log file (see bellow)\n
\tinit: perform a check an compute the sha256sum.txt.prev file and a status.log file (see bellow)\n
\tcheck: run a basic check - get http answer and compare index checksums to the first run\n
\tclean: remove the workdir\n
\t--help: print this help and exit.\n
......@@ -205,10 +205,7 @@ OUTPUT_STATUS="${OUTPUT_DIR}/${DATE}_status.json"
if [[ "$1" == "compare" ]]; then
cd ${WORKDIR}
for file in `ls *.html`; do
mv $file $file.orig
done
ext2compare="orig"
ext2compare="prev"
COMPARE=true
elif [[ "$1" == "compare2init" ]]; then
cd ${WORKDIR}
......@@ -216,11 +213,13 @@ elif [[ "$1" == "compare2init" ]]; then
ext2compare="init"
elif [[ "$1" == "init" ]]; then
rm -rf ${WORKDIR}
CHECK=true
FIRST=true
elif [[ "$1" == "clean" ]];then
rm -rf ${WORKDIR}
exit 0
else
CHECK=true
if [[ "$1" != "check" ]]; then
echo -e $usage
exit 0
......@@ -231,23 +230,22 @@ fi
if [ ! -d ${WORKDIR} ]; then
mkdir ${WORKDIR}
fi
cd ${WORKDIR}
rm status.log *.html sha256sum.txt 2>/dev/null
while read name website type product plugins activity owner
do
if [ ${CHECK} ]; then
rm status.log *.html sha256sum.txt 2>/dev/null
while read name website type product plugins activity owner
do
if [[ "$name" =~ [[:space:]]*# ]] || [[ "$name" == "" ]] || [[ "$activity" == "inactive" ]]; then
continue
else
echo -n $website" " >> status.log && $CURL $CURL_OPTS $website|head -1 >> status.log
$WGET -O "${name}.index.html" $website 2>/dev/null
fi
done < ${URL_LIST}
/usr/bin/dos2unix status.log 2>/dev/null
remove_dyn_elems ${URL_LIST}
done < ${URL_LIST}
/usr/bin/dos2unix status.log 2>/dev/null
remove_dyn_elems ${URL_LIST}
fi
if [ -z ${FIRST} ]; then
$SHA256SUM *.html > sha256sum.txt
......@@ -255,30 +253,30 @@ if [ -z ${FIRST} ]; then
# to check from the file
# sha256sum -c sha256sum.txt
# or the previous one
# sha256sum -c sha256sum.txt.orig
# sha256sum -c sha256sum.txt.prev
if [ ${COMPARE} ]; then
${DIFF} sha256sum.txt sha256sum.txt.orig |tee modified_websites.txt
${DIFF} sha256sum.txt sha256sum.txt.prev |tee modified_websites.txt
websites_modified=`awk '/index.html/ {print $3}' modified_websites.txt|uniq`
for website_modified in $websites_modified
do
orig_file=`echo "${website_modified}.${ext2compare}"`
prev_file=`echo "${website_modified}.${ext2compare}"`
echo ""
echo "#################################################################"
echo "## Checking if there is any file to compare with new result... ##"
echo "#################################################################"
echo ""
if [ -f $website_modified ] && [ -f $orig_file ]; then
echo $DIFF -Ebw workdir/$orig_file workdir/$website_modified
if [ -f $website_modified ] && [ -f $prev_file ]; then
echo $DIFF -Ebw $prev_file $website_modified
echo ""
$DIFF $orig_file $website_modified
$DIFF $prev_file $website_modified
fi
done
#rm modified_websites.txt
else
$DIFF -q sha256sum.txt sha256sum.txt.orig 1>/dev/null 2>&1
$DIFF -q sha256sum.txt sha256sum.txt.prev 1>/dev/null 2>&1
if [ $? -eq 1 ]; then
$DIFF sha256sum.txt sha256sum.txt.orig|$AWK '/[az]/ {print $3}' |uniq
$DIFF sha256sum.txt sha256sum.txt.prev|$AWK '/[az]/ {print $3}' |uniq
fi
fi
......@@ -295,23 +293,29 @@ if [ -z ${FIRST} ]; then
done < ${URL_LIST} |sort >> checksums.json
echo "}" >> checksums.json
else
$SHA256SUM *.html > sha256sum.txt.orig
$SHA256SUM *.html > sha256sum.txt.prev
for file in `ls *.html`; do
cp $file $file.init
done
fi
total=`cat status.log|wc -l`
rm status.json 2>/dev/null
echo "{" > status.json
$AWK -v total=$total '{ if ($NF=="200") {$NF="OK";} if (NR==total) {print "\t\""$1"\": \""$NF"\"";} else {print "\t\""$1"\": \""$NF"\",";} }' status.log >> status.json
echo "}" >> status.json
if [ ${CHECK} ]; then
total=`cat status.log|wc -l`
rm status.json 2>/dev/null
echo "{" > status.json
$AWK -v total=$total '{ if ($NF=="200") {$NF="OK";} if (NR==total) {print "\t\""$1"\": \""$NF"\"";} else {print "\t\""$1"\": \""$NF"\",";} }' status.log >> status.json
echo "}" >> status.json
for file in `ls *.html`; do
cp $file $file.prev
done
# change current directory to $DIR b/c of issues with relative paths in profile.conf
cd ${DIR}
cp ${WORKDIR}/status.json ${OUTPUT_STATUS}
if [ -f ${WORKDIR}/checksums.json ]; then
# change current directory to $DIR b/c of issues with relative paths in profile.conf
cd ${DIR}
cp ${WORKDIR}/status.json ${OUTPUT_STATUS}
if [ -f ${WORKDIR}/checksums.json ]; then
cp ${WORKDIR}/checksums.json ${OUTPUT_CHECKSUMS}
fi
fi
cd ${CURDIR}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment