compare and compare2init are not permfrming a scan anymore; just a comparison of checksums / diff

README.md

@@ -19,6 +19,18 @@ Then, edit `host_https_list.txt` and `url_list.txt` to fit your needs.
 
 ## How to install
 
+### Requirements
+
+Requirements are really basics linux softwares:
+
+  - gnu make,
+  - awk & sed,
+  - sudo (only needed for a system wide install),
+  - curl & wget,
+  - sha256sum to perform checksums
+
+___
+
 After editing the 3 configuration files (`profile.conf` and listings `host_https_list.txt`, `url_list.txt`), just run:
 
 ```bash

check_urls.sh

@@ -4,7 +4,7 @@
 ###### Ce script verifie l'etat des sites web listés dans url_list.txt
 ###### en recuperant la page d'index et leur statut par curl
 ###### les pages d'index permettent de produire des checksums
-###### qui sont comparees a une version historique sha256sum.txt.orig
+###### qui sont comparees a une version historique sha256sum.txt.prev
 ######
 ######
 ######
@@ -48,7 +48,7 @@ DIFF=`command -v diff`
 usage="$0 [compare|init|check|clean|--help]\n\n
     \tcompare: display more informations on differences since previous run\n
     \tcompare2init: display more informations on differences since first run\n
-    \tinit: build the sha256sum.txt.orig file and a status.log file (see bellow)\n
+    \tinit: perform a check an compute the sha256sum.txt.prev file and a status.log file (see bellow)\n
     \tcheck: run a basic check - get http answer and compare index checksums to the first run\n
     \tclean: remove the workdir\n
     \t--help: print this help and exit.\n
@@ -205,10 +205,7 @@ OUTPUT_STATUS="${OUTPUT_DIR}/${DATE}_status.json"
 
 if [[ "$1" == "compare" ]]; then
     cd ${WORKDIR}
-    for file in `ls *.html`; do 
-        mv $file $file.orig
-    done
-    ext2compare="orig"
+    ext2compare="prev"
     COMPARE=true
 elif [[ "$1" == "compare2init" ]]; then
     cd ${WORKDIR}
@@ -216,11 +213,13 @@ elif [[ "$1" == "compare2init" ]]; then
     ext2compare="init"
 elif [[ "$1" == "init" ]]; then
     rm -rf ${WORKDIR}
+    CHECK=true
     FIRST=true
 elif [[ "$1" == "clean" ]];then
     rm -rf ${WORKDIR}
     exit 0
 else
+    CHECK=true
     if [[ "$1" != "check" ]]; then
         echo -e $usage
         exit 0
@@ -231,23 +230,22 @@ fi
 if [ ! -d ${WORKDIR} ]; then
     mkdir ${WORKDIR}
 fi
-
 cd ${WORKDIR}
-rm status.log *.html sha256sum.txt 2>/dev/null
 
-while read name website type product plugins activity owner
-do
+if [ ${CHECK} ]; then
+  rm status.log *.html sha256sum.txt 2>/dev/null
+  while read name website type product plugins activity owner
+  do
     if [[ "$name" =~ [[:space:]]*# ]] || [[ "$name" == "" ]] || [[ "$activity" == "inactive" ]]; then
       continue
     else
       echo -n $website" " >> status.log && $CURL $CURL_OPTS $website|head -1 >> status.log
       $WGET -O "${name}.index.html" $website 2>/dev/null
     fi
-done < ${URL_LIST}
-
-/usr/bin/dos2unix status.log 2>/dev/null
-remove_dyn_elems ${URL_LIST}
-
+  done < ${URL_LIST}
+  /usr/bin/dos2unix status.log 2>/dev/null
+  remove_dyn_elems ${URL_LIST}
+fi
 
 if [ -z ${FIRST} ]; then
     $SHA256SUM *.html > sha256sum.txt
@@ -255,30 +253,30 @@ if [ -z ${FIRST} ]; then
     # to check from the file
     # sha256sum -c sha256sum.txt
     # or the previous one
-    # sha256sum -c sha256sum.txt.orig
+    # sha256sum -c sha256sum.txt.prev
     
     if [ ${COMPARE} ]; then
-        ${DIFF} sha256sum.txt sha256sum.txt.orig |tee modified_websites.txt
+        ${DIFF} sha256sum.txt sha256sum.txt.prev |tee modified_websites.txt
         websites_modified=`awk '/index.html/ {print $3}' modified_websites.txt|uniq`
         for website_modified in $websites_modified
         do
-            orig_file=`echo "${website_modified}.${ext2compare}"`
+            prev_file=`echo "${website_modified}.${ext2compare}"`
             echo ""
             echo "#################################################################"
             echo "## Checking if there is any file to compare with new result... ##"
             echo "#################################################################"            
             echo ""
-            if [ -f $website_modified ] && [ -f $orig_file ]; then
-                echo $DIFF -Ebw workdir/$orig_file workdir/$website_modified
+            if [ -f $website_modified ] && [ -f $prev_file ]; then
+                echo $DIFF -Ebw $prev_file $website_modified
                 echo ""
-                $DIFF $orig_file $website_modified
+                $DIFF $prev_file $website_modified
             fi
         done
         #rm modified_websites.txt
     else
-        $DIFF -q sha256sum.txt sha256sum.txt.orig 1>/dev/null 2>&1
+        $DIFF -q sha256sum.txt sha256sum.txt.prev 1>/dev/null 2>&1
         if [ $? -eq 1 ]; then
-            $DIFF sha256sum.txt sha256sum.txt.orig|$AWK '/[az]/ {print $3}' |uniq
+            $DIFF sha256sum.txt sha256sum.txt.prev|$AWK '/[az]/ {print $3}' |uniq
         fi
     fi
     
@@ -295,23 +293,29 @@ if [ -z ${FIRST} ]; then
     done < ${URL_LIST} |sort >> checksums.json
     echo "}" >> checksums.json
 else
-    $SHA256SUM *.html > sha256sum.txt.orig
+    $SHA256SUM *.html > sha256sum.txt.prev
     for file in `ls *.html`; do 
         cp $file $file.init
     done
 fi
 
-total=`cat status.log|wc -l`
-rm status.json 2>/dev/null
-echo "{" > status.json
-$AWK -v total=$total '{ if ($NF=="200") {$NF="OK";} if (NR==total) {print "\t\""$1"\": \""$NF"\"";} else {print "\t\""$1"\": \""$NF"\",";} }' status.log >> status.json
-echo "}" >> status.json
+if [ ${CHECK} ]; then
+  total=`cat status.log|wc -l`
+  rm status.json 2>/dev/null
+  echo "{" > status.json
+  $AWK -v total=$total '{ if ($NF=="200") {$NF="OK";} if (NR==total) {print "\t\""$1"\": \""$NF"\"";} else {print "\t\""$1"\": \""$NF"\",";} }' status.log >> status.json
+  echo "}" >> status.json
+
+  for file in `ls *.html`; do 
+    cp $file $file.prev
+  done
 
-# change current directory to $DIR b/c of issues with relative paths in profile.conf
-cd ${DIR}
-cp ${WORKDIR}/status.json ${OUTPUT_STATUS}
-if [ -f ${WORKDIR}/checksums.json ]; then
+  # change current directory to $DIR b/c of issues with relative paths in profile.conf
+  cd ${DIR}
+  cp ${WORKDIR}/status.json ${OUTPUT_STATUS}
+  if [ -f ${WORKDIR}/checksums.json ]; then
     cp ${WORKDIR}/checksums.json ${OUTPUT_CHECKSUMS}
+  fi
 fi
 
 cd ${CURDIR}