add -c option to check a given list of packages

README.md

@@ -59,6 +59,10 @@ There are basically two ways to use R package checker :
 # Usage
 ---
 
+Basically you can either check a specific package OR just give a list of packages to run R CMD CHECK on them.
+
+The "-c" Bash script option allows you to directly give a list of packages to check. This option is not compatible with giving a package to test ("-p" option).
+
 ## Direct Bash call
 
 This is the simple way to use the R package checker. The script prints current steps in stdout and all R output files in the output directory (-d|--dest DESTINATION_DIRECTORY). 
@@ -66,9 +70,10 @@ This is the simple way to use the R package checker. The script prints current s
 Here are the script options :
 
 ```
-rtest.sh PACKAGE_TO_TEST [-l|--libuser R_USER_LIB_VALUE] [-d|--dest DESTINATION_DIRECTORY] [-m|--mail MAIL_TO] [-u|--urlresult URL] [-r|--repo PACKAGE_TO_TEST_REPO] [-x | --xvfb XVFB-RUN PATH] [-n | --nbcores NBCORES] [-s | --skipinstall] [-h | --html]
+rtest.sh [-p|--package NAME_OF_PACKAGE_TO_TEST || -c|--checklist CHECKLIST_FILE] [-l|--libuser R_USER_LIB_VALUE] [-d|--dest DESTINATION_DIRECTORY] [-m|--mail MAIL_TO] [-u|--urlresult URL] [-r|--repo PACKAGE_TO_TEST_REPO] [-x|--xvfb XVFB-RUN PATH] [-n|--nbcores NBCORES] [-s|--skipinstall] [-h|--html]
 
-PACKAGE_TO_TEST : package to test
+-p | --package NAME_OF_PACKAGE_TO_TEST : name of package to test (case sensible), all packages depending on this package are going to be checked by their R CMD CHECK
+-c | --checklist CHECKLIST_FILE : file listing packages you want to check (each line should look like : PKG_NAME,PKG_VERSION,anything)
 -r | --repo PACKAGE_TO_TEST_REPO : repository of package you want to test (DEFAULT : R-CRAN)
 -l | --libuser R_LIBS_USER : directory where packages depending on PACKAGE_TO_TEST are going to be installed (DEFAULT : R will choose it)
                              This directory can be used several times for several tests. Using same directory several times will save time.

rtest.sh

@@ -205,9 +205,10 @@ function printUsage(){
 echo "`basename $0` is a tool to test that a R package (PACKAGE_TO_TEST) does not break packages that depend on it by running 'R CMD check' for each of them.
 
 usage :
-$0 PACKAGE_TO_TEST [-l|--libuser R_USER_LIB_VALUE] [-d|--dest DESTINATION_DIRECTORY] [-m|--mail MAIL_TO] [-u|--urlresult URL] [-r|--repo PACKAGE_TO_TEST_REPO] [-x | --xvfb XVFB-RUN PATH] [-n | --nbcores NBCORES] [-s | --skipinstall] [-h | --html]
+$0 [-p|--package NAME_OF_PACKAGE_TO_TEST || -c|--checklist CHECKLIST_FILE] [-l|--libuser R_USER_LIB_VALUE] [-d|--dest DESTINATION_DIRECTORY] [-m|--mail MAIL_TO] [-u|--urlresult URL] [-r|--repo PACKAGE_TO_TEST_REPO] [-x|--xvfb XVFB-RUN PATH] [-n|--nbcores NBCORES] [-s|--skipinstall] [-h|--html]
 
-PACKAGE_TO_TEST : package to test
+-p | --package NAME_OF_PACKAGE_TO_TEST : name of package to test (case sensible), all packages depending on this package are going to be checked by their R CMD CHECK
+-c | --checklist CHECKLIST_FILE : file listing packages you want to check (each line should look like : PKG_NAME,PKG_VERSION,anything)
 -r | --repo PACKAGE_TO_TEST_REPO : repository of package you want to test (DEFAULT : R-CRAN)
 -l | --libuser R_LIBS_USER : directory where packages depending on PACKAGE_TO_TEST are going to be installed (DEFAULT : R will choose it)
                              This directory can be used several times for several tests. Using same directory several times will save time.
@@ -225,7 +226,7 @@ Don't forget to set JAVA_HOME if some dependencies need java, which is quite pro
 "
 }
 
-ARGS=$(getopt -o r:d:m:l:x:n:shu: -l "repo:mail:libuser:dest:xvfb:nbcores:skipinstall,html,urlresult:" -n "$0" -- "$@");
+ARGS=$(getopt -o c:p:r:d:m:l:x:n:shu: -l "checklist:package:repo:mail:libuser:dest:xvfb:nbcores:skipinstall,html,urlresult:" -n "$0" -- "$@");
 
 #Bad arguments
 if [ $? -ne 0 ] || [ $# -eq 0 ];
@@ -237,6 +238,7 @@ REPOSITORYFLAG=0
 MAILTOFLAG=0
 DESTINATIONFLAG=0
 PACKAGE_TO_TEST=""
+CHECKLIST_FILE=""
 NBCORES=1
 SKIPINSTALLFLAG=0
 HTMLFLAG=0
@@ -326,6 +328,40 @@ while true; do
             fi
             shift;
             ;;
+        -p|--package)
+            shift;
+            if [ -n "$1" ]; then
+                PACKAGE_TO_TEST=$1
+                MAIL_SUBJECT_BEGIN="Package \"$PACKAGE_TO_TEST\""
+            else
+                echo "You have to set a non-empty PACKAGE_TO_TEST
+
+                "
+                printUsage
+                exit
+            fi
+            shift;
+            ;;
+        -c|--checklist)
+            shift;
+            if [ -n "$1" ]; then
+                CHECKLIST_FILE="$1"
+                MAIL_SUBJECT_BEGIN="List of packages \"$CHECKLIST_FILE\""
+                if ! [ -f $CHECKLIST_FILE ]; then
+                    echo "List file $CHECKLIST_FILE not found...
+
+                    "
+                    exit
+                fi
+            else
+                echo "You have to set a non-empty CHECKLIST_FILE path
+
+                "
+                printUsage
+                exit
+            fi
+            shift;
+            ;;
         -x|--xvfb)
             shift;
             if [ -n "$1" ]; then
@@ -376,20 +412,31 @@ while true; do
     esac
 done
 
-# getting non-option arguments : THE MAIN ONE : PACKAGE_TO_TEST !
-# only last non-option argument will be taken
-shift;
-while [ -n "$1" ]; do
-    PACKAGE_TO_TEST=$1
-    shift
-done
+## getting non-option arguments : THE MAIN ONE : PACKAGE_TO_TEST !
+## only last non-option argument will be taken
+#shift;
+#while [ -n "$1" ]; do
+#    PACKAGE_TO_TEST=$1
+#    shift
+#done
 
 if ! [ -n "$PACKAGE_TO_TEST" ]; then
-    echo "There must be a package to test
+    if ! [ -n "$CHECKLIST_FILE" ]; then
+        echo "
+There must be a package to test OR a CHECKLIST_FILE.
 
         "
-    printUsage
         exit
+    fi
+else
+    if [ -n "$CHECKLIST_FILE" ]; then
+        echo "
+There must choose between testing a specific package (R CMD CHECK on all packages depending on it) OR checking a given list of packages.
+
+In other words : choose between -p and -c option
+"
+        exit
+    fi
 fi
 
 #### DEFAULT VALUES management
@@ -403,7 +450,11 @@ if [ $RESULTURLFLAG == 0 ]; then
 fi
 
 # RECAP OF ALL PARAMS
-echo "PACKAGE TO TEST : $PACKAGE_TO_TEST"
+if [ -n "$PACKAGE_TO_TEST" ]; then
+    echo "PACKAGE TO TEST : $PACKAGE_TO_TEST . Last version of $PACKAGE_TO_TEST and all packages depending on $PACKAGE_TO_TEST will be installed and checked"
+else
+    echo "LIST OF PACKAGES TO INSTALL AND CHECK : $CHECKLIST_FILE"
+fi
 if [ -n "$R_LIBS_USER" ]; then
     echo "R_LIBS_USER : $R_LIBS_USER"
 else
@@ -442,7 +493,7 @@ $RESULT_URL
 
 Your test directory is named `basename $DESTINATION`
 
-" | mail -s "Package \"$PACKAGE_TO_TEST\" R check started (`basename $DESTINATION`)" $MAILTO
+" | mail -s "$MAIL_SUBJECT_BEGIN R check started (`basename $DESTINATION`)" $MAILTO
 fi
 
 ######## FILE management
@@ -457,22 +508,24 @@ if [[ $HTMLFLAG == 1 ]]; then
     echo
 fi
 
-echo
-printLeftPurple "Generation of depends.R ... started at `date '+%H:%M:%S'` ... "
-# generate depends.R
-echo 'if (!"tools" %in% installed.packages(lib.loc=Sys.getenv("R_LIBS_USER"))) {
-    install.packages("tools",repos=c("'$DEFAULT_REPOSITORY'"), lib=Sys.getenv("R_LIBS_USER"), dependencies=c("Depends", "Imports", "LinkingTo", "Suggests"), Ncpus=$NBCORES)
-}
+# if no package to test but a list : do not generate depends.R and do not install
+if [ -n "$PACKAGE_TO_TEST" ]; then
+    echo
+    printLeftPurple "Generation of depends.R ... started at `date '+%H:%M:%S'` ... "
+    # generate depends.R
+    echo 'if (!"tools" %in% installed.packages(lib.loc=Sys.getenv("R_LIBS_USER"))) {
+        install.packages("tools",repos=c("'$DEFAULT_REPOSITORY'"), lib=Sys.getenv("R_LIBS_USER"), dependencies=c("Depends", "Imports", "LinkingTo", "Suggests"), Ncpus='$NBCORES')
+    }
 
-old <- getOption("defaultPackages"); r <- getOption("repos")
-r["CRAN"] <- "'$DEFAULT_REPOSITORY'"
-options(defaultPackages = c(old, "MASS"), repos = r)
-require("tools")
+    old <- getOption("defaultPackages"); r <- getOption("repos")
+    r["CRAN"] <- "'$DEFAULT_REPOSITORY'"
+    options(defaultPackages = c(old, "MASS"), repos = r)
+    require("tools")
 
-reverse_dependencies_with_maintainers <-
-function(packages, which = c("Depends", "Imports", "LinkingTo"),
+    reverse_dependencies_with_maintainers <-
+    function(packages, which = c("Depends", "Imports", "LinkingTo"),
              recursive = FALSE)
-{
+    {
         contrib.url(getOption("repos")["CRAN"], "source") # trigger chooseCRANmirror() if required
         description <- sprintf("%s/web/packages/packages.rds",
                                getOption("repos")["CRAN"])
@@ -491,61 +544,63 @@ function(packages, which = c("Depends", "Imports", "LinkingTo"),
         pos <- match(rdepends, db[, "Package"], nomatch = 0L)
 
         db[pos, c("Package", "Version", "Maintainer")]
-}
+    }
 
-Sys.setenv(http_proxy = "")
-a <- reverse_dependencies_with_maintainers("'$PACKAGE_TO_TEST'", which = c("Depends", "Imports", "Suggests", "LinkingTo"))
-a <- a[order(a[,1]),]
-ff <- file("depends.txt", open = "w")
-cat( paste(sep=",",a[,1],sub("\n","",a[,2]),sub("\n"," ",a[,3]), collapse = "\n"), "\n", file=ff)
-close(ff)
-' > $DESTINATION/depends.R
+    Sys.setenv(http_proxy = "")
+    a <- reverse_dependencies_with_maintainers("'$PACKAGE_TO_TEST'", which = c("Depends", "Imports", "Suggests", "LinkingTo"))
+    a <- a[order(a[,1]),]
+    ff <- file("depends.txt", open = "w")
+    cat( paste(sep=",",a[,1],sub("\n","",a[,2]),sub("\n"," ",a[,3]), collapse = "\n"), "\n", file=ff)
+    close(ff)
+    ' > $DESTINATION/depends.R
 
-printRightGreen "`date '+%H:%M:%S'` : OK"
+    printRightGreen "`date '+%H:%M:%S'` : OK"
 
-################ INSTALL of package to test ####################
+    ################ INSTALL of package to test ####################
 
-printLeftPurple "Installing $PACKAGE_TO_TEST ... started at `date '+%H:%M:%S'` ... "
+    printLeftPurple "Installing $PACKAGE_TO_TEST ... started at `date '+%H:%M:%S'` ... "
 
-REPO_R_PARAM=""
-if [[ $REPOSITORYFLAG == 1 ]]; then
+    REPO_R_PARAM=""
+    if [[ $REPOSITORYFLAG == 1 ]]; then
         REPO_R_PARAM=", repos=c(\"$REPOSITORY\")"
-else
+    else
         REPO_R_PARAM=", repos=c(\"$DEFAULT_REPOSITORY\")"
-fi
-R_LIBS_USER_PARAM=""
-if [ -n "$R_LIBS_USER" ]; then
+    fi
+    R_LIBS_USER_PARAM=""
+    if [ -n "$R_LIBS_USER" ]; then
         R_LIBS_USER_PARAM=", lib=Sys.getenv(\"R_LIBS_USER\")"
-fi
-echo '
-install.packages("'$PACKAGE_TO_TEST'"'$REPO_R_PARAM''$R_LIBS_USER_PARAM', dependencies=c("Depends", "Imports", "LinkingTo", "Suggests"), Ncpus=$NBCORES)
+    fi
+    echo '
+    install.packages("'$PACKAGE_TO_TEST'"'$REPO_R_PARAM''$R_LIBS_USER_PARAM', dependencies=c("Depends", "Imports", "LinkingTo", "Suggests"), Ncpus='$NBCORES')
 
-ff <- file("pkg.version", open = "w")
-cat( paste(sep=",",packageVersion("'$PACKAGE_TO_TEST'")), file=ff)
-close(ff)
-' > $DESTINATION/install_pkg.R
-R CMD BATCH install_pkg.R
+    ff <- file("pkg.version", open = "w")
+    cat( paste(sep=",",packageVersion("'$PACKAGE_TO_TEST'")), file=ff)
+    close(ff)
+    ' > $DESTINATION/install_pkg.R
+    R CMD BATCH install_pkg.R
 
-if [[ $HTMLFLAG == 1 ]]; then
+    if [[ $HTMLFLAG == 1 ]]; then
         generate_html_report $DESTINATION
-fi
+    fi
 
-if [[ $? == 0 ]]; then
+    if [[ $? == 0 ]]; then
         touch install_pkg_ok
         printRightGreen "`date '+%H:%M:%S'` : OK"
-else
+    else
         touch install_pkg_problem
         printRightRed "`date '+%H:%M:%S'` : FAILED"
-fi
+    fi
 
-############### GENERATION OF DEPENDENCY LIST ###################
+    ############### GENERATION OF DEPENDENCY LIST ###################
 
-printLeftPurple "Generating dependency list (should take less than a minute)... started at `date '+%H:%M:%S'` ... "
-R CMD BATCH --no-timing depends.R
-#cat depends.txt | grep -v structSSI | grep -v SigTree | grep -v SeqFeatR | grep -v gWidgetsRGtk2 | grep -v GrammR | grep -v RADami > list_dependencies.txt
-cat depends.txt > list_dependencies.txt
-printRightGreen "`date '+%H:%M:%S'` : OK"
+    printLeftPurple "Generating dependency list (should take less than a minute)... started at `date '+%H:%M:%S'` ... "
+    R CMD BATCH --no-timing depends.R
+    #cat depends.txt | grep -v structSSI | grep -v SigTree | grep -v SeqFeatR | grep -v gWidgetsRGtk2 | grep -v GrammR | grep -v RADami > list_dependencies.txt
+    cat depends.txt > list_dependencies.txt
+    printRightGreen "`date '+%H:%M:%S'` : OK"
 
+# END of big if "there is a package to test"
+fi
 #################### UPDATE of R packages in R_LIBS_USER #####################
 
 if [[ "$SKIPINSTALLFLAG" == "0" ]]; then
@@ -556,7 +611,7 @@ if [[ "$SKIPINSTALLFLAG" == "0" ]]; then
     update.packages(lib.loc=Sys.getenv("R_LIBS_USER"), repos=c("'$DEFAULT_REPOSITORY'"),ask=FALSE)
 
     if (!"XML" %in% installed.packages(lib.loc=Sys.getenv("R_LIBS_USER"))) {
-        install.packages("XML",repos=c("'$DEFAULT_REPOSITORY'"), lib=Sys.getenv("R_LIBS_USER"), dependencies=c("Depends", "Imports", "LinkingTo", "Suggests"), Ncpus=$NBCORES)
+        install.packages("XML",repos=c("'$DEFAULT_REPOSITORY'"), lib=Sys.getenv("R_LIBS_USER"), dependencies=c("Depends", "Imports", "LinkingTo", "Suggests"), Ncpus='$NBCORES')
     }
     source("http://bioconductor.org/biocLite.R")
     biocLite("Biostrings", ask=FALSE, lib.loc=Sys.getenv("R_LIBS_USER"), suppressUpdates=TRUE)
@@ -589,10 +644,16 @@ if [[ "$SKIPINSTALLFLAG" == "0" ]]; then
 fi
 
 ################ INSTALL packages if they are not installed in R_LIBS_USER ###################
+if [ -n "$PACKAGE_TO_TEST" ]; then
+    PACKAGES_TO_CHECK_LIST_FILE="list_dependencies.txt"
+else
+    cp $CHECKLIST_FILE ./list_packages_to_check.txt
+    PACKAGES_TO_CHECK_LIST_FILE=$CHECKLIST_FILE
+fi
 
 if [[ "$SKIPINSTALLFLAG" == "0" ]]; then
     echo "
-    Installing `wc -l list_dependencies.txt` packages in R_LIBS_USER (may be long if launched for the first time in this R_LIBS_USER)
+    Installing `wc -l $PACKAGES_TO_CHECK_LIST_FILE` packages in R_LIBS_USER (may be long if launched for the first time in this R_LIBS_USER)
     "
     while read line; do
         pname=`echo $line | cut -d ',' -f 1`
@@ -602,7 +663,7 @@ if [[ "$SKIPINSTALLFLAG" == "0" ]]; then
         echo 'chooseBioCmirror(ind=3)
 setRepositories(ind=c(1, 2, 5, 6))
 if (!"'$pname'" %in% installed.packages(lib.loc=Sys.getenv("R_LIBS_USER"))) {
-    install.packages("'$pname'",repos=c("'$DEFAULT_REPOSITORY'"), lib=Sys.getenv("R_LIBS_USER"), dependencies=c("Depends", "Imports", "LinkingTo", "Suggests"), Ncpus=$NBCORES)
+    install.packages("'$pname'",repos=c("'$DEFAULT_REPOSITORY'"), lib=Sys.getenv("R_LIBS_USER"), dependencies=c("Depends", "Imports", "LinkingTo", "Suggests"), Ncpus='$NBCORES')
 }
 ' > ${pname}_install.R
 
@@ -621,7 +682,7 @@ if (!"'$pname'" %in% installed.packages(lib.loc=Sys.getenv("R_LIBS_USER"))) {
         if [[ $HTMLFLAG == 1 ]]; then
             generate_html_report $DESTINATION
         fi
-    done < list_dependencies.txt
+    done < $PACKAGES_TO_CHECK_LIST_FILE
 fi
 
 ###################### R CMD check #############################
@@ -653,7 +714,7 @@ if [[ $NBCORES == 1 ]]; then
         if [[ $HTMLFLAG == 1 ]]; then
             generate_html_report $DESTINATION
         fi
-    done < list_dependencies.txt
+    done < $PACKAGES_TO_CHECK_LIST_FILE
 else
     # MULTICORE PROCESSING
 
@@ -726,7 +787,7 @@ else
         versions[$i]=$pversion
         status[$i]="tocheck"
         let "i = $i + 1"
-    done < list_dependencies.txt
+    done < $PACKAGES_TO_CHECK_LIST_FILE
     NB_PKG=$i
     echo "Number of packages : $NB_PKG"
     echo
@@ -805,5 +866,5 @@ if [[ $MAILTOFLAG == 1 ]]; then
 $RESULT_URL
 
 Your test directory is named `basename $DESTINATION`
-" | mail -s "Package \"$PACKAGE_TO_TEST\" R check DONE (`basename $DESTINATION`)" $MAILTO
+" | mail -s "$MAIL_SUBJECT_BEGIN R check DONE (`basename $DESTINATION`)" $MAILTO
 fi

www/start.php

@@ -22,9 +22,9 @@ else{
 
     $nbli = exec("ps aux | grep rtest.sh | grep -v grep | grep -v 'vi rtest.sh' | wc -l");
     if ($nbli === "0"){
-        echo shell_exec("nohup bash $rtest_location $package_to_test -r \"$package_to_test_repo\" -l $rlib_location  -m $email_addr -u \"".$base_url."view.php?check=check_".$package_to_test."_".$date_now."\" -x $xvfbrun_location  -n $nb_threads -d ".$all_results_dir."/check_".$package_to_test."_".$date_now." > ".$all_results_dir.$date_now.".log 2>&1 & echo $!");
+        echo shell_exec("nohup bash $rtest_location -p $package_to_test -r \"$package_to_test_repo\" -l $rlib_location  -m $email_addr -u \"".$base_url."view.php?check=check_".$package_to_test."_".$date_now."\" -x $xvfbrun_location  -n $nb_threads -d ".$all_results_dir."/check_".$package_to_test."_".$date_now." > ".$all_results_dir.$date_now.".log 2>&1 & echo $!");
         echo "Process started<br/><br/>";
-    echo "nohup bash $rtest_location $package_to_test -r \"$package_to_test_repo\" -l $rlib_location  -m $email_addr -u \"".$base_url."view.php?check=check_".$package_to_test."_".$date_now."\" -x $xvfbrun_location  -n $nb_threads -d ".$all_results_dir."check_".$package_to_test."_".$date_now." > ".$all_results_dir.$date_now.".log 2>&1 & echo $! <br/><br/>";
+    echo "nohup bash $rtest_location -p $package_to_test -r \"$package_to_test_repo\" -l $rlib_location  -m $email_addr -u \"".$base_url."view.php?check=check_".$package_to_test."_".$date_now."\" -x $xvfbrun_location  -n $nb_threads -d ".$all_results_dir."check_".$package_to_test."_".$date_now." > ".$all_results_dir.$date_now.".log 2>&1 & echo $! <br/><br/>";
         echo " <a href='".$base_url."view.php?check=check_".$package_to_test."_$date_now'>VIEW REAL TIME RESULTS</a><br/><br/>";
         echo " <a href='".$base_url."start.php?action=stop'>KILL THE PROCESS</a>";
     }