Commit 4ee08501 authored by julien veyssier's avatar julien veyssier
Browse files

add -c option to check a given list of packages

parent 3cf9bbc2
......@@ -59,6 +59,10 @@ There are basically two ways to use R package checker :
# Usage
---
Basically you can either check a specific package OR just give a list of packages to run R CMD CHECK on them.
The "-c" Bash script option allows you to directly give a list of packages to check. This option is not compatible with giving a package to test ("-p" option).
## Direct Bash call
This is the simple way to use the R package checker. The script prints current steps in stdout and all R output files in the output directory (-d|--dest DESTINATION_DIRECTORY).
......@@ -66,9 +70,10 @@ This is the simple way to use the R package checker. The script prints current s
Here are the script options :
```
rtest.sh PACKAGE_TO_TEST [-l|--libuser R_USER_LIB_VALUE] [-d|--dest DESTINATION_DIRECTORY] [-m|--mail MAIL_TO] [-u|--urlresult URL] [-r|--repo PACKAGE_TO_TEST_REPO] [-x | --xvfb XVFB-RUN PATH] [-n | --nbcores NBCORES] [-s | --skipinstall] [-h | --html]
rtest.sh [-p|--package NAME_OF_PACKAGE_TO_TEST || -c|--checklist CHECKLIST_FILE] [-l|--libuser R_USER_LIB_VALUE] [-d|--dest DESTINATION_DIRECTORY] [-m|--mail MAIL_TO] [-u|--urlresult URL] [-r|--repo PACKAGE_TO_TEST_REPO] [-x|--xvfb XVFB-RUN PATH] [-n|--nbcores NBCORES] [-s|--skipinstall] [-h|--html]
PACKAGE_TO_TEST : package to test
-p | --package NAME_OF_PACKAGE_TO_TEST : name of package to test (case sensible), all packages depending on this package are going to be checked by their R CMD CHECK
-c | --checklist CHECKLIST_FILE : file listing packages you want to check (each line should look like : PKG_NAME,PKG_VERSION,anything)
-r | --repo PACKAGE_TO_TEST_REPO : repository of package you want to test (DEFAULT : R-CRAN)
-l | --libuser R_LIBS_USER : directory where packages depending on PACKAGE_TO_TEST are going to be installed (DEFAULT : R will choose it)
This directory can be used several times for several tests. Using same directory several times will save time.
......
......@@ -205,9 +205,10 @@ function printUsage(){
echo "`basename $0` is a tool to test that a R package (PACKAGE_TO_TEST) does not break packages that depend on it by running 'R CMD check' for each of them.
usage :
$0 PACKAGE_TO_TEST [-l|--libuser R_USER_LIB_VALUE] [-d|--dest DESTINATION_DIRECTORY] [-m|--mail MAIL_TO] [-u|--urlresult URL] [-r|--repo PACKAGE_TO_TEST_REPO] [-x | --xvfb XVFB-RUN PATH] [-n | --nbcores NBCORES] [-s | --skipinstall] [-h | --html]
$0 [-p|--package NAME_OF_PACKAGE_TO_TEST || -c|--checklist CHECKLIST_FILE] [-l|--libuser R_USER_LIB_VALUE] [-d|--dest DESTINATION_DIRECTORY] [-m|--mail MAIL_TO] [-u|--urlresult URL] [-r|--repo PACKAGE_TO_TEST_REPO] [-x|--xvfb XVFB-RUN PATH] [-n|--nbcores NBCORES] [-s|--skipinstall] [-h|--html]
PACKAGE_TO_TEST : package to test
-p | --package NAME_OF_PACKAGE_TO_TEST : name of package to test (case sensible), all packages depending on this package are going to be checked by their R CMD CHECK
-c | --checklist CHECKLIST_FILE : file listing packages you want to check (each line should look like : PKG_NAME,PKG_VERSION,anything)
-r | --repo PACKAGE_TO_TEST_REPO : repository of package you want to test (DEFAULT : R-CRAN)
-l | --libuser R_LIBS_USER : directory where packages depending on PACKAGE_TO_TEST are going to be installed (DEFAULT : R will choose it)
This directory can be used several times for several tests. Using same directory several times will save time.
......@@ -225,7 +226,7 @@ Don't forget to set JAVA_HOME if some dependencies need java, which is quite pro
"
}
ARGS=$(getopt -o r:d:m:l:x:n:shu: -l "repo:mail:libuser:dest:xvfb:nbcores:skipinstall,html,urlresult:" -n "$0" -- "$@");
ARGS=$(getopt -o c:p:r:d:m:l:x:n:shu: -l "checklist:package:repo:mail:libuser:dest:xvfb:nbcores:skipinstall,html,urlresult:" -n "$0" -- "$@");
#Bad arguments
if [ $? -ne 0 ] || [ $# -eq 0 ];
......@@ -237,6 +238,7 @@ REPOSITORYFLAG=0
MAILTOFLAG=0
DESTINATIONFLAG=0
PACKAGE_TO_TEST=""
CHECKLIST_FILE=""
NBCORES=1
SKIPINSTALLFLAG=0
HTMLFLAG=0
......@@ -326,6 +328,40 @@ while true; do
fi
shift;
;;
-p|--package)
shift;
if [ -n "$1" ]; then
PACKAGE_TO_TEST=$1
MAIL_SUBJECT_BEGIN="Package \"$PACKAGE_TO_TEST\""
else
echo "You have to set a non-empty PACKAGE_TO_TEST
"
printUsage
exit
fi
shift;
;;
-c|--checklist)
shift;
if [ -n "$1" ]; then
CHECKLIST_FILE="$1"
MAIL_SUBJECT_BEGIN="List of packages \"$CHECKLIST_FILE\""
if ! [ -f $CHECKLIST_FILE ]; then
echo "List file $CHECKLIST_FILE not found...
"
exit
fi
else
echo "You have to set a non-empty CHECKLIST_FILE path
"
printUsage
exit
fi
shift;
;;
-x|--xvfb)
shift;
if [ -n "$1" ]; then
......@@ -376,20 +412,31 @@ while true; do
esac
done
# getting non-option arguments : THE MAIN ONE : PACKAGE_TO_TEST !
# only last non-option argument will be taken
shift;
while [ -n "$1" ]; do
PACKAGE_TO_TEST=$1
shift
done
## getting non-option arguments : THE MAIN ONE : PACKAGE_TO_TEST !
## only last non-option argument will be taken
#shift;
#while [ -n "$1" ]; do
# PACKAGE_TO_TEST=$1
# shift
#done
if ! [ -n "$PACKAGE_TO_TEST" ]; then
echo "There must be a package to test
if ! [ -n "$CHECKLIST_FILE" ]; then
echo "
There must be a package to test OR a CHECKLIST_FILE.
"
printUsage
exit
fi
else
if [ -n "$CHECKLIST_FILE" ]; then
echo "
There must choose between testing a specific package (R CMD CHECK on all packages depending on it) OR checking a given list of packages.
In other words : choose between -p and -c option
"
exit
fi
fi
#### DEFAULT VALUES management
......@@ -403,7 +450,11 @@ if [ $RESULTURLFLAG == 0 ]; then
fi
# RECAP OF ALL PARAMS
echo "PACKAGE TO TEST : $PACKAGE_TO_TEST"
if [ -n "$PACKAGE_TO_TEST" ]; then
echo "PACKAGE TO TEST : $PACKAGE_TO_TEST . Last version of $PACKAGE_TO_TEST and all packages depending on $PACKAGE_TO_TEST will be installed and checked"
else
echo "LIST OF PACKAGES TO INSTALL AND CHECK : $CHECKLIST_FILE"
fi
if [ -n "$R_LIBS_USER" ]; then
echo "R_LIBS_USER : $R_LIBS_USER"
else
......@@ -442,7 +493,7 @@ $RESULT_URL
Your test directory is named `basename $DESTINATION`
" | mail -s "Package \"$PACKAGE_TO_TEST\" R check started (`basename $DESTINATION`)" $MAILTO
" | mail -s "$MAIL_SUBJECT_BEGIN R check started (`basename $DESTINATION`)" $MAILTO
fi
######## FILE management
......@@ -457,22 +508,24 @@ if [[ $HTMLFLAG == 1 ]]; then
echo
fi
echo
printLeftPurple "Generation of depends.R ... started at `date '+%H:%M:%S'` ... "
# generate depends.R
echo 'if (!"tools" %in% installed.packages(lib.loc=Sys.getenv("R_LIBS_USER"))) {
install.packages("tools",repos=c("'$DEFAULT_REPOSITORY'"), lib=Sys.getenv("R_LIBS_USER"), dependencies=c("Depends", "Imports", "LinkingTo", "Suggests"), Ncpus=$NBCORES)
}
# if no package to test but a list : do not generate depends.R and do not install
if [ -n "$PACKAGE_TO_TEST" ]; then
echo
printLeftPurple "Generation of depends.R ... started at `date '+%H:%M:%S'` ... "
# generate depends.R
echo 'if (!"tools" %in% installed.packages(lib.loc=Sys.getenv("R_LIBS_USER"))) {
install.packages("tools",repos=c("'$DEFAULT_REPOSITORY'"), lib=Sys.getenv("R_LIBS_USER"), dependencies=c("Depends", "Imports", "LinkingTo", "Suggests"), Ncpus='$NBCORES')
}
old <- getOption("defaultPackages"); r <- getOption("repos")
r["CRAN"] <- "'$DEFAULT_REPOSITORY'"
options(defaultPackages = c(old, "MASS"), repos = r)
require("tools")
old <- getOption("defaultPackages"); r <- getOption("repos")
r["CRAN"] <- "'$DEFAULT_REPOSITORY'"
options(defaultPackages = c(old, "MASS"), repos = r)
require("tools")
reverse_dependencies_with_maintainers <-
function(packages, which = c("Depends", "Imports", "LinkingTo"),
reverse_dependencies_with_maintainers <-
function(packages, which = c("Depends", "Imports", "LinkingTo"),
recursive = FALSE)
{
{
contrib.url(getOption("repos")["CRAN"], "source") # trigger chooseCRANmirror() if required
description <- sprintf("%s/web/packages/packages.rds",
getOption("repos")["CRAN"])
......@@ -491,61 +544,63 @@ function(packages, which = c("Depends", "Imports", "LinkingTo"),
pos <- match(rdepends, db[, "Package"], nomatch = 0L)
db[pos, c("Package", "Version", "Maintainer")]
}
}
Sys.setenv(http_proxy = "")
a <- reverse_dependencies_with_maintainers("'$PACKAGE_TO_TEST'", which = c("Depends", "Imports", "Suggests", "LinkingTo"))
a <- a[order(a[,1]),]
ff <- file("depends.txt", open = "w")
cat( paste(sep=",",a[,1],sub("\n","",a[,2]),sub("\n"," ",a[,3]), collapse = "\n"), "\n", file=ff)
close(ff)
' > $DESTINATION/depends.R
Sys.setenv(http_proxy = "")
a <- reverse_dependencies_with_maintainers("'$PACKAGE_TO_TEST'", which = c("Depends", "Imports", "Suggests", "LinkingTo"))
a <- a[order(a[,1]),]
ff <- file("depends.txt", open = "w")
cat( paste(sep=",",a[,1],sub("\n","",a[,2]),sub("\n"," ",a[,3]), collapse = "\n"), "\n", file=ff)
close(ff)
' > $DESTINATION/depends.R
printRightGreen "`date '+%H:%M:%S'` : OK"
printRightGreen "`date '+%H:%M:%S'` : OK"
################ INSTALL of package to test ####################
################ INSTALL of package to test ####################
printLeftPurple "Installing $PACKAGE_TO_TEST ... started at `date '+%H:%M:%S'` ... "
printLeftPurple "Installing $PACKAGE_TO_TEST ... started at `date '+%H:%M:%S'` ... "
REPO_R_PARAM=""
if [[ $REPOSITORYFLAG == 1 ]]; then
REPO_R_PARAM=""
if [[ $REPOSITORYFLAG == 1 ]]; then
REPO_R_PARAM=", repos=c(\"$REPOSITORY\")"
else
else
REPO_R_PARAM=", repos=c(\"$DEFAULT_REPOSITORY\")"
fi
R_LIBS_USER_PARAM=""
if [ -n "$R_LIBS_USER" ]; then
fi
R_LIBS_USER_PARAM=""
if [ -n "$R_LIBS_USER" ]; then
R_LIBS_USER_PARAM=", lib=Sys.getenv(\"R_LIBS_USER\")"
fi
echo '
install.packages("'$PACKAGE_TO_TEST'"'$REPO_R_PARAM''$R_LIBS_USER_PARAM', dependencies=c("Depends", "Imports", "LinkingTo", "Suggests"), Ncpus=$NBCORES)
fi
echo '
install.packages("'$PACKAGE_TO_TEST'"'$REPO_R_PARAM''$R_LIBS_USER_PARAM', dependencies=c("Depends", "Imports", "LinkingTo", "Suggests"), Ncpus='$NBCORES')
ff <- file("pkg.version", open = "w")
cat( paste(sep=",",packageVersion("'$PACKAGE_TO_TEST'")), file=ff)
close(ff)
' > $DESTINATION/install_pkg.R
R CMD BATCH install_pkg.R
ff <- file("pkg.version", open = "w")
cat( paste(sep=",",packageVersion("'$PACKAGE_TO_TEST'")), file=ff)
close(ff)
' > $DESTINATION/install_pkg.R
R CMD BATCH install_pkg.R
if [[ $HTMLFLAG == 1 ]]; then
if [[ $HTMLFLAG == 1 ]]; then
generate_html_report $DESTINATION
fi
fi
if [[ $? == 0 ]]; then
if [[ $? == 0 ]]; then
touch install_pkg_ok
printRightGreen "`date '+%H:%M:%S'` : OK"
else
else
touch install_pkg_problem
printRightRed "`date '+%H:%M:%S'` : FAILED"
fi
fi
############### GENERATION OF DEPENDENCY LIST ###################
############### GENERATION OF DEPENDENCY LIST ###################
printLeftPurple "Generating dependency list (should take less than a minute)... started at `date '+%H:%M:%S'` ... "
R CMD BATCH --no-timing depends.R
#cat depends.txt | grep -v structSSI | grep -v SigTree | grep -v SeqFeatR | grep -v gWidgetsRGtk2 | grep -v GrammR | grep -v RADami > list_dependencies.txt
cat depends.txt > list_dependencies.txt
printRightGreen "`date '+%H:%M:%S'` : OK"
printLeftPurple "Generating dependency list (should take less than a minute)... started at `date '+%H:%M:%S'` ... "
R CMD BATCH --no-timing depends.R
#cat depends.txt | grep -v structSSI | grep -v SigTree | grep -v SeqFeatR | grep -v gWidgetsRGtk2 | grep -v GrammR | grep -v RADami > list_dependencies.txt
cat depends.txt > list_dependencies.txt
printRightGreen "`date '+%H:%M:%S'` : OK"
# END of big if "there is a package to test"
fi
#################### UPDATE of R packages in R_LIBS_USER #####################
if [[ "$SKIPINSTALLFLAG" == "0" ]]; then
......@@ -556,7 +611,7 @@ if [[ "$SKIPINSTALLFLAG" == "0" ]]; then
update.packages(lib.loc=Sys.getenv("R_LIBS_USER"), repos=c("'$DEFAULT_REPOSITORY'"),ask=FALSE)
if (!"XML" %in% installed.packages(lib.loc=Sys.getenv("R_LIBS_USER"))) {
install.packages("XML",repos=c("'$DEFAULT_REPOSITORY'"), lib=Sys.getenv("R_LIBS_USER"), dependencies=c("Depends", "Imports", "LinkingTo", "Suggests"), Ncpus=$NBCORES)
install.packages("XML",repos=c("'$DEFAULT_REPOSITORY'"), lib=Sys.getenv("R_LIBS_USER"), dependencies=c("Depends", "Imports", "LinkingTo", "Suggests"), Ncpus='$NBCORES')
}
source("http://bioconductor.org/biocLite.R")
biocLite("Biostrings", ask=FALSE, lib.loc=Sys.getenv("R_LIBS_USER"), suppressUpdates=TRUE)
......@@ -589,10 +644,16 @@ if [[ "$SKIPINSTALLFLAG" == "0" ]]; then
fi
################ INSTALL packages if they are not installed in R_LIBS_USER ###################
if [ -n "$PACKAGE_TO_TEST" ]; then
PACKAGES_TO_CHECK_LIST_FILE="list_dependencies.txt"
else
cp $CHECKLIST_FILE ./list_packages_to_check.txt
PACKAGES_TO_CHECK_LIST_FILE=$CHECKLIST_FILE
fi
if [[ "$SKIPINSTALLFLAG" == "0" ]]; then
echo "
Installing `wc -l list_dependencies.txt` packages in R_LIBS_USER (may be long if launched for the first time in this R_LIBS_USER)
Installing `wc -l $PACKAGES_TO_CHECK_LIST_FILE` packages in R_LIBS_USER (may be long if launched for the first time in this R_LIBS_USER)
"
while read line; do
pname=`echo $line | cut -d ',' -f 1`
......@@ -602,7 +663,7 @@ if [[ "$SKIPINSTALLFLAG" == "0" ]]; then
echo 'chooseBioCmirror(ind=3)
setRepositories(ind=c(1, 2, 5, 6))
if (!"'$pname'" %in% installed.packages(lib.loc=Sys.getenv("R_LIBS_USER"))) {
install.packages("'$pname'",repos=c("'$DEFAULT_REPOSITORY'"), lib=Sys.getenv("R_LIBS_USER"), dependencies=c("Depends", "Imports", "LinkingTo", "Suggests"), Ncpus=$NBCORES)
install.packages("'$pname'",repos=c("'$DEFAULT_REPOSITORY'"), lib=Sys.getenv("R_LIBS_USER"), dependencies=c("Depends", "Imports", "LinkingTo", "Suggests"), Ncpus='$NBCORES')
}
' > ${pname}_install.R
......@@ -621,7 +682,7 @@ if (!"'$pname'" %in% installed.packages(lib.loc=Sys.getenv("R_LIBS_USER"))) {
if [[ $HTMLFLAG == 1 ]]; then
generate_html_report $DESTINATION
fi
done < list_dependencies.txt
done < $PACKAGES_TO_CHECK_LIST_FILE
fi
###################### R CMD check #############################
......@@ -653,7 +714,7 @@ if [[ $NBCORES == 1 ]]; then
if [[ $HTMLFLAG == 1 ]]; then
generate_html_report $DESTINATION
fi
done < list_dependencies.txt
done < $PACKAGES_TO_CHECK_LIST_FILE
else
# MULTICORE PROCESSING
......@@ -726,7 +787,7 @@ else
versions[$i]=$pversion
status[$i]="tocheck"
let "i = $i + 1"
done < list_dependencies.txt
done < $PACKAGES_TO_CHECK_LIST_FILE
NB_PKG=$i
echo "Number of packages : $NB_PKG"
echo
......@@ -805,5 +866,5 @@ if [[ $MAILTOFLAG == 1 ]]; then
$RESULT_URL
Your test directory is named `basename $DESTINATION`
" | mail -s "Package \"$PACKAGE_TO_TEST\" R check DONE (`basename $DESTINATION`)" $MAILTO
" | mail -s "$MAIL_SUBJECT_BEGIN R check DONE (`basename $DESTINATION`)" $MAILTO
fi
......@@ -22,9 +22,9 @@ else{
$nbli = exec("ps aux | grep rtest.sh | grep -v grep | grep -v 'vi rtest.sh' | wc -l");
if ($nbli === "0"){
echo shell_exec("nohup bash $rtest_location $package_to_test -r \"$package_to_test_repo\" -l $rlib_location -m $email_addr -u \"".$base_url."view.php?check=check_".$package_to_test."_".$date_now."\" -x $xvfbrun_location -n $nb_threads -d ".$all_results_dir."/check_".$package_to_test."_".$date_now." > ".$all_results_dir.$date_now.".log 2>&1 & echo $!");
echo shell_exec("nohup bash $rtest_location -p $package_to_test -r \"$package_to_test_repo\" -l $rlib_location -m $email_addr -u \"".$base_url."view.php?check=check_".$package_to_test."_".$date_now."\" -x $xvfbrun_location -n $nb_threads -d ".$all_results_dir."/check_".$package_to_test."_".$date_now." > ".$all_results_dir.$date_now.".log 2>&1 & echo $!");
echo "Process started<br/><br/>";
echo "nohup bash $rtest_location $package_to_test -r \"$package_to_test_repo\" -l $rlib_location -m $email_addr -u \"".$base_url."view.php?check=check_".$package_to_test."_".$date_now."\" -x $xvfbrun_location -n $nb_threads -d ".$all_results_dir."check_".$package_to_test."_".$date_now." > ".$all_results_dir.$date_now.".log 2>&1 & echo $! <br/><br/>";
echo "nohup bash $rtest_location -p $package_to_test -r \"$package_to_test_repo\" -l $rlib_location -m $email_addr -u \"".$base_url."view.php?check=check_".$package_to_test."_".$date_now."\" -x $xvfbrun_location -n $nb_threads -d ".$all_results_dir."check_".$package_to_test."_".$date_now." > ".$all_results_dir.$date_now.".log 2>&1 & echo $! <br/><br/>";
echo " <a href='".$base_url."view.php?check=check_".$package_to_test."_$date_now'>VIEW REAL TIME RESULTS</a><br/><br/>";
echo " <a href='".$base_url."start.php?action=stop'>KILL THE PROCESS</a>";
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment