Commit 31f9efb6 authored by jlopez's avatar jlopez
Browse files

Add compute data

parent 3ac188cf
library(dplyr)
library(openxlsx)
library(Rcpp)
source("tools.R")
sourceCpp("exstract.cpp")
data_mandrills <- "./mandrill.csv"
mandrills_uuid <- read.csv(data_mandrills, header = TRUE)
mtime <- 30 # 10 min
time_compute <- mtime * 60
# Temperature ext
#dossier meteo avec les températures
root_ext_path <- "./data/meteo"
meteos <- list.files(root_ext_path, full.names = T)
path_output_ext <- paste0("./data", "/meteos.csv")
CPPparseExternTemperature(meteos, path_output_ext)
exts_temp <- readCSVTemperature(path_output_ext) %>% arrange(date)
exts_temp$ext <- as.numeric(as.character(exts_temp$ext))
exts_temp <- exts_temp %>% na.omit()
write.table(exts_temp, file = path_output_ext, row.names = FALSE, sep = ";", dec = ".")
path_output_ext2 <- paste0("./data", "/meteos_compute.csv")
CPPcomputeExternTemperature(path_output_ext, path_output_ext2, as.character(exts_temp$date[1]),time_compute)
exts_temp <- readCSVTemperature(path_output_ext2)
#ALL week
colliers <- "/home/jimmy/Téléchargements/dossier_colliers"
path_colliers <- list.files(colliers,full.names = T)
# Temperature
for(p in path_colliers) {
name <- basename(p)
if(nchar(name) == 14) {
print("========================================================")
print(name)
root_path <- p
start_date <- substr(name, 1, 10)
start_time <- paste0(start_date, " 00:00:00")
print(start_time)
path_temperature <- paste0(root_path, "/logLoRaTemp.csv")
#you cant comment if you have already precompute raw data
preTraitementTemperature(root_path, path_temperature, mandrills_uuid$uuid, "", "", TRUE)
path_temperature_all <- paste0(root_path, "/temperature_all.csv")
path_temperature_compute <- paste0(root_path, "/temperature_compute.csv")
CPPcomputeTemperature(path_temperature_all, path_temperature_compute, mandrills_uuid$uuid, start_time, time_compute)
}
}
# Mouvement
#si jamais cela bug pour par reprendre depuis le depuis
#lastD <- as.POSIXlt("2019-06-10 00:00:00", "GMT")
lastD <- as.POSIXlt("2000-01-01 00:00:00", "GMT")
for(p in path_colliers) {
name <- basename(p)
if(nchar(name) == 14) {
root_path <- p
start_date <- substr(name, 1, 10)
start_time <- paste0(start_date, " 00:00:00")
lD <- as.POSIXlt(start_time, "GMT")
if(lD >= lastD) {
print("========================================================")
print(name)
print(start_time)
path_mouvement <- paste0(root_path, "/logLoRaMov.csv")
#you cant comment if you have already precompute raw data
preTraitementMouvement(root_path, path_mouvement, mandrills_uuid$uuid, "", "", TRUE)
path_mouvement_all <- paste0(root_path, "/mouvement_all.csv")
path_mouvement_compute <- paste0(root_path, "/mouvement_compute.csv")
CPPcomputeMouvement(path_mouvement_all, path_mouvement_compute, mandrills_uuid$uuid, start_time, time_compute)
}
}
}
# Compute All
for(p in path_colliers) {
name <- basename(p)
if(nchar(name) == 14) {
root_path <- p
start_date <- substr(name, 1, 10)
start_time <- paste0(start_date, " 00:00:00")
print("========================================================")
print(name)
print(start_time)
path_temperature_compute <- paste0(root_path, "/temperature_compute.csv")
path_mouvement_compute <- paste0(root_path, "/mouvement_compute.csv")
mandrill_temperature <- readCSVTemperature(path_temperature_compute)
mandrill_mouvement <- readCSVMouvement(path_mouvement_compute)
mandrill_temperature2 <- merge(mandrill_temperature,exts_temp, by=c("date"))
mandrill_merge <- merge(mandrill_temperature2, mandrill_mouvement, by=c("date", "uuid"))
path_mouvement_all <- paste0(root_path, "/compute_",mtime,".csv")
write.table(mandrill_merge, file = path_mouvement_all, row.names = FALSE, sep = ";", dec = ".", col.names = FALSE)
}
}
#Only Linux
cmd <- "cat "
for(p in path_colliers) {
name <- basename(p)
if(nchar(name) == 14) {
root_path <- p
cmd <- paste0(cmd, p, "/compute_",mtime,".csv ")
}
}
path_all_compute <- paste0("/home/jimmy/Téléchargements/dossier_colliers/mandills_compute_",mtime,".csv")
cmd <- paste0(cmd, " > ", path_all_compute)
system(cmd)
#Add this line in head of compute file
"date";"uuid";"temperature";"ext";"mouvement";"iteration_mouvement"
......@@ -10,6 +10,53 @@ using namespace Rcpp;
std::map<std::string, std::string> months { {"Jan", "01"} , {"Feb", "02"}, {"Mar", "03"}, {"Apr", "04"}, {"May", "05"}, {"Jun", "06"}, {"Jul", "07"}, {"Aug", "08"}, {"Sep", "09"}, {"Oct", "10"}, {"Nov", "11"}, {"Dec", "12"} };
class MandrillT {
public:
int iteration = 0;
double temperature = 0.0;
MandrillT() = default;
~MandrillT() = default;
double getComputeT() {
return temperature / (double)iteration;
}
void addTemperature(double tmp) {
iteration++;
temperature += tmp;
}
void rezet() {
iteration = 0;
temperature = 0.0;
}
};
class MandrillM {
public:
int iteration = 0;
long long mouvement = 0;
MandrillM() = default;
~MandrillM() = default;
void addMouvement(int mouv) {
if(mouv > 0) {
iteration++;
}
mouvement += mouv;
}
void rezet() {
iteration = 0;
mouvement = 0.0;
}
};
//[[Rcpp::plugins(cpp11)]]
......@@ -76,6 +123,15 @@ time_t dateToint(std::string v) {
return t;
}
time_t dateToint2(std::string v) {
struct tm tm;
time_t t;
strptime(v.c_str(), "%d/%m/%Y %H:%M:%S", &tm);
tm.tm_isdst = -1;
t = mktime(&tm);
return t;
}
std::string intTodate(time_t t) {
struct tm * timeinfo;
timeinfo = localtime ( &t );
......@@ -102,11 +158,143 @@ std::string intTodate(time_t t) {
}
// [[Rcpp::export]]
void CPPparseAcceleration(std::string data_path, std::string output_path, StringVector mandrill_uuid, std::string timeS, std::string timeE) {
void CPPparseExternTemperature(StringVector paths, std::string output_path) {
std::vector<std::string> ro;
for(int i=0; i < paths.size(); i++){
//std::cout << paths[i] << '\n';
int index = 0;
std::ifstream input( paths[i] );
for( std::string line; getline( input, line ); )
{
if(index >= 2) {
vector<string> vline = split(line, "\t");
std::string date1 = vline[0];
std::string date2 = vline[1];
std::string date = "20"+intTodate(dateToint2(date1 + " " + date2 + ":00"));
auto tempa = date + ";" + vline[2] + "\n";
ro.push_back(tempa);
}
index++;
}
input.close();
}
std::ofstream output(output_path);
output<<"\"date\";\"ext\""<< '\n';
for(auto li : ro) {
output << li;
}
output.close();
}
// [[Rcpp::export]]
void CPPcomputeExternTemperature(std::string data_path, std::string output_path, std::string date_start, int compute_time) {
std::cout << "Start compute ext temperature" << '\n';
time_t timeDateStart = dateToint(date_start);
std::vector<std::string> ro;
std::ifstream input( data_path );
bool first = true;
bool second = true;
double lastTemperature = 0.0;
for( std::string line; getline( input, line ); )
{
if(first) {
first = false;
} else {
vector<string> vline = split(line, ";");
std::string date = vline[0];
double temperature = std::stod(vline[1]);
time_t intD = dateToint(date);
if(timeDateStart >= intD ) {
//std::cout << intTodate(timeDateStart) << '\n';
std::string dateCompute = intTodate(timeDateStart);
std::string stemp = to_string(temperature);
std::string result = dateCompute + ";" + stemp.substr(0, stemp.find(".") + 2) + "\n";
//output<<result<< '\n';
ro.push_back(result);
timeDateStart += compute_time;
lastTemperature = temperature;
} else {
while(timeDateStart < intD) {
std::string dateCompute = intTodate(timeDateStart);
std::string result;
if(timeDateStart + 60*60*1 < intD) {
result = dateCompute + ";" + "0.0" + "\n";
} else {
std::string stemp = to_string(lastTemperature);
result = dateCompute + ";" + stemp.substr(0, stemp.find(".") + 2) + "\n";
}
//output<<result<< '\n';
ro.push_back(result);
timeDateStart += compute_time;
}
lastTemperature = temperature;
}
}
}
input.close();
std::ofstream output(output_path);
output<<"\"date\";\"uuid\";\"x\";\"y\";\"z\""<< endl;
output<<"\"date\";\"ext\""<< '\n';
for(auto li : ro) {
output << li;
}
std::cout << "End compute ext temperature" << '\n';
output.close();
}
// [[Rcpp::export]]
void CPPparseAcceleration(std::string data_path, std::string output_path, StringVector mandrill_uuid, std::string timeS, std::string timeE) {
std::vector<std::string> ro;
std::ifstream input( data_path );
......@@ -175,6 +363,14 @@ void CPPparseAcceleration(std::string data_path, std::string output_path, String
time_t ts = dateToint(timeS);
time_t te = dateToint(timeE);
struct tm * timeinfo;
timeinfo = localtime ( &t );
if(timeinfo->tm_hour == 0 && timeinfo->tm_min == 0 && timeinfo->tm_sec == 0) {
std::cout << intTodate(t) << '\n';
}
if(t >= ts) {
if(t <= te) {
......@@ -205,11 +401,15 @@ void CPPparseAcceleration(std::string data_path, std::string output_path, String
if(t2 <= te) {
std::string sacceleration = std::to_string(ix) + ";" + std::to_string(iy) + ";" + std::to_string(iz);
std::string newdate = intTodate(t2);
std::string result = newdate + ";\"" + uuid + "\";" + sacceleration;
output<<result<< endl;
std::string result = newdate + ";\"" + uuid + "\";" + sacceleration + "\n";
//output<<result<< '\n';
ro.push_back(result);
} else {
return;
}
}
//}
......@@ -218,32 +418,136 @@ void CPPparseAcceleration(std::string data_path, std::string output_path, String
}
}
} else {
return;
}
}
}
}
}
input.close();
std::ofstream output(output_path);
output<<"\"date\";\"uuid\";\"x\";\"y\";\"z\""<< '\n';
for(auto li : ro) {
output << li;
}
output.close();
}
// [[Rcpp::export]]
void CPPparseMouvement(std::string data_path, std::string output_path, StringVector mandrill_uuid, std::string timeS, std::string timeE) {
void CPPcomputeMouvement(std::string data_path, std::string output_path, StringVector mandrills_uuid, std::string date_start, int compute_time) {
std::cout << "Start compute mouvement" << '\n';
std::map<std::string, MandrillM> mandrills;
time_t timeDateStart = dateToint(date_start);
for(int i=0; i < mandrills_uuid.size(); i++){
std::string key = Rcpp::as< std::string >(mandrills_uuid(i));
mandrills[key] = MandrillM();
}
std::vector<std::string> ro;
std::ifstream input( data_path );
bool first = true;
bool initDate = true;
for( std::string line; getline( input, line ); )
{
if(first) {
first = false;
} else {
vector<string> vline = split(line, ";");
std::string date = vline[0];
std::string uuid = vline[1];
uuid = uuid.substr(1,uuid.size()-2);
double mouvement = std::stod(vline[2]);
time_t intD = dateToint(date);
if(initDate) {
while(intD >= (timeDateStart + compute_time)) {
timeDateStart += compute_time;
}
initDate = false;
mandrills[uuid].addMouvement(mouvement);
}
else {
if(intD > timeDateStart + compute_time) {
timeDateStart += compute_time;
struct tm * timeinfo;
timeinfo = localtime ( &timeDateStart );
if(timeinfo->tm_hour == 0 && timeinfo->tm_min == 0 && timeinfo->tm_sec == 0) {
std::cout << intTodate(timeDateStart) << '\n';
}
std::string dateCompute = intTodate(timeDateStart);
for (auto x : mandrills)
{
std::string result = dateCompute + ";\"" + x.first + "\";"+ std::to_string(x.second.mouvement) + ";" + std::to_string(x.second.iteration) + "\n";
//output<<result;
ro.push_back(result);
//cout << result << '\n';
x.second.rezet();
}
} else {
mandrills[uuid].addMouvement(mouvement);
}
}
}
}
input.close();
std::ofstream output(output_path);
output<<"\"date\";\"uuid\";\"mouvement\""<< endl;
output<<"\"date\";\"uuid\";\"mouvement\";\"iteration_mouvement\""<< '\n';
for(auto li : ro) {
output << li;
}
std::cout << "End compute mouvement" << '\n';
output.close();
}
// [[Rcpp::export]]
void CPPparseMouvement(std::string data_path, std::string output_path, StringVector mandrill_uuid, std::string timeS, std::string timeE) {
std::ifstream input( data_path );
std::vector<std::string> ro;
for( std::string line; getline( input, line ); )
{
//cout << line << "\n";
vector<string> vline = split(line, "_");
int size = vline.size();
bool exact = true;
bool exact = false;
if(size >= 23) {
exact = true;
......@@ -303,6 +607,13 @@ void CPPparseMouvement(std::string data_path, std::string output_path, StringVec
time_t t = dateToint(realDate);
time_t ts = dateToint(timeS);
time_t te = dateToint(timeE);
struct tm * timeinfo;
timeinfo = localtime ( &t );
if(timeinfo->tm_hour == 0 && timeinfo->tm_min == 0 && timeinfo->tm_sec == 0) {
std::cout << intTodate(t) << '\n';
}
if(t >= ts) {
if(t <= te) {
......@@ -319,10 +630,14 @@ void CPPparseMouvement(std::string data_path, std::string output_path, StringVec
if(t2 <= te) {
std::string smouv = std::to_string(mouvement);
std::string newdate = intTodate(t2);
std::string result = newdate + ";\"" + uuid + "\";" + smouv;
output<<result<< endl;
std::string result = newdate + ";\"" + uuid + "\";" + smouv + "\n";
//output<<result<< '\n';
ro.push_back(result);
//cout << result << "\n";
} else {
return;
}
}
......@@ -330,42 +645,161 @@ void CPPparseMouvement(std::string data_path, std::string output_path, StringVec
}
}
} else {
return;
}
}
}
}
}
input.close();
std::ofstream output(output_path);
output<<"\"date\";\"uuid\";\"mouvement\""<< '\n';
for(auto li : ro) {
output << li;
}
output.close();
}
// [[Rcpp::export]]
void CPPparseTemperature(std::string data_path, std::string output_path, StringVector mandrill_uuid, std::string timeS, std::string timeE) {
void CPPcomputeTemperature(std::string data_path, std::string output_path, StringVector mandrills_uuid, std::string date_start, int compute_time) {
std::cout << "Start compute temperature" << '\n';
//std::cout << mandrills_uuid.size() << '\n';
std::map<std::string, MandrillT> mandrills;
time_t timeDateStart = dateToint(date_start);
for(int i=0; i < mandrills_uuid.size(); i++){
std::string key = Rcpp::as< std::string >(mandrills_uuid(i));
mandrills[key] = MandrillT();
}
<