Commit 9cc8e7e7 authored by RomainFeron's avatar RomainFeron
Browse files

Moved processed marker log to processing functions instead of table parser

parent 7e2d349d
......@@ -75,6 +75,9 @@ void processor(MarkersQueue& markers_queue, Parameters& parameters, std::mutex&
std::vector<Marker> batch;
bool keep_going = true;
uint marker_processed_tick = static_cast<uint>(markers_queue.n_markers / 100);
uint64_t n_processed_markers = 0;
while (keep_going) {
// Get a batch of markers from the queue
......@@ -85,10 +88,14 @@ void processor(MarkersQueue& markers_queue, Parameters& parameters, std::mutex&
for (auto& marker: batch) {
for (uint i=0; i<marker.individuals.size(); ++i) {
if (marker.n_individuals == n_individuals) depths[i].push_back(marker.individuals[i]); // Only consider markers present in all individuals
if (marker.individuals[i] > 0) ++n_markers[i]; // Increment total number of markers for this individual
}
if (++n_processed_markers % (10 * marker_processed_tick) == 0) std::cerr << "Processed " << n_processed_markers << " markers (" << n_processed_markers / (marker_processed_tick) << " %)" << std::endl;
}
} else {
......
......@@ -18,15 +18,12 @@ void table_parser(Parameters& parameters, const Popmap& popmap, MarkersQueue& ma
std::getline(input_file, temp);
// Check if first line is a comment line with total number of markers
uint n_markers = 0;
if (temp[0] == '#') {
header = split(temp, " : ");
if (header.size() == 2) n_markers = static_cast<uint>(std::stoi(header[1]));
if (header.size() == 2) markers_queue.n_markers = static_cast<uint>(std::stoi(header[1]));
std::getline(input_file, temp); // Load the second line which contains the header
}
uint marker_processed_tick = static_cast<uint>(n_markers / 100);
// Next line (or first if no comment line) is the header
header = split(temp, "\t");
......@@ -89,14 +86,14 @@ void table_parser(Parameters& parameters, const Popmap& popmap, MarkersQueue& ma
queue_mutex.lock();
for (auto& tmp_marker: tmp_queue) {
markers_queue.markers.push(tmp_marker);
++markers_queue.n_markers;
++markers_queue.loaded_markers;
// Reset marker attributes
tmp_marker.reset(no_seq);
}
queue_mutex.unlock();
tmp_queue_real_size = 0;
}
if (marker_n % (10 * marker_processed_tick) == 0) std::cerr << "Loaded " << marker_n << " markers (" << marker_n / (marker_processed_tick) << " %)" << std::endl;
// Reset variables
temp = "";
field_n = 0;
......
......@@ -44,6 +44,7 @@ struct MarkersQueue {
std::queue<Marker> markers;
uint64_t n_markers = 0;
uint64_t loaded_markers = 0;
};
......
......@@ -55,6 +55,9 @@ void processor(MarkersQueue& markers_queue, Parameters& parameters, std::mutex&
std::vector<Marker> batch;
bool keep_going = true;
uint marker_processed_tick = static_cast<uint>(markers_queue.n_markers / 100);
uint64_t n_processed_markers = 0;
while (keep_going) {
// Get a batch of markers from the queue
......@@ -62,7 +65,12 @@ void processor(MarkersQueue& markers_queue, Parameters& parameters, std::mutex&
if (batch.size() > 0) { // Batch not empty
for (auto marker: batch) ++results[marker.groups[parameters.group1]][marker.groups[parameters.group2]].first;
for (auto marker: batch) {
++results[marker.groups[parameters.group1]][marker.groups[parameters.group2]].first;
if (++n_processed_markers % (10 * marker_processed_tick) == 0) std::cerr << "Processed " << n_processed_markers << " markers (" << n_processed_markers / (marker_processed_tick) << " %)" << std::endl;
}
} else {
......
......@@ -45,6 +45,9 @@ void processor(MarkersQueue& markers_queue, std::mutex& queue_mutex, std::vector
std::vector<Marker> batch;
bool keep_going = true;
uint marker_processed_tick = static_cast<uint>(markers_queue.n_markers / 100);
uint64_t n_processed_markers = 0;
while (keep_going) {
// Get a batch of markers from the queue
......@@ -52,7 +55,11 @@ void processor(MarkersQueue& markers_queue, std::mutex& queue_mutex, std::vector
if (batch.size() > 0) { // Batch not empty
for (auto& marker: batch) ++frequencies[marker.n_individuals];
for (auto& marker: batch) {
++frequencies[marker.n_individuals];
if (++n_processed_markers % (10 * marker_processed_tick) == 0) std::cerr << "Processed " << n_processed_markers << " markers (" << n_processed_markers / (marker_processed_tick) << " %)" << std::endl;
}
} else {
......
......@@ -90,25 +90,20 @@ void processor(MarkersQueue& markers_queue, Parameters& parameters, Popmap& popm
int best_alignment[3] {0, -1, 0}; // Info about best alignment: index, score, count
AlignedMarker seq;
uint x[4] = {0, 0, 0, 0};
uint marker_processed_tick = static_cast<uint>(markers_queue.n_markers / 100);
uint64_t n_processed_markers = 0;
while (keep_going) {
// Get a batch of markers from the queue
batch = get_batch(markers_queue, queue_mutex, batch_size);
++x[2];
if (batch.size() > 0) { // Batch not empty
++x[3];
for (auto& marker: batch) {
++x[0];
if (marker.n_individuals >= min_individuals) {
++x[1];
ar = mem_align1(opt, index->bwt, index->bns, index->pac, marker.sequence.size(), marker.sequence.c_str()); // Align the marker to the reference
for (j = 0; j < ar.n; ++j) { // Loop through alignments to find best scoring one
......@@ -153,6 +148,7 @@ void processor(MarkersQueue& markers_queue, Parameters& parameters, Popmap& popm
best_alignment[1] = -1;
best_alignment[2] = 0;
if (++n_processed_markers % (10 * marker_processed_tick) == 0) std::cerr << "Processed " << n_processed_markers << " markers (" << n_processed_markers / (marker_processed_tick) << " %)" << std::endl;
}
} else {
......
......@@ -55,6 +55,9 @@ void processor(MarkersQueue& markers_queue, Popmap& popmap, Parameters& paramete
double chi_squared = 0;
uint marker_processed_tick = static_cast<uint>(markers_queue.n_markers / 100);
uint64_t n_processed_markers = 0;
while (keep_going) {
// Get a batch of markers from the queue
......@@ -74,6 +77,8 @@ void processor(MarkersQueue& markers_queue, Popmap& popmap, Parameters& paramete
}
if (++n_processed_markers % (10 * marker_processed_tick) == 0) std::cerr << "Processed " << n_processed_markers << " markers (" << n_processed_markers / (marker_processed_tick) << " %)" << std::endl;
}
} else {
......
......@@ -33,6 +33,9 @@ void processor(MarkersQueue& markers_queue, Popmap& popmap, Parameters& paramete
double chi_squared = 0;
uint marker_processed_tick = static_cast<uint>(markers_queue.n_markers / 100);
uint64_t n_processed_markers = 0;
while (keep_going) {
// Get a batch of markers from the queue
......@@ -42,21 +45,19 @@ void processor(MarkersQueue& markers_queue, Popmap& popmap, Parameters& paramete
for (auto& marker: batch) {
if (marker.n_individuals > 0) {
if (marker.groups[parameters.group1] >= parameters.subset_min_group1 and marker.groups[parameters.group1] <= parameters.subset_max_group1 and
marker.groups[parameters.group2] >= parameters.subset_min_group2 and marker.groups[parameters.group2] <= parameters.subset_max_group2 and
marker.n_individuals >= parameters.subset_min_individuals and marker.n_individuals <= parameters.subset_max_individuals) {
if (marker.groups[parameters.group1] >= parameters.subset_min_group1 and marker.groups[parameters.group1] <= parameters.subset_max_group1 and
marker.groups[parameters.group2] >= parameters.subset_min_group2 and marker.groups[parameters.group2] <= parameters.subset_max_group2 and
marker.n_individuals >= parameters.subset_min_individuals and marker.n_individuals <= parameters.subset_max_individuals) {
chi_squared = get_chi_squared(marker.groups[parameters.group1], marker.groups[parameters.group2], popmap.counts[parameters.group1], popmap.counts[parameters.group2]);
marker.p = get_chi_squared_p(chi_squared);
chi_squared = get_chi_squared(marker.groups[parameters.group1], marker.groups[parameters.group2], popmap.counts[parameters.group1], popmap.counts[parameters.group2]);
marker.p = get_chi_squared_p(chi_squared);
parameters.output_fasta ? marker.output_fasta(output_file, parameters.min_depth) : marker.output_table(output_file);
}
parameters.output_fasta ? marker.output_fasta(output_file, parameters.min_depth) : marker.output_table(output_file);
}
if (++n_processed_markers % (10 * marker_processed_tick) == 0) std::cerr << "Processed " << n_processed_markers << " markers (" << n_processed_markers / (marker_processed_tick) << " %)" << std::endl;
}
} else {
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment