Commit 1fd33233 by Celine Mercier

Columns: elements names informations are now kept in a memory arena of

adapted size in the header, and added a boolean in the header indicating
whether the values should be evaluated (typically character strings to
be evaluated in Python)
parent 2df5932b
......@@ -32,6 +32,7 @@ cdef extern from "obidmscolumn.h" nogil:
OBIType_t returned_data_type
OBIType_t stored_data_type
bint tuples
bint to_eval
time_t creation_date
obiversion_t version
obiversion_t cloned_from
......
......@@ -86,7 +86,9 @@ cdef extern from "obiview.h" nogil:
index_t nb_lines,
index_t nb_elements_per_line,
char* elements_names,
bint elt_names_formatted,
bint tuples,
bint to_eval,
const_char_p indexer_name,
const_char_p associated_column_name,
obiversion_t associated_column_version,
......
......@@ -56,6 +56,9 @@ cdef class Column_str(Column_idx):
obi_errno_to_exception(obi_errno, line_nb=line_nb, elt_id=None, error_message="Problem getting a value from a column")
if value == OBIStr_NA :
result = None
# For columns containing character strings that should be evaluated:
elif self.to_eval:
result = eval(value)
else :
result = <bytes> value # NOTE: value is not freed because the pointer points to a mmapped region in an AVL data file.
return result
......
......@@ -330,6 +330,7 @@ cdef class View(OBIWrapper) :
line = self[idx]
for k in item :
# If setting from another View Line and the column doesn't exist, create it based on the informations from the other View
# TODO use clone_column
if isinstance(item, Line) and tostr(k) not in self:
col = item.view[k]
Column.new_column(self,
......@@ -337,6 +338,8 @@ cdef class View(OBIWrapper) :
col.data_type_int,
nb_elements_per_line = col.nb_elements_per_line,
elements_names = col.elements_names,
tuples = col.tuples,
to_eval = col.to_eval,
comments = col.comments,
alias=k
)
......
......@@ -154,35 +154,35 @@ static int create_alignment_output_columns(Obiview_p output_view,
bool normalize, int reference, bool similarity_mode)
{
// Create the column for the ids of the 1st sequence aligned
if (obi_view_add_column(output_view, ID1_COLUMN_NAME, -1, NULL, OBI_STR, 0, 1, NULL, false, id1_indexer_name, NULL, -1, ID1_COLUMN_COMMENTS, true) < 0)
if (obi_view_add_column(output_view, ID1_COLUMN_NAME, -1, NULL, OBI_STR, 0, 1, NULL, false, false, false, id1_indexer_name, NULL, -1, ID1_COLUMN_COMMENTS, true) < 0)
{
obidebug(1, "\nError creating the first column for the sequence ids when aligning");
return -1;
}
// Create the column for the ids of the 2nd sequence aligned
if (obi_view_add_column(output_view, ID2_COLUMN_NAME, -1, NULL, OBI_STR, 0, 1, NULL, false, id2_indexer_name, NULL, -1, ID2_COLUMN_COMMENTS, true) < 0)
if (obi_view_add_column(output_view, ID2_COLUMN_NAME, -1, NULL, OBI_STR, 0, 1, NULL, false, false, false, id2_indexer_name, NULL, -1, ID2_COLUMN_COMMENTS, true) < 0)
{
obidebug(1, "\nError creating the second column for the sequence ids when aligning");
return -1;
}
// Create the column for the index (in the input view) of the first sequences aligned
if (obi_view_add_column(output_view, IDX1_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, IDX1_COLUMN_COMMENTS, true) < 0)
if (obi_view_add_column(output_view, IDX1_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, false, false, NULL, NULL, -1, IDX1_COLUMN_COMMENTS, true) < 0)
{
obidebug(1, "\nError creating the first column for the sequence indices when aligning");
return -1;
}
// Create the column for the index (in the input view) of the second sequences aligned
if (obi_view_add_column(output_view, IDX2_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, IDX2_COLUMN_COMMENTS, true) < 0)
if (obi_view_add_column(output_view, IDX2_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, false, false, NULL, NULL, -1, IDX2_COLUMN_COMMENTS, true) < 0)
{
obidebug(1, "\nError creating the second column for the sequence indices when aligning");
return -1;
}
// Create the column for the LCS length
if (obi_view_add_column(output_view, LCS_LENGTH_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, LCS_LENGTH_COLUMN_COMMENTS, true) < 0)
if (obi_view_add_column(output_view, LCS_LENGTH_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, false, false, NULL, NULL, -1, LCS_LENGTH_COLUMN_COMMENTS, true) < 0)
{
obidebug(1, "\nError creating the column for the LCS length when aligning");
return -1;
......@@ -191,7 +191,7 @@ static int create_alignment_output_columns(Obiview_p output_view,
// Create the column for the alignment length if it is computed
if ((reference == ALILEN) && (normalize || !similarity_mode))
{
if (obi_view_add_column(output_view, ALI_LENGTH_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, ALI_LENGTH_COLUMN_COMMENTS, true) < 0)
if (obi_view_add_column(output_view, ALI_LENGTH_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, false, false, NULL, NULL, -1, ALI_LENGTH_COLUMN_COMMENTS, true) < 0)
{
obidebug(1, "\nError creating the column for the alignment length when aligning");
return -1;
......@@ -200,7 +200,7 @@ static int create_alignment_output_columns(Obiview_p output_view,
// Create the column for the alignment score
if (normalize)
{
if (obi_view_add_column(output_view, SCORE_COLUMN_NAME, -1, NULL, OBI_FLOAT, 0, 1, NULL, false, NULL, NULL, -1, SCORE_COLUMN_NAME, true) < 0)
if (obi_view_add_column(output_view, SCORE_COLUMN_NAME, -1, NULL, OBI_FLOAT, 0, 1, NULL, false, false, false, NULL, NULL, -1, SCORE_COLUMN_NAME, true) < 0)
{
obidebug(1, "\nError creating the column for the score when aligning");
return -1;
......@@ -208,7 +208,7 @@ static int create_alignment_output_columns(Obiview_p output_view,
}
else
{
if (obi_view_add_column(output_view, SCORE_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, SCORE_COLUMN_NAME, true) < 0)
if (obi_view_add_column(output_view, SCORE_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, false, false, NULL, NULL, -1, SCORE_COLUMN_NAME, true) < 0)
{
obidebug(1, "\nError creating the column for the score when aligning");
return -1;
......@@ -218,14 +218,14 @@ static int create_alignment_output_columns(Obiview_p output_view,
if (print_seq)
{
// Create the column for the first sequences aligned
if (obi_view_add_column(output_view, SEQ1_COLUMN_NAME, -1, NULL, OBI_SEQ, 0, 1, NULL, false, seq1_indexer_name, NULL, -1, SEQ1_COLUMN_COMMENTS, true) < 0)
if (obi_view_add_column(output_view, SEQ1_COLUMN_NAME, -1, NULL, OBI_SEQ, 0, 1, NULL, false, false, false, seq1_indexer_name, NULL, -1, SEQ1_COLUMN_COMMENTS, true) < 0)
{
obidebug(1, "\nError creating the first column for the sequences when aligning");
return -1;
}
// Create the column for the second sequences aligned
if (obi_view_add_column(output_view, SEQ2_COLUMN_NAME, -1, NULL, OBI_SEQ, 0, 1, NULL, false, seq2_indexer_name, NULL, -1, SEQ2_COLUMN_COMMENTS, true) < 0)
if (obi_view_add_column(output_view, SEQ2_COLUMN_NAME, -1, NULL, OBI_SEQ, 0, 1, NULL, false, false, false, seq2_indexer_name, NULL, -1, SEQ2_COLUMN_COMMENTS, true) < 0)
{
obidebug(1, "\nError creating the second column for the sequences when aligning");
return -1;
......@@ -234,14 +234,14 @@ static int create_alignment_output_columns(Obiview_p output_view,
// if (print_count) // TODO count columns not implemented yet
// {
// // Create the column for the count of the first sequences aligned
// if (obi_view_add_column(output_view, COUNT1_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, COUNT1_COLUMN_COMMENTS, true) < 0)
// if (obi_view_add_column(output_view, COUNT1_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, false, false, NULL, NULL, -1, COUNT1_COLUMN_COMMENTS, true) < 0)
// {
// obidebug(1, "\nError creating the first column for the sequence counts when aligning");
// return -1;
// }
//
// // Create the column for the count of the second sequences aligned
// if (obi_view_add_column(output_view, COUNT2_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, COUNT2_COLUMN_COMMENTS, true) < 0)
// if (obi_view_add_column(output_view, COUNT2_COLUMN_NAME, -1, NULL, OBI_INT, 0, 1, NULL, false, false, false, NULL, NULL, -1, COUNT2_COLUMN_COMMENTS, true) < 0)
// {
// obidebug(1, "\nError creating the second column for the sequence counts when aligning");
// return -1;
......
......@@ -1036,8 +1036,8 @@ obiversion_t obi_import_column(const char* dms_path_1, const char* dms_path_2, c
// Create new column
column_2 = obi_create_column(dms_2, column_name, header_1->returned_data_type, header_1->line_count,
header_1->nb_elements_per_line, header_1->elements_names, true, header_1->tuples,
new_avl_name, (header_1->associated_column).column_name, (header_1->associated_column).version,
header_1->comments);
header_1->to_eval, new_avl_name, (header_1->associated_column).column_name,
(header_1->associated_column).version, header_1->comments);
if (column_2 == NULL)
{
......@@ -1254,6 +1254,8 @@ int obi_import_view(const char* dms_path_1, const char* dms_path_2, const char*
0,
NULL,
false,
false,
false,
NULL,
NULL,
-1,
......
......@@ -28,9 +28,10 @@
#include "obiblob_indexer.h"
#define ELEMENTS_NAMES_MAX (20480) /**< The maximum length of the list of elements names. // TODO Discuss
// TODO delete useless ones, add default nb?
#define ELEMENTS_NAMES_MAX (1000000) /**< The maximum length of the list of elements names. // TODO Discuss
*/
#define NB_ELTS_MAX_IF_DEFAULT_NAME (20480) /**< The maximum number of elements per line if the default element names
#define NB_ELTS_MAX_IF_DEFAULT_NAME (1000000) /**< The maximum number of elements per line if the default element names
* are used ("0\01\02\0...\0n"), considering ELEMENTS_NAMES_MAX. // TODO not up to date
*/
#define COLUMN_GROWTH_FACTOR (2) /**< The growth factor when a column is enlarged.
......@@ -70,16 +71,6 @@ typedef struct OBIDMS_column_header {
*/
index_t nb_elements_per_line; /**< Number of elements per line.
*/
char elements_names[ELEMENTS_NAMES_MAX+1]; /**< Names of the line elements with '\0' as separator
* and '\0\0' as terminal flag.
* (default are the indices: "0\01\02\0...\0n\0\0").
*/
int elements_names_length; /**< Length of the character array where the elements names are stored.
*/
int elements_names_idx[NB_ELTS_MAX_IF_DEFAULT_NAME]; /**< Index for the start of each element name in elements_names.
*/
int sorted_elements_idx[NB_ELTS_MAX_IF_DEFAULT_NAME]; /**< Index for the sorted element names in elements_names_idx.
*/
OBIType_t returned_data_type; /**< Type of the data that is returned when getting an
* element from the column.
*/
......@@ -88,6 +79,9 @@ typedef struct OBIDMS_column_header {
*/
bool tuples; /**< A boolean indicating whether the column contains indices referring to indexed tuples.
*/
bool to_eval; /**< A boolean indicating whether the column contains expressions that should be evaluated
* (typically OBI_STR columns containing character strings to be evaluated by Python).
*/
time_t creation_date; /**< Date of creation of the file.
*/
obiversion_t version; /**< Version of the column.
......@@ -106,6 +100,18 @@ typedef struct OBIDMS_column_header {
*/
char comments[COMMENTS_MAX_LENGTH+1]; /**< Comments stored as a classical zero end C string.
*/
int64_t elements_names_length; /**< Length of the character array where the elements names are stored.
*/
char* elements_names; /**< Pointer in mem_arnea on the names of the line elements with '\0' as separator
* and '\0\0' as terminal flag.
* (default are the indices: "0\01\02\0...\0n\0\0").
*/
int64_t* elements_names_idx; /**< Pointer in mem_arnea on the index for the start of each element name in elements_names.
*/
int64_t* sorted_elements_idx; /**< Index for the sorted element names in elements_names_idx.
*/
byte_t mem_arena[]; /**< Memory array where the elements names, the elements names index and the sorted elements index are stored.
*/
} OBIDMS_column_header_t, *OBIDMS_column_header_p;
......@@ -212,17 +218,19 @@ obiversion_t obi_column_get_latest_version_from_name(OBIDMS_p dms, const char* c
/**
* @brief Returns the header size in bytes of a column on this platform.
* @brief Returns the header size in bytes of a column.
*
* The header size is defined as a multiple of the memory page size.
* As of now the header size is defined as one time the page size.
* The header size is rounded to a multiple of the memory page size.
*
* @param nb_elements_per_line The number of elements per line.
* @param elts_names_length The length of elements_names including the two terminal '\0's.
*
* @returns The header size in bytes.
*
* @since May 2015
* @author Eric Coissac (eric.coissac@metabarcoding.org)
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
size_t obi_get_platform_header_size();
size_t obi_calculate_header_size(index_t nb_elements_per_line, int64_t elts_names_length);
/**
......@@ -242,6 +250,8 @@ size_t obi_get_platform_header_size();
* NULL or "" if the default names are to be used ("0\01\02\0...\0n").
* @param elt_names_formatted Whether the separator for the elements names is ';' (false), or '\0' (true, as formatted by format_elements_names()).
* @param tuples A boolean indicating whether the column should contain indices referring to indexed tuples.
* @param to_eval A boolean indicating whether the column contains expressions that should be evaluated
* (typically OBI_STR columns containing character strings to be evaluated by Python).
* @param indexer_name The name of the indexer if there is one associated with the column.
* If NULL or "", the indexer name is set as the column name.
* @param associated_column_name The name of the associated column if there is one (otherwise NULL or "").
......@@ -262,6 +272,7 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
char* elements_names,
bool elt_names_formatted,
bool tuples,
bool to_eval,
const char* indexer_name,
const char* associated_column_name,
obiversion_t associated_column_version,
......
......@@ -1646,7 +1646,7 @@ Obiview_p obi_new_view(OBIDMS_p dms, const char* view_name, Obiview_p view_to_cl
// If there is a new line selection, build it by combining it with the one from the view to clone if there is one
else if (line_selection != NULL)
{
view->line_selection = obi_create_column(view->dms, LINES_COLUMN_NAME, OBI_IDX, 0, 1, NULL, false, false, NULL, NULL, -1, NULL);
view->line_selection = obi_create_column(view->dms, LINES_COLUMN_NAME, OBI_IDX, 0, 1, NULL, false, false, false, NULL, NULL, -1, NULL);
if ((view->line_selection) == NULL)
{
obidebug(1, "\nError creating a column corresponding to a line selection");
......@@ -1799,6 +1799,8 @@ Obiview_p obi_new_view(OBIDMS_p dms, const char* view_name, Obiview_p view_to_cl
0,
NULL,
false,
false,
false,
NULL,
NULL,
-1,
......@@ -1866,19 +1868,19 @@ Obiview_p obi_new_view_nuc_seqs(OBIDMS_p dms, const char* view_name, Obiview_p v
if (view_to_clone == NULL)
{
// Adding sequence column
if (obi_view_add_column(view, NUC_SEQUENCE_COLUMN, -1, NULL, OBI_SEQ, 0, 1, NULL, false, NULL, NULL, -1, "Nucleotide sequences", true) < 0) // discuss using same indexer "NUC_SEQ_INDEXER"
if (obi_view_add_column(view, NUC_SEQUENCE_COLUMN, -1, NULL, OBI_SEQ, 0, 1, NULL, false, false, false, NULL, NULL, -1, "Nucleotide sequences", true) < 0) // discuss using same indexer "NUC_SEQ_INDEXER"
{
obidebug(1, "Error adding an obligatory column in a nucleotide sequences view");
return NULL;
}
// Adding id column
if (obi_view_add_column(view, ID_COLUMN, -1, NULL, OBI_STR, 0, 1, NULL, false, NULL, NULL, -1, "Sequence identifiers", true) < 0)
if (obi_view_add_column(view, ID_COLUMN, -1, NULL, OBI_STR, 0, 1, NULL, false, false, false, NULL, NULL, -1, "Sequence identifiers", true) < 0)
{
obidebug(1, "Error adding an obligatory column in a nucleotide sequences view");
return NULL;
}
// Adding definition column
if (obi_view_add_column(view, DEFINITION_COLUMN, -1, NULL, OBI_STR, 0, 1, NULL, false, NULL, NULL, -1, "Definitions", true) < 0)
if (obi_view_add_column(view, DEFINITION_COLUMN, -1, NULL, OBI_STR, 0, 1, NULL, false, false, false, NULL, NULL, -1, "Definitions", true) < 0)
{
obidebug(1, "Error adding an obligatory column in a nucleotide sequences view");
return NULL;
......@@ -1887,7 +1889,7 @@ Obiview_p obi_new_view_nuc_seqs(OBIDMS_p dms, const char* view_name, Obiview_p v
if (quality_column)
{
associated_nuc_column = obi_view_get_column(view, NUC_SEQUENCE_COLUMN);
if (obi_view_add_column(view, QUALITY_COLUMN, -1, NULL, OBI_QUAL, 0, 1, NULL, false, NULL, (associated_nuc_column->header)->name, (associated_nuc_column->header)->version, "Sequence qualities", true) < 0) // TODO discuss automatic association
if (obi_view_add_column(view, QUALITY_COLUMN, -1, NULL, OBI_QUAL, 0, 1, NULL, false, false, false, NULL, (associated_nuc_column->header)->name, (associated_nuc_column->header)->version, "Sequence qualities", true) < 0) // TODO discuss automatic association
{
obidebug(1, "Error adding an obligatory column in a nucleotide sequences view");
return NULL;
......@@ -2215,7 +2217,9 @@ int obi_view_add_column(Obiview_p view,
index_t nb_lines,
index_t nb_elements_per_line,
char* elements_names,
bool elt_names_formatted,
bool tuples,
bool to_eval,
const char* indexer_name,
const char* associated_column_name,
obiversion_t associated_column_version,
......@@ -2294,7 +2298,7 @@ int obi_view_add_column(Obiview_p view,
// Open or create the column
if (create)
{ // Create column
column = obi_create_column(view->dms, column_name, data_type, nb_lines, nb_elements_per_line, elements_names, false, tuples, indexer_name, associated_column_name, associated_column_version, comments);
column = obi_create_column(view->dms, column_name, data_type, nb_lines, nb_elements_per_line, elements_names, elt_names_formatted, tuples, to_eval, indexer_name, associated_column_name, associated_column_version, comments);
if (column == NULL)
{
obidebug(1, "\nError creating a column to add to a view");
......@@ -2801,7 +2805,7 @@ int obi_create_auto_count_column(Obiview_p view)
return -1;
}
if (obi_view_add_column(view, COUNT_COLUMN, -1, NULL, OBI_INT, 0, 1, NULL, false, NULL, NULL, -1, "Sequence counts", true) < 0)
if (obi_view_add_column(view, COUNT_COLUMN, -1, NULL, OBI_INT, 0, 1, NULL, false, false, false, NULL, NULL, -1, "Sequence counts", true) < 0)
{
obidebug(1, "Error adding an automatic count column in a view");
return -1;
......@@ -2853,7 +2857,7 @@ int obi_create_auto_id_column(Obiview_p view, const char* prefix)
}
// Create the new ID column
if (obi_view_add_column(view, ID_COLUMN, -1, NULL, OBI_STR, 0, 1, NULL, false, NULL, NULL, -1, "Sequence identifiers", true) < 0)
if (obi_view_add_column(view, ID_COLUMN, -1, NULL, OBI_STR, 0, 1, NULL, false, false, false, NULL, NULL, -1, "Sequence identifiers", true) < 0)
{
obidebug(1, "Error adding an automatic ID column in a view");
return -1;
......
......@@ -361,7 +361,10 @@ Obiview_p obi_open_view(OBIDMS_p dms, const char* view_name);
* @param nb_elements_per_line The number of elements per line, if the column is created.
* @param elements_names The names of the elements with ';' as separator (no terminal ';'),
* if the column is created; NULL or "" if the default names are to be used ("0\01\02\0...\0n").
* @param elt_names_formatted Whether the separator for the elements names is ';' (false), or '\0' (true, as formatted by format_elements_names()).
* @param tuples A boolean indicating whether the column should contain indices referring to indexed tuples.
* @param to_eval A boolean indicating whether the column contains expressions that should be evaluated
* (typically OBI_STR columns containing character strings to be evaluated by Python).
* @param indexer_name The name of the indexer if there is one associated with the column, if the column is created.
* If NULL or "", the indexer name is set as the column name.
* @param associated_column_name The name of the associated column if there is one (otherwise NULL or ""), if the column is created.
......@@ -384,7 +387,9 @@ int obi_view_add_column(Obiview_p view,
index_t nb_lines,
index_t nb_elements_per_line,
char* elements_names,
bool elt_names_formatted,
bool tuples,
bool to_eval,
const char* indexer_name,
const char* associated_column_name,
obiversion_t associated_column_version,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment