Commit 1586956d authored by Celine Mercier's avatar Celine Mercier

Added the lists of opened columns and arrays in the OBIDMS structure,

and a counter in the OBIDMS column structure; fixed some bugs and
created tests for referring columns that are bound to disappear anyway.
parent b45b496b
......@@ -79,7 +79,6 @@ cdef class OBIDMS :
cpdef close(self) :
#TODO close all columns (needs to be discussed)
if (obi_close_dms(self.pointer)) < 0 :
raise Exception("Problem closing an OBIDMS")
......
......@@ -12,7 +12,6 @@ LINE_COUNT_FOR_TEST_COLUMN = 10000 # TODO randomize?
SMALLER_LINE_COUNT_FOR_TEST_COLUMN = 1000 # TODO randomize?
NB_ELEMENTS_PER_LINE = 10 # TODO randomize?
DMS_NAME = "unit_test_dms"
DATA_TYPES = ['OBI_INT', 'OBI_FLOAT', 'OBI_BOOL', 'OBI_CHAR', 'OBI_STR', 'OBI_SEQ']
def create_test_obidms():
......@@ -22,26 +21,24 @@ def create_test_obidms():
return (dms, dms_name, dms_dir_name)
def create_test_column(dms, data_type_code, multiple_elements_per_line=False):
data_types = DATA_TYPES
data_type_code = data_type_code
data_type_str = data_types[data_type_code-1]
col_name = "unit_test_"+data_type_str
def create_test_column(dms, data_type, multiple_elements_per_line=False):
col_name = "unit_test_"+data_type
if multiple_elements_per_line :
elts_names = elements_names()
col = dms.open_column(col_name,
create=True,
data_type=data_type_code,
type=data_type,
nb_elements_per_line=NB_ELEMENTS_PER_LINE,
elements_names=elts_names)
return (col, col_name, elts_names, data_type_str)
return (col, col_name, elts_names)
else :
col = dms.open_column(col_name,
create=True,
data_type=data_type_code)
return (col, col_name, data_type_str)
type=data_type)
return (col, col_name)
def elements_names():
......@@ -74,10 +71,10 @@ class OBIDMS_Column_TestCase(unittest.TestCase):
self.dms.close()
shutil.rmtree(self.dms_dir_name, ignore_errors=True)
def test_OBIDMS_column_type(self):
assert self.col.get_data_type() == self.data_type_str, 'Wrong data type associated with column'
assert self.col.get_data_type() == self.data_type, 'Wrong data type associated with column'
def test_OBIDMS_column_cloning(self):
for i in range(LINE_COUNT_FOR_TEST_COLUMN):
self.col[i]= random_obivalue(self.data_type_str)
self.col[i]= random_obivalue(self.data_type)
self.col.close()
clone = self.dms.open_column(self.col_name, clone=True)
self.col = self.dms.open_column(self.col_name)
......@@ -89,10 +86,21 @@ class OBIDMS_Column_TestCase(unittest.TestCase):
clone.close()
def test_OBIDMS_column_set_and_get(self):
for i in range(LINE_COUNT_FOR_TEST_COLUMN):
v = random_obivalue(self.data_type_str)
v = random_obivalue(self.data_type)
self.col[i] = v
assert self.col[i] == v, "Different value than the set value"
assert self.col[i] is not None, "None value"
def test_OBIDMS_referring_column(self):
for i in range(LINE_COUNT_FOR_TEST_COLUMN):
self.col[i] = random_obivalue(self.data_type)
ref_col = self.dms.open_column(self.col_name, referring=True)
j = 0
for i in range(LINE_COUNT_FOR_TEST_COLUMN):
if i%2 : # TODO randomize
ref_col.grep_line(i)
assert ref_col[j] == self.col[i], "Different value in original column and returned by referring column"
assert ref_col[j] is not None, "None value"
j+=1
class OBIDMS_Column_multiple_elements_TestCase(OBIDMS_Column_TestCase):
......@@ -101,7 +109,7 @@ class OBIDMS_Column_multiple_elements_TestCase(OBIDMS_Column_TestCase):
for i in range(SMALLER_LINE_COUNT_FOR_TEST_COLUMN):
v = {}
for e in self.elts_names :
v[e] = random_obivalue(self.data_type_str)
v[e] = random_obivalue(self.data_type)
self.col[i] = v
self.col.close()
clone = self.dms.open_column(self.col_name, clone=True)
......@@ -115,7 +123,7 @@ class OBIDMS_Column_multiple_elements_TestCase(OBIDMS_Column_TestCase):
def test_OBIDMS_column_set_and_get_with_elements_names(self):
for i in range(SMALLER_LINE_COUNT_FOR_TEST_COLUMN):
for e in range(NB_ELEMENTS_PER_LINE) :
v = random_obivalue(self.data_type_str)
v = random_obivalue(self.data_type)
self.col.set_item(i, self.elts_names[e], v)
assert self.col.get_item(i, self.elts_names[e]) == v, "Different value than the set value"
assert self.col.get_item(i, self.elts_names[e]) is not None, "None value"
......@@ -123,165 +131,168 @@ class OBIDMS_Column_multiple_elements_TestCase(OBIDMS_Column_TestCase):
for i in range(SMALLER_LINE_COUNT_FOR_TEST_COLUMN):
v = {}
for e in self.elts_names :
v[e] = random_obivalue(self.data_type_str)
v[e] = random_obivalue(self.data_type)
self.col[i] = v
assert self.col[i] == v, "Different value than the set value"
assert self.col[i] is not None, "None value"
def test_OBIDMS_referring_column(self):
for i in range(SMALLER_LINE_COUNT_FOR_TEST_COLUMN):
v = {}
for e in self.elts_names :
v[e] = random_obivalue(self.data_type)
self.col[i] = v
ref_col = self.dms.open_column(self.col_name, referring=True)
j = 0
for i in range(SMALLER_LINE_COUNT_FOR_TEST_COLUMN):
if i%2 : # TODO randomize
ref_col.grep_line(i)
assert ref_col[j] == self.col[i], "Different value in original column and returned by referring column"
assert ref_col[j] is not None, "None value"
j+=1
ref_col.close()
class OBIDMS_Column_OBI_INT_TestCase(OBIDMS_Column_TestCase):
def setUp(self):
self.data_type_code = 1
self.data_type = 'OBI_INT'
self.dms, \
self.dms_name, \
self.dms_dir_name = create_test_obidms()
self.col, \
self.col_name, \
self.data_type_str = create_test_column(self.dms,
self.data_type_code)
self.col_name = create_test_column(self.dms,
self.data_type)
class OBIDMS_Column_OBI_INT_multiple_elements_TestCase(OBIDMS_Column_multiple_elements_TestCase):
def setUp(self):
self.data_type_code = 1
self.data_type = 'OBI_INT'
self.dms, \
self.dms_name, \
self.dms_dir_name = create_test_obidms()
self.col, \
self.col_name, \
self.elts_names, \
self.data_type_str = create_test_column(self.dms,
self.data_type_code,
multiple_elements_per_line=True)
self.elts_names = create_test_column(self.dms,
self.data_type,
multiple_elements_per_line=True)
class OBIDMS_Column_OBI_FLOAT_TestCase(OBIDMS_Column_TestCase):
def setUp(self):
self.data_type_code = 2
self.data_type = 'OBI_FLOAT'
self.dms, \
self.dms_name, \
self.dms_dir_name = create_test_obidms()
self.col, \
self.col_name, \
self.data_type_str = create_test_column(self.dms,
self.data_type_code)
self.col_name = create_test_column(self.dms,
self.data_type)
class OBIDMS_Column_OBI_FLOAT_multiple_elements_TestCase(OBIDMS_Column_multiple_elements_TestCase):
def setUp(self):
self.data_type_code = 2
self.data_type = 'OBI_FLOAT'
self.dms, \
self.dms_name, \
self.dms_dir_name = create_test_obidms()
self.col, \
self.col_name, \
self.elts_names, \
self.data_type_str = create_test_column(self.dms,
self.data_type_code,
multiple_elements_per_line=True)
self.elts_names = create_test_column(self.dms,
self.data_type,
multiple_elements_per_line=True)
class OBIDMS_Column_OBI_BOOL_TestCase(OBIDMS_Column_TestCase):
def setUp(self):
self.data_type_code = 3
self.data_type = 'OBI_BOOL'
self.dms, \
self.dms_name, \
self.dms_dir_name = create_test_obidms()
self.col, \
self.col_name, \
self.data_type_str = create_test_column(self.dms,
self.data_type_code)
self.col_name = create_test_column(self.dms,
self.data_type)
class OBIDMS_Column_OBI_BOOL_multiple_elements_TestCase(OBIDMS_Column_multiple_elements_TestCase):
def setUp(self):
self.data_type_code = 3
self.data_type = 'OBI_BOOL'
self.dms, \
self.dms_name, \
self.dms_dir_name = create_test_obidms()
self.col, \
self.col_name, \
self.elts_names, \
self.data_type_str = create_test_column(self.dms,
self.data_type_code,
multiple_elements_per_line=True)
self.elts_names = create_test_column(self.dms,
self.data_type,
multiple_elements_per_line=True)
class OBIDMS_Column_OBI_CHAR_TestCase(OBIDMS_Column_TestCase):
def setUp(self):
self.data_type_code = 4
self.data_type = 'OBI_CHAR'
self.dms, \
self.dms_name, \
self.dms_dir_name = create_test_obidms()
self.col, \
self.col_name, \
self.data_type_str = create_test_column(self.dms,
self.data_type_code)
self.col_name = create_test_column(self.dms,
self.data_type)
class OBIDMS_Column_OBI_CHAR_multiple_elements_TestCase(OBIDMS_Column_multiple_elements_TestCase):
def setUp(self):
self.data_type_code = 4
self.data_type = 'OBI_CHAR'
self.dms, \
self.dms_name, \
self.dms_dir_name = create_test_obidms()
self.col, \
self.col_name, \
self.elts_names, \
self.data_type_str = create_test_column(self.dms,
self.data_type_code,
multiple_elements_per_line=True)
self.elts_names = create_test_column(self.dms,
self.data_type,
multiple_elements_per_line=True)
class OBIDMS_Column_OBI_STR_TestCase(OBIDMS_Column_TestCase):
def setUp(self):
self.data_type_code = 5
self.data_type = 'OBI_STR'
self.dms, \
self.dms_name, \
self.dms_dir_name = create_test_obidms()
self.col, \
self.col_name, \
self.data_type_str = create_test_column(self.dms,
self.data_type_code)
self.col_name = create_test_column(self.dms,
self.data_type)
class OBIDMS_Column_OBI_STR_multiple_elements_TestCase(OBIDMS_Column_multiple_elements_TestCase):
def setUp(self):
self.data_type_code = 5
self.data_type = 'OBI_STR'
self.dms, \
self.dms_name, \
self.dms_dir_name = create_test_obidms()
self.col, \
self.col_name, \
self.elts_names, \
self.data_type_str = create_test_column(self.dms,
self.data_type_code,
multiple_elements_per_line=True)
self.elts_names = create_test_column(self.dms,
self.data_type,
multiple_elements_per_line=True)
class OBIDMS_Column_OBI_SEQ_TestCase(OBIDMS_Column_TestCase):
def setUp(self):
self.data_type_code = 6
self.data_type = 'OBI_SEQ'
self.dms, \
self.dms_name, \
self.dms_dir_name = create_test_obidms()
self.col, \
self.col_name, \
self.data_type_str = create_test_column(self.dms,
self.data_type_code)
self.col_name = create_test_column(self.dms,
self.data_type)
class OBIDMS_Column_OBI_SEQ_multiple_elements_TestCase(OBIDMS_Column_multiple_elements_TestCase):
def setUp(self):
self.data_type_code = 6
self.data_type = 'OBI_SEQ'
self.dms, \
self.dms_name, \
self.dms_dir_name = create_test_obidms()
self.col, \
self.col_name, \
self.elts_names, \
self.data_type_str = create_test_column(self.dms,
self.data_type_code,
multiple_elements_per_line=True)
self.elts_names = create_test_column(self.dms,
self.data_type,
multiple_elements_per_line=True)
if __name__ == '__main__':
unittest.main(verbosity=2, defaultTest=["OBIDMS_Column_OBI_INT_TestCase",
......
......@@ -233,9 +233,20 @@ static char* build_array_data_file_name(const char* array_name)
}
size_t get_array_header_size()
{
return getpagesize() * 1;
size_t header_size;
size_t rounded_header_size;
double multiple;
header_size = sizeof(OBIDMS_array_header_t);
multiple = ceil((double) header_size / (double) getpagesize());
rounded_header_size = multiple * getpagesize();
return rounded_header_size;
}
......@@ -247,7 +258,17 @@ size_t get_initial_array_size()
size_t get_array_data_header_size()
{
return getpagesize() * 1;
size_t header_size;
size_t rounded_header_size;
double multiple;
header_size = sizeof(OBIDMS_array_data_header_t);
multiple = ceil((double) header_size / (double) getpagesize());
rounded_header_size = multiple * getpagesize();
return rounded_header_size;
}
......@@ -261,17 +282,17 @@ int close_array_data(OBIDMS_array_data_p array_data)
{
int ret_val = 0;
if (munmap(array_data->header, (array_data->header)->header_size) < 0)
if (munmap(array_data->data, (array_data->header)->data_size_max) < 0)
{
obi_set_errno(OBI_ARRAY_ERROR);
obidebug(1, "\nError munmapping the header of an array data file");
obidebug(1, "\nError munmapping the data of an array data file");
ret_val = -1;
}
if (munmap(array_data->data, (array_data->header)->data_size_max) < 0)
if (munmap(array_data->header, (array_data->header)->header_size) < 0)
{
obi_set_errno(OBI_ARRAY_ERROR);
obidebug(1, "\nError munmapping the data of an array data file");
obidebug(1, "\nError munmapping the header of an array data file");
ret_val = -1;
}
......@@ -737,6 +758,7 @@ OBIDMS_array_p obi_create_array(OBIDMS_p dms, const char* array_name)
return NULL;
}
array->dms = dms;
array->data = array_data;
array->directory = dms->array_directory;
array->dir_fd = array_dir_file_descriptor;
......@@ -753,6 +775,11 @@ OBIDMS_array_p obi_create_array(OBIDMS_p dms, const char* array_name)
close(array_file_descriptor);
// Add in the list of opened arrays
*(((dms->opened_arrays)->arrays)+((dms->opened_arrays)->nb_opened_arrays)) = array;
((dms->opened_arrays)->nb_opened_arrays)++;
array->counter = 1;
return array;
}
......@@ -767,6 +794,17 @@ OBIDMS_array_p obi_open_array(OBIDMS_p dms, const char* array_name)
int array_dir_file_descriptor;
OBIDMS_array_data_p array_data;
OBIDMS_array_p array;
size_t i;
// Check if the array is already in the list of opened arrays
for (i=0; i < ((dms->opened_arrays)->nb_opened_arrays); i++)
{
if (!strcmp(((*(((dms->opened_arrays)->arrays)+i))->header)->array_name, array_name))
{ // Found the array already opened
((*(((dms->opened_arrays)->arrays)+i))->counter)++;
return *(((dms->opened_arrays)->arrays)+i);
}
}
// Open the data file
......@@ -923,37 +961,65 @@ OBIDMS_array_p obi_open_array(OBIDMS_p dms, const char* array_name)
return NULL;
}
array->dms = dms;
array->data = array_data;
array->directory = dms->array_directory;
array->dir_fd = array_dir_file_descriptor;
close(array_file_descriptor);
// Add in the list of opened arrays
*(((dms->opened_arrays)->arrays)+((dms->opened_arrays)->nb_opened_arrays)) = array;
((dms->opened_arrays)->nb_opened_arrays)++;
array->counter = 1;
return array;
}
int obi_close_array(OBIDMS_array_p array) // TODO when do we call it? (as multiple threads can be using the same array)
int obi_close_array(OBIDMS_array_p array)
{
int ret_val = 0;
size_t i;
Opened_arrays_list_p arrays_list;
OBIDMS_p dms;
ret_val = close_array_data(array->data);
dms = array->dms;
if (munmap(array->first, (((array->header)->nb_items_max) * sizeof(index_t))) < 0)
{
obi_set_errno(OBI_ARRAY_ERROR);
obidebug(1, "\nError munmapping the array of an array file");
ret_val = -1;
}
arrays_list = dms->opened_arrays;
(array->counter)--;
if (munmap(array->header, (array->header)->header_size) < 0)
if (array->counter == 0)
{
obi_set_errno(OBI_ARRAY_ERROR);
obidebug(1, "\nError munmapping the header of an array file");
ret_val = -1;
}
// Delete from the list of opened arrays
for (i=0; i < (arrays_list->nb_opened_arrays); i++)
{
if (!strcmp(((*((arrays_list->arrays)+i))->header)->array_name, (array->header)->array_name))
{ // Found the array. Rearrange list
(arrays_list->nb_opened_arrays)--;
(arrays_list->arrays)[i] = (arrays_list->arrays)[arrays_list->nb_opened_arrays];
}
}
ret_val = close_array_data(array->data);
if (munmap(array->first, (((array->header)->nb_items_max) * sizeof(index_t))) < 0)
{
obi_set_errno(OBI_ARRAY_ERROR);
obidebug(1, "\nError munmapping the array of an array file");
ret_val = -1;
}
free(array);
if (munmap(array->header, (array->header)->header_size) < 0)
{
obi_set_errno(OBI_ARRAY_ERROR);
obidebug(1, "\nError munmapping the header of an array file");
ret_val = -1;
}
free(array);
}
return ret_val;
}
......
......@@ -91,6 +91,8 @@ typedef struct OBIDMS_array_header {
* @brief OBIDMS array structure.
*/
typedef struct OBIDMS_array {
OBIDMS_p dms; /**< A pointer to the OBIDMS structure to which the array belongs.
*/
OBIDMS_array_header_p header; /**< A pointer to the header of the array.
*/
index_t* first; /**< A pointer to the beginning of the array itself.
......@@ -104,6 +106,8 @@ typedef struct OBIDMS_array {
int dir_fd; /**< The file descriptor of the directory entry
* usable to refer and scan the array directory.
*/
size_t counter; /**< Indicates by how many threads/programs (TODO) the array is used.
*/
} OBIDMS_array_t, *OBIDMS_array_p;
......
......@@ -413,6 +413,16 @@ OBIDMS_p obi_open_dms(const char* dms_name)
return NULL;
}
// Initialize the list of opened columns
dms->opened_columns = (Opened_columns_list_p) malloc(sizeof(Opened_columns_list_t));
(dms->opened_columns)->columns = (OBIDMS_column_p*) malloc(MAX_NB_OPENED_COLUMNS*sizeof(OBIDMS_column_p));
(dms->opened_columns)->nb_opened_columns = 0;
// Initialize the list of opened arrays
dms->opened_arrays = (Opened_arrays_list_p) malloc(sizeof(Opened_arrays_list_t));
(dms->opened_arrays)->arrays = (OBIDMS_array_p*) malloc(MAX_NB_OPENED_ARRAYS*sizeof(OBIDMS_array_p));
(dms->opened_arrays)->nb_opened_arrays = 0;
return dms;
}
......@@ -440,6 +450,11 @@ int obi_close_dms(OBIDMS_p dms)
{
if (dms != NULL)
{
// Close all columns
while ((dms->opened_columns)->nb_opened_columns > 0)
obi_close_column(*((dms->opened_columns)->columns));
// Close dms and array directories
if (closedir(dms->directory) < 0)
{
obi_set_errno(OBIDMS_MEMORY_ERROR);
......
......@@ -25,10 +25,30 @@
#include "obierrno.h"
#define OBIDMS_MAX_NAME (2048) /**< The maximum length of an OBIDMS name.
*/
#define ARRAYS_DIR_NAME "arrays" /**< The name of the arrays directory.
*/
#define OBIDMS_MAX_NAME (2048) /**< The maximum length of an OBIDMS name.
*/
#define ARRAYS_DIR_NAME "arrays" /**< The name of the arrays directory.
*/
#define MAX_NB_OPENED_COLUMNS (100) /**< The maximum number of columns open at the same time.
*/
#define MAX_NB_OPENED_ARRAYS (100) /**< The maximum number of arrays open at the same time.
*/
struct OBIDMS_column; // TODO
typedef struct Opened_columns_list {
size_t nb_opened_columns;
struct OBIDMS_column** columns;
} Opened_columns_list_t, *Opened_columns_list_p;
struct OBIDMS_array; // TODO
typedef struct Opened_arrays_list {
size_t nb_opened_arrays;
struct OBIDMS_array** arrays;
} Opened_arrays_list_t, *Opened_arrays_list_p;
/**
......@@ -38,23 +58,27 @@
* and opening of an OBITools Data Management System (DMS)
*/
typedef struct OBIDMS {
char directory_name[OBIDMS_MAX_NAME+1]; /**< The name of the directory
* containing the DMS.
*/
DIR* directory; /**< A directory entry usable to
* refer and scan the database directory.
*/
int dir_fd; /**< The file descriptor of the directory entry
* usable to refer and scan the database directory.
*/
DIR* array_directory; /**< A directory entry usable to
* refer and scan the array directory.
*/
int array_dir_fd; /**< The file descriptor of the directory entry
* usable to refer and scan the array directory.
*/
bool little_endian; /**< Endianness of the database.
*/
char directory_name[OBIDMS_MAX_NAME+1]; /**< The name of the directory
* containing the DMS.
*/
DIR* directory; /**< A directory entry usable to
* refer and scan the database directory.
*/
int dir_fd; /**< The file descriptor of the directory entry