Added the lists of opened columns and arrays in the OBIDMS structure,

and a counter in the OBIDMS column structure; fixed some bugs and
created tests for referring columns that are bound to disappear anyway.

python/obitools3/obidms/_obidms.pyx

@@ -79,7 +79,6 @@ cdef class OBIDMS :
 
 
     cpdef close(self) :
-        #TODO close all columns (needs to be discussed)
         if (obi_close_dms(self.pointer)) < 0 :
             raise Exception("Problem closing an OBIDMS")
 

python/obitools3/unit_tests.py

@@ -12,7 +12,6 @@ LINE_COUNT_FOR_TEST_COLUMN = 10000         # TODO randomize?
 SMALLER_LINE_COUNT_FOR_TEST_COLUMN = 1000  # TODO randomize?
 NB_ELEMENTS_PER_LINE = 10                  # TODO randomize?
 DMS_NAME = "unit_test_dms"
-DATA_TYPES = ['OBI_INT', 'OBI_FLOAT', 'OBI_BOOL', 'OBI_CHAR', 'OBI_STR', 'OBI_SEQ']
 
 
 def create_test_obidms():
@@ -22,26 +21,24 @@ def create_test_obidms():
     return (dms, dms_name, dms_dir_name)
 
 
-def create_test_column(dms, data_type_code, multiple_elements_per_line=False):
-    data_types = DATA_TYPES
-    data_type_code = data_type_code
-    data_type_str = data_types[data_type_code-1]
-    col_name = "unit_test_"+data_type_str
-        
+def create_test_column(dms, data_type, multiple_elements_per_line=False):
+    
+    col_name = "unit_test_"+data_type
+
     if multiple_elements_per_line :
         elts_names = elements_names()
         col = dms.open_column(col_name, 
                               create=True, 
-                              data_type=data_type_code, 
+                              type=data_type, 
                               nb_elements_per_line=NB_ELEMENTS_PER_LINE,
                               elements_names=elts_names)
-        return (col, col_name, elts_names, data_type_str)
+        return (col, col_name, elts_names)
 
     else :
         col = dms.open_column(col_name, 
                               create=True, 
-                              data_type=data_type_code)
-        return (col, col_name, data_type_str)
+                              type=data_type)
+        return (col, col_name)
 
 
 def elements_names():
@@ -74,10 +71,10 @@ class OBIDMS_Column_TestCase(unittest.TestCase):
         self.dms.close()
         shutil.rmtree(self.dms_dir_name, ignore_errors=True)
     def test_OBIDMS_column_type(self):
-        assert self.col.get_data_type() == self.data_type_str, 'Wrong data type associated with column'
+        assert self.col.get_data_type() == self.data_type, 'Wrong data type associated with column'
     def test_OBIDMS_column_cloning(self):
         for i in range(LINE_COUNT_FOR_TEST_COLUMN):
-            self.col[i]= random_obivalue(self.data_type_str)
+            self.col[i]= random_obivalue(self.data_type)
         self.col.close()
         clone = self.dms.open_column(self.col_name, clone=True)
         self.col = self.dms.open_column(self.col_name)
@@ -89,10 +86,21 @@ class OBIDMS_Column_TestCase(unittest.TestCase):
         clone.close()
     def test_OBIDMS_column_set_and_get(self):
         for i in range(LINE_COUNT_FOR_TEST_COLUMN):
-            v = random_obivalue(self.data_type_str)
+            v = random_obivalue(self.data_type)
             self.col[i] = v
             assert self.col[i] == v, "Different value than the set value"
             assert self.col[i] is not None, "None value"
+    def test_OBIDMS_referring_column(self):
+        for i in range(LINE_COUNT_FOR_TEST_COLUMN):
+            self.col[i] = random_obivalue(self.data_type)
+        ref_col = self.dms.open_column(self.col_name, referring=True)
+        j = 0
+        for i in range(LINE_COUNT_FOR_TEST_COLUMN):
+            if i%2 :    # TODO randomize
+                ref_col.grep_line(i)
+                assert ref_col[j] == self.col[i], "Different value in original column and returned by referring column"
+                assert ref_col[j] is not None, "None value"
+                j+=1
 
 
 class OBIDMS_Column_multiple_elements_TestCase(OBIDMS_Column_TestCase):
@@ -101,7 +109,7 @@ class OBIDMS_Column_multiple_elements_TestCase(OBIDMS_Column_TestCase):
         for i in range(SMALLER_LINE_COUNT_FOR_TEST_COLUMN):
             v = {}
             for e in self.elts_names :
-                v[e] = random_obivalue(self.data_type_str)
+                v[e] = random_obivalue(self.data_type)
             self.col[i] = v
         self.col.close()
         clone = self.dms.open_column(self.col_name, clone=True)
@@ -115,7 +123,7 @@ class OBIDMS_Column_multiple_elements_TestCase(OBIDMS_Column_TestCase):
     def test_OBIDMS_column_set_and_get_with_elements_names(self):
         for i in range(SMALLER_LINE_COUNT_FOR_TEST_COLUMN):
             for e in range(NB_ELEMENTS_PER_LINE) :
-                v = random_obivalue(self.data_type_str)
+                v = random_obivalue(self.data_type)
                 self.col.set_item(i, self.elts_names[e], v)
                 assert self.col.get_item(i, self.elts_names[e]) == v, "Different value than the set value"
                 assert self.col.get_item(i, self.elts_names[e]) is not None, "None value"
@@ -123,165 +131,168 @@ class OBIDMS_Column_multiple_elements_TestCase(OBIDMS_Column_TestCase):
         for i in range(SMALLER_LINE_COUNT_FOR_TEST_COLUMN):
             v = {}
             for e in self.elts_names :
-                v[e] = random_obivalue(self.data_type_str)
+                v[e] = random_obivalue(self.data_type)
             self.col[i] = v
             assert self.col[i] == v, "Different value than the set value"
             assert self.col[i] is not None, "None value"
+    def test_OBIDMS_referring_column(self):
+        for i in range(SMALLER_LINE_COUNT_FOR_TEST_COLUMN):
+            v = {}
+            for e in self.elts_names :
+                v[e] = random_obivalue(self.data_type)
+            self.col[i] = v
+        ref_col = self.dms.open_column(self.col_name, referring=True)
+        j = 0
+        for i in range(SMALLER_LINE_COUNT_FOR_TEST_COLUMN):
+            if i%2 :    # TODO randomize
+                ref_col.grep_line(i)
+                assert ref_col[j] == self.col[i], "Different value in original column and returned by referring column"
+                assert ref_col[j] is not None, "None value"
+                j+=1
+        ref_col.close()
 
 
 class OBIDMS_Column_OBI_INT_TestCase(OBIDMS_Column_TestCase):
     def setUp(self):
-        self.data_type_code = 1
+        self.data_type = 'OBI_INT'
         self.dms, \
         self.dms_name, \
         self.dms_dir_name = create_test_obidms()
         self.col, \
-        self.col_name, \
-        self.data_type_str = create_test_column(self.dms, 
-                                                self.data_type_code)
+        self.col_name = create_test_column(self.dms, 
+                                           self.data_type)
        
 
 class OBIDMS_Column_OBI_INT_multiple_elements_TestCase(OBIDMS_Column_multiple_elements_TestCase):
     def setUp(self):
-        self.data_type_code = 1
+        self.data_type = 'OBI_INT'
         self.dms, \
         self.dms_name, \
         self.dms_dir_name = create_test_obidms()
         self.col, \
         self.col_name, \
-        self.elts_names, \
-        self.data_type_str = create_test_column(self.dms, 
-                                                self.data_type_code,
-                                                multiple_elements_per_line=True)
+        self.elts_names = create_test_column(self.dms, 
+                                             self.data_type,
+                                             multiple_elements_per_line=True)
 
 
 class OBIDMS_Column_OBI_FLOAT_TestCase(OBIDMS_Column_TestCase):
     def setUp(self):
-        self.data_type_code = 2
+        self.data_type = 'OBI_FLOAT'
         self.dms, \
         self.dms_name, \
         self.dms_dir_name = create_test_obidms()
         self.col, \
-        self.col_name, \
-        self.data_type_str = create_test_column(self.dms, 
-                                                self.data_type_code)
+        self.col_name = create_test_column(self.dms, 
+                                           self.data_type)
        
 
 class OBIDMS_Column_OBI_FLOAT_multiple_elements_TestCase(OBIDMS_Column_multiple_elements_TestCase):
     def setUp(self):
-        self.data_type_code = 2
+        self.data_type = 'OBI_FLOAT'
         self.dms, \
         self.dms_name, \
         self.dms_dir_name = create_test_obidms()
         self.col, \
         self.col_name, \
-        self.elts_names, \
-        self.data_type_str = create_test_column(self.dms, 
-                                                self.data_type_code,
-                                                multiple_elements_per_line=True)
+        self.elts_names = create_test_column(self.dms, 
+                                             self.data_type,
+                                             multiple_elements_per_line=True)
 
 
 class OBIDMS_Column_OBI_BOOL_TestCase(OBIDMS_Column_TestCase):
     def setUp(self):
-        self.data_type_code = 3
+        self.data_type = 'OBI_BOOL'
         self.dms, \
         self.dms_name, \
         self.dms_dir_name = create_test_obidms()
         self.col, \
-        self.col_name, \
-        self.data_type_str = create_test_column(self.dms, 
-                                                self.data_type_code)
+        self.col_name = create_test_column(self.dms, 
+                                           self.data_type)
        
 
 class OBIDMS_Column_OBI_BOOL_multiple_elements_TestCase(OBIDMS_Column_multiple_elements_TestCase):
     def setUp(self):
-        self.data_type_code = 3
+        self.data_type = 'OBI_BOOL'
         self.dms, \
         self.dms_name, \
         self.dms_dir_name = create_test_obidms()
         self.col, \
         self.col_name, \
-        self.elts_names, \
-        self.data_type_str = create_test_column(self.dms, 
-                                                self.data_type_code,
-                                                multiple_elements_per_line=True)
+        self.elts_names = create_test_column(self.dms, 
+                                             self.data_type,
+                                             multiple_elements_per_line=True)
 
 
 class OBIDMS_Column_OBI_CHAR_TestCase(OBIDMS_Column_TestCase):
     def setUp(self):
-        self.data_type_code = 4
+        self.data_type = 'OBI_CHAR'
         self.dms, \
         self.dms_name, \
         self.dms_dir_name = create_test_obidms()
         self.col, \
-        self.col_name, \
-        self.data_type_str = create_test_column(self.dms, 
-                                                self.data_type_code)
+        self.col_name = create_test_column(self.dms, 
+                                                self.data_type)
        
 
 class OBIDMS_Column_OBI_CHAR_multiple_elements_TestCase(OBIDMS_Column_multiple_elements_TestCase):
     def setUp(self):
-        self.data_type_code = 4
+        self.data_type = 'OBI_CHAR'
         self.dms, \
         self.dms_name, \
         self.dms_dir_name = create_test_obidms()
         self.col, \
         self.col_name, \
-        self.elts_names, \
-        self.data_type_str = create_test_column(self.dms, 
-                                                self.data_type_code,
-                                                multiple_elements_per_line=True)
+        self.elts_names = create_test_column(self.dms, 
+                                             self.data_type,
+                                             multiple_elements_per_line=True)
 
 
 class OBIDMS_Column_OBI_STR_TestCase(OBIDMS_Column_TestCase):
     def setUp(self):
-        self.data_type_code = 5
+        self.data_type = 'OBI_STR'
         self.dms, \
         self.dms_name, \
         self.dms_dir_name = create_test_obidms()
         self.col, \
-        self.col_name, \
-        self.data_type_str = create_test_column(self.dms, 
-                                                self.data_type_code)
+        self.col_name = create_test_column(self.dms, 
+                                           self.data_type)
        
 
 class OBIDMS_Column_OBI_STR_multiple_elements_TestCase(OBIDMS_Column_multiple_elements_TestCase):
     def setUp(self):
-        self.data_type_code = 5
+        self.data_type = 'OBI_STR'
         self.dms, \
         self.dms_name, \
         self.dms_dir_name = create_test_obidms()
         self.col, \
         self.col_name, \
-        self.elts_names, \
-        self.data_type_str = create_test_column(self.dms, 
-                                                self.data_type_code,
-                                                multiple_elements_per_line=True)
+        self.elts_names = create_test_column(self.dms, 
+                                             self.data_type,
+                                             multiple_elements_per_line=True)
 
 class OBIDMS_Column_OBI_SEQ_TestCase(OBIDMS_Column_TestCase):
     def setUp(self):
-        self.data_type_code = 6
+        self.data_type = 'OBI_SEQ'
         self.dms, \
         self.dms_name, \
         self.dms_dir_name = create_test_obidms()
         self.col, \
-        self.col_name, \
-        self.data_type_str = create_test_column(self.dms, 
-                                                self.data_type_code)
+        self.col_name = create_test_column(self.dms, 
+                                           self.data_type)
        
 
 class OBIDMS_Column_OBI_SEQ_multiple_elements_TestCase(OBIDMS_Column_multiple_elements_TestCase):
     def setUp(self):
-        self.data_type_code = 6
+        self.data_type = 'OBI_SEQ'
         self.dms, \
         self.dms_name, \
         self.dms_dir_name = create_test_obidms()
         self.col, \
         self.col_name, \
-        self.elts_names, \
-        self.data_type_str = create_test_column(self.dms, 
-                                                self.data_type_code,
-                                                multiple_elements_per_line=True)
+        self.elts_names = create_test_column(self.dms, 
+                                             self.data_type,
+                                             multiple_elements_per_line=True)
 
 if __name__ == '__main__': 
      unittest.main(verbosity=2, defaultTest=["OBIDMS_Column_OBI_INT_TestCase",

src/obiarray.c

@@ -233,9 +233,20 @@ static char* build_array_data_file_name(const char* array_name)
 }
 
 
+
 size_t get_array_header_size()
 {
-	return getpagesize() * 1;
+	size_t header_size;
+	size_t rounded_header_size;
+	double multiple;
+
+	header_size = sizeof(OBIDMS_array_header_t);
+
+	multiple = 	ceil((double) header_size / (double) getpagesize());
+
+	rounded_header_size = multiple * getpagesize();
+
+	return rounded_header_size;
 }
 
 
@@ -247,7 +258,17 @@ size_t get_initial_array_size()
 
 size_t get_array_data_header_size()
 {
-	return getpagesize() * 1;
+	size_t header_size;
+	size_t rounded_header_size;
+	double multiple;
+
+	header_size = sizeof(OBIDMS_array_data_header_t);
+
+	multiple = 	ceil((double) header_size / (double) getpagesize());
+
+	rounded_header_size = multiple * getpagesize();
+
+	return rounded_header_size;
 }
 
 
@@ -261,17 +282,17 @@ int close_array_data(OBIDMS_array_data_p array_data)
 {
 	int ret_val = 0;
 
-	if (munmap(array_data->header, (array_data->header)->header_size) < 0)
+	if (munmap(array_data->data, (array_data->header)->data_size_max) < 0)
 	{
 		obi_set_errno(OBI_ARRAY_ERROR);
-		obidebug(1, "\nError munmapping the header of an array data file");
+		obidebug(1, "\nError munmapping the data of an array data file");
 		ret_val = -1;
 	}
 
-	if (munmap(array_data->data, (array_data->header)->data_size_max) < 0)
+	if (munmap(array_data->header, (array_data->header)->header_size) < 0)
 	{
 		obi_set_errno(OBI_ARRAY_ERROR);
-		obidebug(1, "\nError munmapping the data of an array data file");
+		obidebug(1, "\nError munmapping the header of an array data file");
 		ret_val = -1;
 	}
 
@@ -737,6 +758,7 @@ OBIDMS_array_p obi_create_array(OBIDMS_p dms, const char* array_name)
 		return NULL;
 	}
 
+	array->dms = dms;
 	array->data = array_data;
 	array->directory = dms->array_directory;
 	array->dir_fd = array_dir_file_descriptor;
@@ -753,6 +775,11 @@ OBIDMS_array_p obi_create_array(OBIDMS_p dms, const char* array_name)
 
 	close(array_file_descriptor);
 
+	// Add in the list of opened arrays
+	*(((dms->opened_arrays)->arrays)+((dms->opened_arrays)->nb_opened_arrays)) = array;
+	((dms->opened_arrays)->nb_opened_arrays)++;
+	array->counter = 1;
+
 	return array;
 }
 
@@ -767,6 +794,17 @@ OBIDMS_array_p obi_open_array(OBIDMS_p dms, const char* array_name)
 	int					array_dir_file_descriptor;
 	OBIDMS_array_data_p array_data;
 	OBIDMS_array_p		array;
+	size_t i;
+
+	// Check if the array is already in the list of opened arrays
+	for (i=0; i < ((dms->opened_arrays)->nb_opened_arrays); i++)
+	{
+		if (!strcmp(((*(((dms->opened_arrays)->arrays)+i))->header)->array_name, array_name))
+		{	// Found the array already opened
+			((*(((dms->opened_arrays)->arrays)+i))->counter)++;
+			return *(((dms->opened_arrays)->arrays)+i);
+		}
+	}
 
 	// Open the data file
 
@@ -923,37 +961,65 @@ OBIDMS_array_p obi_open_array(OBIDMS_p dms, const char* array_name)
 		return NULL;
 	}
 
+	array->dms = dms;
 	array->data = array_data;
 	array->directory = dms->array_directory;
 	array->dir_fd = array_dir_file_descriptor;
 
 	close(array_file_descriptor);
 
+	// Add in the list of opened arrays
+	*(((dms->opened_arrays)->arrays)+((dms->opened_arrays)->nb_opened_arrays)) = array;
+	((dms->opened_arrays)->nb_opened_arrays)++;
+	array->counter = 1;
+
 	return array;
 }
 
 
-int obi_close_array(OBIDMS_array_p array)	// TODO when do we call it? (as multiple threads can be using the same array)
+int obi_close_array(OBIDMS_array_p array)
 {
 	int ret_val = 0;
+	size_t i;
+	Opened_arrays_list_p arrays_list;
+	OBIDMS_p dms;
 
-	ret_val = close_array_data(array->data);
+	dms = array->dms;
 
-	if (munmap(array->first, (((array->header)->nb_items_max) * sizeof(index_t))) < 0)
-	{
-		obi_set_errno(OBI_ARRAY_ERROR);
-		obidebug(1, "\nError munmapping the array of an array file");
-		ret_val = -1;
-	}
+	arrays_list = dms->opened_arrays;
+
+	(array->counter)--;
 
-	if (munmap(array->header, (array->header)->header_size) < 0)
+	if (array->counter == 0)
 	{
-		obi_set_errno(OBI_ARRAY_ERROR);
-		obidebug(1, "\nError munmapping the header of an array file");
-		ret_val = -1;
-	}
+		// Delete from the list of opened arrays
+		for (i=0; i < (arrays_list->nb_opened_arrays); i++)
+		{
+			if (!strcmp(((*((arrays_list->arrays)+i))->header)->array_name, (array->header)->array_name))
+			{	// Found the array. Rearrange list
+				(arrays_list->nb_opened_arrays)--;
+				(arrays_list->arrays)[i] = (arrays_list->arrays)[arrays_list->nb_opened_arrays];
+			}
+		}
+
+		ret_val = close_array_data(array->data);
+
+		if (munmap(array->first, (((array->header)->nb_items_max) * sizeof(index_t))) < 0)
+		{
+			obi_set_errno(OBI_ARRAY_ERROR);
+			obidebug(1, "\nError munmapping the array of an array file");
+			ret_val = -1;
+		}
 
-	free(array);
+		if (munmap(array->header, (array->header)->header_size) < 0)
+		{
+			obi_set_errno(OBI_ARRAY_ERROR);
+			obidebug(1, "\nError munmapping the header of an array file");
+			ret_val = -1;
+		}
+
+		free(array);
+	}
 
 	return ret_val;
 }

src/obiarray.h

@@ -91,6 +91,8 @@ typedef struct OBIDMS_array_header {
  * @brief OBIDMS array structure.
  */
 typedef struct OBIDMS_array {
+	OBIDMS_p                	dms;			 	/**< A pointer to the OBIDMS structure to which the array belongs.
+	 	 	 	 	 	 	 	 	 	 	 	 	 */
 	OBIDMS_array_header_p		header; 		 	/**< A pointer to the header of the array.
 	 	 	 	 	 	 	 	 	 	 	 	 	 */
 	index_t*                 	first;   		 	/**< A pointer to the beginning of the array itself.
@@ -104,6 +106,8 @@ typedef struct OBIDMS_array {
 	int							dir_fd;				/**< The file descriptor of the directory entry
 													 *    usable to refer and scan the array directory.
 													 */
+	size_t						counter;			/**< Indicates by how many threads/programs (TODO) the array is used.
+													 */
 } OBIDMS_array_t, *OBIDMS_array_p;
 
 

src/obidms.c

@@ -413,6 +413,16 @@ OBIDMS_p obi_open_dms(const char* dms_name)
 		return NULL;
 	}
 
+	// Initialize the list of opened columns
+	dms->opened_columns = (Opened_columns_list_p) malloc(sizeof(Opened_columns_list_t));
+	(dms->opened_columns)->columns = (OBIDMS_column_p*) malloc(MAX_NB_OPENED_COLUMNS*sizeof(OBIDMS_column_p));
+	(dms->opened_columns)->nb_opened_columns = 0;
+
+	// Initialize the list of opened arrays
+	dms->opened_arrays = (Opened_arrays_list_p) malloc(sizeof(Opened_arrays_list_t));
+	(dms->opened_arrays)->arrays = (OBIDMS_array_p*) malloc(MAX_NB_OPENED_ARRAYS*sizeof(OBIDMS_array_p));
+	(dms->opened_arrays)->nb_opened_arrays = 0;
+
 	return dms;
 }
 
@@ -440,6 +450,11 @@ int obi_close_dms(OBIDMS_p dms)
 {
 	if (dms != NULL)
 	{
+		// Close all columns
+		while ((dms->opened_columns)->nb_opened_columns > 0)
+			obi_close_column(*((dms->opened_columns)->columns));
+
+		// Close dms and array directories
 		if (closedir(dms->directory) < 0)
 		{
 			obi_set_errno(OBIDMS_MEMORY_ERROR);

src/obidms.h

@@ -25,10 +25,30 @@
 #include "obierrno.h"
 
 
-#define OBIDMS_MAX_NAME (2048) 	   /**< The maximum length of an OBIDMS name.
-                                	*/
-#define ARRAYS_DIR_NAME "arrays"   /**< The name of the arrays directory.
-                                	*/
+#define OBIDMS_MAX_NAME (2048) 	    /**< The maximum length of an OBIDMS name.
+                                	 */
+#define ARRAYS_DIR_NAME "arrays"    /**< The name of the arrays directory.
+                                	 */
+#define MAX_NB_OPENED_COLUMNS (100) /**< The maximum number of columns open at the same time.
+                                	 */
+#define MAX_NB_OPENED_ARRAYS (100)  /**< The maximum number of arrays open at the same time.
+                                	 */
+
+
+struct OBIDMS_column;		// TODO
+
+typedef struct Opened_columns_list {
+	size_t	 		 		nb_opened_columns;
+	struct OBIDMS_column** 	columns;
+} Opened_columns_list_t, *Opened_columns_list_p;
+
+
+struct OBIDMS_array;		// TODO
+
+typedef struct Opened_arrays_list {
+	size_t	 		 		nb_opened_arrays;
+	struct OBIDMS_array** 	arrays;
+} Opened_arrays_list_t, *Opened_arrays_list_p;
 
 
 /**
@@ -38,23 +58,27 @@
  * and opening of an OBITools Data Management System (DMS)
  */
 typedef struct OBIDMS {
-	char	directory_name[OBIDMS_MAX_NAME+1];	/**< The name of the directory
-	                                     	 	 *   containing the DMS.
-	                                     	 	 */
-	DIR*	directory;							/**< A directory entry usable to
-										 	 	 *   refer and scan the database directory.
-										 	 	 */
-	int     dir_fd;								/**< The file descriptor of the directory entry
-												 *   usable to refer and scan the database directory.
-												 */
-	DIR*	array_directory;					/**< A directory entry usable to
-										 	 	 *   refer and scan the array directory.
-										 	 	 */
-	int		array_dir_fd;						/**< The file descriptor of the directory entry
-												 *   usable to refer and scan the array directory.
-												 */
-	bool	little_endian;						/**< Endianness of the database.
-												 */
+	char					directory_name[OBIDMS_MAX_NAME+1];	/**< The name of the directory
+																 *   containing the DMS.
+																 */
+	DIR*					directory;							/**< A directory entry usable to
+																 *   refer and scan the database directory.
+																 */
+	int     				dir_fd;								/**< The file descriptor of the directory entry