Commit 456551ff by Celine Mercier

obi arrays that don't work because of cython bug passing wrong pointers

parent ecb9d97a
/build/
/doxygen/
/build_dir.txt
/.DS_Store
/.gitignore
......@@ -51,7 +51,7 @@ source_suffix = '.rst'
#source_encoding = 'utf-8-sig'
# The master toctree document.
master_doc = 'index'
master_doc = 'source/index'
# General information about the project.
project = u'OBITools3'
......
# Ignore everything in this directory
*
# Except this file
!.gitignore
\ No newline at end of file
......@@ -7,11 +7,14 @@ OBITypes
:download:`html version of the OBITypes UML file <UML/OBITypes_UML.class.violet.html>`
.. image:: ./UML/Obicolumn_classes_UML.png
:download:`html version of the OBIDMS classes UML file <UML/Obicolumn_classes_UML.class.violet.html>`
.. toctree::
:maxdepth: 2
The elementary types <elementary>
The containers <containers>
Special values <specialvalues>
......@@ -12,3 +12,5 @@
../../../src/obitypes.c
../../../src/private_at_functions.h
../../../src/private_at_functions.c
../../../src/obiarray.h
../../../src/obiarray.c
......@@ -23,7 +23,8 @@ cdef class OBIDMS:
OBIType_t data_type=*,
size_t nb_lines=*,
size_t nb_elements_per_line=*,
list elements_names=*)
list elements_names=*,
str array_name=*)
cdef class OBIDMS_column:
......
......@@ -42,10 +42,10 @@ from ._obidmscolumn_char cimport OBIDMS_column_char, \
OBIDMS_column_char_multi_elts, \
OBIDMS_column_char_multi_elts_writable
# from ._obidmscolumn_idx cimport OBIDMS_column_idx, \
# OBIDMS_column_idx_writable, \
# OBIDMS_column_idx_multi_elts, \
# OBIDMS_column_idx_multi_elts_writable
from ._obidmscolumn_str cimport OBIDMS_column_str, \
OBIDMS_column_str_writable, \
OBIDMS_column_str_multi_elts, \
OBIDMS_column_str_multi_elts_writable
cdef class OBIDMS :
......@@ -119,7 +119,8 @@ cdef class OBIDMS :
OBIType_t data_type= <OBIType_t> 0,
size_t nb_lines=0,
size_t nb_elements_per_line=0,
list elements_names=None):
list elements_names=None,
str array_name="default_obiarray"):
# Declarations
cdef OBIDMS_column column
......@@ -202,17 +203,17 @@ cdef class OBIDMS :
subclass = OBIDMS_column_char
else :
subclass = OBIDMS_column_char_multi_elts
# elif data_type == 5 :
# if (create or clone) :
# if nb_elements_per_line == 1 :
# subclass = OBIDMS_column_idx_writable
# else :
# subclass = OBIDMS_column_idx_multi_elts_writable
# else :
# if nb_elements_per_line == 1 :
# subclass = OBIDMS_column_idx
# else :
# subclass = OBIDMS_column_idx_multi_elts
elif data_type == 5 :
if (create or clone) :
if nb_elements_per_line == 1 :
subclass = OBIDMS_column_str_writable
else :
subclass = OBIDMS_column_str_multi_elts_writable
else :
if nb_elements_per_line == 1 :
subclass = OBIDMS_column_str
else :
subclass = OBIDMS_column_str_multi_elts
else :
raise Exception("Problem with the data type")
......@@ -220,7 +221,7 @@ cdef class OBIDMS :
create, clone, clone_data,
version_number, data_type,
nb_lines, nb_elements_per_line,
elements_names)
elements_names, array_name)
return column
......@@ -238,11 +239,13 @@ cdef class OBIDMS_column :
OBIType_t type,
size_t nb_lines,
size_t nb_elements_per_line,
list elements_names):
list elements_names,
str array_name):
# Declarations
cdef bytes column_name_b
cdef bytes dms_name_b
cdef bytes array_name_b
cdef bytes elements_names_b
# Fill structure
......@@ -255,6 +258,7 @@ cdef class OBIDMS_column :
# Format the character strings to send them to C functions
column_name_b = str2bytes(column_name)
dms_name_b = str2bytes(self.dms.dms_name)
array_name_b = str2bytes(array_name)
# Create, clone or open column
if create :
......@@ -262,7 +266,9 @@ cdef class OBIDMS_column :
elements_names_b = column_name_b
else :
elements_names_b = str2bytes(";".join(elements_names))
self.pointer = obi_create_column(self.dms.pointer, column_name_b, type, nb_lines, nb_elements_per_line, elements_names_b)
self.pointer = obi_create_column(self.dms.pointer, column_name_b, type,
nb_lines, nb_elements_per_line,
elements_names_b, array_name_b)
else :
if clone :
self.pointer = obi_clone_column(self.dms.pointer, column_name_b, version_number, clone_data)
......
......@@ -14,3 +14,5 @@
../../../src/obitypes.c
../../../src/private_at_functions.h
../../../src/private_at_functions.c
../../../src/obiarray.h
../../../src/obiarray.c
......@@ -14,3 +14,5 @@
../../../src/obitypes.c
../../../src/private_at_functions.h
../../../src/private_at_functions.c
../../../src/obiarray.h
../../../src/obiarray.c
......@@ -14,3 +14,5 @@
../../../src/obitypes.c
../../../src/private_at_functions.h
../../../src/private_at_functions.c
../../../src/obiarray.h
../../../src/obiarray.c
......@@ -14,3 +14,5 @@
../../../src/obitypes.c
../../../src/private_at_functions.h
../../../src/private_at_functions.c
../../../src/obiarray.h
../../../src/obiarray.c
../../../src/obidmscolumn_idx.c
../../../src/obidmscolumn_idx.h
../../../src/obidmscolumn_str.c
../../../src/obidmscolumn_str.h
../../../src/obidmscolumn.h
../../../src/obidmscolumn.c
../../../src/obidmscolumndir.h
......@@ -14,3 +14,5 @@
../../../src/obitypes.c
../../../src/private_at_functions.h
../../../src/private_at_functions.c
../../../src/obiarray.h
../../../src/obiarray.c
#cython: language_level=3
from .capi.obitypes cimport obiidx_t
from ._obidms cimport OBIDMS_column
cdef class OBIDMS_column_idx(OBIDMS_column):
cdef class OBIDMS_column_str(OBIDMS_column):
cpdef object get_line(self, size_t line_nb)
cpdef set_line(self, size_t line_nb, object value)
cpdef close(self)
cdef class OBIDMS_column_idx_writable(OBIDMS_column_idx):
cdef class OBIDMS_column_str_writable(OBIDMS_column_str):
cpdef set_line(self, size_t line_nb, object value)
cpdef close(self)
cdef class OBIDMS_column_idx_multi_elts(OBIDMS_column_idx):
cdef class OBIDMS_column_str_multi_elts(OBIDMS_column_str):
cpdef object get_item(self, size_t line_nb, str element_name)
cpdef object get_line(self, size_t line_nb)
cpdef set_item(self, size_t line_nb, str element_name, obiidx_t value)
cpdef set_item(self, size_t line_nb, str element_name, str value)
cpdef set_line(self, size_t line_nb, object values)
cdef class OBIDMS_column_idx_multi_elts_writable(OBIDMS_column_idx_multi_elts):
cpdef set_item(self, size_t line_nb, str element_name, obiidx_t value)
cdef class OBIDMS_column_str_multi_elts_writable(OBIDMS_column_str_multi_elts):
cpdef set_item(self, size_t line_nb, str element_name, str value)
cpdef set_line(self, size_t line_nb, object values)
cpdef close(self)
......@@ -2,30 +2,46 @@
from .capi.obidmscolumn cimport obi_close_column,\
obi_truncate_and_close_column, \
obi_column_get_obiidx_with_elt_name, \
obi_column_get_obiidx_with_elt_idx, \
obi_column_set_obiidx_with_elt_name, \
obi_column_set_obiidx_with_elt_idx
obi_column_get_obistr_with_elt_name, \
obi_column_get_obistr_with_elt_idx, \
obi_column_set_obistr_with_elt_name, \
obi_column_set_obistr_with_elt_idx
from .capi.obierrno cimport obi_errno
from .capi.obitypes cimport OBIIdx_NA
from .capi.obitypes cimport OBIIdx_NA, const_char_p
from obitools3.utils cimport str2bytes
from libc.string cimport strlen
from obitools3.utils cimport str2bytes, bytes2str
from cpython.int cimport PyInt_FromSsize_t
cdef class OBIDMS_column_idx(OBIDMS_column):
cdef class OBIDMS_column_str(OBIDMS_column):
cpdef object get_line(self, size_t line_nb):
cdef obiidx_t value
cdef const_char_p cvalue
cdef bytes value
cdef object result
value = obi_column_get_obiidx_with_elt_idx(self.pointer, line_nb, 0)
cvalue = obi_column_get_obistr_with_elt_idx(self.pointer, line_nb, 0)
print('test 1')
print(hex(<int> cvalue))
print(strlen(cvalue))
print("test")
value = <bytes> cvalue
print(value)
if obi_errno > 0 :
raise IndexError(line_nb)
if value == OBIIdx_NA :
result = None
else :
result = PyInt_FromSsize_t(value)
print(value)
result = bytes2str(value)
return result
cpdef set_line(self, size_t line_nb, object value):
......@@ -36,10 +52,10 @@ cdef class OBIDMS_column_idx(OBIDMS_column):
raise Exception("Problem closing a column")
cdef class OBIDMS_column_idx_writable(OBIDMS_column_idx):
cdef class OBIDMS_column_str_writable(OBIDMS_column_str):
cpdef set_line(self, size_t line_nb, object value):
if obi_column_set_obiidx_with_elt_idx(self.pointer, line_nb, 0, <obiidx_t> value) < 0:
if obi_column_set_obistr_with_elt_idx(self.pointer, line_nb, 0, str2bytes(value)) < 0:
raise Exception("Problem setting a value in a column")
cpdef close(self):
......@@ -47,55 +63,55 @@ cdef class OBIDMS_column_idx_writable(OBIDMS_column_idx):
raise Exception("Problem closing a column")
cdef class OBIDMS_column_idx_multi_elts(OBIDMS_column_idx):
cdef class OBIDMS_column_str_multi_elts(OBIDMS_column_str):
cpdef object get_item(self, size_t line_nb, str element_name):
cdef obiidx_t value
cdef bytes value
cdef object result
value = obi_column_get_obiidx_with_elt_name(self.pointer, line_nb, str2bytes(element_name))
value = obi_column_get_obistr_with_elt_name(self.pointer, line_nb, str2bytes(element_name))
if obi_errno > 0 :
raise IndexError(line_nb, element_name)
if value == OBIIdx_NA :
result = None
else :
result = PyInt_FromSsize_t(value)
result = bytes2str(value)
return result
cpdef object get_line(self, size_t line_nb) :
cdef obiidx_t value
cdef bytes value
cdef object result
cdef size_t i
cdef bint all_NA
result = {}
all_NA = True
for i in range(self.nb_elements_per_line) :
value = obi_column_get_obiidx_with_elt_idx(self.pointer, line_nb, i)
value = obi_column_get_obistr_with_elt_idx(self.pointer, line_nb, i)
if obi_errno > 0 :
raise IndexError(line_nb)
result[self.elements_names[i]] = PyInt_FromSsize_t(value)
result[self.elements_names[i]] = bytes2str(value)
if all_NA and (value != OBIIdx_NA) :
all_NA = False
if all_NA :
result = None
return result
cpdef set_item(self, size_t line_nb, str element_name, obiidx_t value):
cpdef set_item(self, size_t line_nb, str element_name, str value):
raise Exception("Column is read-only")
cpdef set_line(self, size_t line_nb, object values):
raise Exception("Column is read-only")
cdef class OBIDMS_column_idx_multi_elts_writable(OBIDMS_column_idx_multi_elts):
cdef class OBIDMS_column_str_multi_elts_writable(OBIDMS_column_str_multi_elts):
cpdef set_item(self, size_t line_nb, str element_name, obiidx_t value):
if obi_column_set_obiidx_with_elt_name(self.pointer, line_nb, str2bytes(element_name), value) < 0:
cpdef set_item(self, size_t line_nb, str element_name, str value):
if obi_column_set_obistr_with_elt_name(self.pointer, line_nb, str2bytes(element_name), str2bytes(value)) < 0:
raise Exception("Problem setting a value in a column")
cpdef set_line(self, size_t line_nb, object values):
cdef obiidx_t value
cdef str value
for element_name in values :
value = <obiidx_t> values[element_name]
value = values[element_name]
self.set_item(line_nb, element_name, value)
cpdef close(self):
......
......@@ -8,7 +8,6 @@ from ..capi.obitypes cimport const_char_p, \
obibool_t, \
obichar_t, \
obifloat_t, \
obiidx_t, \
time_t
......@@ -26,6 +25,7 @@ cdef extern from "obidmscolumn.h" nogil:
obiversion_t version
obiversion_t cloned_from
const_char_p name
const_char_p array_name
const_char_p comments
ctypedef OBIDMS_column_header_t* OBIDMS_column_header_p
......@@ -40,7 +40,8 @@ cdef extern from "obidmscolumn.h" nogil:
OBIType_t type,
size_t nb_lines,
size_t nb_elements_per_line,
const_char_p elements_names)
const_char_p elements_names,
const_char_p array_name)
OBIDMS_column_p obi_open_column(OBIDMS_p dms,
const_char_p column_name,
......@@ -146,23 +147,23 @@ cdef extern from "obidmscolumn_float.h" nogil:
size_t line_nb,
size_t element_idx)
cdef extern from "obidmscolumn_idx.h" nogil:
cdef extern from "obidmscolumn_str.h" nogil:
int obi_column_set_obiidx_with_elt_name(OBIDMS_column_p column,
int obi_column_set_obistr_with_elt_name(OBIDMS_column_p column,
size_t line_nb,
const_char_p element_name,
obiidx_t value)
char* value)
int obi_column_set_obiidx_with_elt_idx(OBIDMS_column_p column,
int obi_column_set_obistr_with_elt_idx(OBIDMS_column_p column,
size_t line_nb,
size_t element_idx,
obiidx_t value)
char* value)
obiidx_t obi_column_get_obiidx_with_elt_name(OBIDMS_column_p column,
size_t line_nb,
const_char_p element_name)
const_char_p obi_column_get_obistr_with_elt_name(OBIDMS_column_p column,
size_t line_nb,
const_char_p element_name)
obiidx_t obi_column_get_obiidx_with_elt_idx(OBIDMS_column_p column,
size_t line_nb,
size_t element_idx)
const_char_p obi_column_get_obistr_with_elt_idx(OBIDMS_column_p column,
size_t line_nb,
size_t element_idx)
This diff is collapsed. Click to expand it.
/****************************************************************************
* OBIDMS array header file *
****************************************************************************/
/**
* @file array.h
* @author Celine Mercier
* @date October 19th 2015
* @brief Header file for handling arrays for storing and retrieving data arrays (i.e. character strings).
*/
#ifndef ARRAY_H_
#define ARRAY_H_
#include <stdlib.h>
#include <stdio.h>
#include <stdint.h>
#include <time.h>
#include <sys/types.h>
#include <dirent.h>
#include "obidms.h"
#include "obitypes.h"
#define ARRAY_MAX_NAME (2048) /**< The maximum length of an array name.
*/
#define ARRAY_GROWTH_FACTOR (2) /**< The growth factor when an array is enlarged.
*/
typedef int32_t index_t; /**< Type of the array indices.
*/
typedef char byte_t; /**< Defining byte type since data is stored in bits
* and char (stored on one byte) is the smallest addressable unit.
*/
/**
* @brief OBIDMS array data header structure.
*/
typedef struct OBIDMS_array_data_header {
int header_size; /**< Size of the header in bytes.
*/
int64_t data_size_used; /**< Size of the data used in bytes. // TODO not sure about type
*/
int64_t data_size_max; /**< Max size of the data in bytes. // TODO not sure about type
*/
index_t nb_items; /**< Number of items.
*/
char array_name[ARRAY_MAX_NAME+1]; /**< The array name as a NULL terminated string.
*/
time_t creation_date; /**< Date of creation of the file.
*/
} OBIDMS_array_data_header_t, *OBIDMS_array_data_header_p;
/**
* @brief OBIDMS array data structure.
*/
typedef struct OBIDMS_array_data {
OBIDMS_array_data_header_p header; /**< A pointer to the header of the array data.
*/
byte_t* data; /**< A pointer to the beginning of the data.
*/
} OBIDMS_array_data_t, *OBIDMS_array_data_p;
/**
* @brief OBIDMS array header structure.
*/
typedef struct OBIDMS_array_header {
int header_size; /**< Size of the header in bytes.
*/
index_t nb_items; /**< Number of items in the array.
*/
index_t nb_items_max; /**< Maximum number of items in the array before it has to be enlarged.
*/
char array_name[ARRAY_MAX_NAME+1]; /**< The array name as a NULL terminated string.
*/
time_t creation_date; /**< Date of creation of the file.
*/
} OBIDMS_array_header_t, *OBIDMS_array_header_p;
/**
* @brief OBIDMS array structure.
*/
typedef struct OBIDMS_array {
OBIDMS_array_header_p header; /**< A pointer to the header of the array.
*/
index_t* first; /**< A pointer to the beginning of the array itself.
*/
OBIDMS_array_data_p data; /**< A pointer to the structure containing the data
* that the array references.
*/
DIR* directory; /**< A directory entry usable to
* refer and scan the array directory.
*/
} OBIDMS_array_t, *OBIDMS_array_p;
/**
*
*/
int obi_array_exists(OBIDMS_p dms, const char* array_name);
/**
*
*/
OBIDMS_array_p obi_array(OBIDMS_p dms, const char* array_name);
/**
*
*/
OBIDMS_array_p obi_create_array(OBIDMS_p dms, const char* array_name);
/**
*
*/
OBIDMS_array_p obi_open_array(OBIDMS_p dms, const char* array_name);
/**
*
*/
int obi_close_array(OBIDMS_array_p array);
/**
*
*/
//int obi_delete_array(OBIDMS_p dms, const char* array_name);
/**
* The header is already in the byte array
*/
index_t obi_array_add(OBIDMS_array_p array, byte_t* value);
/**
* Returns the entire array including the header
*/
byte_t* obi_array_get(OBIDMS_array_p array, index_t index);
/**
*
*/
index_t obi_array_search(OBIDMS_array_p array, byte_t* value);
/**
*
*/
byte_t* obi_str_to_obibytes(char* value);
/**
*
*/
const char* obi_obibytes_to_str(byte_t* value_b);
/**
* TODO LATER
*/
// OBIDMS_array_p obi_column_arrayify(FILE, encoding);
#endif /* ARRAY_H_ */
......@@ -14,6 +14,7 @@
#include <stdio.h>
#include <string.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/types.h>
#include <dirent.h>
......@@ -21,6 +22,7 @@
#include "obierrno.h"
#include "obidebug.h"
#include "obidmscolumn.h"
#include "private_at_functions.h"
#define DEBUG_LEVEL 0 // TODO has to be defined somewhere else (cython compil flag?)
......@@ -113,7 +115,9 @@ int obi_dms_exists(const char* dms_name)
OBIDMS_p obi_create_dms(const char* dms_name)
{
char *directory_name;
char* directory_name;
DIR* dms_dir;
int dms_file_descriptor;
// Build and check the directory name
directory_name = build_directory_name(dms_name);
......@@ -132,6 +136,35 @@ OBIDMS_p obi_create_dms(const char* dms_name)
return NULL;
}
// Get file descriptor of DMS directory to create the arrays directory
dms_dir = opendir(directory_name);
if (dms_dir == NULL)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nProblem opening a newly created OBIDMS directory");
free(directory_name);
return NULL;
}
dms_file_descriptor = dirfd(dms_dir);
if (dms_file_descriptor < 0)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nProblem getting the file descriptor of a newly created OBIDMS directory");
free(directory_name);
return NULL;
}
// Create the arrays directory
if (mkdirat(dms_file_descriptor, ARRAY_DIR_NAME, 00777) < 0)
{
obi_set_errno(OBI_ARRAY_ERROR);
obidebug(1, "\nProblem creating an arrays directory");
free(directory_name);
return NULL;
}
// TODO close file descriptor?
free(directory_name);
return obi_open_dms(dms_name);
......@@ -143,6 +176,7 @@ OBIDMS_p obi_open_dms(const char* dms_name)
OBIDMS_p dms;
char* directory_name;
DIR* directory;
int dms_file_descriptor;
dms = NULL;
......@@ -191,8 +225,30 @@ OBIDMS_p obi_open_dms(const char* dms_name)
strcpy(dms->directory_name, directory_name);
dms->directory = directory;
// Get file descriptor of DMS directory to open the arrays directory
dms_file_descriptor = dirfd(directory);
if (dms_file_descriptor < 0)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError getting the file descriptor for a newly created OBIDMS directory");
free(directory_name);
return NULL;
}
// Open the arrays directory
dms->array_directory = private_opendirat(dms_file_descriptor, ARRAY_DIR_NAME);
if (dms->array_directory == NULL)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError opening the arrays directory");
free(directory_name);
return NULL;
}
free(directory_name);
// TODO test if close file descriptor
return dms;
}
......@@ -223,7 +279,14 @@ int obi_close_dms(OBIDMS_p dms)
if (closedir(dms->directory) < 0)
{
obi_set_errno(OBIDMS_MEMORY_ERROR);
obidebug(1, "\nError closing an OBIDSM directory");
obidebug(1, "\nError closing an OBIDMS directory");
free(dms);
return -1;
}
if (closedir(dms->array_directory) < 0)
{
obi_set_errno(OBI_ARRAY_ERROR);
obidebug(1, "\nError closing an array directory");
free(dms);
return -1;
}
......
......@@ -24,8 +24,10 @@
#include "obierrno.h"
#define OBIDMS_MAX_NAME (2048) /**< The maximum length of an OBIDMS name
*/
#define OBIDMS_MAX_NAME (2048) /**< The maximum length of an OBIDMS name.
*/
#define ARRAY_DIR_NAME "arrays" /**< The name of the arrays directory.
*/
/**
......@@ -41,6 +43,9 @@ typedef struct OBIDMS {
DIR* directory; /**< A directory entry usable to
* refer and scan the database directory.
*/
DIR* array_directory; /**< A directory entry usable to
* refer and scan the array directory.
*/
} OBIDMS_t, *OBIDMS_p;
......
......@@ -16,6 +16,7 @@
#include <sys/types.h>
#include <dirent.h>
#include <unistd.h>
#include <time.h>
#include <fcntl.h>
#include <stdbool.h>
#include <math.h>
......@@ -28,6 +29,7 @@
#include "obierrno.h"
#include "obidebug.h"
#include "obilittlebigman.h"
#include "obiarray.h"
#define DEBUG_LEVEL 0 // TODO has to be defined somewhere else (cython compil flag?)
......@@ -151,35 +153,35 @@ static size_t get_line_count_per_page(OBIType_t data_type, size_t nb_elements_pe
************************************************************************/