Commit dfd51939 by Celine Mercier

Views are now rollbacked if an error occurs, and unfinished views and

columns are deleted when an OBIDMS is opened.
parent 1ae634d5
......@@ -102,6 +102,10 @@ cdef extern from "obiview.h" nogil:
int obi_save_and_close_view(Obiview_p view)
int obi_clean_unfinished_views(OBIDMS_p dms)
int obi_rollback_view(Obiview_p view)
# OBI_INT
int obi_set_int_with_elt_name_and_col_p_in_view(Obiview_p view,
......
......@@ -28,11 +28,11 @@ from .object import OBIWrapper
cdef class DMS(OBIWrapper):
cdef inline OBIDMS_p pointer(self):
cdef inline OBIDMS_p pointer(self) :
return <OBIDMS_p>(self._pointer)
@staticmethod
def obi_atexit():
def obi_atexit() :
atexit(obi_close_atexit)
@staticmethod
......@@ -50,7 +50,7 @@ cdef class DMS(OBIWrapper):
return dms
@staticmethod
def exists(object dms_name):
def exists(object dms_name) :
cdef bytes dms_name_b = tobytes(dms_name)
cdef int rep
rep = obi_dms_exists(dms_name_b)
......@@ -90,7 +90,6 @@ cdef class DMS(OBIWrapper):
The `close` method is automatically called by the object destructor.
'''
cdef OBIDMS_p pointer = self.pointer()
if self.active() :
OBIWrapper.close(self)
if (obi_close_dms(pointer)) < 0 :
......
from .view import View # @UnresolvedImport
from .view import Line_selection # @UnresolvedImport
from .view import RollbackException # @UnresolvedImport
\ No newline at end of file
......@@ -10,6 +10,7 @@ from ..capi.obiview cimport Alias_column_pair_p, \
obi_new_view, \
obi_open_view, \
obi_clone_view, \
obi_rollback_view, \
obi_save_and_close_view, \
obi_view_get_pointer_on_column_in_view, \
obi_view_delete_column, \
......@@ -69,7 +70,7 @@ cdef class View(OBIWrapper) :
cdef bytes view_name_b = tobytes(view_name)
cdef bytes comments_b
cdef str message
cdef str message
cdef void* pointer
cdef View view # @DuplicatedSignature
......@@ -103,7 +104,7 @@ cdef class View(OBIWrapper) :
cdef bytes view_name_b = tobytes(view_name)
cdef bytes comments_b
cdef void* pointer
cdef View view
cdef View view
if not self.active() :
raise OBIDeactivatedInstanceError()
......@@ -231,8 +232,8 @@ cdef class View(OBIWrapper) :
# Remove the column from the view which closes the C structure
if obi_view_delete_column(self.pointer(), column_name_b) < 0 :
raise Exception("Problem deleting column %s from a view",
bytes2str(column_name_b))
raise RollbackException("Problem deleting column %s from a view",
bytes2str(column_name_b), self)
cpdef rename_column(self,
......@@ -249,9 +250,9 @@ cdef class View(OBIWrapper) :
if (obi_view_create_column_alias(self.pointer(),
tobytes(current_name_b),
tobytes(new_name_b)) < 0) :
raise Exception("Problem in renaming column %s to %s" % (
raise RollbackException("Problem in renaming column %s to %s" % (
bytes2str(current_name_b),
bytes2str(new_name_b)))
bytes2str(new_name_b)), self)
# TODO warning, not multithreading compliant
......@@ -434,7 +435,7 @@ cdef class Line :
if column_name_b not in self._view :
if value == None :
raise Exception("Trying to create a column from a None value (can't guess type)")
raise RollbackException("Trying to create a column from a None value (can't guess type)", self)
value_type = type(value)
if value_type == int :
value_obitype = OBI_INT
......@@ -454,7 +455,7 @@ cdef class Line :
elif (len(value) > 1) :
value_obitype = OBI_STR
else :
raise Exception("Could not guess the type of a value to create a new column")
raise RollbackException("Could not guess the type of a value to create a new column", self)
Column.new_column(self._view, column_name_b, value_obitype)
......@@ -622,6 +623,19 @@ cdef class Line_selection(list):
#############################################################
class RollbackException(Exception):
def __init__(self, message, View view):
super(RollbackException, self).__init__(message)
if obi_rollback_view(<Obiview_p>(view.pointer())) < 0 :
raise Exception("Error rollbacking view")
if view.active() :
view._dms.unregister(view)
OBIWrapper.close(view)
#############################################################
cdef register_view_class(bytes view_type_name,
type view_class):
'''
......
......@@ -587,6 +587,30 @@ OBIDMS_p obi_open_dms(const char* dms_path)
return NULL;
}
// Clean unfinished views
if (obi_clean_unfinished_views(dms) < 0)
{
obidebug(1, "\nError cleaning unfinished views when opening an OBIDMS");
closedir(dms->indexer_directory);
closedir(dms->tax_directory);
closedir(dms->view_directory);
closedir(dms->directory);
free(dms);
return NULL;
}
// Clean unfinished columns
if (obi_clean_unfinished_columns(dms) < 0)
{
obidebug(1, "\nError cleaning unfinished columns when opening an OBIDMS");
closedir(dms->indexer_directory);
closedir(dms->tax_directory);
closedir(dms->view_directory);
closedir(dms->directory);
free(dms);
return NULL;
}
// Initialize the list of opened columns
dms->opened_columns = (Opened_columns_list_p) malloc(sizeof(Opened_columns_list_t));
(dms->opened_columns)->nb_opened_columns = 0;
......
......@@ -275,7 +275,6 @@ static char* build_column_file_name(const char* column_name, obiversion_t versio
}
static char* build_version_file_name(const char* column_name)
{
char* file_name;
......@@ -300,7 +299,6 @@ static char* build_version_file_name(const char* column_name)
}
static obiversion_t obi_get_new_version_number(OBIDMS_column_directory_p column_directory, bool block)
{
off_t loc_size;
......@@ -425,7 +423,6 @@ static obiversion_t obi_get_new_version_number(OBIDMS_column_directory_p column_
}
static obiversion_t create_version_file(OBIDMS_column_directory_p column_directory)
{
off_t loc_size;
......@@ -714,6 +711,71 @@ static index_t get_line_count_per_page(OBIType_t data_type, index_t nb_elements_
**********************************************************************/
char* obi_version_file_full_path(OBIDMS_p dms, const char* column_name)
{
char* version_file_name;
char* column_dir_name;
char* relative_path;
char* full_path;
version_file_name = build_version_file_name(column_name);
if (version_file_name == NULL)
return NULL;
column_dir_name = obi_build_column_directory_name(column_name);
if (column_dir_name == NULL)
return NULL;
relative_path = (char*) malloc(strlen(version_file_name) + strlen(column_dir_name) + 2);
strcpy(relative_path, column_dir_name);
strcat(relative_path, "/");
strcat(relative_path, version_file_name);
// Build path relative to DMS
full_path = obi_dms_get_full_path(dms, relative_path);
free(version_file_name);
free(column_dir_name);
free(relative_path);
return full_path;
}
char* obi_column_full_path(OBIDMS_p dms, const char* column_name, obiversion_t version_number)
{
char* column_file_name;
char* column_dir_name;
char* relative_path;
char* full_path;
column_file_name = build_column_file_name(column_name, version_number);
if (column_file_name == NULL)
return NULL;
column_dir_name = obi_build_column_directory_name(column_name);
if (column_dir_name == NULL)
return NULL;
relative_path = (char*) malloc(strlen(column_file_name) + strlen(column_dir_name) + 2);
strcpy(relative_path, column_dir_name);
strcat(relative_path, "/");
strcat(relative_path, column_file_name);
// Build path relative to DMS
full_path = obi_dms_get_full_path(dms, relative_path);
free(column_file_name);
free(column_dir_name);
free(relative_path);
return full_path;
}
obiversion_t obi_get_latest_version_number(OBIDMS_column_directory_p column_directory)
{
off_t loc_size;
......@@ -1046,6 +1108,7 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
header->creation_date = time(NULL);
header->version = version_number;
header->cloned_from = -1;
header->finished = false;
set_elements_names(new_column, elements_names, elts_names_length);
......@@ -1833,7 +1896,6 @@ index_t obi_column_get_element_index_from_name(OBIDMS_column_p column, const cha
}
// TODO doc, returns elements names with ; as separator (discuss maybe char**)
char* obi_get_elements_names(OBIDMS_column_p column)
{
char* elements_names;
......@@ -1921,3 +1983,183 @@ int obi_column_prepare_to_get_value(OBIDMS_column_p column, index_t line_nb)
return 0;
}
int obi_clean_unfinished_columns(OBIDMS_p dms)
{
struct dirent* dms_dirent;
struct dirent* col_dirent;
DIR* col_dir;
int i,j;
char* column_file_path;
char* column_dir_path;
char* col_name;
char* col_version_str;
char* version_file;
obiversion_t col_version;
OBIDMS_column_header_p col_header;
int n;
char* col_to_delete[1000];
char* dir_to_delete[1000];
int ddir;
int dcol;
int d;
int ret_value;
ret_value = 0;
// Find column directories
ddir = 0;
while ((dms_dirent = readdir(dms->directory)) != NULL)
{
if ((dms_dirent->d_name)[0] == '.')
continue;
i=0;
while (((dms_dirent->d_name)[i] != '.') && (i < strlen(dms_dirent->d_name)))
i++;
if ((i != strlen(dms_dirent->d_name)) && (strcmp((dms_dirent->d_name)+i, ".obicol") == 0)) // Found a column directory
{
column_dir_path = obi_dms_get_full_path(dms, dms_dirent->d_name);
if (column_dir_path == NULL)
{
obidebug(1, "\nError getting a column directory path when deleting unfinished columns");
ret_value = -1;
}
col_name = (char*) malloc(strlen(dms_dirent->d_name) * sizeof(char));
if (col_name == NULL)
{
obi_set_errno(OBI_MALLOC_ERROR);
obidebug(1, "\nError allocating memory for a column name when deleting unfinished columns: directory %s", dms_dirent->d_name);
ret_value = -1;
continue;
}
strncpy(col_name, dms_dirent->d_name, i);
col_name[i] = '\0';
col_dir = opendir_in_dms(dms, dms_dirent->d_name);
if (col_dir == NULL)
{
obidebug(1, "\nError opening a column directory when deleting unfinished columns");
ret_value = -1;
continue;
}
// Iteration on files of this column directory
dcol = 0;
while ((col_dirent = readdir(col_dir)) != NULL)
{
if ((col_dirent->d_name)[0] == '.')
continue;
i=0;
j=0;
while (((col_dirent->d_name)[i] != '@') && ((col_dirent->d_name)[i] != '.'))
i++;
if ((col_dirent->d_name)[i] == '@') // Found a column file
{
j=i;
while ((col_dirent->d_name)[j] != '.')
j++;
col_version_str = (char*) malloc(strlen(col_dirent->d_name) * sizeof(char));
if (col_version_str == NULL)
{
obi_set_errno(OBI_MALLOC_ERROR);
obidebug(1, "\nError allocating memory for a column version when deleting unfinished columns: directory %s", dms_dirent->d_name);
ret_value = -1;
continue;
}
strncpy(col_version_str, (col_dirent->d_name)+i, j-i);
col_version_str[j-i] = '\0';
col_version = atoi(col_version_str);
free(col_version_str);
col_header = obi_column_get_header_from_name(dms, col_name, col_version);
if (col_header == NULL) // TODO discuss if delete file or not
{
obidebug(1, "\nError reading a column header when deleting unfinished columns: file %s", col_dirent->d_name);
ret_value = -1;
continue;
}
// Check if the column is finished and delete it if not
if (col_header->finished == false)
{
// Build file and dir paths
column_file_path = obi_column_full_path(dms, col_name, col_version);
if (column_file_path == NULL)
{
obidebug(1, "\nError getting a column file path when deleting unfinished columns");
ret_value = -1;
continue;
}
// Add the column path in the list of files to delete (can't delete while in loop)
col_to_delete[dcol] = column_file_path;
dcol++;
}
// Close the header
if (obi_close_header(col_header) < 0)
ret_value = -1;
}
}
// Delete all column files in to_delete list
for (d=0; d<dcol; d++)
{
if (remove(col_to_delete[d]) < 0)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
obidebug(1, "\nError deleting a column file when deleting unfinished columns: file %s", col_to_delete[d]);
ret_value = -1;
}
free(col_to_delete[d]);
}
// Close column directory
if (closedir(col_dir) < 0)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
obidebug(1, "\nError closing a column directory when deleting unfinished columns");
ret_value = -1;
continue;
}
// Add column dir in list to delete if it's empty
n = count_dir(column_dir_path);
if (n == 1) // Only file left is the version file
{
// Delete the version file
version_file = obi_version_file_full_path(dms, col_name);
if (version_file == NULL)
{
obidebug(1, "\nError getting a version file path when deleting unfinished columns");
ret_value = -1;
continue;
}
if (remove(version_file) < 0)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
obidebug(1, "\nError deleting a version file when deleting unfinished columns: file %s", version_file);
ret_value = -1;
}
free(version_file);
dir_to_delete[ddir] = column_dir_path;
ddir++;
}
free(col_name);
}
}
// Delete all column dir in to_delete list
for (d=0; d<ddir; d++)
{
if (remove(dir_to_delete[d]) < 0)
{
obi_set_errno(OBICOL_UNKNOWN_ERROR);
obidebug(1, "\nError deleting a column directory when deleting unfinished columns: directory %s", dir_to_delete[d]);
ret_value = -1;
}
free(dir_to_delete[d]);
}
return ret_value;
}
......@@ -100,6 +100,8 @@ typedef struct OBIDMS_column_header {
*/
Column_reference_t associated_column; /**< If there is one, the reference to the associated column.
*/
bool finished; /**< A boolean indicating whether the column was properly closed by the view that created it. TODO
*/
char comments[COMMENTS_MAX_LENGTH+1]; /**< Comments stored as a classical zero end C string.
*/
} OBIDMS_column_header_t, *OBIDMS_column_header_p;
......@@ -142,6 +144,42 @@ typedef struct OBIDMS_column {
} OBIDMS_column_t, *OBIDMS_column_p;
/**
* @brief Function building the full path to the version file of a column in an OBIDMS.
*
* @warning The returned pointer has to be freed by the caller.
*
* @param dms A pointer on the OBIDMS.
* @param column_name The name of the OBIDMS column file.
*
* @returns A pointer to the version file name.
* @retval NULL if an error occurred.
*
* @since October 2017
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
char* obi_version_file_full_path(OBIDMS_p dms, const char* column_name);
/**
* @brief Function building the full path to the version file of a column in an OBIDMS.
*
* @warning The returned pointer has to be freed by the caller.
*
* @param dms A pointer on the OBIDMS.
* @param column_name The name of the OBIDMS column file.
* @param version_number The version number of the OBIDMS column file.
*
* @returns A pointer to the version file name.
* @retval NULL if an error occurred.
*
* @since October 2017
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
char* obi_column_full_path(OBIDMS_p dms, const char* column_name, obiversion_t version_number);
/**
* @brief Returns the latest version number of a column in a column directory using the column directory structure.
*
......@@ -277,6 +315,8 @@ int obi_clone_column_indexer(OBIDMS_column_p column);
/**
* @brief Truncates a column to the number of lines used if it is not read-only and closes it.
*
* @warning This function does not flag the column as finished, only finish_view() in the obiview source file does that.
*
* @param column A pointer on an OBIDMS column.
*
* @retval 0 if the operation was successfully completed.
......@@ -424,4 +464,21 @@ int obi_column_prepare_to_set_value(OBIDMS_column_p column, index_t line_nb, ind
int obi_column_prepare_to_get_value(OBIDMS_column_p column, index_t line_nb);
/**
* @brief Goes through all the column files of a DMS and deletes columns that have
* not been flagged as finished (done by the finish_view() function in the
* obiview source file).
*
* @param dms A pointer on an OBIDMS.
*
* @returns A value indicating the success of the operation.
* @retval 0 if the operation was successfully completed.
* @retval -1 if an error occurred.
*
* @since October 2017
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
int obi_clean_unfinished_columns(OBIDMS_p dms);
#endif /* OBIDMSCOLUMN_H_ */
......@@ -28,39 +28,15 @@
#define DEBUG_LEVEL 0 // TODO has to be defined somewhere else (cython compil flag?)
/**************************************************************************
*
* D E C L A R A T I O N O F T H E P R I V A T E F U N C T I O N S
*
**************************************************************************/
/**
* Internal function building the column directory name from an OBIDMS column name.
*
* The function builds the directory name corresponding to an OBIDMS column directory.
* It checks also that the name is not too long.
*
* @warning The returned pointer has to be freed by the caller.
*
* @param column_name The name of the OBIDMS column.
/**********************************************************************
*
* @returns A pointer to the OBIDMS column directory name.
* @retval NULL if an error occurred.
* D E F I N I T I O N O F T H E P U B L I C F U N C T I O N S
*
* @since June 2015
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
static char* build_column_directory_name(const char* column_name);
**********************************************************************/
/************************************************************************
*
* D E F I N I T I O N O F T H E P R I V A T E F U N C T I O N S
*
************************************************************************/
static char* build_column_directory_name(const char* column_name)
char* obi_build_column_directory_name(const char* column_name)
{
char* column_directory_name;
......@@ -86,12 +62,6 @@ static char* build_column_directory_name(const char* column_name)
}
/**********************************************************************
*
* D E F I N I T I O N O F T H E P U B L I C F U N C T I O N S
*
**********************************************************************/
int obi_column_directory_exists(OBIDMS_p dms, const char* column_name)
{
struct stat buffer;
......@@ -100,7 +70,7 @@ int obi_column_directory_exists(OBIDMS_p dms, const char* column_name)
int check_dir;
// Build and check the directory name
column_directory_name = build_column_directory_name(column_name);
column_directory_name = obi_build_column_directory_name(column_name);
if (column_directory_name == NULL)
return -1;
......@@ -131,7 +101,7 @@ OBIDMS_column_directory_p obi_create_column_directory(OBIDMS_p dms, const char*
char* column_directory_name;
// Build and check the directory name
column_directory_name = build_column_directory_name(column_name);
column_directory_name = obi_build_column_directory_name(column_name);
if (column_directory_name == NULL)
{
obi_set_errno(OBICOLDIR_UNKNOWN_ERROR);
......@@ -165,7 +135,7 @@ OBIDMS_column_directory_p obi_open_column_directory(OBIDMS_p dms, const char* co
column_directory = NULL;
// Build and check the directory name
column_directory_name = build_column_directory_name(column_name);
column_directory_name = obi_build_column_directory_name(column_name);
if (column_directory_name == NULL)
return NULL;
......
......@@ -50,6 +50,25 @@ typedef struct OBIDMS_column_directory {
/**
* Function building the column directory name from an OBIDMS column name.
*
* The function builds the directory name corresponding to an OBIDMS column directory.
* It checks also that the name is not too long.
*
* @warning The returned pointer has to be freed by the caller.
*
* @param column_name The name of the OBIDMS column.
*
* @returns A pointer to the OBIDMS column directory name.
* @retval NULL if an error occurred.
*
* @since June 2015
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
char* obi_build_column_directory_name(const char* column_name);
/**
* @brief Checks if an OBIDMS column directory exists.
*
* @param dms A pointer to an OBIDMS as returned by obi_create_dms() or obi_open_dms().
......
......@@ -1062,7 +1062,9 @@ static int rename_finished_view(Obiview_p view)
static int finish_view(Obiview_p view)
{
char* predicates;
char* predicates;
int i;
OBIDMS_column_p column;
// Check that the view is not read-only
if (view->read_only)
......@@ -1111,6 +1113,19 @@ static int finish_view(Obiview_p view)
if (rename_finished_view(view) < 0)
return -1;
// Flag the columns as finished
for (i=0; i < ((view->infos)->column_count); i++)
{
column = *((OBIDMS_column_p*)ll_get(view->columns, i));
if (column == NULL)
{
obi_set_errno(OBIVIEW_ERROR);
obidebug(1, "\nError getting a column to flag it as finished when finishing a view");
return -1;
}
(column->header)->finished = true;
}
// Flag the view as finished
(view->infos)->finished = true;
......@@ -2502,6 +2517,271 @@ int obi_save_and_close_view(Obiview_p view)
}
int obi_clean_unfinished_views(OBIDMS_p dms)
{
struct dirent* dp;
int i;
char* full_path;
char* relative_path;
Obiview_infos_p view_infos;
char* view_name;
int ret_value;
char* to_delete[1000];
int d;
ret_value = 0;
d = 0;
// Look for unfinished views and delete them
while ((dp = readdir(dms->view_directory)) != NULL)
{
if ((dp->d_name)[0] == '.')
continue;
i=0;
while ((dp->d_name)[i] != '.')
i++;
relative_path = (char*) malloc(strlen(VIEW_DIR_NAME) + strlen(dp->d_name) + 2);
strcpy(relative_path, VIEW_DIR_NAME);
strcat(relative_path, "/");
strcat(relative_path, dp->d_name);
full_path = obi_dms_get_full_path(dms, relative_path);
free(relative_path);
if (full_path == NULL)
{
obidebug(1, "\nError getting the full path to a view file when cleaning unfinished views");
ret_value = -1;
continue;
}
if (strcmp((dp->d_name)+i, ".obiview_unfinished") == 0)
{
// Add to the list of files to delete (deleting in loop not safe)
to_delete[d] = full_path;
d++;
}
else if (strcmp((dp->d_name)+i, ".obiview") == 0)
{ // Check if the view was properly flagged as finished
view_name = (char*) malloc((i+1) * sizeof(char));
if (view_name == NULL)
{
obi_set_errno(OBI_MALLOC_ERROR);
obidebug(1, "\nError allocating memory for a view name when deleting unfinished views: file %s", dp->d_name);
ret_value = -1;
continue;
}
strncpy(view_name, dp->d_name, i);
view_name[i] = '\0';
view_infos = obi_view_map_file(dms, view_name, true);
if (view_infos == NULL)
{
obidebug(1, "\nError reading a view file when deleting unfinished views: file %s", dp->d_name);
ret_value = -1;
continue;
}