Commit 2736a926 authored by Celine Mercier's avatar Celine Mercier

DMS: implemented full information file with JSON formatted comments

parent 79f41857
......@@ -20,6 +20,7 @@
#include <dirent.h>
#include <unistd.h>
#include <libgen.h> /* <EC> : Added July 28th 2017 to include basename */
#include <sys/mman.h>
#include "obidms.h"
#include "obierrno.h"
......@@ -29,6 +30,7 @@
#include "obiblob_indexer.h"
#include "utils.h"
#include "obilittlebigman.h"
#include "libjson/json_utils.h"
#define DEBUG_LEVEL 0 // TODO has to be defined somewhere else (cython compil flag?)
......@@ -75,7 +77,7 @@ static char* build_directory_name(const char* dms_name);
*
* @param dms_name The name of the OBIDMS.
*
* @returns A pointer to the file name.
* @returns A pointer on the file name.
* @retval NULL if an error occurred.
*
* @since November 2015
......@@ -84,16 +86,73 @@ static char* build_directory_name(const char* dms_name);
static char* build_infos_file_name(const char* dms_name);
/**
* Internal function calculating the initial size of the file where the informations about a DMS are stored.
*
* @returns The initial size of the file in bytes, rounded to a multiple of page size.
*
* @since September 2018
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
static size_t get_platform_infos_file_size();
/**
* @brief Internal function enlarging a DMS information file.
*
* @param dms A pointer on the DMS.
* @param new_size The new size needed, in bytes (not necessarily rounded to a page size multiple).
*
* @retval 0 if the operation was successfully completed.
* @retval -1 if an error occurred.
*
* @since September 2018
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
static int enlarge_infos_file(OBIDMS_p dms, size_t new_size);
/**
* @brief Internal function mapping a DMS information file and returning the mapped structure stored in it.
*
* The function checks that endianness of the platform and of the DMS match.
*
* @param dms_file_descriptor The file descriptor of the DMS directory.
* @param dms_name The base name of the DMS.
*
* @returns A pointer on the mapped DMS infos structure.
* @retval NULL if an error occurred.
*
* @since September 2018
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
static OBIDMS_infos_p map_infos_file(int dms_file_descriptor, const char* dms_name);
/**
* @brief Unmaps a DMS information file.
*
* @param dms A pointer on the OBIDMS.
*
* @returns A value indicating the success of the operation.
* @retval 0 if the operation was successfully completed.
* @retval -1 if an error occurred.
*
* @since October 2018
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
static int unmap_infos_file(OBIDMS_p dms);
/**
* Internal function creating the file containing basic informations on the OBIDMS.
*
* This file contains:
* - The endianness of the platform
* This file contains an OBIDMS_infos structure.
*
* @warning The returned pointer has to be freed by the caller.
*
* @param dms_file_descriptor The file descriptor for the OBIDMS directory.
* @param dms_name The name of the OBIDMS.
* @param dms_file_descriptor The file descriptor of the DMS directory.
* @param dms_name The base name of the DMS.
*
* @retval 0 if the operation was successfully completed.
* @retval -1 if an error occurred.
......@@ -101,7 +160,7 @@ static char* build_infos_file_name(const char* dms_name);
* @since November 2015
* @author Celine Mercier (celine.mercier@metabarcoding.org)
*/
int create_dms_infos_file(int dms_file_descriptor, const char* dms_name);
static int create_dms_infos_file(int dms_file_descriptor, const char* dms_name);
/**
......@@ -211,16 +270,246 @@ static char* build_infos_file_name(const char* dms_name)
}
int create_dms_infos_file(int dms_file_descriptor, const char* dms_name)
static size_t get_platform_infos_file_size()
{
char* file_name;
int infos_file_descriptor;
off_t file_size;
bool little_endian;
size_t infos_size;
size_t rounded_infos_size;
double multiple;
file_size = sizeof(bool);
infos_size = sizeof(OBIDMS_infos_t);
multiple = ceil((double) (infos_size) / (double) getpagesize());
rounded_infos_size = multiple * getpagesize();
return rounded_infos_size;
}
static int enlarge_infos_file(OBIDMS_p dms, size_t new_size)
{
int infos_file_descriptor;
double multiple;
size_t rounded_new_size;
char* file_name;
// Create file name
file_name = build_infos_file_name(dms->dms_name);
if (file_name == NULL)
return -1;
// Open infos file
infos_file_descriptor = openat(dms->dir_fd, file_name, O_RDWR, 0777);
if (infos_file_descriptor < 0)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError opening a DMS information file");
free(file_name);
return -1;
}
free(file_name);
// Round new size to a multiple of page size // TODO make function in utils
multiple = ceil((double) new_size / (double) getpagesize());
rounded_new_size = multiple * getpagesize();
// Enlarge the file
if (ftruncate(infos_file_descriptor, rounded_new_size) < 0)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError enlarging a DMS information file");
close(infos_file_descriptor);
return -1;
}
// Unmap and remap the file
if (munmap(dms->infos, (dms->infos)->file_size) < 0)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError munmapping a DMS information file when enlarging");
close(infos_file_descriptor);
return -1;
}
dms->infos = mmap(NULL,
rounded_new_size,
PROT_READ | PROT_WRITE,
MAP_SHARED,
infos_file_descriptor,
0
);
if (dms->infos == MAP_FAILED)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError re-mmapping a DMS information file after enlarging the file");
close(infos_file_descriptor);
return -1;
}
// Set new size
(dms->infos)->file_size = rounded_new_size;
if (close(infos_file_descriptor) < 0)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError closing a DMS information file");
return -1;
}
return 0;
}
static OBIDMS_infos_p map_infos_file(int dms_file_descriptor, const char* dms_name)
{
char* file_name;
OBIDMS_infos_p dms_infos;
int file_descriptor;
size_t min_size;
size_t full_size;
bool little_endian_platform;
// Build file name
file_name = build_infos_file_name(dms_name);
if (file_name == NULL)
return NULL;
// Open file
file_descriptor = openat(dms_file_descriptor,
file_name,
O_RDWR, 0777);
if (file_descriptor < 0)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError opening a DMS information file (DMS %s)", dms_name);
free(file_name);
return NULL;
}
free(file_name);
// Map minimum size to read endianness and file size
// (fread() fails to read properly from the structure in the file because of padding)
min_size = get_platform_infos_file_size();
dms_infos = mmap(NULL,
min_size,
PROT_READ | PROT_WRITE,
MAP_SHARED,
file_descriptor,
0
);
if (dms_infos == NULL)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError mapping a DMS information file");
return NULL;
}
// Check endianness of the platform and DMS
little_endian_platform = obi_is_little_endian();
if (little_endian_platform != dms_infos->little_endian)
{
obi_set_errno(OBIDMS_BAD_ENDIAN_ERROR);
obidebug(1, "\nError: The DMS and the platform have different endianness");
close(file_descriptor);
return NULL;
}
// Read actual file size
full_size = dms_infos->file_size;
// Unmap the minimum size and remap the file with the full size
if (munmap(dms_infos, min_size) < 0)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError munmapping a DMS information file");
close(file_descriptor);
return NULL;
}
dms_infos = mmap(NULL,
full_size,
PROT_READ | PROT_WRITE,
MAP_SHARED,
file_descriptor,
0
);
if (dms_infos == NULL)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError mapping a DMS information file");
return NULL;
}
if (close(file_descriptor) < 0)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError closing a DMS information file");
return NULL;
}
return dms_infos;
}
static int unmap_infos_file(OBIDMS_p dms)
{
OBIDMS_infos_p dms_infos;
char* file_name;
int file_descriptor;
size_t file_size;
dms_infos = dms->infos;
// Build file name
file_name = build_infos_file_name(dms->dms_name);
if (file_name == NULL)
return -1;
// Open file
file_descriptor = openat(dms->dir_fd,
file_name,
O_RDWR, 0777);
if (file_descriptor < 0)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError opening a DMS information file (DMS %s) to unmap it", dms->dms_name);
free(file_name);
return -1;
}
free(file_name);
// Unmap the DMS infos structure
file_size = dms_infos->file_size;
if (munmap(dms_infos, file_size) < 0)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError unmapping a DMS information file");
close(file_descriptor);
return -1;
}
if (close(file_descriptor) < 0)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError closing a DMS information file");
return -1;
}
return 0;
}
static int create_dms_infos_file(int dms_file_descriptor, const char* dms_name)
{
char* file_name;
int infos_file_descriptor;
size_t file_size;
// Build file name
file_name = build_infos_file_name(dms_name);
if (file_name == NULL)
return -1;
......@@ -228,41 +517,56 @@ int create_dms_infos_file(int dms_file_descriptor, const char* dms_name)
// Create file
infos_file_descriptor = openat(dms_file_descriptor,
file_name,
O_RDWR | O_CREAT | O_EXCL, 0777);
O_RDWR | O_CREAT | O_EXCL,
0777);
if (infos_file_descriptor < 0)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError creating an informations file");
obidebug(1, "\nError creating a DMS information file (DMS %s)", dms_name);
free(file_name);
return -1;
}
free(file_name);
// Truncate the infos file to the right size
// Truncate file to the initial size
file_size = get_platform_infos_file_size();
if (ftruncate(infos_file_descriptor, file_size) < 0)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError truncating an informations file");
obidebug(1, "\nError truncating a DMS information file to the right size");
close(infos_file_descriptor);
return -1;
}
// Write endianness
little_endian = obi_is_little_endian();
if (write(infos_file_descriptor, &little_endian, sizeof(bool)) < ((ssize_t) sizeof(bool)))
// Map the DMS infos structure
OBIDMS_infos_p infos = mmap(NULL,
file_size,
PROT_READ | PROT_WRITE,
MAP_SHARED,
infos_file_descriptor,
0
);
// Initialize values
infos->little_endian = obi_is_little_endian();
infos->file_size = file_size;
infos->used_size = 0;
infos->comments[0] = '\0';
// Unmap the infos file
if (munmap(infos, file_size) < 0)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError writing the endianness in an informations file");
close(infos_file_descriptor);
obi_set_errno(OBIVIEW_ERROR);
obidebug(1, "\nError munmapping a DMS information file");
return -1;
}
// Close file
if (close(infos_file_descriptor) < 0)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError closing a DMS information file");
obidebug(1, "\nError closing a view file");
return -1;
}
......@@ -371,9 +675,10 @@ int obi_dms_exists(const char* dms_path)
OBIDMS_p obi_create_dms(const char* dms_path)
{
char* directory_name;
DIR* dms_dir;
int dms_file_descriptor;
char* directory_name;
DIR* dms_dir;
int dms_file_descriptor;
OBIDMS_p dms;
// Build and check the directory name
directory_name = build_directory_name(dms_path);
......@@ -439,22 +744,26 @@ OBIDMS_p obi_create_dms(const char* dms_path)
return NULL;
}
/*
// Isolate the dms name
j = 0;
for (i=0; i<strlen(dms_path); i++)
// Create the information file
if (create_dms_infos_file(dms_file_descriptor, basename((char*)dms_path)) < 0)
return NULL;
// Open DMS
dms = obi_open_dms(dms_path);
if (dms == NULL)
{
if (dms_path[i] == '/')
j = i+1;
i++;
obidebug(1, "\nProblem opening a DMS");
return NULL;
}
*/
// Create the informations file
if (create_dms_infos_file(dms_file_descriptor, basename((char*)dms_path)) < 0)
// Write empty json string in comments (Python json lib doesn't like empty "")
if (obi_dms_write_comments(dms, "{}") < 0)
{
obidebug(1, "\nProblem initializing empty json string in new DMS comments");
return NULL;
}
return obi_open_dms(dms_path);
return dms;
}
......@@ -463,10 +772,6 @@ OBIDMS_p obi_open_dms(const char* dms_path)
OBIDMS_p dms;
char* relative_dms_path;
char* absolute_dms_path;
char* infos_file_name;
int infos_file_descriptor;
bool little_endian_dms;
bool little_endian_platform;
dms = NULL;
......@@ -534,7 +839,7 @@ OBIDMS_p obi_open_dms(const char* dms_path)
return NULL;
}
// Get and store file descriptor of DMS directory to open the informations file
// Get and store file descriptor of DMS directory
dms->dir_fd = dirfd(dms->directory);
if (dms->dir_fd < 0)
{
......@@ -545,56 +850,23 @@ OBIDMS_p obi_open_dms(const char* dms_path)
return NULL;
}
// Open informations file to check endianness
infos_file_name = build_infos_file_name(dms->dms_name);
infos_file_descriptor = openat(dms->dir_fd, infos_file_name, O_RDONLY, 0777);
if (infos_file_descriptor < 0)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError opening an informations file");
closedir(dms->directory);
free(dms);
return NULL;
}
free(infos_file_name);
// Check endianness of the platform and DMS
little_endian_platform = obi_is_little_endian();
if (read(infos_file_descriptor, &little_endian_dms, sizeof(bool)) < ((ssize_t) sizeof(bool)))
// Open the information file
dms->infos = map_infos_file(dms->dir_fd, dms->dms_name);
if (dms->infos == NULL)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError reading the endianness in an informations file");
close(infos_file_descriptor);
closedir(dms->directory);
free(dms);
return NULL;
}
if (little_endian_platform != little_endian_dms)
{
obi_set_errno(OBIDMS_BAD_ENDIAN_ERROR);
obidebug(1, "\nError: The DMS and the platform have different endianness");
close(infos_file_descriptor);
obidebug(1, "\nError opening a DMS information file");
closedir(dms->directory);
free(dms);
return NULL;
}
if (close(infos_file_descriptor) < 0)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError closing a DMS information file");
return NULL;
}
dms->little_endian = little_endian_dms;
// Open the indexer directory
dms->indexer_directory = opendir_in_dms(dms, INDEXER_DIR_NAME);
if (dms->indexer_directory == NULL)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError opening the indexer directory");
unmap_infos_file(dms);
closedir(dms->directory);
free(dms);
return NULL;
......@@ -606,6 +878,7 @@ OBIDMS_p obi_open_dms(const char* dms_path)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError getting the file descriptor of the indexer directory");
unmap_infos_file(dms);
closedir(dms->indexer_directory);
closedir(dms->directory);
free(dms);
......@@ -618,6 +891,7 @@ OBIDMS_p obi_open_dms(const char* dms_path)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError opening the view directory");
unmap_infos_file(dms);
closedir(dms->indexer_directory);
closedir(dms->directory);
free(dms);
......@@ -630,6 +904,7 @@ OBIDMS_p obi_open_dms(const char* dms_path)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError getting the file descriptor of the view directory");
unmap_infos_file(dms);
closedir(dms->view_directory);
closedir(dms->directory);
free(dms);
......@@ -642,6 +917,7 @@ OBIDMS_p obi_open_dms(const char* dms_path)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError opening the taxonomy directory");
unmap_infos_file(dms);
closedir(dms->indexer_directory);
closedir(dms->view_directory);
closedir(dms->directory);
......@@ -655,6 +931,7 @@ OBIDMS_p obi_open_dms(const char* dms_path)
{
obi_set_errno(OBIDMS_UNKNOWN_ERROR);
obidebug(1, "\nError getting the file descriptor of the taxonomy directory");
unmap_infos_file(dms);
closedir(dms->indexer_directory);
closedir(dms->tax_directory);
closedir(dms->view_directory);
......@@ -667,6 +944,7 @@ OBIDMS_p obi_open_dms(const char* dms_path)
if (obi_clean_unfinished_views(dms) < 0)
{
obidebug(1, "\nError cleaning unfinished views when opening an OBIDMS");
unmap_infos_file(dms);
closedir(dms->indexer_directory);
closedir(dms->tax_directory);
closedir(dms->view_directory);
......@@ -679,6 +957,7 @@ OBIDMS_p obi_open_dms(const char* dms_path)
if (obi_clean_unfinished_columns(dms) < 0)
{
obidebug(1, "\nError cleaning unfinished columns when opening an OBIDMS");
unmap_infos_file(dms);
closedir(dms->indexer_directory);
closedir(dms->tax_directory);
closedir(dms->view_directory);
......@@ -699,6 +978,7 @@ OBIDMS_p obi_open_dms(const char* dms_path)
if (list_dms(dms) < 0)
{
obidebug(1, "\nError cleaning unfinished columns when opening an OBIDMS");
unmap_infos_file(dms);
closedir(dms->indexer_directory);
closedir(dms->tax_directory);
closedir(dms->view_directory);
......@@ -774,6 +1054,14 @@ int obi_close_dms(OBIDMS_p dms, bool force)
if (dms != NULL)
{
// Unmap information file
if (unmap_infos_file(dms) < 0)
{
obidebug(1, "\nError unmaping a DMS information file while closing a DMS");
free(dms);
return -1;
}
// Close all columns
while ((dms->opened_columns)->nb_opened_columns > 0)
obi_close_column(*((dms->opened_columns)->columns));
......@@ -823,6 +1111,53 @@ int obi_close_dms(OBIDMS_p dms, bool force)
}
int obi_dms_write_comments(OBIDMS_p dms, const char* comments)
{
size_t new_size;
if (comments == NULL)
return 0; // TODO or error? discuss
new_size = sizeof(OBIDMS_infos_t) + strlen(comments) + 1;