Commit 40859043 by Celine Mercier

Merge branch 'multiple_avls_bloom'

parents b04b4b59 3681cecb
......@@ -22,3 +22,7 @@
../../../src/obidmscolumn_idx.c
../../../src/obidms_taxonomy.c
../../../src/obidms_taxonomy.h
../../../src/bloom.c
../../../src/bloom.h
../../../src/MurmurHash2.c
../../../src/murmurhash2.h
\ No newline at end of file
//-----------------------------------------------------------------------------
// MurmurHash2, by Austin Appleby
// Note - This code makes a few assumptions about how your machine behaves -
// 1. We can read a 4-byte value from any address without crashing
// 2. sizeof(int) == 4
// And it has a few limitations -
// 1. It will not work incrementally.
// 2. It will not produce the same results on little-endian and big-endian
// machines.
unsigned int murmurhash2(const void * key, int len, const unsigned int seed)
{
// 'm' and 'r' are mixing constants generated offline.
// They're not really 'magic', they just happen to work well.
const unsigned int m = 0x5bd1e995;
const int r = 24;
// Initialize the hash to a 'random' value
unsigned int h = seed ^ len;
// Mix 4 bytes at a time into the hash
const unsigned char * data = (const unsigned char *)key;
while(len >= 4)
{
unsigned int k = *(unsigned int *)data;
k *= m;
k ^= k >> r;
k *= m;
h *= m;
h ^= k;
data += 4;
len -= 4;
}
// Handle the last few bytes of the input array
switch(len)
{
case 3: h ^= data[2] << 16;
case 2: h ^= data[1] << 8;
case 1: h ^= data[0];
h *= m;
};
// Do a few final mixes of the hash to ensure the last few
// bytes are well-incorporated.
h ^= h >> 13;
h *= m;
h ^= h >> 15;
return h;
}
/*
* Copyright (c) 2012-2015, Jyri J. Virkki
* All rights reserved.
*
* This file is under BSD license. See LICENSE file.
*/
/*
* Refer to bloom.h for documentation on the public interfaces.
*/
#include <assert.h>
#include <fcntl.h>
#include <math.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include "bloom.h"
#include "murmurhash2.h"
#define MAKESTRING(n) STRING(n)
#define STRING(n) #n
#ifdef __linux__
unsigned detect_bucket_size(unsigned fallback_size);
#endif
static int test_bit_set_bit(unsigned char * buf, unsigned int x, int set_bit)
{
register uint32_t * word_buf = (uint32_t *)buf;
register unsigned int offset = x >> 5;
register uint32_t word = word_buf[offset];
register unsigned int mask = 1 << (x % 32);
if (word & mask) {
return 1;
} else {
if (set_bit) {
word_buf[offset] = word | mask;
}
return 0;
}
}
static int bloom_check_add(struct bloom * bloom,
const void * buffer, int len, int add)
{
if (bloom->ready == 0) {
(void)printf("bloom at %p not initialized!\n", (void *)bloom);
return -1;
}
int hits = 0;
register unsigned int a = murmurhash2(buffer, len, 0x9747b28c);
register unsigned int b = murmurhash2(buffer, len, a);
register unsigned int x;
register unsigned int i;
unsigned bucket_index = (a % bloom->buckets);
unsigned char * bucket_ptr =
(bloom->bf + (bucket_index << bloom->bucket_bytes_exponent));
for (i = 0; i < bloom->hashes; i++) {
x = (a + i*b) & bloom->bucket_bits_fast_mod_operand;
if (test_bit_set_bit(bucket_ptr, x, add)) {
hits++;
}
}
if (hits == bloom->hashes) {
return 1; // 1 == element already in (or collision)
}
return 0;
}
static void setup_buckets(struct bloom * bloom, unsigned int cache_size)
{
// If caller passed a non-zero cache_size, use it as given, otherwise
// either compute it or use built-in default
if (cache_size == 0) {
#ifdef __linux__
cache_size = detect_bucket_size(BLOOM_BUCKET_SIZE_FALLBACK);
#else
cache_size = BLOOM_BUCKET_SIZE_FALLBACK;
#endif
}
bloom->buckets = (bloom->bytes / cache_size);
bloom->bucket_bytes = cache_size;
// make sure bloom buffer bytes and bucket_bytes are even
int not_even_by = (bloom->bytes % bloom->bucket_bytes);
if (not_even_by) {
// adjust bytes
bloom->bytes += (bloom->bucket_bytes - not_even_by);
assert((bloom->bytes % bloom->bucket_bytes) == 0); // Should get even
// adjust bits
bloom->bits = bloom->bytes * 8;
// adjust bits per element
bloom->bpe = bloom->bits*1. / bloom->entries;
// adjust buckets
bloom->buckets++;
}
bloom->bucket_bytes_exponent = __builtin_ctz(cache_size);
bloom->bucket_bits_fast_mod_operand = (cache_size * 8 - 1);
}
int bloom_init_size(struct bloom * bloom, int entries, double error,
unsigned int cache_size)
{
bloom->ready = 0;
if (entries < 1 || error == 0) {
return 1;
}
bloom->entries = entries;
bloom->error = error;
double num = log(bloom->error);
double denom = 0.480453013918201; // ln(2)^2
bloom->bpe = -(num / denom);
double dentries = (double)entries;
bloom->bits = (int)(dentries * bloom->bpe);
if (bloom->bits % 8) {
bloom->bytes = (bloom->bits / 8) + 1;
} else {
bloom->bytes = bloom->bits / 8;
}
bloom->hashes = (int)ceil(0.693147180559945 * bloom->bpe); // ln(2)
setup_buckets(bloom, cache_size);
bloom->bf = (unsigned char *)calloc(bloom->bytes, sizeof(unsigned char));
if (bloom->bf == NULL) {
return 1;
}
bloom->ready = 1;
return 0;
}
int bloom_init(struct bloom * bloom, int entries, double error)
{
return bloom_init_size(bloom, entries, error, 0);
}
int bloom_check(struct bloom * bloom, const void * buffer, int len)
{
return bloom_check_add(bloom, buffer, len, 0);
}
int bloom_add(struct bloom * bloom, const void * buffer, int len)
{
return bloom_check_add(bloom, buffer, len, 1);
}
void bloom_print(struct bloom * bloom)
{
(void)printf("bloom at %p\n", (void *)bloom);
(void)printf(" ->entries = %d\n", bloom->entries);
(void)printf(" ->error = %f\n", bloom->error);
(void)printf(" ->bits = %d\n", bloom->bits);
(void)printf(" ->bits per elem = %f\n", bloom->bpe);
(void)printf(" ->bytes = %d\n", bloom->bytes);
(void)printf(" ->buckets = %u\n", bloom->buckets);
(void)printf(" ->bucket_bytes = %u\n", bloom->bucket_bytes);
(void)printf(" ->bucket_bytes_exponent = %u\n",
bloom->bucket_bytes_exponent);
(void)printf(" ->bucket_bits_fast_mod_operand = 0%o\n",
bloom->bucket_bits_fast_mod_operand);
(void)printf(" ->hash functions = %d\n", bloom->hashes);
}
void bloom_free(struct bloom * bloom)
{
if (bloom->ready) {
free(bloom->bf);
}
bloom->ready = 0;
}
const char * bloom_version()
{
return MAKESTRING(BLOOM_VERSION);
}
/*
* Copyright (c) 2012-2015, Jyri J. Virkki
* All rights reserved.
*
* This file is under BSD license. See LICENSE file.
*/
#ifndef _BLOOM_H
#define _BLOOM_H
/** ***************************************************************************
* On Linux, the code attempts to compute a bucket size based on CPU cache
* size info, if available. If that fails for any reason, this fallback size
* is used instead.
*
* On non-Linux systems, this is the bucket size always used unless the
* caller overrides it (see bloom_init_size()).
*
*/
#define BLOOM_BUCKET_SIZE_FALLBACK (32 * 1024)
/** ***************************************************************************
* It was found that using multiplier x0.5 for CPU L1 cache size is
* more effective in terms of CPU usage and, surprisingly, collisions
* number.
*
* Feel free to tune this constant the way it will work for you.
*
*/
#define BLOOM_L1_CACHE_SIZE_DIV 1
/** ***************************************************************************
* Structure to keep track of one bloom filter. Caller needs to
* allocate this and pass it to the functions below. First call for
* every struct must be to bloom_init().
*
*/
struct bloom
{
// These fields are part of the public interface of this structure.
// Client code may read these values if desired. Client code MUST NOT
// modify any of these.
int entries;
double error;
int bits;
int bytes;
int hashes;
// Fields below are private to the implementation. These may go away or
// change incompatibly at any moment. Client code MUST NOT access or rely
// on these.
unsigned buckets;
unsigned bucket_bytes;
// x86 CPU divide by/multiply by operation optimization helpers
unsigned bucket_bytes_exponent;
unsigned bucket_bits_fast_mod_operand;
double bpe;
unsigned char * bf;
int ready;
};
/** ***************************************************************************
* Initialize the bloom filter for use.
*
* The filter is initialized with a bit field and number of hash functions
* according to the computations from the wikipedia entry:
* http://en.wikipedia.org/wiki/Bloom_filter
*
* Optimal number of bits is:
* bits = (entries * ln(error)) / ln(2)^2
*
* Optimal number of hash functions is:
* hashes = bpe * ln(2)
*
* Parameters:
* -----------
* bloom - Pointer to an allocated struct bloom (see above).
* entries - The expected number of entries which will be inserted.
* error - Probability of collision (as long as entries are not
* exceeded).
*
* Return:
* -------
* 0 - on success
* 1 - on failure
*
*/
int bloom_init(struct bloom * bloom, int entries, double error);
/** ***************************************************************************
* Initialize the bloom filter for use.
*
* See comments above for general information.
*
* This is the same as bloom_init() but allows the caller to pass in a
* cache_size to override the internal value (which is either computed
* or the default of BLOOM_BUCKET_SIZE_FALLBACK). Mostly useful for
* experimenting.
*
* See misc/bucketsize for a script which can help identify a good value
* for cache_size.
*
*/
int bloom_init_size(struct bloom * bloom, int entries, double error,
unsigned int cache_size);
/** ***************************************************************************
* Check if the given element is in the bloom filter. Remember this may
* return false positive if a collision occured.
*
* Parameters:
* -----------
* bloom - Pointer to an allocated struct bloom (see above).
* buffer - Pointer to buffer containing element to check.
* len - Size of 'buffer'.
*
* Return:
* -------
* 0 - element is not present
* 1 - element is present (or false positive due to collision)
* -1 - bloom not initialized
*
*/
int bloom_check(struct bloom * bloom, const void * buffer, int len);
/** ***************************************************************************
* Add the given element to the bloom filter.
* The return code indicates if the element (or a collision) was already in,
* so for the common check+add use case, no need to call check separately.
*
* Parameters:
* -----------
* bloom - Pointer to an allocated struct bloom (see above).
* buffer - Pointer to buffer containing element to add.
* len - Size of 'buffer'.
*
* Return:
* -------
* 0 - element was not present and was added
* 1 - element (or a collision) had already been added previously
* -1 - bloom not initialized
*
*/
int bloom_add(struct bloom * bloom, const void * buffer, int len);
/** ***************************************************************************
* Print (to stdout) info about this bloom filter. Debugging aid.
*
*/
void bloom_print(struct bloom * bloom);
/** ***************************************************************************
* Deallocate internal storage.
*
* Upon return, the bloom struct is no longer usable. You may call bloom_init
* again on the same struct to reinitialize it again.
*
* Parameters:
* -----------
* bloom - Pointer to an allocated struct bloom (see above).
*
* Return: none
*
*/
void bloom_free(struct bloom * bloom);
/** ***************************************************************************
* Returns version string compiled into library.
*
* Return: version string
*
*/
const char * bloom_version();
#endif
#ifndef _BLOOM_MURMURHASH2
#define _BLOOM_MURMURHASH2
unsigned int murmurhash2(const void * key, int len, const unsigned int seed);
#endif
......@@ -19,6 +19,9 @@
#include <fcntl.h>
#include <math.h>
//#include <libbloom.h>
#include "bloom.h"
#include "obiavl.h"
#include "obierrno.h"
#include "obitypes.h"
......@@ -30,6 +33,7 @@
#define DEBUG_LEVEL 0 // TODO has to be defined somewhere else (cython compil flag?)
/**************************************************************************
*
* D E C L A R A T I O N O F T H E P R I V A T E F U N C T I O N S
......@@ -481,21 +485,23 @@ int grow_avl(OBIDMS_avl_p avl) // TODO Lock when needed
int avl_file_descriptor;
char* avl_file_name;
// Get the avl file name
avl_file_name = build_avl_file_name((avl->header)->avl_name);
if (avl_file_name == NULL)
return -1;
// Open the avl file
avl_file_descriptor = openat(avl->dir_fd, avl_file_name, O_RDWR);
if (avl_file_descriptor < 0)
{
obi_set_errno(OBI_AVL_ERROR);
obidebug(1, "\nError opening an AVL tree file");
free(avl_file_name);
return -1;
}
free(avl_file_name);
avl_file_descriptor = avl->avl_fd;
// // Get the avl file name
// avl_file_name = build_avl_file_name((avl->header)->avl_name);
// if (avl_file_name == NULL)
// return -1;
//
// // Open the avl file
// avl_file_descriptor = openat(avl->dir_fd, avl_file_name, O_RDWR);
// if (avl_file_descriptor < 0)
// {
// obi_set_errno(OBI_AVL_ERROR);
// obidebug(1, "\nError opening an AVL tree file");
// free(avl_file_name);
// return -1;
// }
// free(avl_file_name);
// Calculate the new file size
old_data_size = (avl->header)->avl_size;
......@@ -544,7 +550,7 @@ int grow_avl(OBIDMS_avl_p avl) // TODO Lock when needed
// Set the new avl size
(avl->header)->avl_size = new_data_size;
close(avl_file_descriptor);
//close(avl_file_descriptor);
return 0;
}
......@@ -559,21 +565,23 @@ int grow_avl_data(OBIDMS_avl_p avl) // TODO Lock when needed
int avl_data_file_descriptor;
char* avl_data_file_name;
// Get the avl data file name
avl_data_file_name = build_avl_data_file_name((avl->header)->avl_name);
if (avl_data_file_name == NULL)
return -1;
// Open the avl data file
avl_data_file_descriptor = openat(avl->dir_fd, avl_data_file_name, O_RDWR);
if (avl_data_file_descriptor < 0)
{
obi_set_errno(OBI_AVL_ERROR);
obidebug(1, "\nError opening an AVL tree data file");
free(avl_data_file_name);
return -1;
}
free(avl_data_file_name);
avl_data_file_descriptor = avl->data_fd;
// // Get the avl data file name
// avl_data_file_name = build_avl_data_file_name((avl->header)->avl_name);
// if (avl_data_file_name == NULL)
// return -1;
//
// // Open the avl data file
// avl_data_file_descriptor = openat(avl->dir_fd, avl_data_file_name, O_RDWR);
// if (avl_data_file_descriptor < 0)
// {
// obi_set_errno(OBI_AVL_ERROR);
// obidebug(1, "\nError opening an AVL tree data file");
// free(avl_data_file_name);
// return -1;
// }
// free(avl_data_file_name);
// Calculate the new file size
old_data_size = ((avl->data)->header)->data_size_max;
......@@ -619,10 +627,12 @@ int grow_avl_data(OBIDMS_avl_p avl) // TODO Lock when needed
// Set new data size
((avl->data)->header)->data_size_max = new_data_size;
//fprintf(stderr, "\nGrowing AVL, new data size = %lld, count = %ld\n", new_data_size, (avl->header)->nb_items);
// Initialize new data to 0
memset(((avl->data)->data)+old_data_size, 0, new_data_size - old_data_size);
close(avl_data_file_descriptor);
//close(avl_data_file_descriptor);
return 0;
}
......@@ -996,6 +1006,74 @@ OBIDMS_avl_p obi_avl(OBIDMS_p dms, const char* avl_name)
}
OBIDMS_avl_group_p obi_create_avl_group(OBIDMS_p dms, const char* avl_name)
{
OBIDMS_avl_group_p avl_group;
char* avl_name_with_idx;
avl_group = (OBIDMS_avl_group_p) malloc(sizeof(OBIDMS_avl_group_t));
// Create 1st avl
asprintf(&avl_name_with_idx,"%s_%u", avl_name, 0);
(avl_group->sub_avls)[0] = obi_create_avl(dms, avl_name_with_idx);
avl_group->current_avl_idx = 0;
strcpy(avl_group->avl_name, avl_name);
avl_group->dms = dms;
return avl_group;
}
int unmap_an_avl(OBIDMS_avl_p avl)
{
if (munmap((avl->data)->data, ((avl->data)->header)->data_size_max) < 0)
return -1;
if (munmap(avl->tree, (((avl->header)->nb_items_max) * sizeof(AVL_node_t))) < 0)
return -1;
return 0;
}
int remap_an_avl(OBIDMS_avl_p avl)
{
(avl->data)->data = mmap(NULL,
((avl->data)->header)->data_size_max,
PROT_READ | PROT_WRITE,
MAP_SHARED,
avl->data_fd,
((avl->data)->header)->header_size);
if ((avl->data)->data == NULL)
return -1;
avl->tree = mmap(NULL,
((avl->header)->nb_items_max) * sizeof(AVL_node_t),
PROT_READ | PROT_WRITE,
MAP_SHARED,
avl->avl_fd,
(avl->header)->header_size);
if (avl->tree == NULL)
return -1;
return 0;
}
int obi_add_new_avl_in_group(OBIDMS_avl_group_p avl_group) // TODO check for errors
{
char* avl_name_with_idx;
// unmap older
unmap_an_avl((avl_group->sub_avls)[avl_group->current_avl_idx]);
(avl_group->current_avl_idx)++;
asprintf(&avl_name_with_idx,"%s_%u", avl_group->avl_name, avl_group->current_avl_idx);
(avl_group->sub_avls)[avl_group->current_avl_idx] = obi_create_avl(avl_group->dms, avl_name_with_idx);
return 0;
}
OBIDMS_avl_p obi_create_avl(OBIDMS_p dms, const char* avl_name)
{
char* avl_file_name;
......@@ -1098,7 +1176,7 @@ OBIDMS_avl_p obi_create_avl(OBIDMS_p dms, const char* avl_name)
// Initialize all bits to 0
memset(avl_data->data, 0, (avl_data->header)->data_size_max);
close(avl_data_file_descriptor);
//close(avl_data_file_descriptor);
// Create the AVL tree file
......@@ -1198,7 +1276,13 @@ OBIDMS_avl_p obi_create_avl(OBIDMS_p dms, const char* avl_name)
(avl->header)->creation_date = time(NULL);
strcpy((avl->header)->avl_name, avl_name);
close(avl_file_descriptor);
avl->avl_fd = avl_file_descriptor;
avl->data_fd = avl_data_file_descriptor;
// Bloom filter
bloom_init(&((avl->header)->bloom_filter), 2000000, 0.001); // TODO use macros
//close(avl_file_descriptor);
// Add in the list of opened AVL trees
*(((dms->opened_avls)->avls)+((dms->opened_avls)->nb_opened_avls)) = avl;
......@@ -1305,7 +1389,7 @@ OBIDMS_avl_p obi_open_avl(OBIDMS_p dms, const char* avl_name)
return NULL;
}
close(avl_data_file_descriptor);
//close(avl_data_file_descriptor);
// Open the AVL tree file
......@@ -1391,7 +1475,10 @@ OBIDMS_avl_p obi_open_avl(OBIDMS_p dms, const char* avl_name)
avl->directory = dms->avl_directory;
avl->dir_fd = avl_dir_file_descriptor;
close(avl_file_descriptor);
avl->avl_fd = avl_file_descriptor;
avl->data_fd = avl_data_file_descriptor;
//close(avl_file_descriptor);
// Add in the list of opened AVL trees
*(((dms->opened_avls)->avls)+((dms->opened_avls)->nb_opened_avls)) = avl;
......@@ -1456,6 +1543,53 @@ byte_t* obi_avl_get(OBIDMS_avl_p avl, index_t idx)
}
int maybe_in_avl(OBIDMS_avl_p avl, byte_t* value)
{
return (bloom_check(&((avl->header)->bloom_filter), value, (BYTE_ARRAY_HEADER_SIZE + *((int32_t*)(value+1)))));
}
index_t insert_in_avl_group(OBIDMS_avl_group_p avl_group, byte_t* value) // TODO won't be index_t
{
index_t index_if_already_in;
int i;
if (maybe_in_avl((avl_group->sub_avls)[avl_group->current_avl_idx], value))
{
//fprintf(stderr, "\nyah maybe");
index_if_already_in = obi_avl_find((avl_group->sub_avls)[avl_group->current_avl_idx], value);
if (index_if_already_in >= 0)
return index_if_already_in;
}
// else
// fprintf(stderr, "\nnah");
for (i=0; i < (avl_group->current_avl_idx); i++)
{
if (maybe_in_avl((avl_group->sub_avls)[i], value))
{
//fprintf(stderr, "\nyah maybe");
if (remap_an_avl((avl_group->sub_avls)[i]) < 0)
return -1;
index_if_already_in = obi_avl_find((avl_group->sub_avls)[i], value);
if (unmap_an_avl((avl_group->sub_avls)[i]) < 0)
return -1;
if (index_if_already_in >= 0)
return index_if_already_in;
}
// else
// fprintf(stderr, "\nnah");
}