Commit 3681cecb by Celine Mercier

Multiple AVLs with bloom filters (very raw test version)

parent 545ed811
......@@ -22,3 +22,7 @@
../../../src/obidmscolumn_idx.c
../../../src/obidms_taxonomy.c
../../../src/obidms_taxonomy.h
../../../src/bloom.c
../../../src/bloom.h
../../../src/MurmurHash2.c
../../../src/murmurhash2.h
\ No newline at end of file
//-----------------------------------------------------------------------------
// MurmurHash2, by Austin Appleby
// Note - This code makes a few assumptions about how your machine behaves -
// 1. We can read a 4-byte value from any address without crashing
// 2. sizeof(int) == 4
// And it has a few limitations -
// 1. It will not work incrementally.
// 2. It will not produce the same results on little-endian and big-endian
// machines.
unsigned int murmurhash2(const void * key, int len, const unsigned int seed)
{
// 'm' and 'r' are mixing constants generated offline.
// They're not really 'magic', they just happen to work well.
const unsigned int m = 0x5bd1e995;
const int r = 24;
// Initialize the hash to a 'random' value
unsigned int h = seed ^ len;
// Mix 4 bytes at a time into the hash
const unsigned char * data = (const unsigned char *)key;
while(len >= 4)
{
unsigned int k = *(unsigned int *)data;
k *= m;
k ^= k >> r;
k *= m;
h *= m;
h ^= k;
data += 4;
len -= 4;
}
// Handle the last few bytes of the input array
switch(len)
{
case 3: h ^= data[2] << 16;
case 2: h ^= data[1] << 8;
case 1: h ^= data[0];
h *= m;
};
// Do a few final mixes of the hash to ensure the last few
// bytes are well-incorporated.
h ^= h >> 13;
h *= m;
h ^= h >> 15;
return h;
}
/*
* Copyright (c) 2012-2015, Jyri J. Virkki
* All rights reserved.
*
* This file is under BSD license. See LICENSE file.
*/
/*
* Refer to bloom.h for documentation on the public interfaces.
*/
#include <assert.h>
#include <fcntl.h>
#include <math.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include "bloom.h"
#include "murmurhash2.h"
#define MAKESTRING(n) STRING(n)
#define STRING(n) #n
#ifdef __linux__
unsigned detect_bucket_size(unsigned fallback_size);
#endif
static int test_bit_set_bit(unsigned char * buf, unsigned int x, int set_bit)
{
register uint32_t * word_buf = (uint32_t *)buf;
register unsigned int offset = x >> 5;
register uint32_t word = word_buf[offset];
register unsigned int mask = 1 << (x % 32);
if (word & mask) {
return 1;
} else {
if (set_bit) {
word_buf[offset] = word | mask;
}
return 0;
}
}
static int bloom_check_add(struct bloom * bloom,
const void * buffer, int len, int add)
{
if (bloom->ready == 0) {
(void)printf("bloom at %p not initialized!\n", (void *)bloom);
return -1;
}
int hits = 0;
register unsigned int a = murmurhash2(buffer, len, 0x9747b28c);
register unsigned int b = murmurhash2(buffer, len, a);
register unsigned int x;
register unsigned int i;
unsigned bucket_index = (a % bloom->buckets);
unsigned char * bucket_ptr =
(bloom->bf + (bucket_index << bloom->bucket_bytes_exponent));
for (i = 0; i < bloom->hashes; i++) {
x = (a + i*b) & bloom->bucket_bits_fast_mod_operand;
if (test_bit_set_bit(bucket_ptr, x, add)) {
hits++;
}
}
if (hits == bloom->hashes) {
return 1; // 1 == element already in (or collision)
}
return 0;
}
static void setup_buckets(struct bloom * bloom, unsigned int cache_size)
{
// If caller passed a non-zero cache_size, use it as given, otherwise
// either compute it or use built-in default
if (cache_size == 0) {
#ifdef __linux__
cache_size = detect_bucket_size(BLOOM_BUCKET_SIZE_FALLBACK);
#else
cache_size = BLOOM_BUCKET_SIZE_FALLBACK;
#endif
}
bloom->buckets = (bloom->bytes / cache_size);
bloom->bucket_bytes = cache_size;
// make sure bloom buffer bytes and bucket_bytes are even
int not_even_by = (bloom->bytes % bloom->bucket_bytes);
if (not_even_by) {
// adjust bytes
bloom->bytes += (bloom->bucket_bytes - not_even_by);
assert((bloom->bytes % bloom->bucket_bytes) == 0); // Should get even
// adjust bits
bloom->bits = bloom->bytes * 8;
// adjust bits per element
bloom->bpe = bloom->bits*1. / bloom->entries;
// adjust buckets
bloom->buckets++;
}
bloom->bucket_bytes_exponent = __builtin_ctz(cache_size);
bloom->bucket_bits_fast_mod_operand = (cache_size * 8 - 1);
}
int bloom_init_size(struct bloom * bloom, int entries, double error,
unsigned int cache_size)
{
bloom->ready = 0;
if (entries < 1 || error == 0) {
return 1;
}
bloom->entries = entries;
bloom->error = error;
double num = log(bloom->error);
double denom = 0.480453013918201; // ln(2)^2
bloom->bpe = -(num / denom);
double dentries = (double)entries;
bloom->bits = (int)(dentries * bloom->bpe);
if (bloom->bits % 8) {
bloom->bytes = (bloom->bits / 8) + 1;
} else {
bloom->bytes = bloom->bits / 8;
}
bloom->hashes = (int)ceil(0.693147180559945 * bloom->bpe); // ln(2)
setup_buckets(bloom, cache_size);
bloom->bf = (unsigned char *)calloc(bloom->bytes, sizeof(unsigned char));
if (bloom->bf == NULL) {
return 1;
}
bloom->ready = 1;
return 0;
}
int bloom_init(struct bloom * bloom, int entries, double error)
{
return bloom_init_size(bloom, entries, error, 0);
}
int bloom_check(struct bloom * bloom, const void * buffer, int len)
{
return bloom_check_add(bloom, buffer, len, 0);
}
int bloom_add(struct bloom * bloom, const void * buffer, int len)
{
return bloom_check_add(bloom, buffer, len, 1);
}
void bloom_print(struct bloom * bloom)
{
(void)printf("bloom at %p\n", (void *)bloom);
(void)printf(" ->entries = %d\n", bloom->entries);
(void)printf(" ->error = %f\n", bloom->error);
(void)printf(" ->bits = %d\n", bloom->bits);
(void)printf(" ->bits per elem = %f\n", bloom->bpe);
(void)printf(" ->bytes = %d\n", bloom->bytes);
(void)printf(" ->buckets = %u\n", bloom->buckets);
(void)printf(" ->bucket_bytes = %u\n", bloom->bucket_bytes);
(void)printf(" ->bucket_bytes_exponent = %u\n",
bloom->bucket_bytes_exponent);
(void)printf(" ->bucket_bits_fast_mod_operand = 0%o\n",
bloom->bucket_bits_fast_mod_operand);
(void)printf(" ->hash functions = %d\n", bloom->hashes);
}
void bloom_free(struct bloom * bloom)
{
if (bloom->ready) {
free(bloom->bf);
}
bloom->ready = 0;
}
const char * bloom_version()
{
return MAKESTRING(BLOOM_VERSION);
}
/*
* Copyright (c) 2012-2015, Jyri J. Virkki
* All rights reserved.
*
* This file is under BSD license. See LICENSE file.
*/
#ifndef _BLOOM_H
#define _BLOOM_H
/** ***************************************************************************
* On Linux, the code attempts to compute a bucket size based on CPU cache
* size info, if available. If that fails for any reason, this fallback size
* is used instead.
*
* On non-Linux systems, this is the bucket size always used unless the
* caller overrides it (see bloom_init_size()).
*
*/
#define BLOOM_BUCKET_SIZE_FALLBACK (32 * 1024)
/** ***************************************************************************
* It was found that using multiplier x0.5 for CPU L1 cache size is
* more effective in terms of CPU usage and, surprisingly, collisions
* number.
*
* Feel free to tune this constant the way it will work for you.
*
*/
#define BLOOM_L1_CACHE_SIZE_DIV 1
/** ***************************************************************************
* Structure to keep track of one bloom filter. Caller needs to
* allocate this and pass it to the functions below. First call for
* every struct must be to bloom_init().
*
*/
struct bloom
{
// These fields are part of the public interface of this structure.
// Client code may read these values if desired. Client code MUST NOT
// modify any of these.
int entries;
double error;
int bits;
int bytes;
int hashes;
// Fields below are private to the implementation. These may go away or
// change incompatibly at any moment. Client code MUST NOT access or rely
// on these.
unsigned buckets;
unsigned bucket_bytes;
// x86 CPU divide by/multiply by operation optimization helpers
unsigned bucket_bytes_exponent;
unsigned bucket_bits_fast_mod_operand;
double bpe;
unsigned char * bf;
int ready;
};
/** ***************************************************************************
* Initialize the bloom filter for use.
*
* The filter is initialized with a bit field and number of hash functions
* according to the computations from the wikipedia entry:
* http://en.wikipedia.org/wiki/Bloom_filter
*
* Optimal number of bits is:
* bits = (entries * ln(error)) / ln(2)^2
*
* Optimal number of hash functions is:
* hashes = bpe * ln(2)
*
* Parameters:
* -----------
* bloom - Pointer to an allocated struct bloom (see above).
* entries - The expected number of entries which will be inserted.
* error - Probability of collision (as long as entries are not
* exceeded).
*
* Return:
* -------
* 0 - on success
* 1 - on failure
*
*/
int bloom_init(struct bloom * bloom, int entries, double error);
/** ***************************************************************************
* Initialize the bloom filter for use.
*
* See comments above for general information.
*
* This is the same as bloom_init() but allows the caller to pass in a
* cache_size to override the internal value (which is either computed
* or the default of BLOOM_BUCKET_SIZE_FALLBACK). Mostly useful for
* experimenting.
*
* See misc/bucketsize for a script which can help identify a good value
* for cache_size.
*
*/
int bloom_init_size(struct bloom * bloom, int entries, double error,
unsigned int cache_size);
/** ***************************************************************************
* Check if the given element is in the bloom filter. Remember this may
* return false positive if a collision occured.
*
* Parameters:
* -----------
* bloom - Pointer to an allocated struct bloom (see above).
* buffer - Pointer to buffer containing element to check.
* len - Size of 'buffer'.
*
* Return:
* -------
* 0 - element is not present
* 1 - element is present (or false positive due to collision)
* -1 - bloom not initialized
*
*/
int bloom_check(struct bloom * bloom, const void * buffer, int len);
/** ***************************************************************************
* Add the given element to the bloom filter.
* The return code indicates if the element (or a collision) was already in,
* so for the common check+add use case, no need to call check separately.
*
* Parameters:
* -----------
* bloom - Pointer to an allocated struct bloom (see above).
* buffer - Pointer to buffer containing element to add.
* len - Size of 'buffer'.
*
* Return:
* -------
* 0 - element was not present and was added
* 1 - element (or a collision) had already been added previously
* -1 - bloom not initialized
*
*/
int bloom_add(struct bloom * bloom, const void * buffer, int len);
/** ***************************************************************************
* Print (to stdout) info about this bloom filter. Debugging aid.
*
*/
void bloom_print(struct bloom * bloom);
/** ***************************************************************************
* Deallocate internal storage.
*
* Upon return, the bloom struct is no longer usable. You may call bloom_init
* again on the same struct to reinitialize it again.
*
* Parameters:
* -----------
* bloom - Pointer to an allocated struct bloom (see above).
*
* Return: none
*
*/
void bloom_free(struct bloom * bloom);
/** ***************************************************************************
* Returns version string compiled into library.
*
* Return: version string
*
*/
const char * bloom_version();
#endif
#ifndef _BLOOM_MURMURHASH2
#define _BLOOM_MURMURHASH2
unsigned int murmurhash2(const void * key, int len, const unsigned int seed);
#endif
......@@ -25,6 +25,10 @@
#include "obidms.h"
#include "obitypes.h"
#include "bloom.h"
#define NB_OF_AVLS (64)
#define MASK (63)
#define AVL_MAX_NAME (1024) /**< The maximum length of an AVL tree name.
*/
......@@ -39,6 +43,8 @@
#define BYTE_ARRAY_HEADER_SIZE (9) /**< The size of the header of a byte array.
*/
typedef struct bloom bloom_t;
/**
* @brief AVL tree node structure.
......@@ -48,7 +54,7 @@ typedef struct AVL_node {
*/
index_t right_child; /**< Index of right greater child node.
*/
int8_t balance_factor; /**< Balance factor of the node.
int8_t balance_factor; /**< Balance factor of the node.
*/
index_t value; /**< Index of the value associated with the node in the data array.
*/
......@@ -103,6 +109,7 @@ typedef struct OBIDMS_avl_header {
*/
time_t creation_date; /**< Date of creation of the file.
*/
bloom_t bloom_filter;
} OBIDMS_avl_header_t, *OBIDMS_avl_header_p;
......@@ -132,10 +139,29 @@ typedef struct OBIDMS_avl {
*/
size_t counter; /**< Indicates by how many threads/programs (TODO) the AVL tree is used.
*/
int avl_fd;
int data_fd;
} OBIDMS_avl_t, *OBIDMS_avl_p;
/**
* @brief OBIDMS AVL tree group structure.
*/
typedef struct OBIDMS_avl_group {
// TODO put each group in a directory later
OBIDMS_avl_p sub_avls[64]; // TODO macro for max
int current_avl_idx;
char avl_name[AVL_MAX_NAME+1];
OBIDMS_p dms;
} OBIDMS_avl_group_t, *OBIDMS_avl_group_p;
OBIDMS_avl_group_p obi_create_avl_group(OBIDMS_p dms, const char* avl_name);
index_t insert_in_avl_group(OBIDMS_avl_group_p avl_group, byte_t* value);
/**
* @brief Checks if an AVL tree already exists or not.
*
* @param dms The OBIDMS to which the AVL tree belongs.
......@@ -340,12 +366,5 @@ byte_t* obi_seq_to_obibytes(char* seq);
const char* obi_obibytes_to_seq(byte_t* value_b);
OBIDMS_avl_p* obi_create_avl_in_64_parts(OBIDMS_p dms, const char* avl_name);
typedef uint8_t crc;
crc compute_crc(const char* s);
#endif /* OBIAVL_H_ */
......@@ -521,7 +521,6 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
OBIDMS_column_p new_column;
OBIDMS_column_directory_p column_directory;
OBIDMS_column_header_p header;
OBIDMS_avl_p* avl;
size_t file_size;
obiversion_t version_number;
char* column_file_name;
......@@ -723,16 +722,15 @@ OBIDMS_column_p obi_create_column(OBIDMS_p dms,
// If the data type is OBI_STR or OBI_SEQ, the associated obi_avl is opened or created
if ((returned_data_type == OBI_STR) || (returned_data_type == OBI_SEQ))
{
avl = obi_create_avl_in_64_parts(dms, avl_name);
if (avl == NULL)
{
obidebug(1, "\nError opening or creating the aVL tree associated with a column");
munmap(new_column->header, header_size);
close(column_file_descriptor);
free(new_column);
return NULL;
}
memcpy(new_column->avl, avl, 64*sizeof(OBIDMS_avl_p));
new_column->avl = obi_create_avl_group(dms, avl_name);
// if (avl == NULL) TODO
// {
// obidebug(1, "\nError opening or creating the aVL tree associated with a column");
// munmap(new_column->header, header_size);
// close(column_file_descriptor);
// free(new_column);
// return NULL;
// }
strncpy(header->avl_name, avl_name, AVL_MAX_NAME);
}
......@@ -756,11 +754,11 @@ OBIDMS_column_p obi_open_column(OBIDMS_p dms,
{
OBIDMS_column_p column;
OBIDMS_column_directory_p column_directory;
OBIDMS_avl_p avl;
char* column_file_name;
int column_file_descriptor;
size_t header_size;
size_t i;
OBIDMS_avl_p avl;
column = NULL;
......
......@@ -98,7 +98,7 @@ typedef struct OBIDMS_column {
*/
OBIDMS_column_header_p header; /**< A pointer to the header of the column.
*/
OBIDMS_avl_p avl[64]; /**< A pointer to the group of AVL trees associated with the column if there is one.
OBIDMS_avl_group_p avl; /**< TODO A pointer to the group of AVL trees associated with the column if there is one.
*/
void* data; /**< A `void` pointer to the beginning of the data.
*
......
......@@ -61,13 +61,13 @@ int obi_column_set_obiseq_with_elt_idx(OBIDMS_column_p column, index_t line_nb,
if (value_b == NULL)
return -1;
if (strlen(value_b) == 0)
fprintf(stderr, "\nPOUIC");
//if (strlen(value_b) == 0)
// fprintf(stderr, "\nPOUIC");
//fprintf(stderr, "\n>%s||%s", value, obi_obibytes_to_seq(value_b));
// Add in the AVL tree
idx = obi_avl_add((column->avl)[compute_crc(value)], value_b);
idx = insert_in_avl_group(column->avl, value_b);
if (idx == -1)
return -1;
......
......@@ -61,7 +61,7 @@ int obi_column_set_obistr_with_elt_idx(OBIDMS_column_p column, index_t line_nb,
return -1;
// Add in the AVL tree
idx = obi_avl_add((column->avl)[compute_crc(value)], value_b);
idx = insert_in_avl_group(column->avl, value_b);
if (idx == -1)
return -1;
......
......@@ -227,6 +227,7 @@ Obiview_p obi_new_view_nuc_seqs(OBIDMS_p dms, const char* view_name, Obiview_p v
if (view== NULL)
return NULL;
fprintf(stderr, "\nmmmm\n");
strcpy(view->view_type, VIEW_TYPE_NUC_SEQS);
if (view_to_clone == NULL)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment