Distributed Hash Table File System for Linux








magma_flare_internals.c File Reference

#include "magma.h"

Defines

#define INCLUDE_FLARE_INTERNALS
#define GC_USAGE_THRESHOLD   -20
#define GC_TIMESTAMP_THRESHOLD   60
#define GC_START_THRESHOLD   0
#define lock_flare(f)
#define trylock_flare(f, result)
#define unlock_flare(f)
#define CHECK_LOAD   1
#define DONT_CHECK_LOAD   0
#define magma_add_to_cache(flare)   magma_add_to_cache_after(flare,magma_cache,CHECK_LOAD)
#define GROW_FILE   1
#define DONT_GROW_FILE   0

Functions

uint8_t check_permission (magma_flare_t *flare, uid_t uid, gid_t gid, uint8_t operations)
int check_filesystem ()
int bootstrap_network ()
void magma_set_state (uint8_t state)
uint8_t magma_get_state ()
char * magma_simplify_path (const char *path)
int flare_system_init (int bootstrap, char *bootserver, int bootport, int start_balancer)
magma_flare_tmagma_new_flare (const char *path)
int magma_cast_to_dir (magma_flare_t *flare)
magma_flare_tmagma_new_dir_flare (const char *path)
int magma_cast_to_file (magma_flare_t *flare)
magma_flare_tmagma_new_file_flare (const char *path)
int magma_cast_to_blockdev (magma_flare_t *flare)
magma_flare_tmagma_new_blockdev_flare (const char *path)
int magma_cast_to_chardev (magma_flare_t *flare)
magma_flare_tmagma_new_chardev_flare (const char *path)
int magma_cast_to_symlink (magma_flare_t *flare)
magma_flare_tmagma_new_symlink_flare (const char *path)
int magma_cast_to_fifo (magma_flare_t *flare)
magma_flare_tmagma_new_fifo_flare (const char *path)
int old_magma_flare_upcast (magma_flare_t *flare)
int magma_destroy_flare (magma_flare_t *flare)
void print_flare (magma_flare_t *flare)
magma_flare_tmagma_search_under_node (const char *hash, magma_flare_t *startnode)
magma_flare_tmagma_search (const char *path)
magma_flare_tmagma_search_by_armour (const char *armoured)
magma_flare_tmagma_search_by_hash (const unsigned char *hash)
magma_flare_tmagma_search_or_create (const char *path)
void magma_increment_flare_load (magma_flare_t *flare)
void magma_decrement_flare_load (magma_flare_t *flare)
magma_flare_tmagma_add_to_cache_after (magma_flare_t *flare, magma_flare_t *node, int checkload)
int magma_remove_from_cache (magma_flare_t *flare)
int magma_init_flare_on_disk (magma_flare_t *flare)
int magma_check_flare (magma_flare_t *flare)
int magma_save_flare (magma_flare_t *flare)
int magma_erase_flare_from_disk (magma_flare_t *flare)
int magma_erase_flare (magma_flare_t *flare)
int magma_load_flare (magma_flare_t *flare)
int magma_touch_flare (magma_flare_t *flare, int mode)
unsigned int magma_get_flare_inode (magma_flare_t *flare)
long long int magma_count_cache_elements_on_node (magma_flare_t *node)
void magma_save_cache_node (magma_flare_t *node, int remove)
int allocate_blank_page ()
int magma_grow_directory (magma_flare_t *dir)
int check_directory_size (char *border, magma_DIR_t *dirp, int res, struct stat *st, int do_grow_file)
char * magma_get_slot (magma_DIR_t *dirp, int do_grow_file)
char * magma_get_empty_slot (magma_DIR_t *dirp)
char * magma_get_full_slot (magma_DIR_t *dirp)
int magma_get_slot_length (magma_DIR_t *dirp, char *slot)
int magma_push_dirent (magma_DIR_t *dirp, char *path)
int magma_pop_dirent (magma_DIR_t *dirp, char *dirent)
int magma_add_flare_to_parent (magma_flare_t *flare)
int magma_remove_flare_from_parent (magma_flare_t *flare)
int magma_dir_is_empty (magma_flare_t *dir)

Variables

uid_t flare_system_uid = 0
gid_t flare_system_gid = 0
const char dht_path [] = DHT_PATH
const char run_path [] = RUN_PATH
magma_destroyable_flare_t * destroyable_flare_queue
pthread_mutex_t destroyable_flare_queue_mutex = PTHREAD_MUTEX_INITIALIZER
sem_t * destroyable_flare_queue_semaphore
magma_flare_tmagma_cache
pthread_mutex_t cache_manipulation_mutex = PTHREAD_MUTEX_INITIALIZER
int pagesize = 0
const uint8_t MAGMA_OFF = 0
const uint8_t MAGMA_RUNNING = 1
const uint8_t MAGMA_DHT_REBUILD = 10
const uint8_t MAGMA_SHUTDOWN = 254
pthread_mutex_t magma_state_mutex = PTHREAD_MUTEX_INITIALIZER
uint8_t magma_state
regex_t simplify_path_single_dot_re
regex_t simplify_path_double_dot_re
regex_t simplify_path_multislash_re
char * blank_page

Define Documentation

#define CHECK_LOAD   1

recalculate load of cache branches

#define DONT_CHECK_LOAD   0

don't recalculate load of cache branches

#define DONT_GROW_FILE   0

#define GC_START_THRESHOLD   0

#define GC_TIMESTAMP_THRESHOLD   60

#define GC_USAGE_THRESHOLD   -20

#define GROW_FILE   1

#define INCLUDE_FLARE_INTERNALS

#define lock_flare (  ) 

Value:

{\
        if (f != NULL) {\
                dbg(LOG_INFO, DEBUG_MUTEX, "[L+] Lock on flare \"%s\" @ %s:%d", f->path, __FILE__, __LINE__);\
                pthread_mutex_lock(&(f->mutex));\
                dbg(LOG_INFO, DEBUG_MUTEX, "[L+] Locked!");\
        }\
}

#define magma_add_to_cache ( flare   )     magma_add_to_cache_after(flare,magma_cache,CHECK_LOAD)

Add to cache a new flare.

Parameters:
flare the flare to be cached

#define trylock_flare ( f,
result   ) 

Value:

{\
        if (f != NULL) {\
                dbg(LOG_INFO, DEBUG_MUTEX, "[L+] (try) Lock on flare \"%s\" @ %s:%d", f->path, __FILE__, __LINE__);\
                result = pthread_mutex_trylock(&(f->mutex));\
                if (result == 0) {\
                        dbg(LOG_INFO, DEBUG_MUTEX, "[L+] (try) Locked!");\
                } else {\
                        dbg(LOG_INFO, DEBUG_MUTEX, "[L!] (try) lock failed!");\
                }\
        }\
}

#define unlock_flare (  ) 

Value:

{\
        if (f != NULL) {\
                pthread_mutex_unlock(&(f->mutex));\
                dbg(LOG_INFO, DEBUG_MUTEX, "[L-] Lock on flare \"%s\" @ %s:%d", f->path, __FILE__, __LINE__);\
        }\
}


Function Documentation

int allocate_blank_page (  ) 

Preallocate a blank page of pagesize bytes used for directory mmap() operations

Returns:
1 on success, 0 on failure

int bootstrap_network (  ) 

Bootstrap a new lava ring network. First checks the filesystem using check_filesystem(). Than init myself node by assigning the whole key space to this node. Finally creates "/" flare directory and "/.dht" flare directory.

int check_directory_size ( char *  border,
magma_DIR_t dirp,
int  res,
struct stat *  st,
int  do_grow_file 
)

Checks if current directory size can hold a new slot of NAME_MAX size. Strange res and st parameters are due to previous coding state when check_directory_size was directly embedded in magma_get_slot(), which is currently the only function to call this one.

Parameters:
border directory ending address
dirp directory handle
res result of previous lstat()
st struct stat filled by previous lstat()
do_grow_file if equals to GROW_FILE, directory is grown by one page if needed
Returns:
0 on success, -1 on failure (errno is set to ENOSPC)

int check_filesystem ( void   ) 

Performs prechecking of filesystem, creating storage directory.

Returns:
always 0. In case of failure, will exit(1) the program.

uint8_t check_permission ( magma_flare_t flare,
uid_t  uid,
gid_t  gid,
uint8_t  operations 
)

Check if a user with specified uid and gid is allowed to perform "operations" operation on flare

Parameters:
flare the flare to be accessed
uid UID accessing the flare
gid GID accessing the flare
operations bitmask field of required operations
Returns:
0 if success, a uint8_t with failing operations otherwise

int flare_system_init ( int  bootstrap,
char *  bootserver,
int  bootport,
int  start_balancer 
)

This call should be used on top of any program using the flare system to reserve resources internally used!

If bootstrap == 1, bootserver and bootport are ignored, so flare_system_init(1,NULL,0,0) is a valid call.

Before calling flare_system_init(), call magma_config_myself(). (see vulcano.h)

Actually it is declared inside magma_flare_internals.c but is exported by magma_flare.h because is needed to initialize the flare system.

Parameters:
bootstrap if true, new lava will be created, if false this node will remote bootstrap
bootserver remote boot server used if bootstrap is false
bootport remote boot port used if bootstrap is false
start_balancer if true, balancer thread will be started
Returns:
Always 0. Return value can be ignored since, in case of failure, exit(N) is called.

int magma_add_flare_to_parent ( magma_flare_t flare  ) 

Adds a flare to parent directory. barely just a wrapper around magma_push_dirent() which also check for existance of new entry

Parameters:
flare the flare to be added
Returns:
0 on success, -1 on failure (errno is set accordingly)

magma_flare_t* magma_add_to_cache_after ( magma_flare_t flare,
magma_flare_t node,
int  checkload 
)

Adds an entry to magma to caching system. should never be used directly. Always use macro magma_add_to_cache() instead. This function use recursion to traverse the entire tree. magma_add_to_cache() simply calls this function with magma_cache as second param.

Parameters:
flare is the flare to be added to cache
node is the flare inside the cache to match for positioning
checkload if equals to CHECK_LOAD, load of parent branch is updated
Returns:
the node added if succesful, NULL otherwise.

int magma_cast_to_blockdev ( magma_flare_t flare  ) 

cast a flare to a block device flare

Parameters:
flare the flare to be casted
Returns:
1 on success, 0 otherwise

int magma_cast_to_chardev ( magma_flare_t flare  ) 

cast a flare to a character device flare

Parameters:
flare the flare to be casted
Returns:
1 on success, 0 otherwise

int magma_cast_to_dir ( magma_flare_t flare  ) 

cast a flare to a directory flare

Parameters:
flare the flare to be casted
Returns:
1 on success, 0 otherwise

int magma_cast_to_fifo ( magma_flare_t flare  ) 

cast a flare to a FIFO flare

Parameters:
flare the flare to be casted
Returns:
1 on success, 0 otherwise

int magma_cast_to_file ( magma_flare_t flare  ) 

cast a flare to a regular file flare

Parameters:
flare the flare to be casted
Returns:
1 on success, 0 otherwise

int magma_cast_to_symlink ( magma_flare_t flare  ) 

cast a flare to a symbolic link flare

Parameters:
flare the flare to be casted
Returns:
1 on success, 0 otherwise

int magma_check_flare ( magma_flare_t flare  ) 

check if flare is still on disk (has not been removed by external agent able to access repository space)

Parameters:
flare flare to be checked
Returns:
0 on success, -1 on failure (errno is set accordingly)

long long int magma_count_cache_elements_on_node ( magma_flare_t node  ) 

recursively counts element below given node

Parameters:
node the node to calculated
Returns:
load of the node

void magma_decrement_flare_load ( magma_flare_t flare  ) 

decrement by 1 load of given flare and do the same recursively on all flares before this one inside the cache.

Parameters:
flare the flare to be reduced

int magma_destroy_flare ( magma_flare_t flare  ) 

free resources used by a flare

Parameters:
flare the flare to be destroyed
Returns:
0 on success, 1 otherwise
Todo:
Why 0 on success? Should be better 1 on success!

int magma_dir_is_empty ( magma_flare_t dir  ) 

Check if a directory is empty.

Parameters:
dir the directory to be test for emptiness
Returns:
0 if directory dir is not empty (there is at least 1 entry before end of last page), 1 otherwise (or dir has some problem like being NULL pointer...)
Todo:
If this check is used while doing a "rmdir" should it return 0 if even if only "." and ".." are listed? Please verify.

int magma_erase_flare ( magma_flare_t flare  ) 

Erase a flare from parent and from disk, basically calling remove_flare_from_parent() and erase_flare_from_disk() in sequence. Flare is also removed from cache.

Parameters:
flare the flare to be saved
Returns:
0 on success, -1 on failure

int magma_erase_flare_from_disk ( magma_flare_t flare  ) 

Erase a flare from disk

Parameters:
flare the flare to be saved
Returns:
0 on success, -1 on failure

char* magma_get_empty_slot ( magma_DIR_t dirp  ) 

Calls magma_get_slot() until a new empty slot (a slot not pointing to a valid entry) is found.

Parameters:
dirp magma_DIR_t pointer used to track offset inside directory
Returns:
a pointer to next free slot inside directory. Modifying this pointer contents means working on directory contents.

unsigned int magma_get_flare_inode ( magma_flare_t flare  ) 

Return a flare inode. that function should be expanded to consider vulcano on which flare is located. inodes should be 64bit integers and should be ORed with vulcano number in MSB to allow a consistent numbering scheme for all nodes. that should result in 16bit for vulcano network (which is 65535 nodes! quite a big SAN) and 48 bits for local inodes.

Parameters:
flare the flare
Returns:
unsigned value rappresenting flare inode
Todo:
this function should still be coded!!!

char* magma_get_full_slot ( magma_DIR_t dirp  ) 

Calls magma_get_slot() until a new valid slot (a slot pointing to a valid entry) is found.

Parameters:
dirp magma_DIR_t pointer used to track offset inside directory
Returns:
a pointer to filled slot. Modifying this pointer contents means working on directory contents.

char* magma_get_slot ( magma_DIR_t dirp,
int  do_grow_file 
)

get a slot inside a directory. a slot is basically a valid position. using a magma_DIR_t pointer, this function can keep track of current position, using offset member. offset should always refer to a valid entry or to empty space.

if it points to a valid entry, magma_get_slot() will move offset forward of length of current entry plus 1 position for \0 null terminator.

if it points to free space, simply forwards offset by 1.

it does not check if next slot is big enough to store a new entry. just check if from current offset position to end of last page there is enough space to store a NAME_MAX entry (usually 255 characters). if not, a new page is allocated using magma_grow_directory(). there is just an exception: if was called with do_grow_file equals to DONT_GROW_FILE, directory will not be grown.

Parameters:
dirp magma_DIR_t pointer used to track current position
do_grow_file if GROW_FILE, directory will be grown, if DONT_GROW_FILE, directory will be unaltered
Returns:
a pointer to found slot. Modifying this pointer contents means changing directory contents.

int magma_get_slot_length ( magma_DIR_t dirp,
char *  slot 
)

Return slot length without risk to have an overflow on mmap()ed directory area.

Parameters:
dirp magma_DIR_t pointer used to track offset inside directory
slot the (valid or not) slot to be mesured
Returns:
slot length

uint8_t magma_get_state (  ) 

get this node state

Returns:
this node current state

int magma_grow_directory ( magma_flare_t dir  ) 

Grow a directory's contents file by 1 page.

Parameters:
dir the directory flare to be grown
Returns:
0 on success, -1 otherwise.
Todo:
Why errno is not set accordingly to return value?

void magma_increment_flare_load ( magma_flare_t flare  ) 

increment by 1 load of given flare and do the same recursively on all flares before this one inside the cache.

Parameters:
flare the flare to be augmented

int magma_init_flare_on_disk ( magma_flare_t flare  ) 

create a flare on disk. that is: making directory, creating metadata and contents files and issuing chmod on everything, using flare->st.st_mode ORed with user read and write permissions.

Parameters:
flare magma_flare_t structure to use as model
Returns:
0 on success, -1 on error (errno is set accordingly)
Todo:
check errno in this function, it's not always set.

int magma_load_flare ( magma_flare_t flare  ) 

Load a flare from disk.

Parameters:
flare the flare to be loaded
Returns:
1 on success, 0 otherwise (errno is set accordingly)
Todo:
Please check if errno is always set accordingly

magma_flare_t* magma_new_blockdev_flare ( const char *  path  ) 

create a new, upcasted block device flare

Parameters:
path the path of the new flare
Returns:
a pointer to the new flare

magma_flare_t* magma_new_chardev_flare ( const char *  path  ) 

create a new, upcasted character device flare

Parameters:
path the path of the new flare
Returns:
a pointer to the new flare

magma_flare_t* magma_new_dir_flare ( const char *  path  ) 

create a new, upcasted directory flare

Parameters:
path the path of new directory flare
Returns:
a pointer to new directory flare

magma_flare_t* magma_new_fifo_flare ( const char *  path  ) 

create a new, upcasted FIFO flare

Parameters:
path the path of the new flare
Returns:
a pointer to the new flare

magma_flare_t* magma_new_file_flare ( const char *  path  ) 

create a new, upcasted regular file flare

Parameters:
path the path of the new flare
Returns:
a pointer to the new flare

magma_flare_t* magma_new_flare ( const char *  path  ) 

Create new uncasted flares.

Parameters:
path the path of new uncasted flare
Returns:
a pointer to new flare

magma_flare_t* magma_new_symlink_flare ( const char *  path  ) 

create a new, upcasted symlink flare

Parameters:
path the path of the new flare
Returns:
a pointer to the new flare

int magma_pop_dirent ( magma_DIR_t dirp,
char *  dirent 
)

Remove an entry from pointed directory. after removing, directory is msync()ed to disk.

Parameters:
dirp magma_DIR_t structure pointing to an open directory
dirent the entry to be removed
Returns:
0.

int magma_push_dirent ( magma_DIR_t dirp,
char *  path 
)

Add a new entry inside a directory. This function should never be called outside.

Parameters:
dirp magma_DIR_t pointer to track offset and access directory
path the new dirent to be added
Returns:
0 on success, -1 otherwise (errno is set accordingly)

int magma_remove_flare_from_parent ( magma_flare_t flare  ) 

Removes a flare from parent directory. barely just a wrapper around magma_pop_dirent() which also do remote operation if necessary

Parameters:
flare the flare to be saved
Returns:
0 on success, -1 on failure (errno is set accordingly)

int magma_remove_from_cache ( magma_flare_t flare  ) 

remove an entry from the cache. flare removal is done under locking mechanism as follows.

first: the manipulation mutex is locked (that means that just a manipulation per time can be done. this place a concurrency problem that will be solved in future.)

second: the flare parent is locked. cache can't be traversed starting by the flare parent.

that prevents access to that sublevel of the cache to magma_search_under_node() and to magma_add_to_cache()

Parameters:
flare the flare to be removed. should be already placed inside the b-tree cache (that's being connected to a parent)
Returns:
1 on success, 0 on failure

void magma_save_cache_node ( magma_flare_t node,
int  remove 
)

sync to disk a cache node. if remove is true, node is also deleted from volatile memory

Parameters:
node the node of the cache to sync
remove if true, the node is also destroyed

int magma_save_flare ( magma_flare_t flare  ) 

save a flare to disk. if flare does not exists, magma_create_flare() is called. then, flare st, path, hash, xlated, parent_path, parent_hash and parent_xlated members are dumped inside metadata.

if flare is a directory, pages are msync()ed to disk.

Parameters:
flare the flare to be saved
Returns:
1 on success, 0 on failure

magma_flare_t* magma_search ( const char *  path  ) 

searches flares from internal cache. if an entry is not available, a null pointer is returned.

if entry is missing, it will *NOT* be created. this function just query cache contents, don't update or add nothing. a more generic function, called magma_search_or_create() exists and should be preferred as main interface to flare subsystem.

Parameters:
path the path to be searched
Returns:
a pointer to the flare if found, NULL otherwise

magma_flare_t* magma_search_by_armour ( const char *  armoured  ) 

searches flares from internal cache. if an entry is not available, a null pointer is returned. cache is searched by armoured version of binary hash

if entry is missing, it will *NOT* be created. this function just query cache contents, don't update or add nothing. a more generic function, called magma_search_or_create() exists and should be preferred as main interface to flare subsystem.

Parameters:
armoured the armoured rapresentation of searched flare
Returns:
a pointer to the flare if found, NULL otherwise

magma_flare_t* magma_search_by_hash ( const unsigned char *  hash  ) 

searches flares from internal cache. if an entry is not available, a null pointer is returned. cache is searched by binary hash

if entry is missing, it will *NOT* be created. this function just query cache contents, don't update or add nothing. a more generic function, called magma_search_or_create() exists and should be preferred as main interface to flare subsystem.

Parameters:
hash the binary hash of the flare to be searched
Returns:
a pointer to the flare if found, NULL otherwise

magma_flare_t* magma_search_or_create ( const char *  path  ) 

searches flares from internal cache. if flare is not found, a new one is created using new_flare() macro.

after a flare has ben got (from cache or from scratch) is checked on disk and, if found, loaded from disk.

in both cases, if flare->is_upcasted is false, flare basically does not exists on disk, or is not loadable.

Parameters:
path the path to be searched
Returns:
a pointer to the flare or NULL if something went wrong

magma_flare_t* magma_search_under_node ( const char *  hash,
magma_flare_t startnode 
)

provide searching facility from the cache. should never be used. use magma_search() or (better) magma_search_or_create() instead.

Note: magma_search_under_node returns flare in locked state. That should allow operation in not destructive ways. Each function getting a flare from magma_search_under_node should than unlock flare.

Parameters:
hash the hash key to be searched
startnode is the node used during this iteration to match
Returns:
a pointer to the flare if found, NULL otherwise

void magma_set_state ( uint8_t  state  ) 

set this node state to one of previous MAGMA_* states

Parameters:
state new state
Returns:
nothing. Always succeed.

char* magma_simplify_path ( const char *  path  ) 

simplify given path by removing /./ and /../ and ////+

Parameters:
path the path to be simplified
Returns:
simplified path

int magma_touch_flare ( magma_flare_t flare,
int  mode 
)

update flare atime, ctime and/or mtime. mode is a ORed combination of TOUCH_ATIME, TOUCH_CTIME and TOUCH_MTIME.

Parameters:
flare the flare to be updated
mode a bitmask of fields to be modified
Returns:
0 on success, -1 on failure (errno is set accordingly)
Todo:
set errno also if flare == NULL

int old_magma_flare_upcast ( magma_flare_t flare  ) 

upcast a flare to proper type if flare has been already marked as dir, file, whateven in its struct stat

Parameters:
flare the flare to be casted
Returns:
1 on success, 0 otherwise

void print_flare ( magma_flare_t flare  ) 

outputs to debugging facility all informations avalable about given flare.

Parameters:
flare the flare to be printed


Variable Documentation

char* blank_page

This is a pointer to a pagesize wide memory region filled with 0. Is used mainly by mmap() operations to grow directory size.

pthread_mutex_t cache_manipulation_mutex = PTHREAD_MUTEX_INITIALIZER

cache manipulation mutex, used only inside magma_remove_from_cache()

magma_destroyable_flare_t* destroyable_flare_queue

the queue that holds flares to be destroyed

pthread_mutex_t destroyable_flare_queue_mutex = PTHREAD_MUTEX_INITIALIZER

this is the mutex that protects destroy queue, used only inside magma_schedule_destroy_flare()

semaphore which controls queue access

const char dht_path[] = DHT_PATH

the path of the distributed hashtable profile directory

gid_t flare_system_gid = 0

GID of group magma got from system files scanning

uid_t flare_system_uid = 0

UID of user magma got from system files scanning

magma internal cache, organized like a b-tree. magma_cache is the starting node.

const uint8_t MAGMA_DHT_REBUILD = 10

this node is rebuilding DHT

const uint8_t MAGMA_OFF = 0

this node is not operating

const uint8_t MAGMA_RUNNING = 1

this node is up'n'running

const uint8_t MAGMA_SHUTDOWN = 254

this node is shutting down

uint8_t magma_state

holds this node state

pthread_mutex_t magma_state_mutex = PTHREAD_MUTEX_INITIALIZER

this mutex protects magma_state

int pagesize = 0

instead of calling getpagesize() every time page size is needed, flare system stores this value one here and than refers to pagesize later in code.

const char run_path[] = RUN_PATH

the path on real filesystem of the directory in which running data (pid, status) are saved

regular expression to eliminate 'something/../' from paths

regular expression to eliminate multiple slash sequences from paths

regular expression to eliminate './' from paths

This file last modified Wednesday, 09-Jan-2008 20:02:38 CET