#include <stdatomic.h>
#include "mda.h"

hash_t database;
hash_t Tabledefinitions;
hash_t instruments;
hash_t pricetypes; /* protected by the lock in queue #0 since that is the only queue that uses this database */

#ifndef WIN32
	pthread_mutex_t	database_lock;
	pthread_mutex_t	instruments_lock;
#else
	CRITICAL_SECTION database_lock;
	CRITICAL_SECTION instruments_lock;
#endif

struct receiver *receivers = NULL;
int receivers_num = 0;

Modulefunctions modfunc;

settings	config;
statistics	*stats;
int		stats_not_mapped;
int		stats_created;
char *application_string;

Queue *queues;

/*Queue queues[0];	// keyratios,performance,quotes,noii,greeks
Queue queues[1];	// trades,news,newscoding,newstags,stats
Queue queues[2];	// instruments,pricehistory,adjustedpricehistory, corporateactions,fundamentals,listmappings,constituents,ticksizes,L10N,estimates,estimateshistory,brokers
Queue queues[3];	// orderbooks,mbo (or news and newscoding if RequestOrderbooks and OrderMBO is disabled)
*/

_Atomic int running = 0;
int	return_status;
_Atomic int runningthreads = 0;

#ifndef WIN32
sem_t	shutdownevent;

#else
HANDLE	shutdownevent;
HANDLE hMap;
SERVICE_STATUS_HANDLE	hStatus;
SERVICE_STATUS	ServiceStatus;
#endif

static int addQueueItem(Queuefield *queuefields,MessageInfo *messageinfo);

static const FACILITYMAP facilityMap[] = {
	{ "LOG_AUTH", LOG_AUTH },
#ifdef LOG_AUTHPRIV
	{ "LOG_AUTHPRIV", LOG_AUTHPRIV },
#endif
	{ "LOG_CRON", LOG_CRON },
	{ "LOG_DAEMON", LOG_DAEMON },
#ifdef LOG_FTP
	{ "LOG_FTP", LOG_FTP },
#endif
	{ "LOG_KERN", LOG_KERN },
	{ "LOG_LOCAL0", LOG_LOCAL0 },
	{ "LOG_LOCAL1", LOG_LOCAL1 },
	{ "LOG_LOCAL2", LOG_LOCAL2 },
	{ "LOG_LOCAL3", LOG_LOCAL3 },
	{ "LOG_LOCAL4", LOG_LOCAL4 },
	{ "LOG_LOCAL5", LOG_LOCAL5 },
	{ "LOG_LOCAL6", LOG_LOCAL6 },
	{ "LOG_LOCAL7", LOG_LOCAL7 },
	{ "LOG_LPR", LOG_LPR },
	{ "LOG_MAIL", LOG_MAIL },
	{ "LOG_NEWS", LOG_NEWS },
	{ "LOG_SYSLOG", LOG_SYSLOG },
	{ "LOG_USER", LOG_USER },
	{ "LOG_UUCP", LOG_UUCP },
	{ NULL, -1 }
};
static int get_facility (const char *name)
{
	int i;
	for (i = 0; facilityMap[i].name != NULL; i++) {
		if (strcmp (facilityMap[i].name, name) == 0)
			return facilityMap[i].facility;
	}

	i = atoi (name);

	if (i == 0 && strcmp (name, "0") == 0)
		return 0;

	if (i <= 123)
		return i * 8;

	return -1;
}

#ifdef WIN32
static void send_service_pending ()
{
	if (hStatus == 0)
		return;

	ServiceStatus.dwCheckPoint++;
	SetServiceStatus (hStatus, &ServiceStatus);
}

static void send_service_running ()
{
	if (hStatus == 0)
		return;

	ServiceStatus.dwCheckPoint = 0;
	ServiceStatus.dwCurrentState = SERVICE_RUNNING;
	SetServiceStatus (hStatus, &ServiceStatus);
}
#endif

#ifdef WIN32
static int getregvalue (HKEY key, char *subkey, char *name, unsigned char **value)
{
	HKEY hkey;
	DWORD len;
	LONG ret = RegOpenKeyEx (key, subkey, 0, KEY_QUERY_VALUE, &hkey);

	if (ret != ERROR_SUCCESS)
		return -1;

	ret = RegQueryValueEx (hkey, name, NULL, NULL, NULL, &len);

	if (ret != ERROR_SUCCESS) {
		RegCloseKey (hkey);

		return 0;
	}

	*value = (unsigned char *) malloc (len);

	ret = RegQueryValueEx (hkey, name, NULL, NULL, *value, &len);

	if (ret != ERROR_SUCCESS) {
		RegCloseKey (hkey);

		return -1;
	}

	RegCloseKey (hkey);

	return len;
}

static void build_application_string ()
{
	char os_string[100];
	char *registry_value;
	SYSTEM_INFO si;
	BOOL is_wow64;
	size_t len;

	if (getregvalue (HKEY_LOCAL_MACHINE, "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion", "ProductName", (unsigned char **) &registry_value) > 0) {
		strcpy (os_string, registry_value);
		free (registry_value);
	} else {
		strcpy (os_string, "Windows");
	}

	GetNativeSystemInfo (&si);

	if (si.wProcessorArchitecture == PROCESSOR_ARCHITECTURE_AMD64)
		strcat (os_string, " AMD64");
	else
		strcat (os_string, " X86");
	
	if (IsWow64Process (GetCurrentProcess(), &is_wow64) && is_wow64 == TRUE)
		strcat (os_string, " WOW64");
	
	len = strlen (os_string) + strlen (MDA_VERSION) + 7;

	application_string = malloc (len);

	if (application_string != NULL)
		sprintf (application_string, "MDA %s [%s]", MDA_VERSION, os_string);
}
#else
static int try_version_file (char *fname, char *prefix, char *needle)
{
	FILE *f = fopen (fname, "r");

	if (f == NULL)
		return 0;

	char *line = NULL;
	size_t len = 0;
	ssize_t ret;

	while ((ret = getline (&line, &len, f)) != -1) {
		while (ret > 0 && isspace ((unsigned char)line[ret - 1]))
			ret--;

		if (ret == 0)
			continue;

		line[ret] = '\0';

		char *p;
		if (needle != NULL) {
			p = strchr (line, '=');

			if (p == NULL)
				continue;

			*p++ = '\0';

			if (strcmp (line, needle) != 0)
				continue;
		} else {
			p = line;
		}

		if (*p == '\'' || *p == '\"') {
			p++;
			line[ret - 1] = '\0';
		}

		struct utsname uts;

		if (uname (&uts) != 0)
			uts.machine[0] = '\0';

		len = strlen (p) + strlen (MDA_VERSION) + strlen (uts.machine) + 10;

		if (prefix != NULL)
			len += strlen (prefix) + 1;

		application_string = malloc (len);

		if (application_string != NULL) {
			if (prefix == NULL)
				sprintf (application_string, "MDA %s [%s %s]", MDA_VERSION, p, uts.machine);
			else
				sprintf (application_string, "MDA %s [%s %s %s]", MDA_VERSION, prefix, p, uts.machine);
		}

		free (line);
		fclose (f);
		return 1;
	}

	free (line);
	fclose (f);
	return 0;
}

static void build_application_string ()
{
	if (try_version_file ("/etc/os-release", NULL, "PRETTY_NAME") ||
	    try_version_file ("/etc/lsb-release", NULL, "DISTRIB_DESCRIPTION") ||
	    try_version_file ("/etc/debian_version", "Debian", NULL) ||
	    try_version_file ("/etc/redhat-release", NULL, NULL))
		return;

	struct utsname uts;

	if (uname (&uts) == 0) {
		size_t len = strlen (uts.sysname) + strlen (uts.release) + strlen (uts.machine) + strlen (MDA_VERSION) + 10;
		application_string = malloc (len);

		if (application_string != NULL) {
			sprintf (application_string, "MDA %s [%s %s %s]", MDA_VERSION, uts.sysname, uts.release, uts.machine);
			return;
		}
	}

	application_string = malloc (strlen (MDA_VERSION) + 7);

	if (application_string != NULL)
		sprintf (application_string, "MDA %s", MDA_VERSION);
}

static void close_pidfile (FILE *f)
{
	if (f == NULL)
		return;

	fclose (f);

	if (config.pidfile != NULL)
		remove (config.pidfile);
}
#endif

static int database_compare ( void *a, void *b)
{
	return *((uint64_t *)a) - *((uint64_t *)b);
}

static void print_help ()
{
	printf ("mda - Millistream Database Application\n");
#ifndef WIN32
	printf ("  --daemon           run as a daemon\n");
	printf ("  --console          run in console\n");
#endif
	printf ("  --status           print status\n");
	printf ("  --map-name=NAME    change the name used to find the memory mapped file used\n");
	printf ("                     to store the statistics data. This must be used to set a\n");
	printf ("                     different name for each instance of more than one MDA is\n");
	printf ("                     running on the same machine.\n");
	printf ("  --check            perform some tests on the config, tabledef and sql server\n");
	printf ("  --check-types      check whether the data types in the sql server corresponds\n");
	printf ("                     with the column definitions in the tabledef file.\n");
	printf ("  --upgrade-db       execute ALTER TABLE on the sql server to add any missing\n");
	printf ("                     columns (must be called with --check or --check-types)\n");
#ifndef WIN32
	printf ("  --config=FNAME     read config from FNAME (default is 'mda.config' in the\n");
	printf ("                     current directory and '/etc/mda.config')\n");
#else
	printf ("  --config=FNAME     read config from FNAME (default is 'mda.config' in the\n");
	printf ("                     install directory [fetched from the registry])\n");
#endif
	printf ("  --dbsuser=USER     use USER for the SQL Authentication instead of the details\n");
	printf ("                     from the config file. The password will be prompted for.\n");
#ifdef WIN32
	printf ("  --domainuser=USER  use USER for the Windows Authentication instead of the\n");
	printf ("                     details from the config file. The format of USER is\n");
	printf ("                     DOMAIN\\USERNAME, if only USERNAME then the local computer\n");
	printf ("                     is used for DOMAIN. The password will be prompted for.\n");
#endif
	printf ("  --import=FNAME     read SQL commands from FNAME and execute them. This is\n");
	printf ("                     used to import sql dumps created with i.e mysqldump on\n");
	printf ("                     incompatible SQL servers, or SQL servers without a good\n");
	printf ("                     import application. Supports Gzipped files (.gz).\n");
	printf ("                     If FNAME is \"-\" then stdin will be read instead of a file.\n");
	printf ("  --help             show this screen\n");
	printf ("\n");
}
static void forceItemOnQueue(Queue *queue,Queueitem *queueitem)
{
Queueitem *ptr = queue->nowaitptr == NULL ? queue->head : queue->nowaitptr;
const time_t	timenow = queueitem->timestamp;

	queue->stats->items++;
	queueitem->timestamp-= (queue->mintimebetweenupdates+1);
	if(queue->head == NULL) {
		queue->head = queueitem;
		queue->tail = queueitem;
		queue->nowaitptr = queueitem;
		return;
	}
	while(ptr != NULL && timenow > (ptr->timestamp + (time_t)(queue->mintimebetweenupdates)) ) {
		ptr = ptr->next;
	}
	if(ptr == NULL) {
		queue->tail->next = queueitem;
		queueitem->previous = queue->tail;
		queue->tail = queueitem;
	}
	else {
		queueitem->next = ptr;
		queueitem->previous = ptr->previous;
		
		if(queueitem->previous != NULL) queueitem->previous->next = queueitem;
		else queue->head = queueitem;
		queueitem->next->previous = queueitem;
	}
	queue->nowaitptr = queueitem;
	return;
}
static void insertItemInQueue(Queue *queue,Queueitem *queueitem)
{
	if (queue->head == NULL ) {
		queue->head = queueitem;
		queue->tail = queueitem;
	}
	else {
		queue->tail->next = queueitem;
		queueitem->previous = queue->tail;
		queue->tail = queueitem;
	}
	if(queueitem->option & MDA_OPTION_EMPTY) {
		queue->stats->empty++;
	}
	else {
		queue->stats->items++;
	}
}
static void deleteItemInQueue(Queue *queue,Queueitem *_workitem)
{
	// move the nowaitptr if its the current item
	if(queue->nowaitptr == _workitem ) queue->nowaitptr = _workitem->next;

	if (_workitem->previous ) _workitem->previous->next = _workitem->next;
	if (_workitem->next ) _workitem->next->previous = _workitem->previous;
	if (_workitem == queue->head ) queue->head = _workitem->next;
	if(queue->tail == _workitem) queue->tail = _workitem->previous;
	if(queue->head == NULL ) queue->tail = NULL;

	if (_workitem->option & MDA_OPTION_HASH && _workitem->key != NULL) {
		hashit_delete (queue->queueindex,_workitem->key);
	}
	if(_workitem->option & MDA_OPTION_EMPTY) {
		queue->stats->empty--;
	}
}

static void	freequeuefields (Queuefield *_queuefields)
{
	Queuefield	*_qf = _queuefields;
	Queuefield	*_qf2;
	while (_qf != NULL) {
		_qf2 = _qf->next;
		free (_qf->value);
		free (_qf);
		_qf = _qf2;
	}
}
static void freequeueitem (Queueitem	*_queueitem)
{
	freequeuefields(_queueitem->fields);
	if(_queueitem->bidorders != NULL) {
		Queueitem	*order = _queueitem->bidorders;
		while(order != NULL) {
			Queueitem *next = order->next;
			freequeueitem(order);
			order = next;	
		}
	}
	if( _queueitem->askorders != NULL ) {
		Queueitem	*order = _queueitem->askorders;
		while(order != NULL) {
			Queueitem *next = order->next;
			freequeueitem(order);
			order = next;
		}
	}
	free ( _queueitem->key);
	free ( _queueitem->orderprice);
	free ( _queueitem->orderyield);
	free ( _queueitem->bidprice);
	free ( _queueitem->bidyield);
	free ( _queueitem );
}

static void clearItemHash(Queue *queue,uint64_t insref,uint32_t messagetype, const int delete)
{
	char _key[256];
	sprintf(_key,"%" PRIu64 ":%" PRIu32,insref,messagetype);

	MUTEX_LOCK(&queue->lock);

	Queueitem *queueitem = hashit_lookup(queue->queueindex,_key);
	if(queueitem != NULL) {
		if (delete == 1) {	/* remove the item from the queue and delete it */
			deleteItemInQueue (queue, queueitem);

			if((queueitem->option & MDA_OPTION_EMPTY) == 0)
				queue->stats->items--;

			freequeueitem (queueitem);
		} else {		/* keep the item in the queue but remove it from the hash */
			hashit_delete (queue->queueindex,queueitem->key);
			queueitem->option &= ~MDA_OPTION_HASH;
		}
	}

	MUTEX_UNLOCK(&queue->lock);

	return;
}

#ifdef VALGRIND
static void	clear_tabeldef()
{
char	**_keys;
char	**_startkey;

	free(config.quotefields);
	free(config.quotebbofields);
	free(config.noiifields);
	free(config.greeksfields);
	free(config.keyratios);

	_keys = (char **) hashit_getkeys(Tabledefinitions);
	if(_keys == NULL) return;
	_startkey = _keys;
	while (*_keys) {
		Tabledef *_def = (Tabledef *) hashit_lookup (Tabledefinitions, *_keys);
		if(_def != NULL ) { 
			hashit_delete(Tabledefinitions,*_keys);
			free(_def->tablename);
			free(_def->fieldname);
			free(_def->fieldname2);
			free(_def->factor);
			free(_def->hashkey);
			free(_def->fieldnumber);
			free(_def->messages);
			free(_def->type);
			free(_def->len);
		}
		_keys++;
		free(_def);
	}
	free(_startkey);
}

static void free_pricetypes()
{
	void **keys = hashit_getkeys(pricetypes);
	void **tmp = keys;

	while (*keys != NULL)
		free (*keys++);

	free (tmp);
}
#endif

static void free_instruments (int do_delete)
{
	uint64_t **keys;
	uint64_t **first;
	MessageInfo mi;
	long unsigned deleted_num = 0;

	if (instruments == NULL)
		return;

	keys = (uint64_t **) hashit_getkeys (instruments);

	if (keys == NULL)
		return;

	first = keys;

	memset (&mi, 0, sizeof mi);

	/* safety measure */
	if (do_delete == 1 && hashit_size (instruments) > config.maxdeletes) {
		write_to_log(LOG_INFO,"Delete instrument safety limit hit, ignoring delete of %lu stale instruments.", hashit_size (instruments));
		do_delete = 0;
	}

	while (*keys) {
		if (do_delete == 1) {
			mi.insref = **keys;
			mi.mdatype = MDA_TYPE_INSTRUMENTDELETE;
			mi.option = MDA_OPTION_NOHASH;	

			addQueueItem (NULL, &mi);
			deleted_num++;
		}

		hashit_delete (instruments, *keys);
		free (*keys);
		keys++;
	}

	free (first);
	hashit_destroy (instruments);
	instruments = NULL;

	if (deleted_num != 0)
		write_to_log(LOG_INFO,"Deleted %lu stale instruments.", deleted_num);
}

#ifdef VALGRIND
static void	clear_queue(Queue *queue)
{
Queueitem	*qi;
	/* 	Free remaining in queue     */
	while (queue->head != NULL) {
		qi = queue->head->next;
		if (queue->head->option & MDA_OPTION_HASH ) {
			hashit_delete (queue->queueindex,queue->head->key);
		}
		freequeueitem(queue->head);
		queue->head = qi;	
	}
}
#endif

static void	clear_orderbook(Instrument *_instrument)
{
Orderbook	*_order;
Orderbook	*_order2;

	_order = _instrument->askorders;
	while (_order != NULL ) {
		free(_order->price);
		_order2 = _order->next;
		free(_order);
		_order = _order2;
	}
	_instrument->askorders = NULL;

	_order = _instrument->bidorders;
	while (_order != NULL ) {
		free(_order->price);
		_order2 = _order->next;
		free(_order);
		_order = _order2;
	}
	_instrument->bidorders = NULL;

	_instrument->numbidorders = 0;
	_instrument->numaskorders = 0;
}

#ifdef VALGRIND
static void	clear_orderbooks()
{
Instrument *_instrument;
uint64_t **_keys;
uint64_t **_startkey;

	_keys = (uint64_t **)hashit_getkeys(database);
	if(_keys == NULL) return;
	_startkey = _keys;
	while (*_keys) {
		_instrument = (Instrument *) hashit_lookup (database, *_keys);
		if(_instrument != NULL ) { 
			clear_orderbook(_instrument);
			hashit_delete(database,&_instrument->insref);
			free (_instrument);
		}
		_keys++;
	}
	free(_startkey);
}
#endif

/* For Windows we have to wait for a key press if the user has
 * selected to run in the console since this could happen from
 * the desktop, and thus the window would dissapear right away
 * on any error */
static void waitforkeypress ()
{
#ifdef WIN32
	if (config.daemonize == 0) {
		printf ("Press any key to continue...\n");
		_getch ();
	}
#endif
}

static int show_status ()
{
#ifdef WIN32
	running = 0;

	hMap = CreateFileMapping (INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0, sizeof *stats, config.map_name);

	if (hMap == NULL) {
		printf ("Cannot create shared memory mapping: %u\n", (unsigned int)GetLastError ());
		return 1;
	}

	stats = MapViewOfFile (hMap, FILE_MAP_READ, 0, 0, 0);

	if (stats == NULL) {
		printf ("Cannot map the shared memory into process: %u\n", (unsigned int)GetLastError ());
		CloseHandle (hMap);
		return 1;
	}
#else
	running = 0;

	int hMap = shm_open (config.map_name, O_RDONLY, 0);

	if (hMap == -1) {
		printf ("Cannot create shared memory mapping: %s\n", strerror(errno));
		return 1;
	}

	stats = mmap (NULL, sizeof *stats, PROT_READ, MAP_SHARED, hMap, 0);
	close (hMap);

	if (stats == NULL) {
		printf ("Cannot map the shared memory into process: %s\n", strerror(errno));
		return 1;
	}

	running = 1;

	if (stats->pid < 1 || (kill (stats->pid, 0) != 0 && errno == ESRCH))
		running = 0;

	if (time (NULL) - stats->last_update_stats > 2)
		running = 0;
#endif
	uint64_t connected_to_feed = stats->connected_to_feed;

	if (running == 0)
		connected_to_feed = 0;

	printf ("received-messages: %" PRIu64 "\n", stats->receivedmessages);
	printf ("connected-to-feed: %" PRIu64 "\n", connected_to_feed);

	uint64_t items = 0, dedup = 0, updates = 0, failed = 0;
	int i;
	for(i = 0; i < NUM_QUEUES;i++) {
		items += stats->queuestat[i].items;
		updates += stats->queuestat[i].updates;
		dedup += stats->queuestat[i].dedup;
		failed += stats->queuestat[i].failed;
	}

	printf ("deduplicated-messages: %" PRIu64 "\n", dedup);
	printf ("database-updates-ok: %" PRIu64 "\n", updates);
	printf ("database-updated-failed: %" PRIu64 "\n", failed);
	printf ("database-updates-queued: %" PRIu64 "\n", items);

	for(i = 0; i < NUM_QUEUES;i++) {
		printf ("q%d-queue: %" PRIu64 "\n",i+1, stats->queuestat[i].items);
		printf ("q%d-empty-items: %" PRIu64 "\n",i+1, stats->queuestat[i].empty);
		printf ("q%d-updated: %" PRIu64 "\n",i+1, stats->queuestat[i].updates);
		printf ("q%d-deduplicated: %" PRIu64 "\n",i+1, stats->queuestat[i].dedup);
		printf ("q%d-failed: %" PRIu64 "\n",i+1, stats->queuestat[i].failed);
	}

#ifdef WIN32
	printf ("instrument-hash-size: %Iu/%Iu\n", stats->database_hashsize, stats->database_hashsize_max);
#else
	printf ("instrument-hash-size: %zu/%zu\n", stats->database_hashsize, stats->database_hashsize_max);
#endif
	printf ("templates-version: %" PRIu64 "/%" PRIu64 "\n", stats->file_templates_ver, stats->feed_templates_ver);
	printf ("running: %d\n", running);
	printf ("version: %s\n",MDA_VERSION);


#ifdef WIN32
	UnmapViewOfFile (stats);
	CloseHandle (hMap);
#else
	munmap (stats, sizeof *stats);
#endif
	return 0;
}


static int compare_tabledefs (const void *a, const void *b)
{
	int ret = strcmp ((*(Tabledef * const *)a)->tablename, (*(Tabledef * const *)b)->tablename);

	if (ret == 0)
		ret = strcmp ((*(Tabledef * const *)a)->fieldname, (*(Tabledef * const *)b)->fieldname);

	return ret;
}

static uint8_t	check_db(Databaseinfo	*_dbinfo)
{
Tabledef *_def;
Tabledef **defs = NULL;
Tabledef **mods = NULL;
int defs_num = 0;
int mods_num = 0;
char **_keys;
char **_startkey;
hash_t added_columns;
char *key = NULL;
char tmpkey[256];
	
	_keys = (char **)hashit_getkeys(Tabledefinitions);
	if(_keys == NULL) {
		write_to_log(LOG_CRIT,"No tabledefinitions. Stopping..");
		return 0;
	}
	added_columns = hashit_create(1024,0,NULL,( int( * ) (void *,void *))strcmp,CHAIN_H);
	_startkey = _keys;

	while (*_keys) {
		_def = (Tabledef *) hashit_lookup (Tabledefinitions, *_keys);
		if(_def != NULL ) { 
			char new_type[256];
			char current_type[256];
			int _found = 0, do_modify = 0;
			Databaseinfo *_tmp = _dbinfo;
			while(_tmp != NULL) {
				if((0 == strcasecmp(_def->tablename,_tmp->tablename)) && (0 == strcasecmp(_def->fieldname,_tmp->fieldname))) {
					if (config.perform_checktypes == 1) {
						if (modfunc.add_column (queues[0].modhandle, current_type, NULL, _tmp->type, (_def->len == NULL && _def->type != NULL && _tmp->type != NULL && strcmp (_def->type, _tmp->type) == 0) ? NULL : _tmp->len, _tmp->nullable, _tmp->_unsigned, 2) == 1) {
							if (modfunc.add_column (queues[0].modhandle, new_type, NULL, _def->type, _def->len, _def->nullable, _def->_unsigned, 2) == 1) {
								if (strcmp (current_type, new_type) != 0) {
									do_modify = 1;
								}
							}
						}
					}

					_found = 1;
					break;
				}
				_tmp = _tmp->next;
			}

			if(_found == 0) {
				sprintf(tmpkey,"%s:%s",_def->tablename,_def->fieldname);
				if(NULL == hashit_lookup(added_columns,tmpkey)) {
					if(config.perform_check == 1 && config.upgradedb == 1) {
						if(_def->type == NULL) {
							write_to_log(LOG_CRIT,"Database upgrade not possible, old mda.tabledef?");
							return 0;
						}
						if( 1 != modfunc.add_column(queues[0].modhandle,_def->tablename,_def->fieldname,_def->type,_def->len,_def->nullable,_def->_unsigned,0)) {
							void *tmp;
							char *errorstring = modfunc.geterrorstring(queues[0].modhandle);

							write_to_log(LOG_CRIT,"Failed to add column %s to table %s. Error:%s Code:%d",_def->fieldname,_def->tablename,errorstring == NULL ? "NULL":errorstring, modfunc.getlasterror(queues[0].modhandle));

							tmp = realloc (defs, sizeof *defs * (defs_num + 1));

							if (tmp == NULL) {
								free (defs);
								free (_startkey);
								write_to_log(LOG_CRIT,"Out of memory. Stopping..");
								return 0;
							}

							defs = tmp;
							defs[defs_num++] = _def;
						}
						else
							write_to_log(LOG_INFO,"Added column %s to table %s.",_def->fieldname,_def->tablename);
					}
					else {
						void *tmp = realloc (defs, sizeof *defs * (defs_num + 1));

						if (tmp == NULL) {
							free (defs);
							free (_startkey);
							write_to_log(LOG_CRIT,"Out of memory. Stopping..");
							return 0;
						}

						defs = tmp;
						defs[defs_num++] = _def;
					}
					key = strdup(tmpkey);
					hashit_insert(added_columns,key,key);
				}
			}

			if (do_modify == 1) {
				sprintf(tmpkey,"%s:%s",_def->tablename,_def->fieldname);
				if(NULL == hashit_lookup(added_columns,tmpkey)) {
					if(config.perform_check == 1 && config.upgradedb == 1) {
						if(_def->type == NULL) {
							write_to_log(LOG_CRIT,"Database upgrade not possible, old mda.tabledef?");
							return 0;
						}
						if( 1 != modfunc.add_column(queues[0].modhandle,_def->tablename,_def->fieldname,_def->type,_def->len,_def->nullable,_def->_unsigned,1)) {
							char *errorstring;
							int errorno;
							void *tmp;
							errorstring = modfunc.geterrorstring(queues[0].modhandle);

							errorno = modfunc.getlasterror(queues[0].modhandle);
							write_to_log(LOG_CRIT,"Failed to modify column '%s.%s' from '%s' to '%s'. Error:%s Code:%d",_def->tablename,_def->fieldname,current_type,new_type,errorstring == NULL ? "NULL":errorstring,errorno);

							tmp = realloc (mods, sizeof *mods * (mods_num + 1));

							if (tmp == NULL) {
								free (mods);
								free (_startkey);
								write_to_log(LOG_CRIT,"Out of memory. Stopping..");
								return 0;
							}

							mods = tmp;
							mods[mods_num++] = _def;
						}
						else
							write_to_log(LOG_INFO,"Modified column '%s.%s' from '%s' to '%s'.",_def->tablename,_def->fieldname,current_type,new_type);
					}
					else {
						void *tmp = realloc (mods, sizeof *mods * (mods_num + 1));

						if (tmp == NULL) {
							free (mods);
							free (_startkey);
							write_to_log(LOG_CRIT,"Out of memory. Stopping..");
							return 0;
						}

						mods = tmp;
						mods[mods_num++] = _def;
					}
					key = strdup(tmpkey);
					hashit_insert(added_columns,key,key);
				}
			}
		}
		_keys++;
	}
	free(_startkey);

	_keys = (char **)hashit_getkeys(added_columns);
	_startkey = _keys ;
	while(*_keys) {
		hashit_delete(added_columns,*_keys);
		free(*_keys);
		_keys++;
	}
	free(_startkey);
	hashit_destroy(added_columns);

	if (defs_num != 0) {
		char *tablename = NULL;
		int i;

		qsort (defs, defs_num, sizeof *defs, compare_tabledefs);

		write_to_log (LOG_CRIT, "");
		write_to_log (LOG_CRIT, "The following columns must be added to the SQL server:");
		write_to_log (LOG_CRIT, "");
		write_to_log (LOG_CRIT, "Table   Column");
		write_to_log (LOG_CRIT, "------------------------------------------------------");

		for (i = 0; i < defs_num; i++) {
			if (tablename == NULL || strcmp (defs[i]->tablename, tablename) != 0) {
				if (tablename != NULL)
					write_to_log (LOG_CRIT, "");

				tablename = defs[i]->tablename;

				write_to_log (LOG_CRIT, "%s", tablename);
			}
			write_to_log (LOG_CRIT, "        %s", defs[i]->fieldname);
		}

		write_to_log (LOG_CRIT, "");
	}

	if (mods_num != 0) {
		int i;

		qsort (mods, mods_num, sizeof *mods, compare_tabledefs);

		write_to_log (LOG_WARNING, "");
		write_to_log (LOG_WARNING, "The following columns should have their definitions changed in the SQL server:");
		write_to_log (LOG_WARNING, "");
		write_to_log (LOG_WARNING, "Column        New Definition");
		write_to_log (LOG_WARNING, "------------------------------------------------------------------------------");

		for (i = 0; i < mods_num; i++) {
			modfunc.add_column (queues[0].modhandle, tmpkey, NULL, mods[i]->type, mods[i]->len, mods[i]->nullable, mods[i]->_unsigned, 2);

			write_to_log (LOG_WARNING, "%s.%s '%s'", mods[i]->tablename, mods[i]->fieldname, tmpkey);
		}

		write_to_log (LOG_WARNING, "");
	}

	if (defs_num != 0 || mods_num != 0) {
		if (mods_num != 0)
			write_to_log(LOG_INFO,"To automatically fix this, run \"mda --check-types --upgrade-db\"");
		else
			write_to_log(LOG_INFO,"To automatically fix this, run \"mda --check --upgrade-db\"");

		write_to_log(LOG_INFO,"You might also have to add \"--dbsuser <username>\" in case the SQL user in mda.config does not have rights to make such changes to the tables.");

		if (defs_num != 0) {
			write_to_log(LOG_CRIT,"The SQL database check failed");
			return 0;
		}

		write_to_log(LOG_WARNING, "The SQL database check had warnings");
	} else if (config.perform_check != 0) {
		write_to_log(LOG_INFO,"The SQL database check was successful");
	}

	return 1;
}

int	main ( int argc, char	**argv )
{
	int i;
	int ret;
#ifdef WIN32
	SERVICE_TABLE_ENTRY _servicetable[2];
#endif
	char	_key[256];
	Tabledef *_def = NULL;

	config.silent = 0;
	config.perform_check = 0;
	config.perform_checktypes = 0;
	config.show_status = 0;
	config.upgradedb = 0;
	config.import_fname = NULL;
	config.map_name = strdup (DEFAULT_MAP_NAME);

#ifndef WIN32
	if(argc < 2) {
		print_help ();
		return 1;
	}
	config.daemonize = 2;
#else
	config.daemonize = 0;
#endif
	for ( i=1; i < argc ;i++ ) {
		char *arg = argv[i];
		char *opt = strchr (arg, '=');

		if (opt != NULL) {
			*opt++ = '\0';
		} else {
			if (i == argc)
				opt = NULL;
			else
				opt = argv[i + 1];
		}

		if ( 0 == strcmp (arg, "--daemon" )) {
			config.daemonize = 1;
			config.silent = 1;
		} else if ( 0 == strcmp (arg, "--debug" )) { /* TODO: should either be removed or actually enable some debugging */
			config.daemonize = 0;
		} else if ( 0 == strcmp (arg, "--console" )) {
			config.daemonize = 0;
		} else if ( 0 == strcmp (arg, "--silent" )) {
			config.silent = 1;
		} else if ( 0 == strcmp (arg, "--config" )) {
			if (opt == NULL ) {
				printf("Missing config filename\n");
				free(config.configfilename);
				return 1;
			}
			free(config.configfilename);
			config.configfilename = strdup(opt);

			if (opt == argv[i + 1])
				i++;
		} else if ( 0 == strcmp (arg, "--status" )) {
			config.show_status = 1;
		} else if ( 0 == strcmp (arg, "--check" )) {
			config.perform_check = 1;
		} else if ( 0 == strcmp (arg, "--check-types" )) {
			config.perform_check = 1;
			config.perform_checktypes = 1;
		} else if ( 0 == strcmp (arg, "--help" )) {
			print_help ();
			return 0;
		} else if ( 0 == strcmp (arg, "--import")) {
			if(opt == NULL) {
				printf("Missing filename for --import\n");
				return 1;
			}
			config.import_fname = opt;
			i++;
		} else if ( 0 == strcmp (arg, "--upgrade-db" )) {
			config.upgradedb = 1;
		} else if ( 0 == strcmp (arg, "--dbsuser")) {
			if(opt == NULL) {
				printf("Missing username\n");
				return 1;
			}
			config.overrideusr = opt;

			if (opt == argv[i + 1])
				i++;
#ifdef WIN32
		} else if ( 0 == strcmp (arg, "--domainuser")) {
			if(opt == NULL) {
				printf("Missing username\n");
				return 1;
			}
			config.overridedomainuser = opt;

			if (opt == argv[i + 1])
				i++;
#endif
		} else if ( 0 == strcmp (arg, "--map-name" )) {
			if (opt == NULL ) {
				printf("Missing name for --map-name\n");
				return 1;
			}
			free(config.map_name);
			config.map_name = strdup(opt);

			if (opt == argv[i + 1])
				i++;
		} else {
			printf ("unknown argument: '%s'\n", arg);
			return 1;
		}
	}

	// set default values
	config.databasekeepdeleted = 1;		// keep deleted, set tradestate = 9 
	config.checkdatabase = 1;
	config.instrumentslimit = UINT64_MAX;
	config.databasehashsize = DATABASE_SIZE;
	config.queuehashsize = QUEUEINDEX_SIZE;
	config.max_timelen = 15;

	if (config.perform_check == 1 || config.import_fname != NULL)
		config.daemonize = 0;

	if (config.show_status == 1)
		return show_status ();

	return_status = 1;

#ifdef WIN32
	_servicetable[0].lpServiceName = "Millistream MDA";
	_servicetable[0].lpServiceProc = ServiceMain;
	_servicetable[1].lpServiceName = NULL;
	_servicetable[1].lpServiceProc = NULL;

	if (StartServiceCtrlDispatcher (_servicetable) == 0 && GetLastError () == ERROR_FAILED_SERVICE_CONTROLLER_CONNECT) {
		ret = init ();

		if (ret != 0)
			return ret;

		SetConsoleCtrlHandler((PHANDLER_ROUTINE)signal_handler,TRUE);
		WaitForSingleObject (shutdownevent,INFINITE);
	} else {
		ret = init ();

		if (ret != 0)
			return ret;
	}
#else
	ret = init ();

	if (ret != 0)
		return ret;

	while ( 0 != sem_wait (  &shutdownevent ) );

# ifdef HAVE_LIBSYSTEMD
	sd_notify (0, "STOPPING=1");
# endif
#endif
	write_to_log (LOG_INFO, "Shutting down, please wait...");

	while (runningthreads > 0 ) {
#ifdef HAVE_LIBSYSTEMD
		if (sd_watchdog_enabled (0, NULL) > 0)
			sd_notify (0, "WATCHDOG=1");
#endif
#ifndef WIN32
		sleep ( 1 );
#else
		Sleep (1000);
#endif
	}

/*	set running  = 0 in stats */
	sprintf(_key,"%d:insref",MDA_TYPE_INTERNALSTATUS);
	if(NULL != (_def = hashit_lookup (Tabledefinitions,_key))) {
		sprintf(_key,"UPDATE %s SET value = '0' WHERE name = 'Running'",_def->tablename);
		if (1 != modfunc.executesql (queues[1].modhandle,_key)) {
			sprintf(_key,"INSERT INTO %s (name,value) VALUES ('Running','0')",_def->tablename);
			modfunc.executesql (queues[1].modhandle,_key);
		}
	}

	for(i = 0 ; i < NUM_QUEUES; i++) {
		modfunc.disconnect(queues[i].modhandle);
		modfunc.destroy(queues[i].modhandle);
	}

	modfunc.exit();

#ifndef WIN32
	for(i = 0 ; i < NUM_QUEUES ; i++) {
		pthread_mutex_destroy ( &queues[i].lock);
	}
	pthread_mutex_destroy (&database_lock);
	pthread_mutex_destroy (&instruments_lock);
	sem_destroy ( &shutdownevent );
	dlclose(config.dbhandle);
	if(config.pidfile) free(config.pidfile);
	if(config.applicationuser) free(config.applicationuser);
	if(config.applicationgroup) free(config.applicationgroup);
#else
	CloseHandle(shutdownevent);
	for(i = 0 ; i < NUM_QUEUES ; i++) {
		DeleteCriticalSection(&queues[i].lock);
	}
	DeleteCriticalSection(&database_lock);
	DeleteCriticalSection(&instruments_lock);
	FreeLibrary(config.dbhandle);
#endif

#ifdef VALGRIND
	clear_orderbooks();	// clear before locks are destroyed
	free(config.millistreamusername);
	free(config.millistreampassword);
	free(config.millistreamserver);
	free(config.insrefs);
	
	for(i = 0 ; i < NUM_QUEUES ; i++) {
		clear_queue(&queues[i]);
		free(queues[i].sqlbuildstring->sqlstring);
		free(queues[i].sqlbuildstring->whereclause);
		free(queues[i].sqlbuildstring);
	}
	
	if(config.fd_sqllog) fclose(config.fd_sqllog);
	if(config.logsqlpath) free(config.logsqlpath);
	
	clear_tabeldef();
	
	free(config.configfilename);
	free(config.tabledeffile);
	free(config.modulefile);

	free_instruments(0);

	hashit_destroy(database);		
	free_pricetypes();
	hashit_destroy(pricetypes);
	for(i = 0 ; i < NUM_QUEUES ; i++) {
		hashit_destroy(queues[i].queueindex);
	}
	free(queues[0].transaction);
	hashit_destroy(Tabledefinitions);
	free(queues);
	free (config.mod_config.username);
	free (config.mod_config.password);
	free (config.mod_config.database);
	free (config.mod_config.host);
	free (config.mod_config.port);
	free (config.mod_config.connectionstring);
	free (config.mod_config.ssl_key);
	free (config.mod_config.ssl_cert);
	free (config.mod_config.ssl_ca);
	free (config.mod_config.ssl_capath);
	free (config.mod_config.ssl_cipher);
#endif

	if (stats_not_mapped == 0) {
#ifdef WIN32
		UnmapViewOfFile (stats);
		CloseHandle (hMap);
#else
		munmap (stats, sizeof *stats);

		if (stats_created == 1)
			shm_unlink (config.map_name);
#endif
	}

	write_to_log (LOG_INFO, "Shutdown complete");
	waitforkeypress ();
	return return_status;
}

#ifndef WIN32
char get_pwdchar()
{
	char buf = 0;
	struct termios old;
	memset (&old, 0, sizeof old);

	if (tcgetattr(0, &old) < 0)
		printf("tcsetattr()");
	old.c_lflag &= ~ICANON;
	old.c_lflag &= ~ECHO;
	old.c_cc[VMIN] = 1;
	old.c_cc[VTIME] = 0;
	if (tcsetattr(0, TCSANOW, &old) < 0)
		printf("tcsetattr ICANON");
	if (read(0, &buf, 1) < 0)
		perror ("read()");

	old.c_lflag |= ICANON;
	old.c_lflag |= ECHO;
	if (tcsetattr(0, TCSADRAIN, &old) < 0)
		printf ("tcsetattr ~ICANON");
	return buf;

}
#endif

int init_queues()
{
	queues = calloc(NUM_QUEUES,sizeof(Queue));
	queues[0].mintimebetweenupdates = config.mintimebetweenupdates;
	queues[1].mintimebetweenupdates = 0;
	queues[2].mintimebetweenupdates = 0;
	queues[3].mintimebetweenupdates = config.mintimebetweenupdates;

	if (config.requestOrderbooks != 1 && config.requestMBO != 1)
		queues[3].mintimebetweenupdates = 0;
	else
		queues[3].mintimebetweenupdates = config.mintimebetweenupdates;

	int i;
	for(i = 0; i <  NUM_QUEUES;i++) {
		queues[i].num_transactions = 0;
		queues[i].sqlbuildstring = calloc(1,sizeof(Sqlstring));
		queues[i].sqlbuildstring->sqlstring = malloc(SQLSTRING_MINSIZE);
		queues[i].sqlbuildstring->sqlsize = SQLSTRING_MINSIZE;
		queues[i].sqlbuildstring->whereclause  = malloc(SQLSTRING_MINSIZE);
		queues[i].sqlbuildstring->wheresize = SQLSTRING_MINSIZE;
		queues[i].queueindex = hashit_create(config.queuehashsize,0,NULL, ( int( * ) (void *,void *))strcmp,CHAIN_H);	// 0.8 MB	// can now be configured
#ifndef WIN32
		pthread_mutex_init ( &queues[i].lock, NULL);
		sem_init ( &queues[i].itemevent, 0, 0 );
#else
		InitializeCriticalSection(&queues[i].lock);
		queues[i].itemevent = CreateSemaphore (NULL,0,LONG_MAX,NULL);
#endif
		queues[i].pos = i + 1;
	}
	return 1;
}

#ifndef WIN32
int change_user (char *user, char *group)
{
	if (user == NULL && group == NULL)
		return 0;

	struct passwd *pwd = NULL;

	if (user != NULL) {
		pwd = getpwnam (user);

		if (pwd == NULL) {
			write_to_log (LOG_CRIT, "Cannot find user: %s : %s", user , strerror(errno));
			return -1;
		}

		if (pwd->pw_uid == 0) {
			write_to_log (LOG_CRIT, "I will not set uid to 0");
			return -1;
		}
	}

	if (group != NULL) {
		struct group *grp = getgrnam (group);

		if (grp == NULL) {
			write_to_log (LOG_CRIT, "Could not find group: %s : %s", group, strerror(errno));
			return -1;
		}

		if (grp->gr_gid == 0) {
			write_to_log (LOG_CRIT, "I will not set gid to 0");
			return -1;
		}

		if (setgid (grp->gr_gid ) == -1) {
			write_to_log (LOG_CRIT, "Unable to change to group: %s : %s", group, strerror(errno));
			return -1;
		}

		if (setgroups ( 0, NULL ) == -1) {
			write_to_log (LOG_CRIT, "Error calling setgroups(): %s", strerror(errno));
			return -1;
		}
	}

	if (pwd != NULL) {
		if (setuid (pwd->pw_uid ) == -1) {
			write_to_log (LOG_CRIT, "Unable to change to user: %s : %s", user, strerror(errno));
			return -11;
		}

		write_to_log (LOG_INFO, "Dropped root privileges");
	}

	return 0;
}
#endif

int init ()
{
char	_key[256];
Tabledef *_def = NULL;
int facility = -1;
mod_result	*modres = NULL;
mod_info        modinfo;

	build_application_string ();
	
	/* if --config is not used we try to open a few "default" places */
	if (config.configfilename == NULL) {
		config.configfilename = strdup("mda.config");

		if (read_config () == -1) {
			free (config.configfilename);
#ifdef WIN32
			if (getregvalue (HKEY_LOCAL_MACHINE, "Software\\Millistream\\MDA", "Install Directory", (unsigned char **) &config.configfilename) > 0) {
				config.configfilename = realloc (config.configfilename, strlen (config.configfilename) + 16);
				strcat (config.configfilename, "\\MDA\\mda.config");
			} else if (getregvalue (HKEY_LOCAL_MACHINE, "Software\\Wow6432Node\\Millistream\\MDA", "Install Directory", (unsigned char **) &config.configfilename) > 0) {
				config.configfilename = realloc (config.configfilename, strlen (config.configfilename) + 16);
				strcat (config.configfilename, "\\MDA\\mda.config");
			} else {
				config.configfilename = strdup("mda.config");
			}
#else
			config.configfilename = strdup("/etc/mda.config");
#endif
			if (read_config () == -1) {
				write_to_log(LOG_CRIT,"Could not open config file '%s'", config.configfilename);
				free(config.configfilename);
				waitforkeypress ();
				return 1;
			}
		}
	} else if (-1 == read_config () ) {
		write_to_log(LOG_CRIT,"Could not open config file '%s'", config.configfilename);
		free(config.configfilename);
		waitforkeypress ();
		return 1;
	}

	if (receivers_num == 0) {
		write_to_log(LOG_CRIT,"No MDF receivers configured");
		free(config.configfilename);
		waitforkeypress ();
		return 1;
	}

#ifndef WIN32
	if (config.perform_check == 1 || config.daemonize == 0) {
		free (config.pidfile);
		config.pidfile = NULL;
	}
#endif
	if (config.perform_check == 1 && config.overrideusr) {
		char _pwd[512];
		int ch = 0;
		int pos =0;
		free(config.mod_config.username);
		config.mod_config.username = strdup(config.overrideusr);
		printf("Enter SQL password: ");
		fflush(stdout);
#ifndef WIN32
		while (0x0a != (ch = get_pwdchar())) {
			_pwd[pos++] = ch;
			if (pos + 1 == sizeof _pwd)
				break;
		}
#else

		while (0x0d != (ch = _getch())) {
			_pwd[pos++] = ch;
			if (pos + 1 == sizeof _pwd)
				break;
		}
#endif
		printf ("\n");
		_pwd[pos] = '\0';
		free(config.mod_config.password);
		if(pos == 0) config.mod_config.password = NULL;
		else config.mod_config.password = strdup(_pwd);
	}
	
#ifdef WIN32
	if (config.perform_check == 1 && config.overridedomainuser) {
		char *p;
		char _pwd[512];
		int ch = 0;
		int pos =0;
		
		free(config.domainuser);
		free(config.domain);
		
		p = strchr (config.overridedomainuser, '\\');
		
		if (p != NULL) {
			*p++ = '\0';
			config.domainuser = strdup(p);
			config.domain = strdup(config.overridedomainuser);
		} else {
			config.domainuser = strdup(config.overridedomainuser);
			config.domain = NULL;
		}
			
		printf("Enter Windows password: ");
		fflush(stdout);

		while (0x0d != (ch = _getch())) {
			_pwd[pos++] = ch;
			if (pos + 1 == sizeof _pwd)
				break;
		}
		printf ("\n");
		
		_pwd[pos] = '\0';
		free(config.domainpassword);
		if(pos == 0) config.domainpassword = NULL;
		else config.domainpassword = strdup(_pwd);
	}
#endif
		
	if (config.logfacility != NULL) {
		facility = get_facility (config.logfacility);

		if (facility == -1 && strcmp (config.logfacility, "-1") != 0) {
			write_to_log (LOG_CRIT, "Defined logging facility, '%s', does not exist", config.logfacility);
			return 1;
		}

		free (config.logfacility);
		config.logfacility = NULL;
	}

	if (facility != -1) {
		openlog ("mda", LOG_PID, facility);
		config.do_syslog = 1;
	}

	if (config.perform_check == 0 && config.import_fname == NULL)
		write_to_log (LOG_INFO, "Starting mda v%s", MDA_VERSION );

	if (-1 == load_module() ) {
		write_to_log(LOG_CRIT,"Error loading module file %s",config.modulefile);
		waitforkeypress ();
		return 1;
	}
	init_queues();
	// logon using the Windows domain if configured to do so
#ifdef WIN32
	if (config.domainuser != NULL) {		
		if (!LogonUser (config.domainuser, config.domain == NULL ? "." : config.domain, config.domainpassword, LOGON32_LOGON_NETWORK, LOGON32_PROVIDER_DEFAULT, &config.domaintoken)) {
			write_to_log(LOG_CRIT,"Unable to logon to Windows domain. Error: %d", GetLastError());
			waitforkeypress ();
			return 1;
		}
		
		if (!ImpersonateLoggedOnUser (config.domaintoken)) {
			write_to_log(LOG_CRIT,"Unable to switch to Windows domain user. Error: %d", GetLastError());
			waitforkeypress ();
			return 1;
		}
	}

	send_service_pending ();
#else
	FILE	*f_pid = NULL;
	if (config.pidfile != NULL) {
		f_pid = fopen ( config.pidfile,"w" );
		if (!f_pid ) {
			write_to_log (LOG_CRIT, "could not create pidfile: %s : %s", config.pidfile, strerror(errno));
			modfunc.exit();
			return 1;
		}
	}

	if (config.change_user_early == 1 && change_user (config.applicationuser, config.applicationgroup) == -1) {
		close_pidfile (f_pid);
		closelog ();
		return 1;
	}
#endif
	if (config.transactions > 1)
		config.mod_config.use_transactions = 1;
	
	// set up databaseconnections for the queues	
	int i;
	for(i = 0 ; i < NUM_QUEUES; i++) {
		queues[i].modhandle = modfunc.create(&config.mod_config);
	}

	if (config.perform_check == 0 && config.import_fname == NULL)
		write_to_log(LOG_INFO,"Connecting to database");
	for(i = 0 ; i < NUM_QUEUES ; i++) {
		if (1 != modfunc.connect(queues[i].modhandle)) {
#ifdef WIN32
			if (config.domainuser != NULL)
				RevertToSelf ();
#endif
			char	*_p = modfunc.geterrorstring(queues[i].modhandle);
			write_to_log (LOG_CRIT, "Connection %d to database failed: %s",queues[i].pos, _p == NULL ? "DB unknown error" : _p);
#ifndef WIN32
			close_pidfile (f_pid);
#endif
			waitforkeypress ();
			modfunc.exit();
			return 1;
		}
	}
	
#ifdef WIN32
	send_service_pending ();

	if (config.domainuser != NULL)
		RevertToSelf ();
#endif

	if (config.perform_check == 0 && config.import_fname == NULL)
		write_to_log(LOG_INFO,"Connected to database");

#ifndef WIN32

	signal( SIGINT, signal_handler );
	signal( SIGHUP, signal_handler );
	signal( SIGTERM, signal_handler );
	signal( SIGQUIT, signal_handler );
	signal( SIGPIPE, SIG_IGN) ;

	if (config.change_user_early == 0 && change_user (config.applicationuser, config.applicationgroup) == -1) {
		close_pidfile (f_pid);
		closelog ();
		modfunc.exit();
		return 1;
	}

	if ( config.daemonize == 1 ) {
		write_to_log (LOG_INFO, "Daemonizing" );
		if (daemon( 1, 0 ) == -1) {
			write_to_log (LOG_CRIT, "Unable to daemonize: %s", strerror(errno));

			close_pidfile (f_pid);
			closelog ();
			modfunc.exit();
			return 1;
		}
	}

	if (f_pid != NULL) {
		fprintf (f_pid, "%d\n", (int)getpid () );
		fclose (f_pid);
	}
#endif	
	pricetypes = hashit_create ( config.databasehashsize, sizeof(uint64_t), NULL, database_compare, CHAIN_H );
	database = hashit_create ( config.databasehashsize, sizeof(uint64_t), NULL, database_compare, CHAIN_H );	// 0.8 MB	// can now be configured

#ifndef WIN32
	pthread_mutex_init (&database_lock, NULL);	
	pthread_mutex_init (&instruments_lock, NULL);	
#else
	InitializeCriticalSection(&database_lock);
	InitializeCriticalSection(&instruments_lock);
#endif

	Tabledefinitions = hashit_create( TABLEDEFINITION_SIZE, 0, NULL, ( int( * ) (void *,void *))strcmp,CHAIN_H);
	if(-1 == read_table_definitions()) {
		write_to_log(LOG_CRIT,"Could not read table definition file");
		waitforkeypress ();
		modfunc.exit();
		return 1;
	}

	if (config.requestOrderbooks != 1 && config.requestMBO != 1)
		write_to_log (LOG_INFO, "Orderbooks and MBO are disabled, handling of news have been moved to Queue #4");

	if (modfunc.get_info (queues[0].modhandle, &modinfo, sizeof modinfo) == -1) {
		modinfo.transaction = NULL;
		write_to_log(LOG_ERR,"Could not get database version");
	} else {
		if (config.perform_check == 0)
			write_to_log(LOG_INFO,"Database version: %s", modinfo.version);
#ifndef WIN32
		free (modinfo.version);
#endif

	}

	if (config.disable_upsert)
		modinfo.upsert = 0;

	if (modinfo.upsert != 0)
		write_to_log(LOG_INFO,"Upsert supported and enabled");

	for(i = 0; i < NUM_QUEUES; i++) {
		queues[i].transaction = modinfo.transaction;
		queues[i].upsert = modinfo.upsert;
	}

	/*	check database table/column names against mda.tabledef */	
	if(config.checkdatabase == 1 || config.perform_check == 1) {
#ifdef WIN32
		send_service_pending ();
#endif
		Databaseinfo *_dbinfo;

		if (config.perform_check != 0)
			write_to_log(LOG_INFO,"Checking SQL database");

		_dbinfo = modfunc.get_databaseinfo(queues[0].modhandle);
		if( NULL == _dbinfo) {
			write_to_log(LOG_CRIT,"Database check failed. Error: '%s', Code: %d", modfunc.geterrorstring(queues[0].modhandle), modfunc.getlasterror(queues[0].modhandle));
			modfunc.exit();
			waitforkeypress();
			return 1;
		}
		if(0 == check_db(_dbinfo)) {
			if (config.perform_check == 0) {
				write_to_log(LOG_CRIT,"System check failed. Terminating");
				waitforkeypress();
			}

			modfunc.free_databaseinfo(_dbinfo);
			modfunc.exit();
			return 1;
		}
		modfunc.free_databaseinfo(_dbinfo);
	}
	else {
		write_to_log(LOG_INFO,"Skipping SQL database check");
	}

	if (config.import_fname != NULL)
		import_sqldump ();

	/* if called with --check or --import we only want to peform the check/import and then exit */
	if (config.perform_check == 1 || config.import_fname != NULL) {
		for(i = 0; i < NUM_QUEUES; i++) {
			modfunc.disconnect(queues[i].modhandle);
			modfunc.destroy(queues[i].modhandle);
			modfunc.exit();
			exit(0);
		}
	}

#ifdef HAVE_LIBSYSTEMD
	if (config.daemonize == 1)
		sd_notifyf (0, "READY=1\nMAINPID=%lu", (unsigned long)getpid ());
	else
		sd_notify (0, "READY=1");
#endif

#ifdef WIN32
	send_service_running ();
#endif

	/* fetch insrefs from instruments table so that we can see if we need to delete some instruments that
	 * where killed since we last connected with the Millistream distribution servers */
	if (config.requestBasicdata == 1 && config.instrumentslimit != 0) {
		sprintf ( _key,"%d:insref", MDA_TYPE_BASICDATA);
		_def = hashit_lookup (Tabledefinitions, _key);

		if (_def != NULL) {
# ifdef HAVE_LIBSYSTEMD
			sd_notifyf (0, "STATUS=Fetching insrefs from table '%s'", _def->tablename);
# endif
			int pos = sprintf (queues[0].sqlbuildstring->sqlstring, "SELECT %s FROM %s ", _def->fieldname, _def->tablename);

			sprintf(_key,"%d:%d", MDA_TYPE_BASICDATA, MDF_F_TRADESTATE);
			_def = hashit_lookup (Tabledefinitions, _key);

			if (_def != NULL) {
				sprintf (queues[0].sqlbuildstring->sqlstring + pos, "WHERE (%s!=9 OR %s IS NULL)", _def->fieldname, _def->fieldname);

				int ret;
				modres = querySql(&queues[0], &ret,__FILE__,__LINE__);

				if (modres != NULL) {
					instruments = hashit_create (1024*100, sizeof (uint64_t), NULL, database_compare, CHAIN_H);

					for (; modres != NULL; modres = modres->next) {
						if (modres->columnvalue != NULL) {
							uint64_t *insrefp;
							uint64_t insref = strtoull (modres->columnvalue, NULL, 10);

							if (insref >= config.instrumentslimit)
								continue;

							insrefp = malloc (sizeof *insrefp);
							*insrefp = strtoull (modres->columnvalue, NULL, 10);

							hashit_insert (instruments, insrefp, insrefp);
						}
					}
				}

				modfunc.freeresult (queues[0].modhandle);
			}
		}
	}

	running = 1;
#ifndef WIN32
	sem_init ( &shutdownevent, 0, 0 );
#else
	shutdownevent = CreateSemaphore (NULL,0,1024,NULL);
#endif
/*	set running and startup time in database */
	sprintf(_key,"%d:insref",MDA_TYPE_INTERNALSTATUS);
	if(NULL != (_def = hashit_lookup (Tabledefinitions,_key))) {
		char _date[30];
		char _time[30];

		get_gmtime(_time,MDA_TYPE_INTERNALSTATUS);
		get_gmdate(_date,MDA_TYPE_INTERNALSTATUS);
		sprintf(_key,"UPDATE %s SET value = '%s %s' WHERE name = 'Startup time'",_def->tablename,_date,_time);
		if (1 != modfunc.executesql (queues[1].modhandle,_key)) {
			sprintf(_key,"INSERT INTO %s (name,value) VALUES ('Startup time','%s %s')",_def->tablename,_date,_time);
			modfunc.executesql (queues[1].modhandle,_key);
		}
		sprintf(_key,"UPDATE %s SET value = '1' WHERE name = 'Running'",_def->tablename);
		if (1 != modfunc.executesql (queues[1].modhandle,_key)) {
			sprintf(_key,"INSERT INTO %s (name,value) VALUES ('Running','1')",_def->tablename);
			modfunc.executesql (queues[1].modhandle,_key);
		}
	}

#ifndef WIN32
	int hMap = shm_open (config.map_name, O_RDWR|O_CREAT|O_EXCL, 0644);

	if (hMap == -1)
		hMap = shm_open (config.map_name, O_RDWR|O_CREAT, 0644);
	else
		stats_created = 1;

	if (hMap != -1 && ftruncate (hMap, sizeof stats) == -1) {
		write_to_log (LOG_ERR, "ftruncate() on shared memory mapping failed: %s", strerror(errno));
		close (hMap);
		hMap = -1;
	}

	if (hMap == -1) {
		write_to_log (LOG_ERR, "Cannot create shared memory for statistics: %s", strerror(errno));
		stats = calloc (1, sizeof *stats);
		stats_not_mapped = 1;
	} else {
		stats = mmap (NULL, sizeof *stats, PROT_WRITE|PROT_READ, MAP_SHARED, hMap, 0);
		close (hMap);

		if (stats == NULL) {
			write_to_log (LOG_ERR, "Cannot map shared memory for statistics: %s", strerror(errno));

			if (stats_created == 1)
				shm_unlink (config.map_name);

			stats = malloc (sizeof *stats);
			stats_not_mapped = 1;
			stats_created = 0;
		}
	}

	memset (stats, 0, sizeof *stats);
	stats->database_hashsize_max = hashit_tablesize (database);
	stats->pid = getpid ();

	pthread_t	threadid;
	pthread_attr_t	threadattr;

	pthread_attr_init (&threadattr);
	pthread_attr_setdetachstate (&threadattr, PTHREAD_CREATE_DETACHED);

	for (i = 0; i < receivers_num; i++) {
		sprintf (_key, "mdf #%d", i + 1);

		pthread_create( &threadid, &threadattr, mdf_receiver_thread, &receivers[i]);
		pthread_setname_np (threadid, _key);
	}

	pthread_create( &threadid, &threadattr, status_thread, ( void* ) NULL );
	pthread_setname_np (threadid, "status");

	for(i = 0; i < NUM_QUEUES; i++) {
		queues[i].stats = stats->queuestat + i;
		pthread_create( &threadid, &threadattr, mda_database_update_thread, &queues[i] );

		sprintf (_key, "queue #%d", i + 1);
		pthread_setname_np (threadid, _key);
	}
#else
	SECURITY_ATTRIBUTES sa;
	sa.nLength = sizeof sa;
	sa.bInheritHandle = TRUE;
	ConvertStringSecurityDescriptorToSecurityDescriptor ("D:(OA;OI;GAFA;;;WD)", SDDL_REVISION_1, &sa.lpSecurityDescriptor, NULL);

	hMap = CreateFileMapping (INVALID_HANDLE_VALUE, &sa, PAGE_READWRITE, 0, sizeof stats, config.map_name);

	if (hMap == NULL) {
		write_to_log (LOG_ERR, "Cannot create shared memory for the performance monitor counters %d", GetLastError ());
		stats = calloc (1, sizeof *stats);
		stats_not_mapped = 1;
	} else {
		stats = MapViewOfFile (hMap, FILE_MAP_WRITE, 0, 0, 0);
	}

	stats->database_hashsize_max = hashit_tablesize (database);

	for (i = 0; i < receivers_num; i++)
		_beginthread (mdf_receiver_thread,0,&receivers[i]);

	_beginthread (status_thread,0,NULL);
	for(i = 0; i < NUM_QUEUES; i++) {
		queues[i].stats = stats->queuestat + i;
		_beginthread (mda_database_update_thread,0,&queues[i]);
	}
#endif
# ifdef HAVE_LIBSYSTEMD
	sd_notify (0, "STATUS=Running");
# endif
	return 0;
}

#ifdef WIN32
void WINAPI ControlHandler (DWORD request)
{
    if (request == SERVICE_CONTROL_STOP || request == SERVICE_CONTROL_SHUTDOWN) {
		int i;
		running = 0;
		for(i = 0; i < NUM_QUEUES; i++) {
			modfunc.set_status(queues[i].modhandle,MOD_STATUS_SHUTDOWN);
			ReleaseSemaphore (queues[i].itemevent, 1, NULL);
		}
	        ReleaseSemaphore (shutdownevent, 1, NULL);
        return;
    }
     SetServiceStatus (hStatus, &ServiceStatus);
}

void WINAPI ServiceMain (unsigned long __attribute__ ((unused))argc, char __attribute__ ((unused)) **argv)
{
	config.daemonize = 1;
	config.silent = 1;
	ServiceStatus.dwServiceType = SERVICE_WIN32;
	ServiceStatus.dwCurrentState = SERVICE_START_PENDING;
	ServiceStatus.dwControlsAccepted = 0;
	ServiceStatus.dwWin32ExitCode = 0;
	ServiceStatus.dwServiceSpecificExitCode = 0;
	ServiceStatus.dwCheckPoint = 0;
	ServiceStatus.dwWaitHint = 5000;

	hStatus = RegisterServiceCtrlHandler ("Millistream MDA", ControlHandler);

	if (hStatus == 0)
		return;

	SetServiceStatus (hStatus, &ServiceStatus);

	if (init () != 0) {
		ServiceStatus.dwCurrentState = SERVICE_STOPPED;
		ServiceStatus.dwWin32ExitCode = -1;
		SetServiceStatus (hStatus, &ServiceStatus);
		return;
	}

	ServiceStatus.dwCurrentState = SERVICE_RUNNING;
	ServiceStatus.dwControlsAccepted = SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_SHUTDOWN;
	SetServiceStatus (hStatus, &ServiceStatus);

	WaitForSingleObject (shutdownevent, INFINITE);

	ServiceStatus.dwControlsAccepted = 0;
	ServiceStatus.dwCheckPoint = 0;

	while (runningthreads > 0 ) {
		ServiceStatus.dwCurrentState = SERVICE_STOP_PENDING;
		SetServiceStatus (hStatus, &ServiceStatus);
		ServiceStatus.dwCheckPoint++;
		Sleep (1000);
	}
	ServiceStatus.dwCurrentState = SERVICE_STOPPED;
	ServiceStatus.dwWin32ExitCode = 0;
	SetServiceStatus (hStatus, &ServiceStatus);
}
BOOL signal_handler(DWORD fdwCtrlType) 
{ 
    switch (fdwCtrlType) 
    { 
        case CTRL_C_EVENT: 
        case CTRL_CLOSE_EVENT: 
        case CTRL_BREAK_EVENT: 
        case CTRL_LOGOFF_EVENT: 
        case CTRL_SHUTDOWN_EVENT: {
		int i;
		return_status = 0;
		running = 0;
		for(i = 0; i < NUM_QUEUES; i++) {
			modfunc.set_status(queues[i].modhandle,MOD_STATUS_SHUTDOWN);
			ReleaseSemaphore (queues[i].itemevent, 1, NULL);
			ReleaseSemaphore(shutdownevent,1,NULL);
		}
		return 1; 
	}
        default: 
		return 0; 
    }
}
#else
void signal_handler ( int sig )
{
	int i;
    switch ( sig ) {
	/* this is the only signal that means a good exit */
        case SIGTERM:
		return_status = 0; 
		__attribute__ ((fallthrough));
        case SIGQUIT:
        case SIGINT:
        	running = 0;
		for(i = 0; i < NUM_QUEUES; i++) {
			modfunc.set_status(queues[i].modhandle,MOD_STATUS_SHUTDOWN);
			sem_post(&queues[i].itemevent);		// Release semaphores on queues to avoid waiting to long
			sem_post ( &shutdownevent );
		}
            break;
        case SIGHUP:
		for (int i = 0; i < receivers_num; i++)
			receivers[i].connected = false;/* we force a disconnect/reconnect */

		stats->connected_to_feed = 0;
	       	break;
        default:
            break;
    }
}
#endif

mod_result *querySql(Queue *queue, int * const ret,const char * const fname, const int line)
{
	mod_result *result;

retry_query:
	result = modfunc.querysql(queue->modhandle,queue->sqlbuildstring->sqlstring,ret);

	if (*ret >= 0) {
		write_sqllog(queue->pos, "%s", queue->sqlbuildstring->sqlstring);
		return result;
	}

	const char *errorString = modfunc.geterrorstring(queue->modhandle);
	write_to_log (LOG_ERR,"SQL Error:'%s' DB:%s: File:%s:%d/%s:%d Code:%d",queue->sqlbuildstring->sqlstring,errorString == NULL ? "Unknown error":errorString,__FILE__,__LINE__,fname,line,modfunc.getlasterror(queue->modhandle));

	if (*ret == -5) { /* in wrong state */
		queue->num_transactions = 0;
		strcpy (queue->sqlbuildstring->sqlstring,"ROLLBACK");
		modfunc.executesql (queue->modhandle,queue->sqlbuildstring->sqlstring);
		*ret = -4;
	} else if (*ret == -3) { /* deadlock */
		queue->num_transactions = 0;
		strcpy (queue->sqlbuildstring->sqlstring,"ROLLBACK");
		modfunc.executesql (queue->modhandle,queue->sqlbuildstring->sqlstring);
	} else if (*ret == -2) { /* lost connection */
		modfunc.reconnect(queue->modhandle);
		if(modfunc.get_status(queue->modhandle) == MOD_STATUS_OK && config.transactions < 2)
			goto retry_query;

		if (config.transactions > 1) {
			queue->num_transactions = 0;
			*ret = -3; /* let caller handle it like a rolled back event */
		}
	} else { /* so severe that there is no reason to retry the SELECT */
		write_sqllog(queue->pos, "** %s : %s",queue->sqlbuildstring->sqlstring, errorString == NULL ? "Unknown error":errorString);
	}

	return NULL;
}

int runSqlQuery(Queue *queue,uint8_t firstTry,const char * const fname, const int line)
{
	int ret;
	char *errorString;
retry_execute:
	ret = modfunc.executesql (queue->modhandle,queue->sqlbuildstring->sqlstring);

	if (ret == -5) { /* in wrong state */
		errorString = modfunc.geterrorstring(queue->modhandle);
		write_sqllog(queue->pos, "** %s : %s",queue->sqlbuildstring->sqlstring, errorString == NULL ? "Unknown error":errorString);
		write_to_log (LOG_ERR,"SQL Error:'%s' DB:%s: File:%s:%d/%s:%d Code:%d",queue->sqlbuildstring->sqlstring,errorString == NULL ? "Unknown error":errorString,__FILE__,__LINE__,fname,line,modfunc.getlasterror(queue->modhandle));
		queue->stats->failed++;
		strcpy (queue->sqlbuildstring->sqlstring,"ROLLBACK");
		modfunc.executesql (queue->modhandle,queue->sqlbuildstring->sqlstring);
		return -4;
	} else if (ret == -4) { /* normal but irreparable error, aka if this is an insert it cannot be fixed by a update and vice versa */
		errorString = modfunc.geterrorstring(queue->modhandle);
		write_sqllog(queue->pos, "** %s : %s",queue->sqlbuildstring->sqlstring, errorString == NULL ? "Unknown error":errorString);
		write_to_log (LOG_ERR,"SQL Error:'%s' DB:%s: File:%s:%d/%s:%d Code:%d",queue->sqlbuildstring->sqlstring,errorString == NULL ? "Unknown error":errorString,__FILE__,__LINE__,fname,line,modfunc.getlasterror(queue->modhandle));
		queue->stats->failed++;
		return ret;
	} else if (ret == -3) { /* deadlock */
		errorString = modfunc.geterrorstring(queue->modhandle);
		write_sqllog(queue->pos, "** %s : %s",queue->sqlbuildstring->sqlstring, errorString == NULL ? "Unknown error":errorString);

		write_to_log (LOG_ERR,"SQL Error:'%s' DB:%s: File:%s:%d/%s:%d Code:%d",queue->sqlbuildstring->sqlstring,errorString == NULL ? "Unknown error":errorString,__FILE__,__LINE__,fname,line,modfunc.getlasterror(queue->modhandle));
		queue->stats->failed++;

		if (queue->num_transactions != 0) {
			queue->num_transactions = 0;
			strcpy (queue->sqlbuildstring->sqlstring,"ROLLBACK");
			modfunc.executesql (queue->modhandle,queue->sqlbuildstring->sqlstring);
		}

		return ret;
	} else if (ret == -2) { /* disconnected */
		errorString = modfunc.geterrorstring(queue->modhandle);
		write_sqllog(queue->pos, "** %s : %s",queue->sqlbuildstring->sqlstring, errorString == NULL ? "Unknown error":errorString);

		write_to_log (LOG_ERR,"SQL Error:'%s' DB:%s: File:%s:%d/%s:%d Code:%d Status:%d",queue->sqlbuildstring->sqlstring,errorString == NULL ? "Unknown error":errorString,__FILE__,__LINE__,fname,line,modfunc.getlasterror(queue->modhandle),modfunc.get_status(queue->modhandle));

		modfunc.reconnect(queue->modhandle);
		if(modfunc.get_status(queue->modhandle) == MOD_STATUS_OK) {
			if(queue->num_transactions != 0) {
				queue->num_transactions = 0;
				return -3; /* let the caller handle it as a rolled back event */
			}

			if (strcmp (queue->sqlbuildstring->sqlstring, "COMMIT") != 0)
				goto retry_execute;
		}

		if(queue->num_transactions != 0) {
			queue->num_transactions = 0;
			return -3; /* let the caller handle it as a rolled back event */
		}

		return ret;
	} else if (ret == -1) { /* Duplicate Key error */
		errorString = modfunc.geterrorstring(queue->modhandle);
		write_sqllog(queue->pos, "** %s : %s",queue->sqlbuildstring->sqlstring, errorString == NULL ? "Unknown error":errorString);

		if(firstTry == MDA_SQL_INSERTFIRST) {
			return ret;
		}

		write_to_log (LOG_ERR,"SQL Error:'%s' DB:%s: File:%s:%d/%s:%d Code:%d",queue->sqlbuildstring->sqlstring,errorString == NULL ? "Unknown error":errorString,__FILE__,__LINE__,fname,line,modfunc.getlasterror(queue->modhandle));
		queue->stats->failed++;
		return ret;
	} else if (ret == 0) { /* no rows affected */
		if (firstTry == MDA_SQL_UPDATEFIRST) {
			errorString = modfunc.geterrorstring(queue->modhandle);
			write_sqllog(queue->pos, "** %s : %s",queue->sqlbuildstring->sqlstring, errorString == NULL ? "Unknown error":errorString);

			// kolla felkod
			return ret;
		} else if (firstTry == MDA_SQL_FAIL_ON_0) {
			errorString = modfunc.geterrorstring(queue->modhandle);
			write_sqllog(queue->pos, "** %s : %s",queue->sqlbuildstring->sqlstring, errorString == NULL ? "Unknown error":errorString);

			write_to_log (LOG_ERR,"SQL Error:'%s' DB:%s: File:%s:%d/%s:%d Code:%d",queue->sqlbuildstring->sqlstring,errorString == NULL ? "Unknown error":errorString,__FILE__,__LINE__,fname,line,modfunc.getlasterror(queue->modhandle));
			queue->stats->failed++;
			return ret;
		}
	}

	queue->stats->updates++;
	write_sqllog(queue->pos, "%s",queue->sqlbuildstring->sqlstring);
	return ret;
}
static int runOrderbook(Queue *queue,Queueitem *queueitem,Tabledef *_def)
{
	Queueitem *orderbookitem = queueitem;
	Queueitem *next = NULL;

	while (orderbookitem != NULL) {
		if (build_orderbookstrings2(queue,orderbookitem,_def) != 0) {
			if (runSqlQuery(queue,MDA_SQL_ONETRY,__FILE__,__LINE__) == -3)
				return -3;
		}

		next = orderbookitem->next;
		orderbookitem = next;
	}

	return 0;
}
static int runMBO(Queue *queue, Queueitem *queueitem, Tabledef *def)
{
	Queueitem *mbo;
	for (mbo = queueitem; mbo != NULL; mbo = mbo->next) {
		if (mbo->option & MDA_OPTION_DELETERECORD) {
			if (build_historicaldelete (queue, mbo, def) == -3)
				return -3;
		} else if (mbo->option & MDA_OPTION_UPDATEFIRST) {
			if ((mbo->option & MDA_OPTION_DELETEFIRST) && build_historicaldelete (queue, mbo, def) == -3)
				return -3;

			if (build_updatestring2 (queue, mbo, def, NULL) == 0)
				continue;

			int ret = runSqlQuery (queue, MDA_SQL_UPDATEFIRST,__FILE__, __LINE__);

			if (ret == -3) {
				return -3;
			} else if (ret == 0) {
				/* no match try insert */
				build_insertstring2 (queue, mbo, def, NULL);

				if (runSqlQuery (queue, MDA_SQL_ONETRY,__FILE__, __LINE__) == -3)
					return -3;
			}
		} else {
			if ((mbo->option & MDA_OPTION_DELETEFIRST) && build_historicaldelete (queue, mbo, def) == -3)
				return -3;

			int ret = build_upsert (queue, mbo, def, NULL);

			if (ret == 0) {
				continue;
			} else if (ret == -1) {
				if (build_insertstring2 (queue, mbo, def, NULL) == 0)
					continue;

				ret = runSqlQuery (queue, MDA_SQL_INSERTFIRST,__FILE__, __LINE__);

				if (ret == -3) {
					return -3;
				} else if (ret == -1) {
					/* insert failed  try update */
					build_updatestring2 (queue, mbo, def, NULL);

					if (runSqlQuery (queue, MDA_SQL_FAIL_ON_0,__FILE__, __LINE__) == -3)
						return -3;
				}
			} else {
				if (runSqlQuery (queue, MDA_SQL_FAIL_ON_0,__FILE__, __LINE__) == -3)
					return -3;
			}
		}
	}

	return 0;
}

#ifndef WIN32
void	*mda_database_update_thread (void	*_arg)
#else
void	mda_database_update_thread (void	*_arg)
#endif
{
Queue *queue = _arg;
char	_key[256];
Queueitem	*_workitem = NULL;
Tabledef *_def = NULL;

#ifdef WIN32
DWORD timeout;
	// logon using the Windows domain if configured to do so
	if (config.domainuser != NULL)
		ImpersonateLoggedOnUser (config.domaintoken);
#endif
	runningthreads++;

	while (running ) {
		MUTEX_LOCK(&queue->lock);

		Queueitem *transactions_head = NULL;
		Queueitem *transactions_tail = NULL;
		uint64_t transactions_items = 0;

		_workitem = queue->head;
		time_t _timenow = time(NULL);

		while (running && _workitem != NULL && _timenow >= (_workitem->timestamp + queue->mintimebetweenupdates ) ) {
			/* if newsblock is not last we have to skip this item for now unless it's a headline message sent after the body with no headline */
			if (_workitem->type == MDA_TYPE_NEWS && (_workitem->option & MDA_OPTION_NEWS_LASTBLOCK) == 0 && (_workitem->option & MDA_OPTION_NEWSHASHEADLINE) == MDA_OPTION_NEWSHASHEADLINE) {
				_workitem = _workitem->next;
				continue;
			}

			deleteItemInQueue(queue,_workitem);

			if (_workitem->option & MDA_OPTION_EMPTY) {
				freequeueitem(_workitem);
				_workitem = queue->head;
				continue;
			}

			/* record keep the item in case we have re re-run the item after a dead-lock */
			if (transactions_head == NULL)
				transactions_head = _workitem;
			else
				transactions_tail->next = _workitem;

			_workitem->next = NULL;
			_workitem->previous = transactions_tail;
			transactions_tail = _workitem;
			transactions_items++;

			time_t then = time (NULL);

			if(queue->mintimebetweenupdates != 0 && _workitem->option & MDA_OPTION_HASH) {
				Queueitem *emptyitem = calloc(1,sizeof(Queueitem));
				emptyitem->timestamp = then;
				emptyitem->option = MDA_OPTION_EMPTY|MDA_OPTION_HASH;
				emptyitem->type = _workitem->type;
				emptyitem->insref = _workitem->insref;
				if(_workitem->key == NULL) {
					if (config.debuglog == 1)
						write_to_log(LOG_CRIT,"NULL key");
				} else {
					emptyitem->key = strdup(_workitem->key);
					hashit_insert(queue->queueindex,emptyitem->key,emptyitem);
				}
				insertItemInQueue(queue,emptyitem);
			}

			/* so we won't redelete the item from the hash if we deadlock and later retry this item */
			_workitem->option &= ~MDA_OPTION_HASH;

			MUTEX_UNLOCK(&queue->lock);

			int deadlocked = 0;

			sprintf (_key,"%" PRIu32 ":insref" ,_workitem->type);
			if(NULL != (_def = hashit_lookup (Tabledefinitions,_key))) {
				if(config.transactions > 1) {
					if(queue->num_transactions++ == 0) {
						if (queue->transaction != NULL) {
							sprintf(queue->sqlbuildstring->sqlstring,"%s",queue->transaction);

							if (runSqlQuery(queue,MDA_SQL_ONETRY,__FILE__,__LINE__) < 0)
								queue->num_transactions = 0;
						}
					} else if(queue->num_transactions > config.transactions || then - _timenow > 2) {
						_timenow = then;

						/* free the kept items since the transaction now is complete */
						while (transactions_head != _workitem) {
							Queueitem *next = transactions_head->next;
							freequeueitem (transactions_head);
							transactions_head = next;
						}

						transactions_head = _workitem;
						transactions_tail = _workitem;
						_workitem->previous = NULL;

						strcpy(queue->sqlbuildstring->sqlstring,"COMMIT");
						runSqlQuery(queue,MDA_SQL_ONETRY,__FILE__,__LINE__);

						MUTEX_LOCK(&queue->lock);
						queue->stats->items -= (transactions_items - 1);
						MUTEX_UNLOCK(&queue->lock);

						transactions_items = 1;

						if (queue->transaction != NULL) {
							strcpy(queue->sqlbuildstring->sqlstring,queue->transaction);

							if (runSqlQuery(queue,MDA_SQL_ONETRY,__FILE__,__LINE__) < 0)
								queue->num_transactions = 0;
							else
								queue->num_transactions = 1;
						} else {
							queue->num_transactions = 1;
						}
					}
				}
				switch(_workitem->type ) {
				case MDA_TYPE_ADJUSTHISTORY: {
					int ret = adjusthistory(queue,_workitem,_def);
					if(ret == 1) {
						if (runSqlQuery(queue,MDA_SQL_ONETRY,__FILE__,__LINE__) == -3)
							deadlocked = 1;
					} else if (ret == -3) {
						deadlocked = 1;
					}
					break;
				}
				case MDA_TYPE_INTERNALSTATUS:
					if (build_internalstatus(queue,_workitem,_def, stats) == -3)
						deadlocked = 1;
					// sql execute in build function
					break;
				case MDA_TYPE_NEWSCODING:
					if (build_newscoding(queue,_workitem,_def) == -3)
						deadlocked = 1;
					// sql executed in build function
					break;
				case MDA_TYPE_NEWSTAGS:
					if (build_newstags(queue,_workitem,_def) == -3)
						deadlocked = 1;
					// sql executed in build function
					break;
				case MDA_TYPE_INSTRUMENTRESET: {
					if ((config.requestQuotes == 1 || config.requestQuotesBBO == 1) && 0 < build_quotereset (queue,_workitem,_def)) {
						if (runSqlQuery(queue,MDA_SQL_ONETRY,__FILE__,__LINE__) == -3)
							deadlocked = 1;
					}

					if (config.requestNOII == 1 && 0 < build_noiireset (queue,_workitem,_def)) {
						if (runSqlQuery(queue,MDA_SQL_ONETRY,__FILE__,__LINE__) == -3)
							deadlocked = 1;
					}

					if (config.requestPerformance == 1 && 0 < clear_performancediffs(queue,_workitem)) {
						if (runSqlQuery(queue,MDA_SQL_ONETRY,__FILE__,__LINE__) == -3)
							deadlocked = 1;
					}

					if (config.requestGreeks == 1 && 0 < build_greeksreset (queue,_workitem,_def)) {
						if (runSqlQuery(queue,MDA_SQL_ONETRY,__FILE__,__LINE__) == -3)
							deadlocked = 1;
					}

					break;
				}
				case MDA_TYPE_ORDERBOOKFLUSH:
					if (build_orderbookflush (queue,_workitem) == -3)
						deadlocked = 1;
					break;
				case MDA_TYPE_QUOTEBBOFLUSH:
					if (0 < build_quotebboreset (queue,_workitem,_def)) {
						if (runSqlQuery(queue,MDA_SQL_ONETRY,__FILE__,__LINE__) == -3)
							deadlocked = 1;
					}
					break;
				case	MDA_TYPE_INSTRUMENTDELETE:
					if (build_instrumentdelete(queue,_workitem) == -3)
						deadlocked = 1;
					/* execute sql in build_instrumentdelete, so not here */
					break;
				case	MDA_TYPE_CONSTITUENTS:
					if (build_tabular(queue,_workitem,_def) == -3)
						deadlocked = 1;
					/* execute sql in build_constituents, so not here */
					break;
				case	MDA_TYPE_TICKSIZES:
					if (build_tabular(queue,_workitem,_def) == -3)
						deadlocked = 1;
					/* execute sql in build_ticksize, so not here */
					break;
				case	MDA_TYPE_BROKERS:
					if (build_tabular(queue,_workitem,_def) == -3)
						deadlocked = 1;
					/* execute sql in build_brokers, so not here */
					break;
				case	MDA_TYPE_UNDERLYINGS:
					if (build_tabular(queue,_workitem,_def) == -3)
						deadlocked = 1;
					/* execute sql in build_brokers, so not here */
					break;
				case	MDA_TYPE_COMBOLEGS:
					if (build_tabular(queue,_workitem,_def) == -3)
						deadlocked = 1;
					/* execute sql in build_brokers, so not here */
					break;
				case	MDA_TYPE_MAPPINGS: /* this is just a place holder for other types so if we ever get here then we have gotten a non handled type */
					break;
				case	MDA_TYPE_SECTORS:
					if (build_sectors(queue,_workitem,_def) == -3)
						deadlocked = 1;
					/* execute sql in build function, so not here */
					break;
				case	MDA_TYPE_LISTMAPPING:
					if (build_listmappings(queue,_workitem,_def) == -3)
						deadlocked = 1;
					/* execute sql in build function, so not here */
					break;
				case	MDA_TYPE_ORDERBOOK: {
					if (runOrderbook(queue,_workitem->bidorders,_def) == -3)
						deadlocked = 1;
					else if (runOrderbook(queue,_workitem->askorders,_def) == -3)
						deadlocked = 1;

					break;
				}
				case MDA_TYPE_MBO :
					if (runMBO (queue, _workitem->bidorders, _def) == -3)
						deadlocked = 1;
					break;
				case MDA_TYPE_AGGTRADE :
					if (runAggTrade (queue, _workitem, _def) == -3)
						deadlocked = 1;
					break;
				case	MDA_TYPE_PRICEHISTORY:	// to update adjustedhistory , unadjusted is handled in default, ie no break after this one.
					sprintf (_key,"%d:insref" ,MDA_TYPE_ADJUSTEDPRICEHISTORY);
					if(NULL != (_def = hashit_lookup (Tabledefinitions,_key))) {
						if (_workitem->option & MDA_OPTION_DELETERECORD) {
							_workitem->type = MDA_TYPE_ADJUSTEDPRICEHISTORY;

							int ret = build_historicaldelete(queue, _workitem, _def);

							if (ret == -3) {
								deadlocked = 1;
							} else if (ret == 1) {
								if (runSqlQuery(queue, MDA_SQL_ONETRY,__FILE__, __LINE__) == -3)
									deadlocked = 1;
							}

							_workitem->type = MDA_TYPE_PRICEHISTORY;
						} else {
							int ret = get_adjustmentfactor(queue,_workitem,_def);

							if (ret == -3) {
								deadlocked = 1;
								break;
							}

							if (ret != 1) {
								_workitem->type = MDA_TYPE_ADJUSTEDPRICEHISTORY;

								if(_def->insertfirst == 1 ) {
									ret = build_upsert (queue, _workitem, _def, NULL);

									if (ret == -1) {
										if (build_insertstring2 (queue, _workitem, _def, NULL) != 0) {
											ret = runSqlQuery (queue, MDA_SQL_INSERTFIRST,__FILE__, __LINE__);

											if (ret == -3) {
												deadlocked = 1;
											} else if (ret != 1) {
												build_updatestring2 (queue, _workitem, _def, NULL);
												if (runSqlQuery(queue,MDA_SQL_ONETRY,__FILE__,__LINE__) == -3)
													deadlocked = 1;
											}
										}
									} else if (ret > 0 && runSqlQuery (queue, MDA_SQL_FAIL_ON_0,__FILE__, __LINE__) == -3) {
										deadlocked = 1;
									}
								} else {
									if (build_updatestring2 (queue, _workitem, _def, NULL) != 0) {
										ret = runSqlQuery (queue, MDA_SQL_UPDATEFIRST,__FILE__, __LINE__);

										if (ret == -3) {
											deadlocked = 1;
										} else if (ret == 0) {
											build_insertstring2(queue,_workitem,_def,NULL);
											if (runSqlQuery(queue,MDA_SQL_ONETRY,__FILE__,__LINE__) == -3)
												deadlocked = 1;
										}
									}
								}
								_workitem->type = MDA_TYPE_PRICEHISTORY;
							}
						}
					}
					if (deadlocked == 1)
						break;
					sprintf (_key,"%d:insref", _workitem->type);
					if(NULL == (_def = hashit_lookup (Tabledefinitions,_key))) break;
					// no break here
					__attribute__ ((fallthrough));
				default:
					if (_workitem->option & MDA_OPTION_DELETERECORD) {
						int ret = build_historicaldelete(queue, _workitem, _def);

						if (ret == -3) {
							deadlocked = 1;
						} else if (ret == 1) {
							if (runSqlQuery(queue, MDA_SQL_ONETRY,__FILE__, __LINE__) == -3)
								deadlocked = 1;
						}

						break;
					}

					/* fetch a lastprice for the KeyRatios that might need them */
					if (_workitem->type == MDA_TYPE_KEYRATIOS && config.keyratios_num != 0) {
						PriceType *pt = hashit_lookup (pricetypes, &_workitem->insref);

						int ret;
						_workitem->orderprice  = sql_getlastprice (queue,_workitem->insref,pt,&ret);	// orderprice is borrowed

						if (ret == -3) {
							deadlocked = 1;
							break;
						}

						/* if quotes was NULL then we fetch Performance.ClosePrice1d */
						if (_workitem->orderprice == NULL) {
							_workitem->orderprice = sql_getcloseprice (queue, _workitem->insref, &ret);

							if (ret == -3) {
								deadlocked = 1;
								break;
							}
						}
					}

					if (_workitem->option & MDA_OPTION_UPDATEFIRST || _def->insertfirst == 0 || (_workitem->type == MDA_TYPE_NEWS && ((_workitem->option & MDA_OPTION_NEWSHASHEADLINE) == 0)) ) {
						PriceType *pt = _workitem->type == MDA_TYPE_QUOTE ? hashit_lookup (pricetypes, &_workitem->insref) : NULL;

						if (build_updatestring2 (queue, _workitem, _def, pt) == 0)
							break;

						int ret = runSqlQuery(queue,MDA_SQL_UPDATEFIRST,__FILE__,__LINE__);

						if (ret == -3) {
							deadlocked = 1;
							break;
						} else if (ret == 0) {
							/* no match try insert */
							build_insertstring2 (queue,_workitem,_def,pt);
							if (runSqlQuery(queue,MDA_SQL_ONETRY,__FILE__,__LINE__) == -3) {
								deadlocked = 1;
								break;
							}
						}
					} else {
						PriceType *pt = _workitem->type == MDA_TYPE_QUOTE ? hashit_lookup (pricetypes, &_workitem->insref) : NULL;

						int ret = build_upsert (queue, _workitem, _def, pt);

						if (ret == 0) {
							break;
						} else if (ret == -1) {
							if (build_insertstring2 (queue, _workitem, _def, pt) == 0)
								break;

							ret = runSqlQuery(queue,MDA_SQL_INSERTFIRST,__FILE__,__LINE__);

							if (ret == -3) {
								deadlocked = 1;
								break;
							} else if (ret == -1) {
								/* insert failed  try update */
								build_updatestring2 (queue,_workitem,_def,pt);
								if (runSqlQuery(queue,MDA_SQL_FAIL_ON_0,__FILE__,__LINE__) == -3) {
									deadlocked = 1;
									break;
								}
							}
						} else {
							if (runSqlQuery (queue, MDA_SQL_FAIL_ON_0,__FILE__, __LINE__) == -3) {
								deadlocked = 1;
								break;
							}
						}
					}

					if (_workitem->type == MDA_TYPE_QUOTE) {
						// has_lastXXX is set in sqlbuilds.c
						if ((_workitem->haslastprice == 1 || _workitem->hasbidprice == 1 || _workitem->haslastyield == 1 || _workitem->hasbidyield == 1) && config.requestPerformance == 1) {
							if(0 < build_diffcalc(queue,_workitem)) {
								if (runSqlQuery(queue,MDA_SQL_ONETRY,__FILE__,__LINE__) == -3) {
									deadlocked = 1;
									break;
								}
							}
						}

						if (_workitem->haslastprice == 1 && config.requestKeyratios == 1) {
							if(0 < build_keyratios_calc (queue, _workitem)) {
								if (runSqlQuery(queue,MDA_SQL_ONETRY,__FILE__,__LINE__) == -3) {
									deadlocked = 1;
									break;
								}
							}
						}
					} else if (_workitem->type == MDA_TYPE_QUOTEEX) {
						// has_lastXXX is set in sqlbuilds.c
						if ((_workitem->haslastprice == 1) && config.requestPerformance == 1) {
							if(0 < build_diffex(queue,_workitem)) {
								if (runSqlQuery(queue,MDA_SQL_ONETRY,__FILE__,__LINE__) == -3) {
									deadlocked = 1;
									break;
								}
							}
						}
					} else if (_workitem->type == MDA_TYPE_PERFORMANCE && (_workitem->option & (MDA_OPTION_CLOSEPRICE|MDA_OPTION_CLOSEBIDPRICE|MDA_OPTION_CLOSEYIELD|MDA_OPTION_CLOSEBIDYIELD) && config.requestQuotes == 1) != 0) {
						int ret;

						if (_workitem->option & MDA_OPTION_CLOSEPRICE) {
							PriceType *pt = hashit_lookup (pricetypes, &_workitem->insref);

							_workitem->orderprice  = sql_getlastprice(queue,_workitem->insref,pt,&ret);	// orderprice is borrowed for diff calc

							if (ret == -3) {
								deadlocked = 1;
								break;
							}
							_workitem->haslastprice = 1;
						}

						if (_workitem->option & MDA_OPTION_CLOSEBIDPRICE) {
							_workitem->bidprice  = sql_getbidprice(queue,_workitem->insref,&ret);

							if (ret == -3) {
								deadlocked = 1;
								break;
							}
							_workitem->hasbidprice = 1;
						}

						if (_workitem->option & MDA_OPTION_CLOSEYIELD) {
							_workitem->orderyield  = sql_getlastyield(queue,_workitem->insref,&ret);	// orderyield is borrowed for diff calc

							if (ret == -3) {
								deadlocked = 1;
								break;
							}
							_workitem->haslastyield = 1;
						}

						if (_workitem->option & MDA_OPTION_CLOSEBIDYIELD) {
							_workitem->bidyield  = sql_getbidyield(queue,_workitem->insref,&ret);

							if (ret == -3) {
								deadlocked = 1;
								break;
							}
							_workitem->hasbidyield = 1;
						}

						if(0 < build_diffcalc(queue,_workitem)) {
							if (runSqlQuery(queue,MDA_SQL_ONETRY,__FILE__,__LINE__) == -3)
								deadlocked = 1;
						}
					}
					break;
				}
			}

			MUTEX_LOCK(&queue->lock);

			/* if we where deadlocked then we put all the items from the transactions back in the queue at the head */
			if (deadlocked == 1) {
				write_to_log(LOG_ERR,"Deadlocked so will rerun %" PRIu64 " queue items", transactions_items);
				queue->num_transactions = 0;

				transactions_tail->next = queue->head;

				if (queue->head == NULL)
					queue->tail = transactions_tail;
				else
					queue->head->previous = transactions_tail;

				queue->head = transactions_head;

				transactions_head = NULL;
				transactions_tail = NULL;
				transactions_items = 0;
			}

			_workitem = queue->head;	/* _workitem->next could be removed so start at head */
		}

		MUTEX_UNLOCK(&queue->lock);

		/* free the kept items since the transaction now is complete */
		while (transactions_head != NULL) {
			Queueitem *next = transactions_head->next;
			freequeueitem (transactions_head);
			transactions_head = next;
		}

		if(queue->num_transactions != 0) {
			queue->num_transactions = 0;
			strcpy(queue->sqlbuildstring->sqlstring,"COMMIT");
			runSqlQuery(queue,MDA_SQL_ONETRY,__FILE__,__LINE__);
		}

		MUTEX_LOCK(&queue->lock);

		queue->stats->items -= transactions_items;

		// check head for news without last block and move to next if so, remove that news if its been in the queue for to long
		_workitem = queue->head;
		while(_workitem != NULL && (_workitem->type == MDA_TYPE_NEWS && !(_workitem->option & MDA_OPTION_NEWS_LASTBLOCK))) {
			// check how long newsitem has been in queue
			if(_workitem->timestamp + 3600 < _timenow) {
				char *newsid = NULL, *newsdate = NULL;

				Queuefield *qf;
				for (qf = _workitem->fields; qf != NULL; qf = qf->next) {
					if (qf->field == MDF_F_NEWSID)
						newsid = qf->value;
					else if (qf->field == MDF_F_DATE)
						newsdate = qf->value;
				}

				if (config.debuglog == 1)
					write_to_log(LOG_ERR,"News: no lastblock for '%s','%s' in time, kicking it from the queue", newsid, newsdate);

				if ((_workitem->option & MDA_OPTION_EMPTY) == 0)
					queue->stats->items--;

				deleteItemInQueue(queue,_workitem);
				freequeueitem(_workitem);
				_workitem = queue->head;
			} else {
				_workitem = _workitem->next;
			}
		}

#ifndef WIN32
		struct timespec ts;
		ts.tv_sec = _workitem == NULL ? time( NULL ) +1 : _workitem->timestamp + queue->mintimebetweenupdates;
		ts.tv_nsec = 0;

		MUTEX_UNLOCK(&queue->lock);

		sem_timedwait(&queue->itemevent,&ts);
#else
		if (_workitem == NULL) {
			timeout = 1000;
		} else {
			time_t timestamp = _workitem->timestamp + queue->mintimebetweenupdates;
			const time_t now = time (NULL);

			/* timestamp overflowed */
			if (timestamp < _workitem->timestamp)
				timestamp = _workitem->timestamp;

			if (timestamp < now) {
				timeout = 0; /* oops we are a little slow, this item should already by pushed */
			} else {
				timestamp -= now;
			
				if (timestamp > 300)
					timeout = 300 * 1000; /* we never wait more than 5 minutes */
				else
					timeout = (DWORD)(timestamp * 1000);
			}
		}
		MUTEX_UNLOCK(&queue->lock);
		WaitForSingleObject (queue->itemevent, timeout);
#endif
	}
	if(queue->num_transactions != 0) {
		queue->num_transactions = 0;
		strcpy(queue->sqlbuildstring->sqlstring,"COMMIT");
		runSqlQuery(queue,MDA_SQL_ONETRY,__FILE__,__LINE__);
	}
	write_to_log(LOG_INFO,"Database thread %d ended",queue->pos);
	runningthreads--;

#ifdef WIN32
	_endthread ();
#else
	pthread_exit(0);
#endif
}
#ifndef WIN32
void	*status_thread (void __attribute__((__unused__)) *_arg)
#else
void	status_thread (void __attribute__((__unused__)) *_arg)
#endif
{
Queue *queue = &queues[1];
time_t	_tt;
time_t	_dbdisconnecttime;
Queueitem	*_queueitem;
int counter = 0;

struct tm _tm;
uint8_t	tmpstatus;
char *errorString;

	runningthreads++;

#ifdef WIN32
	// logon using the Windows domain if configured to do so
	if (config.domainuser != NULL)
		ImpersonateLoggedOnUser (config.domaintoken);
#endif
	config.dbstatus = MOD_STATUS_OK;

	time_t now = time (NULL);

	for (int i = 0; i < receivers_num; i++)
		receivers[i].live = now;

	while(running) {
#ifdef HAVE_LIBSYSTEMD
		/* only send watchdog if receiverthread seams to be alive */
		int alive_num = 0, connected_num = 0;
		now = time (NULL);
		for (int i = 0; i < receivers_num; i++) {
			if (receivers[i].live >= now)
				alive_num++;
			if (receivers[i].connected)
				connected_num++;
		}

		if (alive_num == receivers_num) {
			if (sd_watchdog_enabled (0, NULL) > 0)
				sd_notify (0, "WATCHDOG=1");
		}

		stats->connected_to_feed = connected_num;
#endif
#ifndef WIN32
		sleep(1);
#else
		Sleep (1000);
#endif
		uint64_t items = 0, updates = 0, dedup = 0, failed = 0;

		int i;
		for(i = 0 ; i < NUM_QUEUES; i++) {
			items += stats->queuestat[i].items;
			updates += stats->queuestat[i].updates;
			dedup += stats->queuestat[i].dedup;
			failed += stats->queuestat[i].failed;
		}

		stats->database_hashsize = hashit_size (database);
#ifndef WIN32
		stats->last_update_stats = time (NULL);
#endif
		if (++counter < 5)
			continue;

		counter = 0;
		time ( &_tt );
		localtime_r ( &_tt,&_tm );

		if (config.silent == 0) {
			printf ("%02d:%02d:%02d  T: %" PRIu64 " S:%" PRIu64 " D:%" PRIu64 " F:%" PRIu64 " DQ:%" PRIu64 "\n",_tm.tm_hour,_tm.tm_min,_tm.tm_sec,
				stats->receivedmessages, dedup, updates, failed, items);
			for(i = 0; i < NUM_QUEUES;i++) {
				printf ("%02d:%02d:%02d  Q%d: Queue: %" PRIu64 " Deduplicated:%" PRIu64 " Updates:%" PRIu64 " Empty:%" PRIu64 "\n",_tm.tm_hour,_tm.tm_min,_tm.tm_sec,i+1,
					stats->queuestat[i].items, stats->queuestat[i].dedup, stats->queuestat[i].updates, stats->queuestat[i].empty);
			}
		}
		
		tmpstatus = MOD_STATUS_OK;
		errorString = NULL;
		for(i = 0; i < NUM_QUEUES;i++) {
			const uint8_t status = modfunc.get_status(queues[i].modhandle);

			if (status != MOD_STATUS_OK) {
				if (status != MOD_STATUS_SHUTDOWN) {
					tmpstatus = status;
					errorString = modfunc.geterrorstring(queues[i].modhandle);
					break;
				} else {
					config.dbstatus = tmpstatus; /* so not to trigger the "Reconnected" logging */
				}
			}
		}

		if (tmpstatus != MOD_STATUS_OK) {
			if (config.dbstatus == MOD_STATUS_OK ) {
				time(&_dbdisconnecttime);
				config.dbstatus = tmpstatus;
				write_to_log(LOG_ERR,"Disconnected from database: \"%s\"",errorString == NULL ? "" : errorString);
			} else if ( (_tt -_dbdisconnecttime) > DATABASE_MAX_DISCONNECTED_TIME) {
				int did_one = 0;
				for (int i = 0; i < receivers_num; i++) {
					if (receivers[i].connected == 1) {
						mdf_disconnect (receivers[i].handle);
						receivers[i].connected = false;
						did_one = 1;
					}
				}

				stats->connected_to_feed = 0;

				if (did_one == 1)
					write_to_log(LOG_ERR,"Database connection down for too long, disconnecting from Millistream");
			}
		} else if (config.dbstatus != MOD_STATUS_OK) {
			write_to_log(LOG_INFO,"Reconnected to database");
			config.dbstatus = MOD_STATUS_OK;
		}

		MUTEX_LOCK(&queue->lock);

		// will not add if its already one waiting first in queue
		if(queue->head == NULL || queue->head->type != MDA_TYPE_INTERNALSTATUS) {
			_queueitem = calloc(1,sizeof(Queueitem));
			_queueitem->type = MDA_TYPE_INTERNALSTATUS;	// timestamp is 0 , so it will not hold the queue...
			insertItemInQueue(queue,_queueitem);
#ifndef WIN32
			sem_post(&queue->itemevent);
#else
			ReleaseSemaphore (queue->itemevent, 1, NULL);			
#endif
		}

		MUTEX_UNLOCK(&queue->lock);
	}
#ifdef WIN32
	if (config.domainuser != NULL)
		RevertToSelf();

	runningthreads--;
	_endthread ();
#else
	runningthreads--;
	pthread_exit(0);
#endif
}
static void	strangle_default (Queueitem	*_queueitem, Queuefield	*_queuefield)
{
Queuefield	*_tmpfield = _queuefield;
	while ( _tmpfield != NULL ) {
		Queuefield *_next = _tmpfield->next ;
		Queuefield *_itemfield = _queueitem->fields;

		while ( _itemfield != NULL ) {
			if( _itemfield->field == _tmpfield->field )
				break;
			_itemfield = _itemfield->next;
		}

		if (_itemfield == NULL ) {	// new field add first 
			_tmpfield->next = _queueitem->fields;
			_tmpfield->previous = NULL;
			if( _queueitem->fields != NULL )
				_queueitem->fields->previous = _tmpfield;
			_queueitem->fields = _tmpfield;
		} else {	// strangle field 
			if( _itemfield->field == MDF_F_TEXTBODY ) {
				if(_tmpfield->value != NULL ) {
					_itemfield->value = realloc(_itemfield->value,strlen(_itemfield->value) + strlen(_tmpfield->value) +10 );
					strcat(_itemfield->value,_tmpfield->value);
					free (_tmpfield->value);
				}
			}
			else {
				free (_itemfield->value);
				_itemfield->value = _tmpfield->value;
			}
			free (_tmpfield);
		}
		_tmpfield = _next;
	}
}

static void strangle_mbo (Queueitem *item, Queuefield *fields, const uint64_t orderid, const uint32_t option)
{
	Queueitem *mbo, *prev = NULL;
	for (mbo = item->bidorders; mbo != NULL; prev = mbo, mbo = mbo->next) {
		if (mbo->orderid == orderid) {
			if (option & MDA_OPTION_DELETERECORD) {
				freequeuefields (mbo->fields);
				mbo->fields = fields;
			} else {
				strangle_default (mbo, fields);
			}

			/* in case we overwrote a Delete we mark this item so that we do perform a delete before
			 * we insert the data that we overwrite the item with */
			if (mbo->option & (MDA_OPTION_DELETERECORD|MDA_OPTION_DELETEFIRST))
				mbo->option = option|MDA_OPTION_DELETEFIRST;
			else
				mbo->option = option;

			return;
		}
	}

	mbo = calloc (1, sizeof *mbo);
	mbo->insref = item->insref;
	mbo->orderid = orderid;
	mbo->option = option;
	mbo->fields = fields;
	mbo->type = item->type;

	if (prev == NULL) {
		item->bidorders = mbo;
	} else {
		prev->next = mbo;
		mbo->previous = prev;
	}
}

static void strangle_orderbooks2(Queueitem *queueitem2,Queuefield *queuefields,MessageInfo *messageinfo)
{
Queueitem	**queueitem,*last;
Queueitem	*orderbook;
char	*sideptr = NULL;
static char bid[] = "bid";
static char ask[] = "ask";

	if(messageinfo->orderprice == NULL) {
		if (config.debuglog == 1 && (messageinfo->option & (MDA_OPTION_BIDORDERINSERT|MDA_OPTION_ASKORDERINSERT)) != 0)
			write_to_log(LOG_ERR,"Orderbook strangle error no price. Insref:%" PRIu64 " Option: %" PRIu32 " File:%s Line:%d",messageinfo->insref,messageinfo->option,__FILE__,__LINE__);

		freequeuefields(queuefields);
		return;
	}
	if(messageinfo->mdatype == MDA_TYPE_BIDORDER) {
		queueitem = &queueitem2->bidorders;
		sideptr = bid;
	}
	else {
		queueitem = &queueitem2->askorders;
		sideptr = ask;
	}
	if(*queueitem == NULL) {
		*queueitem  = calloc(1,sizeof(Queueitem));
		(*queueitem)->fields = queuefields;
		(*queueitem)->orderprice = strdup(messageinfo->orderprice);
		(*queueitem)->insref = queueitem2->insref;
		(*queueitem)->type = messageinfo->mdatype;
		(*queueitem)->option = messageinfo->option;
		return;
	}
	orderbook = *queueitem;
	while(orderbook != NULL) {
		last = orderbook;
		if(0 == strcmp(orderbook->orderprice,messageinfo->orderprice)) break;
		orderbook = orderbook->next;
	}
	if(orderbook == NULL) {
		last->next = calloc (1,sizeof(Queueitem));
		last->next->previous = last;
		last->next->fields = queuefields;
		last->next->orderprice = strdup(messageinfo->orderprice);
		last->next->insref = queueitem2->insref; 
		last->next->type = messageinfo->mdatype;
		last->next->option = messageinfo->option;
		return;
	}
	if(orderbook->option & MDA_OPTION_BIDORDERINSERT || orderbook->option & MDA_OPTION_ASKORDERINSERT ) {	// we have an insert in queye
		if(messageinfo->option & MDA_OPTION_BIDORDERDELETE || messageinfo->option & MDA_OPTION_ASKORDERDELETE) {		// delete the insert
			if(orderbook->next != NULL) orderbook->next->previous = orderbook->previous;
			if(orderbook->previous != NULL) orderbook->previous->next  = orderbook->next;
			else
				*queueitem = orderbook->next;
			freequeueitem(orderbook);
			freequeuefields(queuefields);
			return;
		}
		if(messageinfo->option & MDA_OPTION_BIDORDERUPDATE || messageinfo->option & MDA_OPTION_ASKORDERUPDATE) {				// update the insert
			strangle_default(orderbook,queuefields);
			return;
		}
		if(messageinfo->option & MDA_OPTION_BIDORDERINSERT || messageinfo->option & MDA_OPTION_ASKORDERINSERT) {	/* Should not happen, but just incase */
			if (config.debuglog == 1)
				write_to_log(LOG_ERR,"%sorder insert on insert %" PRIu64 " %s",sideptr,orderbook->insref,orderbook->orderprice == NULL ? "NULL":orderbook->orderprice);
			freequeuefields(queuefields);
			return;
		}
	}
	else
	if(orderbook->option & MDA_OPTION_BIDORDERDELETE || orderbook->option & MDA_OPTION_ASKORDERDELETE) {	// we have a delete in queue
		if(messageinfo->option & MDA_OPTION_BIDORDERINSERT || messageinfo->option & MDA_OPTION_ASKORDERINSERT) {	// change to update
			freequeuefields(orderbook->fields);
			orderbook->fields = queuefields;
			orderbook->option = (orderbook->option & MDA_OPTION_BIDORDERDELETE) ? MDA_OPTION_BIDORDERUPDATE|MDA_OPTION_HASH : MDA_OPTION_ASKORDERUPDATE|MDA_OPTION_HASH;
			free(orderbook->orderprice);
			orderbook->orderprice = strdup(messageinfo->orderprice);
			return;
		}
		if(messageinfo->option & MDA_OPTION_BIDORDERUPDATE || messageinfo->option & MDA_OPTION_ASKORDERUPDATE) {
			freequeuefields(orderbook->fields);
			orderbook->fields = queuefields;
			orderbook->option = (orderbook->option & MDA_OPTION_BIDORDERDELETE) ? MDA_OPTION_BIDORDERUPDATE|MDA_OPTION_HASH : MDA_OPTION_ASKORDERUPDATE|MDA_OPTION_HASH;
			return;
		}
		if(messageinfo->option & MDA_OPTION_BIDORDERDELETE || messageinfo->option & MDA_OPTION_ASKORDERDELETE) {
			if (config.debuglog == 1)
				write_to_log(LOG_ERR,"%sorder delete on delete %" PRIu64 " %s",sideptr,orderbook->insref,orderbook->orderprice == NULL ? "NULL":orderbook->orderprice);
			freequeuefields(queuefields);		/* Should never happen , but just incase */
			return;
		}
	}
	else
	if(orderbook->option & MDA_OPTION_BIDORDERUPDATE || orderbook->option & MDA_OPTION_ASKORDERUPDATE) {	// we have an update in queue
		if(messageinfo->option & MDA_OPTION_BIDORDERINSERT || messageinfo->option & MDA_OPTION_ASKORDERINSERT) {	//change to insert
			freequeuefields(orderbook->fields);
			orderbook->fields = queuefields;
			orderbook->option = (orderbook->option & MDA_OPTION_BIDORDERUPDATE)  ? MDA_OPTION_BIDORDERINSERT|MDA_OPTION_HASH : MDA_OPTION_ASKORDERINSERT|MDA_OPTION_HASH;
			free(orderbook->orderprice);
			orderbook->orderprice = strdup(messageinfo->orderprice);
			return;
		}
		if(messageinfo->option & MDA_OPTION_BIDORDERUPDATE || messageinfo->option & MDA_OPTION_ASKORDERUPDATE) {	// strangle 
			strangle_default(orderbook,queuefields);
			return;
		}
		if(messageinfo->option & MDA_OPTION_BIDORDERDELETE || messageinfo->option & MDA_OPTION_ASKORDERDELETE) {	// change to delete
			freequeuefields(orderbook->fields);
			orderbook->fields = queuefields;
			orderbook->option = (orderbook->option & MDA_OPTION_BIDORDERUPDATE) ? MDA_OPTION_BIDORDERDELETE|MDA_OPTION_HASH : MDA_OPTION_ASKORDERDELETE|MDA_OPTION_HASH;
			return;
		}
	}

	if (config.debuglog == 1)
		write_to_log(LOG_ERR,"Unknown orderbook update File:%s Line:%d",__FILE__,__LINE__);
}
static int addQueueItem(Queuefield *queuefields,MessageInfo *messageinfo)
{
Queueitem *queueitem = NULL;
Queue *queue  = NULL;
/*char key[256];

	sprintf (key,"%" PRIu32 ":insref" ,messageinfo->type);
	if(NULL == hashit_lookup (Tabledefinitions,_key)) return 0;		// do we want to queue items not in tabledef?
*/
	switch (messageinfo->mdatype) {
		case MDA_TYPE_QUOTE:
		case MDA_TYPE_INSTRUMENTRESET:
		case MDA_TYPE_PERFORMANCE:
		case MDA_TYPE_KEYRATIOS:
		case MDA_TYPE_NETORDERIMBALANCE:
		case MDA_TYPE_DIFFCALC:
		case MDA_TYPE_GREEKS:
		case MDA_TYPE_QUOTEBBOFLUSH:
		case MDA_TYPE_QUOTEEX:
			queue = &queues[0];	
			break;

		case MDA_TYPE_NEWS:
		case MDA_TYPE_NEWSCODING:
		case MDA_TYPE_NEWSTAGS:
			if (config.requestOrderbooks != 1 && config.requestMBO != 1)
				queue = &queues[3];
			else
				queue = &queues[1];
			break;

		case MDA_TYPE_INTERNALSTATUS:
		case MDA_TYPE_TRADE:
		case MDA_TYPE_AGGTRADE:
			queue = &queues[1];
			break;

		case MDA_TYPE_ORDERBOOKFLUSH:
		case MDA_TYPE_ORDERBOOK:
		case MDA_TYPE_BIDORDER:
		case MDA_TYPE_ASKORDER:
		case MDA_TYPE_MBO:
			queue = &queues[3];
			break;

		default:
			queue = &queues[2];	// no strangling data perhaps change here to use queue X
			break;
	}

	MUTEX_LOCK(&queue->lock);

	if( messageinfo->option & MDA_OPTION_HASH ) {
		if(messageinfo->mdatype == MDA_TYPE_BIDORDER || messageinfo->mdatype == MDA_TYPE_ASKORDER )
			messageinfo->keylen = sprintf(messageinfo->tmpkey,"%" PRIu64 ":%d",messageinfo->insref,MDA_TYPE_ORDERBOOK);

		queueitem =  hashit_lookup (queue->queueindex,messageinfo->tmpkey);
	}

	if (NULL == queueitem ) {
		if(messageinfo->mdatype == MDA_TYPE_BIDORDER || messageinfo->mdatype == MDA_TYPE_ASKORDER ) {
			if(messageinfo->orderprice != NULL) {
				queueitem = calloc (1,sizeof(Queueitem));
				time(&queueitem->timestamp);

				if(queue->mintimebetweenupdates == 0)
					insertItemInQueue(queue,queueitem);
				else
					forceItemOnQueue(queue,queueitem);

				queueitem->key = strdup(messageinfo->tmpkey);
				hashit_insert ( queue->queueindex, queueitem->key, queueitem );

				queueitem->type = MDA_TYPE_ORDERBOOK;
				queueitem->insref= messageinfo->insref;
				queueitem->option = messageinfo->option | MDA_OPTION_HASH; /* we set HASH just in case since we just added the key to the hash */

				if (messageinfo->mdatype == MDA_TYPE_BIDORDER) {
					queueitem->bidorders = calloc (1,sizeof(Queueitem));
					queueitem = queueitem->bidorders;
				} else {
					queueitem->askorders = calloc(1,sizeof(Queueitem));
					queueitem = queueitem->askorders;
				}

				queueitem->orderprice = strdup(messageinfo->orderprice);
			} else if ((messageinfo->option & (MDA_OPTION_BIDORDERINSERT|MDA_OPTION_ASKORDERINSERT)) != 0) {
				if (config.debuglog == 1)
					write_to_log(LOG_ERR,"addQueueItem error no price. Insref:%" PRIu64 " Option: %" PRIu32 " File:%s Line:%d",messageinfo->insref,messageinfo->option,__FILE__,__LINE__);
			}
		} else {
			queueitem = calloc (1,sizeof(Queueitem));
			time(&queueitem->timestamp);

			if(queue->mintimebetweenupdates == 0)
				insertItemInQueue(queue,queueitem);
			else
				forceItemOnQueue(queue,queueitem);

			if (messageinfo->option & MDA_OPTION_HASH) {
				queueitem->key = strdup(messageinfo->tmpkey);
				hashit_insert ( queue->queueindex, queueitem->key, queueitem );
			}

			if (messageinfo->mdatype == MDA_TYPE_MBO) {
				queueitem->type = messageinfo->mdatype;
				queueitem->insref= messageinfo->insref;
				queueitem->option = messageinfo->option;

				/* we store the actual data on the bidorders member */
				queueitem->bidorders = calloc (1, sizeof (Queueitem));
				queueitem = queueitem->bidorders;
				queueitem->orderid = messageinfo->orderid;
			}
		}

		if(queueitem != NULL) {
			queueitem->insref = messageinfo->insref;
			queueitem->option = messageinfo->option;
			queueitem->fields = queuefields;
			queueitem->type = messageinfo->mdatype;
#ifndef WIN32
			sem_post(&queue->itemevent);
#else
			ReleaseSemaphore (queue->itemevent, 1, NULL);
#endif
		}
	}
	else {
		if(queueitem->option & MDA_OPTION_EMPTY) {
			queue->stats->empty--;
			queue->stats->items++;
			queueitem->option ^= MDA_OPTION_EMPTY;
		}
		else {
			queue->stats->dedup++;
		}
		queueitem->option |= messageinfo->option;
		switch (messageinfo->mdatype) {
		case MDA_TYPE_BIDORDER:
		case MDA_TYPE_ASKORDER:
			strangle_orderbooks2 (queueitem, queuefields,messageinfo);
			break;
		case MDA_TYPE_MBO:
			strangle_mbo (queueitem, queuefields, messageinfo->orderid, messageinfo->option);
			break;
		case MDA_TYPE_NEWS:
			strangle_default (queueitem, queuefields );

			if ((messageinfo->option & MDA_OPTION_NEWSHASHEADLINE) == MDA_OPTION_NEWSHASHEADLINE)
				queueitem->option |=MDA_OPTION_NEWSHASHEADLINE;

			if ((messageinfo->option & MDA_OPTION_NEWS_LASTBLOCK) == MDA_OPTION_NEWS_LASTBLOCK)
				queueitem->option |=MDA_OPTION_NEWS_LASTBLOCK;
#ifndef WIN32
			sem_post(&queue->itemevent);
#else
			ReleaseSemaphore (queue->itemevent, 1, NULL);
#endif
			break;
		case MDA_TYPE_AGGTRADE: {
			strangle_default (queueitem, queuefields );

			queueitem->option &= ~MDA_OPTION_HASH;
			deleteItemInQueue(queue,queueitem);
			queue->stats->items--;

			queueitem->next = NULL;
			queueitem->previous = NULL;
			queueitem->option |= MDA_OPTION_HASH;
			insertItemInQueue(queue,queueitem);
			break;
		}
		default:
			strangle_default (queueitem, queuefields );
			break;
		}
	}

	MUTEX_UNLOCK(&queue->lock);
	return 1;
}
static void	add_newsCodingItem(MessageInfo *mi,Queuefield *fields)
{
Queuefield *qf, *qf_curr;
MessageInfo	messageinfo;

	memset(&messageinfo,0,sizeof(MessageInfo));
	qf = calloc (1,sizeof(Queuefield));
	messageinfo.insref = mi->insref;
	messageinfo.mdatype = MDA_TYPE_NEWSCODING;
	messageinfo.option = MDA_OPTION_NOHASH;
	qf->value = strdup (mi->newsid);
	qf->field = MDF_F_NEWSID;

	qf_curr = qf;
	while(fields) {
		if(fields->field == MDF_F_DATE || fields->field == MDF_F_NEWSCODINGCOMPANY || fields->field == MDF_F_NEWSCODINGISIN) {
			qf_curr->next = calloc(1,sizeof(Queuefield));
			qf_curr->next->field = fields->field;
			qf_curr->next->value = strdup(fields->value);
			qf_curr->next->previous = qf_curr;
			qf_curr = qf_curr->next;
		}
		fields = fields->next;
	}	
	addQueueItem(qf,&messageinfo);
}

static void add_newsTagsItem (MessageInfo *mi, Queuefield *fields)
{
	MessageInfo messageinfo;
	memset(&messageinfo,0,sizeof(MessageInfo));
	messageinfo.insref = mi->insref;
	messageinfo.mdatype = MDA_TYPE_NEWSTAGS;
	messageinfo.option = MDA_OPTION_NOHASH;

	Queuefield *qf = calloc (1,sizeof(Queuefield));
	qf->value = strdup (mi->newsid);
	qf->field = MDF_F_NEWSID;

	Queuefield *qf_curr = qf;
	while(fields) {
		if(fields->field == MDF_F_DATE || fields->field == MDF_F_NEWSCODINGTAGS) {
			qf_curr->next = calloc(1,sizeof(Queuefield));
			qf_curr->next->field = fields->field;
			qf_curr->next->value = strdup(fields->value);
			qf_curr->next->previous = qf_curr;
			qf_curr = qf_curr->next;
		}
		fields = fields->next;
	}	
	addQueueItem(qf,&messageinfo);
}

static int	add_adjusthistory(MessageInfo *mi,Queuefield	*_queuefields)
{
Queuefield	*_fieldptr = _queuefields;
Queuefield	*adjustfields = NULL;
MessageInfo	messageinfo;
char	*_factor = NULL;
char	*_date = NULL;
char	*_time = NULL;
char	*_type = NULL;
char	*_source = NULL;
char 	*_lastmod = NULL;

	while(_fieldptr != NULL ) {
		switch(_fieldptr->field) {
		case MDF_F_ADJUSTMENTFACTOR:
			_factor = _fieldptr->value;
			break;
		case MDF_F_DATE:	
			_date = _fieldptr->value;
			break;
		case MDF_F_TIME:
			_time = _fieldptr->value;
			break;
		case MDF_F_CATYPE:
			_type = _fieldptr->value;
			break;
		case MDF_F_SOURCE:
			_source = _fieldptr->value;
			break;
		case MDF_F_LASTMOD:
			_lastmod = _fieldptr->value;
			break;
		default:
			break;
		}
		_fieldptr = _fieldptr->next;
	}
	if(_factor == NULL || _date == NULL || _time == NULL || _source == NULL || _type == NULL) return 0;
	memset(&messageinfo,0,sizeof(MessageInfo));
	messageinfo.insref = mi->insref;
	messageinfo.option = MDA_OPTION_NOHASH;
	messageinfo.mdatype = MDA_TYPE_ADJUSTHISTORY;
	
	adjustfields = calloc(1,sizeof(Queueitem));
	adjustfields->field = MDF_F_ADJUSTMENTFACTOR;
	adjustfields->value = strdup(_factor);

	adjustfields->next = calloc(1,sizeof(Queuefield));
	adjustfields->next->field = MDF_F_DATE;
	adjustfields->next->value = strdup(_date);
	adjustfields->next->previous = adjustfields;

	adjustfields->next->next = calloc(1,sizeof(Queuefield));
	adjustfields->next->next->field = MDF_F_TIME;
	adjustfields->next->next->value = strdup(_time);
	adjustfields->next->next->previous = adjustfields->next;

	adjustfields->next->next->next = calloc(1,sizeof(Queuefield));
	adjustfields->next->next->next->field = MDF_F_CATYPE;
	adjustfields->next->next->next->value = strdup(_type);
	adjustfields->next->next->next->previous = adjustfields->next->next;

	adjustfields->next->next->next->next = calloc(1,sizeof(Queuefield));
	adjustfields->next->next->next->next->field = MDF_F_SOURCE;
	adjustfields->next->next->next->next->value = strdup(_source);
	adjustfields->next->next->next->next->previous = adjustfields->next->next->next;

	if (_lastmod != NULL) {
		adjustfields->next->next->next->next->next = calloc(1,sizeof(Queuefield));
		adjustfields->next->next->next->next->next->field = MDF_F_LASTMOD;
		adjustfields->next->next->next->next->next->value = strdup(_lastmod);
		adjustfields->next->next->next->next->next->previous = adjustfields->next->next->next->next;
	}

	addQueueItem(adjustfields,&messageinfo);
	return 1;
}
static int orderbookdelete(Orderbook **orderbook,uint32_t	*levels,uint32_t level)
{
Orderbook *order = *orderbook;
uint32_t i;
	if(level > *levels || *orderbook == NULL) return -3;
	for (i = 0; i < level-1; i++) {
		order = order->next;
	}
	if(order == NULL ) {
		return -4;
	}
	if(order->previous != NULL)
		order->previous->next = order->next;
	else
		*orderbook = order->next;

	if(order->next != NULL )
		order->next->previous = order->previous;
	
	free(order->price);
	free(order);
	(*levels)--;
	return 1;
}
static int orderbookinsert(Orderbook **orderbook,uint32_t	*levels,uint32_t level,char *orderprice)
{
Orderbook *order  = *orderbook;
Orderbook *neworderlevel = NULL;
uint32_t i;
	if(level > *levels+1) return -1;
	if(orderprice == NULL) return -2;
	neworderlevel = calloc(1,sizeof(Orderbook));
	neworderlevel->price = strdup(orderprice);
	if(*orderbook == NULL) {
		*orderbook = neworderlevel;
		(*levels)++;
		return 1;
	}
	for(i = 0; i < *levels;i++) {
		if(i == level-1) {
			neworderlevel->next = order;
			neworderlevel->previous = order->previous;
			(*levels)++;
			if(order->previous != NULL)
				order->previous->next = neworderlevel;
			else
				*orderbook = neworderlevel;
			order->previous = neworderlevel;
			return 1;
		}
		if (order->next == NULL ) {
			order->next = neworderlevel;
			neworderlevel->previous = order;
			(*levels)++;
			return 1;
		}
		order = order->next;
	}
	free (neworderlevel);
	return 1;
}
static int	createinternalorderbook(Instrument *_instrument,char *_orderprice,char *level,uint32_t	_option)
{
uint32_t _level;
int ret = 0;
char	*sideptr = NULL;
static char	bid[] = "bid";
static char	ask[] = "ask";
	// updates cant be with price so dont care about them
	if(level == NULL) {
		if (config.debuglog == 1)
			write_to_log(LOG_ERR,"Internal orderbook error no level %" PRIu64 " File:%s Line:%d",_instrument->insref,__FILE__,__LINE__);

		return 0;
	}
	_level = strtol(level,NULL,10);
	if(_option & MDA_OPTION_BIDORDERINSERT ) {
		ret = orderbookinsert(&_instrument->bidorders,&_instrument->numbidorders,_level,_orderprice);
		sideptr = bid;
	}
	else 
	if(_option & MDA_OPTION_ASKORDERINSERT) {
		ret = orderbookinsert(&_instrument->askorders,&_instrument->numaskorders,_level,_orderprice);
		sideptr = ask;
	}
	else
	if(_option & MDA_OPTION_BIDORDERDELETE ) {
		ret = orderbookdelete(&_instrument->bidorders,&_instrument->numbidorders,_level);
		sideptr = bid;
	}
	else
	if(_option & MDA_OPTION_ASKORDERDELETE ) {
		ret = orderbookdelete(&_instrument->askorders,&_instrument->numaskorders,_level);
		sideptr = ask;
	}
	else return 1;	// its an update so return.
	switch(ret) {
		case -1:
			if (config.debuglog == 1)
				write_to_log(LOG_ERR,"Invalid %sorder level insert received internal orderbook error(%" PRIu64 ") level %s line:%d",sideptr,_instrument->insref,level,__LINE__);
			return -1;	
		case -2:
			if (config.debuglog == 1)
				write_to_log(LOG_ERR,"Orderbook %sprice is NULL %" PRIu64 " level:%s line:%d",sideptr,_instrument->insref,level,__LINE__);
			return -1;
		case -3:
			if (config.debuglog == 1)
				write_to_log(LOG_ERR,"Invalid %sorder level DELETE received internal orderbook error(%" PRIu64 ") level %s line:%d",sideptr,_instrument->insref,level,__LINE__);
			return -1;
		case -4:
			if (config.debuglog == 1)
				write_to_log(LOG_ERR,"Orderbook %s delete error line:%d",sideptr,__LINE__);
			return -1;
		default:break;
	}
	return 1;
}

static Queuefield *add_queue_field (Queuefield **head, Queuefield **tail)
{
	Queuefield *qf = calloc (1, sizeof *qf);

	if (*head == NULL) {
		*head = qf;
		*tail = qf;
	} else {
		(*tail)->next = qf;
		qf->previous = *tail;
		*tail = qf;
	}

	return qf;
}

#ifndef WIN32
void	*mdf_receiver_thread (void *_arg)
#else
void	mdf_receiver_thread (void *_arg)
#endif
{
	struct receiver * const receiver = _arg;
	runningthreads++;

	time_t timeoflastupdate = 0;
	time_t timeofconnect = 0; /* if 0 the we have logged on, else we are simply connected */
	int juststarted = 1;
	char *host = NULL;

	while ( running ) {
		int requestfinished = 0;
		while (running == 1 && !receiver->connected ) {
			mdf_disconnect (receiver->handle); /* just in case */

			if(juststarted == 0) {
				receiver->live = time (NULL) + config.reconnectinterval + 1;
#ifndef WIN32
				sleep ( config.reconnectinterval );
#else
				Sleep (config.reconnectinterval * 1000);
#endif
			} else {
				juststarted = 0;
			}

			if(config.dbstatus == MOD_STATUS_OK ) {
				receiver->live = time (NULL) + 6; /* default connect timeout is 5 second */
				if (!mdf_connect (receiver->handle, receiver->server)) {
					MDF_ERROR errorcode;

					mdf_get_property (receiver->handle, MDF_OPT_ERROR, &errorcode);
					write_to_log(LOG_ERR, "Connecting via MDF failed: %d", errorcode);
				} else {
					char *host_tmp;
					mdf_get_property (receiver->handle, MDF_OPT_CONNECTED_HOST, &host_tmp);
					free (host);
					host = strdup (host_tmp);
					write_to_log(LOG_INFO, "Connected to MDF node %s", host);
					receiver->connected = true;
					timeofconnect = time (NULL);
				}
			} else {
				receiver->live = time (NULL) + config.reconnectinterval + 1;
#ifndef WIN32
				sleep ( config.reconnectinterval );
#else
				Sleep (config.reconnectinterval * 1000);
#endif
			}
		}

		if (running == 0)
			break;

		/* Login */
		mdf_message_t mdf_message = mdf_message_create();

		mdf_message_add (mdf_message,0,MDF_M_LOGON);
		mdf_message_add_string (mdf_message, MDF_F_USERNAME, (receiver->username == NULL) ? config.millistreamusername : receiver->username);
		mdf_message_add_string (mdf_message, MDF_F_PASSWORD, (receiver->password == NULL) ? config.millistreampassword : receiver->password);

		if (application_string != NULL)
			mdf_message_add_string (mdf_message, MDF_F_S1, application_string);

		mdf_message_send (receiver->handle, mdf_message);
		mdf_message_destroy (mdf_message);

		while ( running == 1 && receiver->connected ) {
			receiver->live = time (NULL) + 3;
			switch (mdf_consume (receiver->handle, 2)) {
				case -1: {
					MDF_ERROR errorcode;
					mdf_get_property (receiver->handle, MDF_OPT_ERROR, &errorcode);
					write_to_log (LOG_ERR, "Disconnected from MDF node %s: %d", host, errorcode);
					receiver->connected = false;
					mdf_disconnect(receiver->handle);
					break;
				}

				case 0: {	// timeout
					if (timeofconnect != 0) {
						if (difftime (time (NULL), timeofconnect) >= 30.0) {
							receiver->connected = false;
							mdf_disconnect(receiver->handle);
							write_to_log (LOG_ERR, "No reply from MDF node %s in 30 seconds. Reconnecting.", host);
						}
					}
					break;
				}

				case 1: {
					receiver->live = time (NULL) + 30;

					if (requestfinished == 1)
						timeoflastupdate = time (NULL);

					uint64_t insref;
					int _messageclass, _messagereference;
					while (1 == mdf_get_next_message ( receiver->handle, &_messagereference, &_messageclass, &insref ) ) {
						Queuefield *queuefield_head = NULL, *queuefield_tail = NULL;

						Queuefield *datefield = NULL; /* Date */
						Queuefield *datefield2 = NULL; /* Trade Agreement Date */
						Queuefield *timefield = NULL; /* Time or Trade Time */
						Queuefield *timefield2 = NULL; /* Trade Cancel Time */
						Queuefield *timefield3 = NULL; /* Trade Agreement Time */

						MessageInfo messageinfo;
						memset(&messageinfo,0,sizeof(MessageInfo));
						messageinfo.insref = insref;
						messageinfo.option = MDA_OPTION_HASH;

						switch ( _messagereference ) {
						case	MDF_M_MESSAGESREFERENCE:
							config.feed_templates_ver = insref;
							break;
						case 	MDF_M_INSTRUMENTRESET:
							messageinfo.mdatype = MDA_TYPE_INSTRUMENTRESET;
							messageinfo.option = MDA_OPTION_NOHASH;

							MUTEX_LOCK(&database_lock);
							Instrument *_instrument = hashit_lookup ( database,&messageinfo.insref );
							if (_instrument != NULL) clear_orderbook(_instrument);
							MUTEX_UNLOCK(&database_lock);

							// remove quote,noii and performance from queue hash, so new items will end up in 
							clearItemHash(&queues[0],insref,MDA_TYPE_QUOTE, 0);
							clearItemHash(&queues[0],insref,MDA_TYPE_PERFORMANCE, 0);
							clearItemHash(&queues[0],insref,MDA_TYPE_NETORDERIMBALANCE, 0);
							clearItemHash(&queues[0],insref,MDA_TYPE_GREEKS, 0);
							clearItemHash(&queues[3],insref,MDA_TYPE_ORDERBOOK, 1);
							clearItemHash(&queues[3],insref,MDA_TYPE_MBO, 1);

							/* we have to add a OrderbookFlush here since we now handle Orders/MBO messages on a
							 * different queue than the Quotes */
							MessageInfo mi;
							memset(&mi,0,sizeof(MessageInfo));
							mi.insref = messageinfo.insref;
							mi.mdatype = MDA_TYPE_ORDERBOOKFLUSH;
							mi.option = MDA_OPTION_NOHASH|MDA_OPTION_FLUSH_MBL|MDA_OPTION_FLUSH_MBO;
							addQueueItem(NULL,&mi);
							break;
						case	MDF_M_ORDERBOOKFLUSH:
							messageinfo.mdatype = MDA_TYPE_ORDERBOOKFLUSH;
							messageinfo.option = MDA_OPTION_NOHASH|MDA_OPTION_FLUSH_MBL;
							break;
						case 	MDF_M_INSTRUMENTDELETE:
							messageinfo.mdatype = MDA_TYPE_INSTRUMENTDELETE;
							messageinfo.option = MDA_OPTION_NOHASH;

							MUTEX_LOCK(&queues[0].lock);
							PriceType *pt = hashit_lookup (pricetypes, &insref);
							if (pt != NULL) {
								hashit_delete (pricetypes, &insref);
								free (pt);
							}
							MUTEX_UNLOCK(&queues[0].lock);
							break;
						case	MDF_M_QUOTE:
							messageinfo.mdatype = MDA_TYPE_QUOTE;
							break;
						case	MDF_M_QUOTEBBO:
							messageinfo.mdatype = MDA_TYPE_QUOTE;
							break;
						case	MDF_M_QUOTEEX:
							messageinfo.mdatype = MDA_TYPE_QUOTEEX;
							break;
						case	MDF_M_TRADE:
							messageinfo.mdatype = MDA_TYPE_TRADE;
							messageinfo.option = MDA_OPTION_NOHASH;
							break;
						case	MDF_M_BASICDATA:
							messageinfo.mdatype = MDA_TYPE_BASICDATA;

							MUTEX_LOCK(&instruments_lock);

							if (instruments != NULL) {
								uint64_t *insrefp = hashit_lookup (instruments, &insref);

								if (insrefp != NULL) {
									hashit_delete (instruments, insrefp);
									free (insrefp);
								}
							}

							MUTEX_UNLOCK(&instruments_lock);
							break;
						case	MDF_M_PRICEHISTORY:
							messageinfo.mdatype = MDA_TYPE_PRICEHISTORY;
							messageinfo.option = MDA_OPTION_NOHASH;
							break;
						case	MDF_M_NEWSHEADLINE:
							messageinfo.mdatype = MDA_TYPE_NEWS;
							break;
						case	MDF_M_NEWSCONTENT:
							messageinfo.mdatype = MDA_TYPE_NEWS;
							break;
						case	MDF_M_CORPORATEACTION:
							messageinfo.mdatype = MDA_TYPE_CORPORATEACTION;
							messageinfo.option = MDA_OPTION_NOHASH;
							break;
						case	MDF_M_TRADESTATE:
							messageinfo.mdatype = MDA_TYPE_BASICDATA;
							messageinfo.option |= MDA_OPTION_TRADESTATE;
							break;
						case	MDF_M_PERFORMANCE:
							messageinfo.mdatype = MDA_TYPE_PERFORMANCE;
							break;
						case	MDF_M_FUNDAMENTALS:
							messageinfo.mdatype = MDA_TYPE_FUNDAMENTALS;
							messageinfo.option = MDA_OPTION_NOHASH;
							break;
						case	MDF_M_KEYRATIOS:
							messageinfo.mdatype = MDA_TYPE_KEYRATIOS;
							break;
						case	MDF_M_NETORDERIMBALANCE:
							messageinfo.mdatype = MDA_TYPE_NETORDERIMBALANCE;
							break;
						case	MDF_M_ESTIMATES:
							messageinfo.mdatype = MDA_TYPE_ESTIMATES;
							messageinfo.option = MDA_OPTION_NOHASH;
							break;
						case	MDF_M_ESTIMATESHISTORY:
							messageinfo.mdatype = MDA_TYPE_ESTIMATESHISTORY;
							messageinfo.option = MDA_OPTION_NOHASH;
							break;
						case	MDF_M_L10N:
							messageinfo.mdatype = MDA_TYPE_L10N;
							messageinfo.option = MDA_OPTION_NOHASH;
							break;
						case	MDF_M_CI:
							messageinfo.mdatype = MDA_TYPE_CI;
							messageinfo.option = MDA_OPTION_NOHASH;
							break;
						case	MDF_M_CIHISTORY:
							messageinfo.mdatype = MDA_TYPE_CIHISTORY;
							messageinfo.option = MDA_OPTION_NOHASH;
							break;
						case	MDF_M_PRIIP:
							messageinfo.mdatype = MDA_TYPE_PRIIP;
							break;
						case	MDF_M_MIFID:
							messageinfo.mdatype = MDA_TYPE_MIFID;
							break;
						case	MDF_M_MIFIDHISTORY:
							messageinfo.mdatype = MDA_TYPE_MIFIDHISTORY;
							messageinfo.option = MDA_OPTION_NOHASH;
							break;
						case	MDF_M_EET:
							messageinfo.mdatype = MDA_TYPE_EET;
							break;
						case	MDF_M_MAPPINGS:
							messageinfo.mdatype = MDA_TYPE_MAPPINGS;
							messageinfo.option = MDA_OPTION_NOHASH;
							break;
						case	MDF_M_GREEKS:
							messageinfo.mdatype = MDA_TYPE_GREEKS;
							break;
						/*	ORDERBOOKS	*/
						case	MDF_M_BIDLEVELINSERT:
							messageinfo.option |= MDA_OPTION_BIDORDERINSERT;
							messageinfo.mdatype = MDA_TYPE_BIDORDER;
							break;
						case	MDF_M_BIDLEVELDELETE:
							messageinfo.option |= MDA_OPTION_BIDORDERDELETE;
							messageinfo.mdatype = MDA_TYPE_BIDORDER;
							break;
						case	MDF_M_BIDLEVELUPDATE:
							messageinfo.option |= MDA_OPTION_BIDORDERUPDATE;
							messageinfo.mdatype = MDA_TYPE_BIDORDER;
							break;
						case	MDF_M_ASKLEVELINSERT:
							messageinfo.option |= MDA_OPTION_ASKORDERINSERT;
							messageinfo.mdatype = MDA_TYPE_ASKORDER;
							break;
						case	MDF_M_ASKLEVELDELETE:
							messageinfo.option |= MDA_OPTION_ASKORDERDELETE;
							messageinfo.mdatype = MDA_TYPE_ASKORDER;
							break;
						case	MDF_M_ASKLEVELUPDATE:
							messageinfo.option |= MDA_OPTION_ASKORDERUPDATE;
							messageinfo.mdatype = MDA_TYPE_ASKORDER;
							break;
						case	MDF_M_MBOADD:
							messageinfo.mdatype = MDA_TYPE_MBO;
							break;
						case	MDF_M_MBOUPDATE:
							messageinfo.option |= MDA_OPTION_UPDATEFIRST;
							messageinfo.mdatype = MDA_TYPE_MBO;
							break;
						case	MDF_M_MBODELETE:
							messageinfo.option |= MDA_OPTION_DELETERECORD;
							messageinfo.mdatype = MDA_TYPE_MBO;
							break;
						case	MDF_M_LOGOFF: {
							uint32_t _tag;
							char *_value;
							while ( 1 == mdf_get_next_field ( receiver->handle, &_tag, &_value ) ) {
								if(_tag == MDF_F_LOGOFFREASON) {
									const int reason = _value != NULL ? strtol (_value, NULL, 10) : -1;

									if (reason == 401)
										write_to_log(LOG_CRIT,"The MDF node %s didn't accept the supplied username/password combination", host);
									else if (reason == 503)
										write_to_log(LOG_CRIT,"The MDF node %s is shutting down", host);
								}
							}
							mdf_disconnect(receiver->handle);
							receiver->connected = false;
							write_to_log(LOG_INFO,"The MDF node %s went off-line, disconnecting and trying to reconnect", host);
							break;
						}

						case	MDF_M_LOGONGREETING: {
							int pos = 0;
							char tmpbuf[100];
							uint32_t _tag;
							char *_value, *servername = NULL;

							while ( 1 == mdf_get_next_field ( receiver->handle, &_tag, &_value ) ) {
								if(_tag == MDF_F_SERVERNAME) {
									servername = _value == NULL ? NULL : strdup (_value);
									timeoflastupdate = 0;
								}
							}

							write_to_log(LOG_INFO,"Successfully logged on to MDF node %s: %s", host, servername);
							free (servername);
							timeofconnect = 0;

							mdf_message_t mdf_message = mdf_message_create();

							mdf_message_add (mdf_message, 0, MDF_M_REQUEST);
							mdf_message_add_numeric (mdf_message, MDF_F_REQUESTTYPE, MDF_RT_FULL);
							mdf_message_add_uint (mdf_message, MDF_F_REQUESTID, 0, 0);

							if (timeoflastupdate != 0) {
								struct tm tm;
								time_t tmp = timeoflastupdate;
								int timediff;

								/* compensating for clock drift and/or network latency */
								if (!mdf_get_property (receiver->handle, MDF_OPT_TIME_DIFFERENCE, &timediff))
									timediff = 0;

								tmp += timediff;
								tmp--; /* since there can be many messages per second */
								gmtime_r (&tmp, &tm);

								mdf_message_add_date2 (mdf_message, MDF_F_DATE, tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday);
								mdf_message_add_time2 (mdf_message, MDF_F_TIME, tm.tm_hour, tm.tm_min, tm.tm_sec, 0);
								write_to_log(LOG_INFO,"Requesting data from %d-%02d-%02dT%02d:%02d:%02dZ from MDF node %s", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, host);

								/* just to be safe since we don't want the MDA to delete instruments we haven't seen due to a timed request */
								MUTEX_LOCK(&instruments_lock);
								free_instruments (0);
								MUTEX_UNLOCK(&instruments_lock);
							} else {
								write_to_log(LOG_INFO,"Requesting full snapshot from MDF node %s", host);
							}

							if (receiver->r_ignore) {
								if(config.requestBasicdata == 1) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_BASICDATA);
								if(config.requestQuotes == 1) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_QUOTE);
								if(config.requestQuotesBBO == 1) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_QUOTEBBO);
								if(config.requestQuotesEx == 1) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_QUOTEEX);
								if(config.requestPerformance == 1) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_PERFORMANCE);
								if(config.requestOrderbooks == 1) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_ORDER);
								if(config.requestCorporateactions == 1) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_CORPORATEACTION);
								if(config.requestPricehistory == 1) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_PRICEHISTORY);
								if(config.requestTradestate == 1) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_TRADESTATE);
								if(config.requestTrades == 1) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_TRADE);
								if(config.requestFundamentals == 1) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_FUNDAMENTALS);
								if(config.requestKeyratios == 1) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_KEYRATIOS);
								if(config.requestNOII == 1) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_NETORDERIMBALANCE);
								if(config.requestEstimates == 1) {
									pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_ESTIMATES);
									pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_ESTIMATESHISTORY);
								}

								if(config.requestL10N == 1) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_L10N);

								if(config.requestNews == 1) {
									pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_NEWSHEADLINE);
									pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_NEWSCONTENT);
								}

								if(config.requestCI == 1) {
									pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_CI);
									pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_CIHISTORY);
								}

								if(config.requestPRIIP == 1) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_PRIIP);

								if(config.requestMIFID == 1) {
									pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_MIFID);
									pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_MIFIDHISTORY);
								}

								if(config.requestEET == 1) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_EET);
								if(config.requestMappings == 1) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_MAPPINGS);
								if(config.requestGreeks == 1) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_GREEKS);
								if(config.requestMBO == 1)pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_MBO);
							} else {
								if(receiver->r_basic) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_BASICDATA);
								if(receiver->r_quote) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_QUOTE);
								if(receiver->r_quotebbo) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_QUOTEBBO);
								if(receiver->r_quoteex) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_QUOTEEX);
								if(receiver->r_perf) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_PERFORMANCE);
								if(receiver->r_order) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_ORDER);
								if(receiver->r_ca) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_CORPORATEACTION);
								if(receiver->r_ph) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_PRICEHISTORY);
								if(receiver->r_tradestate) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_TRADESTATE);
								if(receiver->r_trades) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_TRADE);
								if(receiver->r_fund) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_FUNDAMENTALS);
								if(receiver->r_keyr) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_KEYRATIOS);
								if(receiver->r_noii) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_NETORDERIMBALANCE);
								if(receiver->r_estimates) {
									pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_ESTIMATES);
									pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_ESTIMATESHISTORY);
								}

								if(receiver->r_l10n) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_L10N);

								if(receiver->r_news) {
									pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_NEWSHEADLINE);
									pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_NEWSCONTENT);
								}

								if(receiver->r_ci) {
									pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_CI);
									pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_CIHISTORY);
								}

								if(receiver->r_ept) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_PRIIP);

								if(receiver->r_emt) {
									pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_MIFID);
									pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_MIFIDHISTORY);
								}

								if(receiver->r_eet) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_EET);
								if(receiver->r_mappings) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_MAPPINGS);
								if(receiver->r_greeks) pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_GREEKS);
								if(receiver->r_mbo)pos += sprintf (tmpbuf + pos, "%s ", MDF_RC_MBO);
							}

							tmpbuf[pos - 1] = '\0';

							mdf_message_add_list (mdf_message, MDF_F_REQUESTCLASS, tmpbuf);

							if (strcmp (config.insrefs, "*") == 0)
								mdf_message_add_string (mdf_message, MDF_F_INSREFLIST, config.insrefs);
							else
								mdf_message_add_list (mdf_message, MDF_F_INSREFLIST, config.insrefs);

							mdf_message_send (receiver->handle, mdf_message);
							mdf_message_destroy (mdf_message);
							break;
						}

						case	MDF_M_REQUESTFINISHED: {
							long status_code = 101;
							uint32_t _tag;
							char *_value;

							while (mdf_get_next_field ( receiver->handle, &_tag, &_value)) {
								if (_tag == MDF_F_REQUESTSTATUS && _value != NULL)
									status_code = strtol (_value, NULL, 10);
							}

							if (status_code == 100) {
								requestfinished = 1;
								write_to_log(LOG_INFO,"All the requested data have been received from MDF node %s (it's not all yet written to SQL though!)", host);

								receiver->received_all_basicdata = true;

								MUTEX_LOCK(&instruments_lock);
								int i;
								for (i = 0; i < receivers_num; i++) {
									if (receivers[i].r_basic && !receivers[i].received_all_basicdata)
										break;
								}

								if (i == receivers_num)
									free_instruments (1);
								MUTEX_UNLOCK(&instruments_lock);
							} else {
								mdf_disconnect(receiver->handle);
								receiver->connected = false;
								write_to_log(LOG_INFO,"The request failed on MDF host %s, disconnecting and trying to reconnect", host);
							}

							break;
						}
					}

					if (messageinfo.mdatype == 0)
						continue;

					stats->receivedmessages++;
					if( messageinfo.mdatype == MDA_TYPE_BIDORDER || messageinfo.mdatype == MDA_TYPE_ASKORDER ) {
						MUTEX_LOCK(&database_lock);
						Instrument *_instrument = hashit_lookup ( database,&messageinfo.insref );
						if( _instrument == NULL ) {
							_instrument = calloc (1,sizeof(Instrument));
							_instrument->insref = messageinfo.insref;
							hashit_insert (database,&_instrument->insref,_instrument);
						}
						MUTEX_UNLOCK(&database_lock);
					}
					messageinfo.keylen = sprintf (messageinfo.tmpkey,"%" PRIu64 ":%" PRIu8 ,messageinfo.insref,messageinfo.mdatype);
					queuefield_head = NULL;
					queuefield_tail = NULL;

					MessageInfo mi_ex; /* used for QuoteEx -> Quote is special flag is set */
					Queuefield *qf_ex_head = NULL, *qf_ex_tail = NULL; /* used for QuoteEx -> Quote is special flag is set */

					uint32_t _tag;
					char *_value;

					while ( 1 == mdf_get_next_field ( receiver->handle, &_tag, &_value ) ) {
						if (_tag == MDF_F_DELETERECORD) {
							messageinfo.option |= MDA_OPTION_DELETERECORD;
							messageinfo.option &= ~MDA_OPTION_HASH;
							continue;
						}

						Queuefield *_queuefield = add_queue_field (&queuefield_head, &queuefield_tail);
						_queuefield->value = _value == NULL ? NULL:strdup (_value);
						_queuefield->field = _tag;

						if (_tag == MDF_F_DATE) datefield = _queuefield;
						else if (_tag == MDF_F_TIME) timefield = _queuefield;
						else if (_tag == MDF_F_TRADETIME) timefield = _queuefield;
						else if (_tag == MDF_F_TRADECANCELTIME) timefield2 = _queuefield;
						else if (_tag == MDF_F_TRADEAGREEMENTTIME) timefield3 = _queuefield;
						else if (_tag == MDF_F_TRADEAGREEMENTDATE) datefield2 = _queuefield;
						else if (_tag == MDF_F_LASTMOD) {
							if (_value != NULL) {
								time_t then = strtoull (_value, NULL, 10);

								struct tm tm;
								if (config.uselocaltime != 0)
									localtime_r (&then, &tm);
								else
									gmtime_r (&then, &tm);

								free (_queuefield->value);
								_queuefield->value = malloc (72);
								sprintf (_queuefield->value, "%d-%02d-%02d %02d:%02d:%02d", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec);
							}
						}

						// check for message specific tags
						switch(messageinfo.mdatype) {
							case	MDA_TYPE_TRADE:
								if(_tag == MDF_F_TRADEREFERENCE) {
									sprintf(messageinfo.tmpkey + messageinfo.keylen,":%s",_value == NULL ? "":_value);
								} else if(_tag == MDF_F_TRADECODE) {
									unsigned long tradecode = _value == NULL ? 0 : strtoul (_value, NULL, 10);

									if (tradecode & MDF_TC_CANCEL)
										messageinfo.option |= MDA_OPTION_CANCELTRADE;

									/* we have to remove the correction flag from the tradecode so insert
									 * will fail if the original trade is in the database for people with
									 * old sql schemas where tradecode is part of the primary key */
									if(tradecode & MDF_TC_CORRECTION) {
										messageinfo.option |= MDA_OPTION_UPDATEFIRST;
										sprintf (_queuefield->value, "%lu", tradecode & ~MDF_TC_CORRECTION);
									}
								} else if (_tag == MDF_F_TRADECANCELTIME && _value != NULL) {
									messageinfo.option |= MDA_OPTION_CANCELTRADE;
								}
								break;
							case	MDA_TYPE_NEWS:
								if(_tag == MDF_F_NEWSISLASTBLOCK)
									messageinfo.option |= MDA_OPTION_NEWS_LASTBLOCK;
								else if(_tag == MDF_F_NEWSID) {
									messageinfo.keylen = sprintf(messageinfo.tmpkey + messageinfo.keylen,":%s",_value == NULL ? "":_value);
									messageinfo.newsid = _queuefield->value;
								} else if(_tag == MDF_F_NEWSCODINGISIN)
									messageinfo.option |= MDA_OPTION_NEWSHAVECODING;
								else if(_tag == MDF_F_NEWSCODINGCOMPANY)
									messageinfo.option |= MDA_OPTION_NEWSHAVECODING;
								else if(_tag == MDF_F_NEWSCODINGTAGS)
									messageinfo.option |= MDA_OPTION_NEWSHAVECODINGTAGS;
								else if(_tag == MDF_F_HEADLINE && _value != NULL)
									messageinfo.option |= MDA_OPTION_NEWSHASHEADLINE;
								break;

							case MDA_TYPE_MAPPINGS:
								if (_tag == MDF_F_SECTORMEMBERS)
									messageinfo.mdatype = MDA_TYPE_SECTORS;
								else if (_tag == MDF_F_LIST)
									messageinfo.mdatype = MDA_TYPE_LISTMAPPING;

								break;

							case	MDA_TYPE_BIDORDER:
							case	MDA_TYPE_ASKORDER:
								if( (_tag == MDF_F_BIDPRICE || _tag == MDF_F_ASKPRICE) && _value != NULL )
									messageinfo.orderprice = strdup(_value);
								else
								if(_tag == MDF_F_ORDERLEVEL && _value != NULL)
									messageinfo.orderlevel = strdup(_value);
								break;
							case	MDA_TYPE_MBO:
								if (_tag == MDF_F_ORDERID && _value != NULL)
									messageinfo.orderid = strtoull (_value, NULL, 10);
								break;
							case	MDA_TYPE_BASICDATA:
								if(_tag == MDF_F_LIST) {
									MessageInfo mi;
									Queuefield *qf;
									memset(&mi,0,sizeof(MessageInfo));
									qf = calloc (1,sizeof(Queuefield));
									mi.insref = messageinfo.insref;
									mi.mdatype = MDA_TYPE_LISTMAPPING;
									mi.option = MDA_OPTION_NOHASH;
									qf->value = _value == NULL ? NULL : strdup (_value);
									qf->field = _tag;
									if(_value && (_value[0] == '=' || _value[0] == '-' || _value[0] == '+') ) {
										_value++;	// remove the sign for list field in instruments...
										free(_queuefield->value);
										_queuefield->value  = strdup(_value);
									}
									addQueueItem(qf,&mi);
								} else if(_tag == MDF_F_CONSTITUENTS) {
									MessageInfo mi;
									Queuefield *qf;
									memset(&mi,0,sizeof(MessageInfo));
									qf = calloc (1,sizeof(Queuefield));
									mi.insref = messageinfo.insref;
									mi.mdatype = MDA_TYPE_CONSTITUENTS;
									mi.option = MDA_OPTION_NOHASH;
									qf->value = _value == NULL ? NULL : strdup (_value);
									qf->field = _tag;
									addQueueItem(qf,&mi);
								} else if(_tag == MDF_F_TICKSIZES) {
									MessageInfo mi;
									Queuefield *qf;
									memset(&mi,0,sizeof(MessageInfo));
									qf = calloc (1,sizeof(Queuefield));
									mi.insref = messageinfo.insref;
									mi.mdatype = MDA_TYPE_TICKSIZES;
									mi.option = MDA_OPTION_NOHASH;
									qf->value = _value == NULL ? NULL : strdup (_value);
									qf->field = _tag;
									addQueueItem(qf,&mi);
								} else if(_tag == MDF_F_BROKERS) {
									MessageInfo mi;
									Queuefield *qf;
									memset(&mi,0,sizeof(MessageInfo));
									qf = calloc (1,sizeof(Queuefield));
									mi.insref = messageinfo.insref;
									mi.mdatype = MDA_TYPE_BROKERS;
									mi.option = MDA_OPTION_NOHASH;
									qf->value = _value == NULL ? NULL : strdup (_value);
									qf->field = _tag;
									addQueueItem(qf,&mi);
								} else if(_tag == MDF_F_UNDERLYINGID) {
									MessageInfo mi;
									Queuefield *qf;
									memset(&mi,0,sizeof(MessageInfo));
									qf = calloc (1,sizeof(Queuefield));
									mi.insref = messageinfo.insref;
									mi.mdatype = MDA_TYPE_UNDERLYINGS;
									mi.option = MDA_OPTION_NOHASH;
									qf->value = _value == NULL ? NULL : strdup (_value);
									qf->field = _tag;
									addQueueItem(qf,&mi);
								} else if(_tag == MDF_F_COMBOLEGS) {
									MessageInfo mi;
									Queuefield *qf;
									memset(&mi,0,sizeof(MessageInfo));
									qf = calloc (1,sizeof(Queuefield));
									mi.insref = messageinfo.insref;
									mi.mdatype = MDA_TYPE_COMBOLEGS;
									mi.option = MDA_OPTION_NOHASH;
									qf->value = _value == NULL ? NULL : strdup (_value);
									qf->field = _tag;
									addQueueItem(qf,&mi);
								} else if (_tag == MDF_F_PRICETYPE) {
									MUTEX_LOCK(&queues[0].lock);

									const int ptype = _value == NULL ? 0 : strtol (_value, NULL, 10);
									PriceType *pt = hashit_lookup (pricetypes, &messageinfo.insref);

									if (pt == NULL) {
										if (ptype == 3) { /* For now we are only interested in NAV */
											pt = malloc (sizeof *pt);
											pt->insref = messageinfo.insref;
											pt->type = ptype;
											hashit_insert (pricetypes, &pt->insref, pt);
										}
									} else if (pt->type != ptype) {
										hashit_delete (pricetypes, &messageinfo.insref);
										free (pt);
									}

									MUTEX_UNLOCK(&queues[0].lock);
								} else if (_tag == MDF_F_SECTOR) {
									if(_value && (_value[0] == '=' || _value[0] == '-' || _value[0] == '+') ) {
										_value++;	// remove the sign for sector field in instruments...
										free(_queuefield->value);
										_queuefield->value  = strdup(_value);
									}
								}
								break;

							case	MDA_TYPE_CORPORATEACTION:
								if(MDF_F_ADJUSTMENTFACTOR == _tag)
									messageinfo.adjustmentfactor =1;
								break;

							case 	MDA_TYPE_PERFORMANCE: {
								switch (_tag) {
									case MDF_F_CLOSEBIDPRICE1D :
									case MDF_F_CLOSEBIDPRICE1W :
										messageinfo.option |= MDA_OPTION_CLOSEBIDPRICE;
										break;

									case MDF_F_CLOSEPRICE1D :
									case MDF_F_CLOSEPRICE1W :
									case MDF_F_CLOSEPRICE2W :
									case MDF_F_CLOSEPRICE1M :
									case MDF_F_CLOSEPRICE3M :
									case MDF_F_CLOSEPRICE6M :
									case MDF_F_CLOSEPRICE9M :
									case MDF_F_CLOSEPRICE1Y :
									case MDF_F_CLOSEPRICE2Y :
									case MDF_F_CLOSEPRICE3Y :
									case MDF_F_CLOSEPRICE4Y :
									case MDF_F_CLOSEPRICE5Y :
									case MDF_F_CLOSEPRICE6Y :
									case MDF_F_CLOSEPRICE7Y :
									case MDF_F_CLOSEPRICE8Y :
									case MDF_F_CLOSEPRICE9Y :
									case MDF_F_CLOSEPRICE10Y :
									case MDF_F_CLOSEPRICELD :
									case MDF_F_CLOSEPRICEWTD :
									case MDF_F_CLOSEPRICEMTD :
									case MDF_F_CLOSEPRICEQTD :
									case MDF_F_CLOSEPRICEYTD :
									case MDF_F_CLOSEPRICEPYTD :
										messageinfo.option |= MDA_OPTION_CLOSEPRICE;
										break;

									case MDF_F_CLOSEBIDYIELD1D :
									case MDF_F_CLOSEBIDYIELD1W :
										messageinfo.option |= MDA_OPTION_CLOSEBIDYIELD;
										break;

									case MDF_F_CLOSEYIELD1D :
									case MDF_F_CLOSEYIELD1W :
									case MDF_F_CLOSEYIELD2W :
									case MDF_F_CLOSEYIELD1M :
									case MDF_F_CLOSEYIELD3M :
									case MDF_F_CLOSEYIELD6M :
									case MDF_F_CLOSEYIELD9M :
									case MDF_F_CLOSEYIELD1Y :
									case MDF_F_CLOSEYIELD2Y :
									case MDF_F_CLOSEYIELD3Y :
									case MDF_F_CLOSEYIELD4Y :
									case MDF_F_CLOSEYIELD5Y :
									case MDF_F_CLOSEYIELD6Y :
									case MDF_F_CLOSEYIELD7Y :
									case MDF_F_CLOSEYIELD8Y :
									case MDF_F_CLOSEYIELD9Y :
									case MDF_F_CLOSEYIELD10Y :
									case MDF_F_CLOSEYIELDLD :
									case MDF_F_CLOSEYIELDWTD :
									case MDF_F_CLOSEYIELDMTD :
									case MDF_F_CLOSEYIELDQTD :
									case MDF_F_CLOSEYIELDYTD :
									case MDF_F_CLOSEYIELDPYTD :
										messageinfo.option |= MDA_OPTION_CLOSEYIELD;
										break;
								}

								break;
							}

							case MDA_TYPE_ORDERBOOKFLUSH : {
								if(MDF_F_I1 == _tag && _value != NULL) {
									messageinfo.option &= ~MDA_OPTION_FLUSH_MBL; /* clear the default value */

									uint64_t i1 = strtoull (_value, NULL, 10);

									if ((i1 & 1) == 1)
										messageinfo.option |= MDA_OPTION_FLUSH_MBL;

									if ((i1 & 2) == 2)
										messageinfo.option |= MDA_OPTION_FLUSH_MBO;

									/* the flush is for the QuoteBBO so we add a new message to Queue #1 since we handle
									 * Quotes and Orders/MBO on different queues, also we only do this if we actually do
									 * request QuoteBBO messages */
									if ((i1 & 4) == 4 && config.requestQuotesBBO == 1) {
										MessageInfo mi;
										memset(&mi,0,sizeof(MessageInfo));
										mi.insref = messageinfo.insref;
										mi.mdatype = MDA_TYPE_QUOTEBBOFLUSH;
										mi.option = MDA_OPTION_NOHASH;
										addQueueItem(NULL,&mi);
									}
								}

								break;
							}

							case MDA_TYPE_QUOTEEX : {
								/* only do this if told to */
								if (config.quoteExspecial == 0)
									break;

								/* never take the prices from QuoteEx if we are in recovery */
								if (requestfinished == 0)
									break;

								/* and only do it for these fields */
								if (_tag != MDF_F_BIDPRICE && _tag != MDF_F_ASKPRICE && _tag != MDF_F_BIDQUANTITY && _tag != MDF_F_ASKQUANTITY)
									break;

								if (qf_ex_head == NULL) {
									memset(&mi_ex, 0, sizeof mi_ex);

									mi_ex.insref = insref;
									mi_ex.keylen = sprintf (mi_ex.tmpkey, "%" PRIu64 ":%" PRIu8, insref, MDA_TYPE_QUOTE);
									mi_ex.option = MDA_OPTION_HASH;
									mi_ex.mdatype = MDA_TYPE_QUOTE;
								}

								Queuefield *qf = add_queue_field (&qf_ex_head, &qf_ex_tail);
								qf->value = (_value == NULL) ? NULL : strdup (_value);
								qf->field = _tag;
								break;
							}

							default:
								break;
						}
					}

					/* we have to handle this down here since we need the data from the MDF_F_I1 field */
					if (messageinfo.mdatype == MDA_TYPE_ORDERBOOKFLUSH) {
						if ((messageinfo.option & MDA_OPTION_FLUSH_MBL) == MDA_OPTION_FLUSH_MBL) {
							MUTEX_LOCK(&database_lock);
							Instrument *_instrument = hashit_lookup (database, &messageinfo.insref );

							if (_instrument != NULL)
								clear_orderbook(_instrument);

							MUTEX_UNLOCK(&database_lock);

							clearItemHash(&queues[3],insref,MDA_TYPE_ORDERBOOK, 1);
						}

						if ((messageinfo.option & MDA_OPTION_FLUSH_MBO) == MDA_OPTION_FLUSH_MBO)
							clearItemHash(&queues[3],insref,MDA_TYPE_MBO, 1);
					}

					/* make sure that the timestamps are within our defined max length */
					if (timefield  && timefield->value != NULL  && strlen (timefield->value) >  (size_t)config.max_timelen) timefield->value[config.max_timelen] = '\0';
					if (timefield2 && timefield2->value != NULL && strlen (timefield2->value) > (size_t)config.max_timelen) timefield2->value[config.max_timelen] = '\0';
					if (timefield3 && timefield3->value != NULL && strlen (timefield3->value) > (size_t)config.max_timelen) timefield3->value[config.max_timelen] = '\0';

					if ((messageinfo.option & MDA_OPTION_DELETERECORD) == 0) {
						/* makes sure that all messages that are not deletes always has a date and a time, not for newscontents though! */
						if (_messagereference != MDF_M_NEWSCONTENT && _messagereference != MDF_M_CI && _messagereference != MDF_M_CIHISTORY && _messagereference != MDF_M_PRIIP && _messagereference != MDF_M_MIFID && _messagereference != MDF_M_MIFIDHISTORY && _messagereference != MDF_M_EET) {
							if (datefield == NULL || datefield->value == NULL || timefield == NULL || timefield->value == NULL) {
								struct tm tm;
								char tmp[36];
								time_t now = time (NULL);

								gmtime_r (&now, &tm);

								if (datefield == NULL) {
									datefield = add_queue_field (&queuefield_head, &queuefield_tail);
									sprintf (tmp, "%04d-%02d-%02d", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday);
									datefield->value = strdup (tmp);
									datefield->field = MDF_F_DATE;
								} else if (datefield->value == NULL) {
									sprintf (tmp, "%04d-%02d-%02d", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday);
									datefield->value = strdup (tmp);
								}

								if (timefield == NULL) {
									timefield = add_queue_field (&queuefield_head, &queuefield_tail);
									sprintf (tmp, "%02d:%02d:%02d", tm.tm_hour, tm.tm_min, tm.tm_sec);
									timefield->value = strdup (tmp);
									timefield->field = MDF_F_TIME;
								} else if (timefield->value == NULL) {
									sprintf (tmp, "%02d:%02d:%02d", tm.tm_hour, tm.tm_min, tm.tm_sec);
									timefield->value = strdup (tmp);
								}
							}
						}
					}
#if !defined(WIN32) && !defined(__sun__)
					if (config.useunixtime == 1 && _messagereference == MDF_M_TRADE && timefield->value != NULL && datefield->value != NULL) {
						struct tm tm;
						if (sscanf (datefield->value, "%d-%d-%d", &tm.tm_year, &tm.tm_mon, &tm.tm_mday) == 3) {
							tm.tm_year -= 1900;
							tm.tm_mon -= 1;

							if (sscanf (timefield->value, "%d:%d:%d", &tm.tm_hour, &tm.tm_min, &tm.tm_sec) == 3) {
								Queuefield *unixfield = add_queue_field (&queuefield_head, &queuefield_tail);

								char tmp[100];
								sprintf (tmp, "%" PRIu64, ((uint64_t)(timegm (&tm))));
								unixfield->value = strdup (tmp);
								unixfield->field = MDF_F_UNIXTIME;
							}
						}
					}
#endif
					if (config.uselocaltime != 0) {
						switch (_messagereference) {
							/* these tables must remain in UTC (or does not have a timestamp) */
							case MDF_M_CORPORATEACTION:
							case MDF_M_PRICEHISTORY:
							case MDF_M_NEWSCONTENT:
							case MDF_M_CI:
							case MDF_M_CIHISTORY:
							case MDF_M_PRIIP:
							case MDF_M_MIFID:
							case MDF_M_MIFIDHISTORY:
							case MDF_M_MAPPINGS:
							case MDF_M_EET:
								break;

							default : {
								if (messageinfo.option & MDA_OPTION_DELETERECORD)
									break;
#ifdef WIN32
								SYSTEMTIME st;
								FILETIME ft1, ft2;
								unsigned long msec = 0;
								memset (&st, 0, sizeof st);

								if (sscanf (timefield->value, "%hu:%hu:%hu.%lu", &st.wHour, &st.wMinute, &st.wSecond, &msec) < 3)
									break;

								if (sscanf (datefield->value, "%hu-%hu-%hu", &st.wYear, &st.wMonth, &st.wDay) != 3)
									break;

								if (SystemTimeToFileTime (&st,&ft1) == 0)
									break;

								if (timefield2 != NULL && timefield2->value != NULL) {
									FILETIME ft3;
									unsigned long msec2 = 0;

									if (sscanf (timefield2->value, "%hu:%hu:%hu.%lu", &st.wHour, &st.wMinute, &st.wSecond, &msec2) >= 3) {
										if (SystemTimeToFileTime (&st,&ft3) != 0) {
											SYSTEMTIME st2;
											FileTimeToLocalFileTime (&ft3, &ft2);
											FileTimeToSystemTime (&ft2, &st2);

											if (msec2 == 0)
												sprintf (timefield2->value, "%02hu:%02hu:%02hu", st2.wHour, st2.wMinute, st2.wSecond);
											else if (msec2 < 1000)
												sprintf (timefield2->value, "%02hu:%02hu:%02hu.%03lu", st2.wHour, st2.wMinute, st2.wSecond, msec2);
											else
												sprintf (timefield2->value, "%02hu:%02hu:%02hu.%06lu", st2.wHour, st2.wMinute, st2.wSecond, msec2);
										}
									}
								}

								if (timefield3 != NULL && timefield3->value != NULL) {
									FILETIME ft3;
									unsigned long msec2 = 0;

									if (sscanf (timefield3->value, "%hu:%hu:%hu.%lu", &st.wHour, &st.wMinute, &st.wSecond, &msec2) >= 3) {
										if (datefield2 != NULL && datefield2->value != NULL && sscanf (datefield2->value, "%hu-%hu-%hu", &st.wYear, &st.wMonth, &st.wDay) != 3)
											break;

										if (SystemTimeToFileTime (&st,&ft3) != 0) {
											FileTimeToLocalFileTime (&ft3, &ft2);
											FileTimeToSystemTime (&ft2, &st);

											if (msec2 == 0)
												sprintf (timefield3->value, "%02hu:%02hu:%02hu", st.wHour, st.wMinute, st.wSecond);
											else if (msec2 < 1000)
												sprintf (timefield3->value, "%02hu:%02hu:%02hu.%03lu", st.wHour, st.wMinute, st.wSecond, msec2);
											else
												sprintf (timefield3->value, "%02hu:%02hu:%02hu.%06lu", st.wHour, st.wMinute, st.wSecond, msec2);

											if (datefield2 != NULL && datefield2->value != NULL)
												sprintf (datefield2->value, "%04d-%02d-%02d", st.wYear, st.wMonth, st.wDay);
										}
									}
								}

								FileTimeToLocalFileTime (&ft1, &ft2);
								FileTimeToSystemTime (&ft2, &st);

								if (msec == 0)
									sprintf (timefield->value, "%02hu:%02hu:%02hu", st.wHour, st.wMinute, st.wSecond);
								else if (msec < 1000)
									sprintf (timefield->value, "%02hu:%02hu:%02hu.%03lu", st.wHour, st.wMinute, st.wSecond, msec);
								else
									sprintf (timefield->value, "%02hu:%02hu:%02hu.%06lu", st.wHour, st.wMinute, st.wSecond, msec);

								sprintf (datefield->value, "%04d-%02d-%02d", st.wYear, st.wMonth, st.wDay);
#else
								unsigned long msec = 0;
								struct tm tm;
								memset (&tm, 0, sizeof tm);

								if (sscanf (timefield->value, "%d:%d:%d.%lu", &tm.tm_hour, &tm.tm_min, &tm.tm_sec, &msec) < 3)
									break;

								if (sscanf (datefield->value, "%d-%d-%d", &tm.tm_year, &tm.tm_mon, &tm.tm_mday) != 3)
									break;

								tm.tm_year -= 1900;
								tm.tm_mon--;

								unsigned long msec2 = 0;
								struct tm tm2;

								if (timefield2 != NULL && timefield2->value != NULL) {
									memcpy (&tm2, &tm, sizeof tm);

									if (sscanf (timefield2->value, "%d:%d:%d.%lu", &tm2.tm_hour, &tm2.tm_min, &tm2.tm_sec, &msec2) < 3)
										timefield2 = NULL;
								}

								unsigned long msec3 = 0;
								struct tm tm3;

								if (timefield3 != NULL && timefield3->value != NULL) {
									memcpy (&tm3, &tm, sizeof tm);

									if (sscanf (timefield3->value, "%d:%d:%d.%lu", &tm3.tm_hour, &tm3.tm_min, &tm3.tm_sec, &msec3) >= 3) {
										if (datefield2 != NULL && datefield2->value != NULL) {
											if (sscanf (datefield2->value, "%d-%d-%d", &tm3.tm_year, &tm3.tm_mon, &tm3.tm_mday) == 3) {
												tm3.tm_year -= 1900;
												tm3.tm_mon--;
											} else {
												timefield3 = NULL;
											}
										}
									} else {
										timefield3 = NULL;
									}
								}
# ifdef __sun__
								char *tz = getenv ("TZ");

								setenv ("TZ", "", 1);
								tzset ();
								const time_t now = mktime (&tm);
								const time_t now2 = timefield2 == NULL || timefield2->value == NULL ? 0 : mktime (&tm2);
								const time_t now3 = timefield3 == NULL || timefield3->value == NULL ? 0 : mktime (&tm3);

								if (tz != NULL)
									setenv ("TZ", tz, 1);
								else
									unsetenv ("TZ");

								tzset ();
# else
								const time_t now = timegm (&tm);
								const time_t now2 = timefield2 == NULL || timefield2->value == NULL ? 0 : timegm (&tm2);
								const time_t now3 = timefield3 == NULL || timefield3->value == NULL ? 0 : timegm (&tm3);
# endif
								localtime_r (&now, &tm);

								if (msec == 0)
									sprintf (timefield->value, "%02d:%02d:%02d", tm.tm_hour, tm.tm_min, tm.tm_sec);
								else if (msec < 1000)
									sprintf (timefield->value, "%02d:%02d:%02d.%03lu", tm.tm_hour, tm.tm_min, tm.tm_sec, msec);
								else
									sprintf (timefield->value, "%02d:%02d:%02d.%06lu", tm.tm_hour, tm.tm_min, tm.tm_sec, msec);

								sprintf (datefield->value, "%04d-%02d-%02d", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday);

								if (now2 != 0) {
									localtime_r (&now2, &tm2);

									if (msec2 == 0)
										sprintf (timefield2->value, "%02d:%02d:%02d", tm2.tm_hour, tm2.tm_min, tm2.tm_sec);
									else if (msec2 < 1000)
										sprintf (timefield2->value, "%02d:%02d:%02d.%03lu", tm2.tm_hour, tm2.tm_min, tm2.tm_sec, msec2);
									else
										sprintf (timefield2->value, "%02d:%02d:%02d.%06lu", tm2.tm_hour, tm2.tm_min, tm2.tm_sec, msec2);
								}

								if (now3 != 0) {
									localtime_r (&now3, &tm3);

									if (msec3 == 0)
										sprintf (timefield3->value, "%02d:%02d:%02d", tm3.tm_hour, tm3.tm_min, tm3.tm_sec);
									else if (msec2 < 1000)
										sprintf (timefield3->value, "%02d:%02d:%02d.%03lu", tm3.tm_hour, tm3.tm_min, tm3.tm_sec, msec3);
									else
										sprintf (timefield3->value, "%02d:%02d:%02d.%06lu", tm3.tm_hour, tm3.tm_min, tm3.tm_sec, msec3);

									if (datefield2 != NULL && datefield2->value != NULL)
										sprintf (datefield2->value, "%04d-%02d-%02d", tm3.tm_year + 1900, tm3.tm_mon + 1, tm3.tm_mday);
								}
#endif							
							}
						}
					}

					if(messageinfo.newsid != NULL && (messageinfo.option & MDA_OPTION_NEWSHAVECODING) == MDA_OPTION_NEWSHAVECODING)
						add_newsCodingItem(&messageinfo,queuefield_head);
					if(messageinfo.newsid != NULL && (messageinfo.option & MDA_OPTION_NEWSHAVECODINGTAGS) == MDA_OPTION_NEWSHAVECODINGTAGS)
						add_newsTagsItem(&messageinfo,queuefield_head);
					if(messageinfo.adjustmentfactor == 1)
						add_adjusthistory(&messageinfo,queuefield_head);

					/* if orderbook delete or update, get the price from internaldatabase */
					if((messageinfo.mdatype == MDA_TYPE_BIDORDER || messageinfo.mdatype == MDA_TYPE_ASKORDER) && messageinfo.orderlevel != NULL ) {
						MUTEX_LOCK(&database_lock);
						Instrument *_instrument = hashit_lookup (database, &messageinfo.insref );

						if (_instrument != NULL) {
							int _level = atoi(messageinfo.orderlevel);
							Orderbook	*_order;
							if(_level > 0 ) {
								if(!(messageinfo.option & MDA_OPTION_BIDORDERINSERT) && !(messageinfo.option & MDA_OPTION_ASKORDERINSERT)) {
									int i = 0;

									if(messageinfo.mdatype == MDA_TYPE_BIDORDER)
										_order = _instrument->bidorders;
									else
										_order = _instrument->askorders;
									for (i=0; i < _level-1;i++) {
										if(_order != NULL) 
											_order = _order->next;
									}
									if (_order != NULL) {
										// if this level is not in queue we need the price so add that field
										Queuefield	*_qf = calloc(1,sizeof(Queuefield));
										_qf->field = messageinfo.mdatype == MDA_TYPE_BIDORDER ? MDF_F_BIDPRICE : MDF_F_ASKPRICE;
										_qf->value = _order->price == NULL ? NULL:strdup(_order->price);
										queuefield_tail->next = _qf;
										_qf->previous = queuefield_tail;
										queuefield_tail = _qf;
										free(messageinfo.orderprice);
										messageinfo.orderprice = _order->price == NULL ? NULL:strdup(_order->price);
										createinternalorderbook(_instrument,messageinfo.orderprice,messageinfo.orderlevel,messageinfo.option);
									}
									else if (config.debuglog == 1) {
										if (messageinfo.option & MDA_OPTION_BIDORDERUPDATE || messageinfo.option & MDA_OPTION_ASKORDERUPDATE)
											write_to_log(LOG_ERR,"Invalid %s update level received: %s (%" PRIu64 ")",messageinfo.mdatype == MDA_TYPE_BIDORDER ? "BID" : "ASK", messageinfo.orderlevel, messageinfo.insref);
										else
											write_to_log(LOG_ERR,"Invalid %s delete level received: %s (%" PRIu64 ")",messageinfo.mdatype == MDA_TYPE_BIDORDER ? "BID" : "ASK", messageinfo.orderlevel, messageinfo.insref);
									}
								} else {
									createinternalorderbook(_instrument,messageinfo.orderprice,messageinfo.orderlevel,messageinfo.option);
								}
							} else if (config.debuglog == 1) {
								write_to_log(LOG_ERR,"Invalid orderlevel :%d (insref:%" PRIu64 " File: %s Line:%d",_level,messageinfo.insref,__FILE__,__LINE__);
							}

							free (messageinfo.orderlevel);
							messageinfo.orderlevel = NULL;
						}

						MUTEX_UNLOCK(&database_lock);
					}

					// add the QuoteEx -> Quote item to the queue (if any)
					if (qf_ex_head != NULL) {
						Queuefield *qf = add_queue_field (&qf_ex_head, &qf_ex_tail);
						qf->value = strdup (datefield->value);
						qf->field = MDF_F_DATE;

						qf = add_queue_field (&qf_ex_head, &qf_ex_tail);
						qf->value = strdup (timefield->value);
						qf->field = MDF_F_TIME;

						addQueueItem (qf_ex_head, &mi_ex);
					}

					Queuefield *qf_head = NULL, *qf_tail = NULL;
					MessageInfo mi;

					if (messageinfo.mdatype == MDA_TYPE_TRADE && config.aggtrades == 1) {
						memset(&mi,0,sizeof(MessageInfo));

						struct tm tm;

						sscanf (datefield->value, "%d-%d-%d", &tm.tm_year, &tm.tm_mon, &tm.tm_mday);
						sscanf (timefield->value, "%d:%d:%d", &tm.tm_hour, &tm.tm_min, &tm.tm_sec);

						tm.tm_year -= 1900;
						tm.tm_mon -= 1;

						time_t then = mktime (&tm) / 60;

						mi.insref = messageinfo.insref;
						mi.mdatype = MDA_TYPE_AGGTRADE;
						mi.keylen = sprintf (mi.tmpkey, "%" PRIu64 ":%" PRIu8 ":%" PRIu64, mi.insref, mi.mdatype, then);
						mi.option = MDA_OPTION_HASH;

						Queuefield *qf;

						qf = add_queue_field (&qf_head, &qf_tail);
						qf->value = strdup (datefield->value);
						qf->field = MDF_F_DATE;

						qf = add_queue_field (&qf_head, &qf_tail);
						qf->value = strdup (timefield->value);
						qf->field = MDF_F_TRADETIME;
					}

					// Add the item to the queue
					addQueueItem(queuefield_head,&messageinfo);
					free(messageinfo.orderprice);

					if (qf_head != NULL)
						addQueueItem(qf_head, &mi);
				}
			}
		}
	}
	}
	receiver->live = time (NULL) + 1;
	mdf_destroy(receiver->handle);
	runningthreads--;
#ifdef WIN32
	_endthread ();
#else
	pthread_exit(0);
#endif
}

#ifdef WIN32
struct tm *gmtime_r (const time_t *clock, struct tm *result)
{
	struct tm *tm = gmtime (clock);

	if (tm == NULL)
		return NULL;

	memcpy (result, tm, sizeof *result);
	return result;
}

struct tm *localtime_r (const time_t *clock, struct tm *result)
{
	struct tm *tm = localtime (clock);

	if (tm == NULL)
		return NULL;

	memcpy (result, tm, sizeof *result);
	return result;
}
#endif
