/*
    Copyright (C) 2008-2024, Millistream Market Data <support@millistream.com>

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Lesser General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU Lesser General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.

*/

#include "common.h"
#include "encode.h"

static void grow_buffer (struct mdf_message_s * const message, const int len_to_add)
{
	if (message->data_used + len_to_add > message->data_size) {
		message->data_size = message->data_used + len_to_add;
		message->data_size += BUFFER_CHUNK_SIZE - (message->data_size % BUFFER_CHUNK_SIZE);

		message->data = (uint8_t *) realloc (message->data, message->data_size);
	}
}

LIBMDF_API mdf_message_t mdf_message_create ()
{
	mdf_message_t message = (struct mdf_message_s *) calloc (1, sizeof (struct mdf_message_s));

	if (message == NULL)
		return NULL;

	message->current_message = -1;
	message->compression_level = Z_BEST_SPEED;
	message->utf8_validate = 1;
	message->delay = MDF_DLY_BEST;
	return message;
}

LIBMDF_API int mdf_message_set_property (mdf_message_t message, MDF_MSG_OPTION option, const int value)
{
	if (message == NULL)
		return 0;

	if (option == MDF_MSG_OPT_UTF8) {
		if (value != 0 && value != 1)
			return 0;

		message->utf8_validate = value;
	} else if (option == MDF_MSG_OPT_COMPRESSION) {
		if (value < Z_NO_COMPRESSION || value > Z_BEST_COMPRESSION)
			return 0;

		message->compression_level = value;
	} else if (option == MDF_MSG_OPT_DELAY) {
		if (value < MDF_DLY_REALTIME || value > MDF_DLY_BEST)
			return 0;

		if (message->current_message != -1)
			message->messages[message->current_message].delay = value;

		message->delay = value;
	} else {
		return 0;
	}

	return 1;
}

LIBMDF_API int mdf_message_add (mdf_message_t message, const uint64_t instrument_reference, const int message_reference)
{
	return mdf_message_add2 (message, instrument_reference, message_reference, message->delay);
}

LIBMDF_API int mdf_message_add2 (mdf_message_t message, const uint64_t insref, const uint16_t mref, const uint8_t delay)
{
	if (message == NULL)
		return 0;

	/* terminate the current message before we proceed */
	if (message->current_message != -1) {
		struct message *m = &(message->messages[message->current_message]);

		/* the current message is not filled at all, reuse! */
		if (m->fields == 0 && m->mref != MDF_M_INSTRUMENTRESET && m->mref != MDF_M_INSTRUMENTDELETE && m->mref != MDF_M_ORDERBOOKFLUSH && m->mref != MDF_M_LOGOFF)
			goto reuse;
	}

	message->current_message++;

	/* grow the message chain if current is over the top */
	if (message->current_message == message->messages_max) {
		message->messages_max += MESSAGES_CHUNK_SIZE;
		message->messages = (struct message *) realloc (message->messages, sizeof *message->messages * message->messages_max);
	}

reuse:
	message->messages[message->current_message].insref = insref;
	message->messages[message->current_message].mref = mref;
	message->messages[message->current_message].fields = 0;
	message->messages[message->current_message].delay = delay;

	return 1;
}

LIBMDF_API int mdf_message_del (mdf_message_t message)
{
	if (message == NULL)
		return 0;

	if (message->current_message == -1)
		return 0;

	if (message->messages[message->current_message].fields != 0) {
		message->fields_num -= message->messages[message->current_message].fields;
		message->data_used = message->fields[message->fields_num].offset; 
	}

	message->current_message--;

	if (message->current_message == -1)
		return 0;

	return 1;
}

LIBMDF_API void mdf_message_reset (mdf_message_t message)
{
	message->current_message = -1;
	message->fields_num = 0;
	message->data_used = 0;
}

LIBMDF_API void mdf_message_destroy (mdf_message_t message)
{
	if (message == NULL)
		return;

	free (message->messages);
	free (message->fields);
	free (message->data);
	free (message->sendbuf);
	free (message->sendfields);
	free (message->sendtags);
	free (message);
}

LIBMDF_API int mdf_message_get_num (mdf_message_t message)
{
	if (message == NULL)
		return 0;

	return message->messages_max;
}

LIBMDF_API int mdf_message_get_num_active (mdf_message_t message)
{
	if (message == NULL)
		return 0;

	return message->current_message + 1;
}

LIBMDF_API int mdf_message_get_num_fields (mdf_message_t message)
{
	if (message == NULL || message->current_message == -1)
		return 0;

	return message->messages[message->current_message].fields;
}

LIBMDF_API int mdf_message_deserialize (const mdf_message_t message, const char * const data)
{
	if (message == NULL || data == NULL)
		return 0;

	size_t len = strlen (data);
#if OPENSSL_VERSION_NUMBER < 0x1000207fL /* 1.0.2g */
	BIO *bio = BIO_new_mem_buf ((void *)data, len);
#else
	BIO *bio = BIO_new_mem_buf (data, len);
#endif

	if (bio == NULL) 
		return 0;

	BIO *b64 = BIO_new (BIO_f_base64 ());

	if (b64 == NULL) {
		BIO_free (bio);
		return 0;
	}

	bio = BIO_push(b64, bio);

	BIO_set_flags(bio, BIO_FLAGS_BASE64_NO_NL);

	uint8_t *buf = malloc ((len * 3) / 4 + 1);

	if (buf == NULL) {
		BIO_free_all(bio);
		return 0;
	}

	const size_t ret = BIO_read (bio, buf, len);

	BIO_free_all(bio);

	int current_message;
	int fields_num;
	size_t data_used;

	if (sizeof current_message + sizeof fields_num + sizeof data_used > ret) {
		free (buf);
		return 0;
	}

	size_t offset = sizeof current_message;
	memcpy (&current_message, buf, offset);
	memcpy (&fields_num, buf + offset, sizeof fields_num); offset += sizeof fields_num;
	memcpy (&data_used, buf + offset, sizeof data_used); offset += sizeof data_used;

	if (offset + (current_message + 1) * sizeof *message->messages + fields_num * sizeof *message->fields + data_used != ret) {
		free (buf);
		return 0;
	}

	message->current_message = current_message;
	message->fields_num = fields_num;
	message->data_used = data_used;

	if (message->current_message + 1 >= message->messages_max) {
		message->messages_max = message->current_message + 1;
		message->messages_max += MESSAGES_CHUNK_SIZE - (message->messages_max % MESSAGES_CHUNK_SIZE);
		message->messages = (struct message *) realloc (message->messages, sizeof *message->messages * message->messages_max);
	}

	memcpy (message->messages, buf + offset, (message->current_message + 1) * sizeof *message->messages); offset += (message->current_message + 1) * sizeof *message->messages;

	if (message->fields_num >= message->fields_max) {
		message->fields_max = message->fields_num;
		message->fields_max += FIELDS_CHUNK_SIZE - (message->fields_max % FIELDS_CHUNK_SIZE);
		message->fields = (struct field *) realloc (message->fields, sizeof *message->fields * message->fields_max);
	}

	memcpy (message->fields, buf + offset, message->fields_num * sizeof *message->fields); offset += message->fields_num * sizeof *message->fields;

	if (message->data_used > message->data_size) {
		message->data_size = message->data_used;
		message->data_size += BUFFER_CHUNK_SIZE - (message->data_size % BUFFER_CHUNK_SIZE);
		free (message->data);
		message->data = (uint8_t *) malloc (message->data_size);
	}

	memcpy (message->data, buf + offset, message->data_used);

	free (buf);
	return 1;
}

LIBMDF_API int mdf_message_serialize (const mdf_message_t message, char **result)
{
	if (message == NULL || message->current_message == -1)
		return 0;

	uint8_t *buf = malloc (sizeof message->current_message + sizeof message->fields_num + sizeof message->data_used + (message->current_message + 1) * sizeof *message->messages + message->fields_num * sizeof *message->fields + message->data_used);

	if (buf == NULL)
		return 0;

	size_t offset = sizeof message->current_message;
	memcpy (buf, &message->current_message, offset);
	memcpy (buf + offset, &message->fields_num, sizeof message->fields_num); offset += sizeof message->fields_num;
	memcpy (buf + offset, &message->data_used, sizeof message->data_used); offset += sizeof message->data_used;
	memcpy (buf + offset, message->messages, (message->current_message + 1) * sizeof *message->messages); offset += (message->current_message + 1) * sizeof *message->messages;
	memcpy (buf + offset, message->fields, message->fields_num * sizeof *message->fields); offset += message->fields_num * sizeof *message->fields;
	memcpy (buf + offset, message->data, message->data_used); offset += message->data_used;

	BIO *b64 = BIO_new (BIO_f_base64 ());

	if (b64 == NULL) {
		free (buf);
		return 0;
	}

	BIO *bio = BIO_new (BIO_s_mem ());

	if (bio == NULL) {
		BIO_free (b64);
		free (buf);
		return 0;
	}

	bio = BIO_push (b64, bio);
	BIO_set_flags(bio, BIO_FLAGS_BASE64_NO_NL);
	BIO_write (bio, buf, offset);
	BIO_flush (bio);
	free (buf);

	BUF_MEM *bptr;
	BIO_get_mem_ptr (bio, &bptr);

	*result = malloc (bptr->length + 1);

	if (*result == NULL) {
		BIO_free_all(bio);
		return 0;
	}

	memcpy (*result, bptr->data, bptr->length);
	(*result)[bptr->length] = '\0';
	BIO_free_all(bio);

	return 1;
}

LIBMDF_API int mdf_message_move (const mdf_message_t message_src, const mdf_message_t message_dst, const uint64_t insref_src, const uint64_t insref_dst)
{
	if (message_src == NULL)
		return 0;

	if (message_src->current_message == -1)
		return 0;

	int current_field = 0;
	for (int i = 0; i <= message_src->current_message; i++) {
		if (!(insref_src == UINT64_MAX && insref_dst == UINT64_MAX) && message_src->messages[i].insref != insref_src) {
			current_field += message_src->messages[i].fields;
			continue;
		}

		if (message_src->messages[i].fields == 0 && message_src->messages[i].mref != MDF_M_INSTRUMENTRESET && message_src->messages[i].mref != MDF_M_INSTRUMENTDELETE && message_src->messages[i].mref != MDF_M_ORDERBOOKFLUSH && message_src->messages[i].mref != MDF_M_LOGOFF)
			continue;

		/* destination and source is same so we simply change the insref in place */
		if (message_src == message_dst || message_dst == NULL) {
			if (!(insref_src == UINT64_MAX && insref_dst == UINT64_MAX))
				message_src->messages[i].insref = insref_dst;

			continue;
		}

		message_dst->current_message++;

		if (message_dst->current_message == message_dst->messages_max) {
			message_dst->messages_max += MESSAGES_CHUNK_SIZE;
			message_dst->messages = (struct message *) realloc (message_dst->messages, sizeof *message_dst->messages * message_dst->messages_max);
		}

		if (insref_src == UINT64_MAX && insref_dst == UINT64_MAX)
			message_dst->messages[message_dst->current_message].insref = message_src->messages[i].insref;
		else
			message_dst->messages[message_dst->current_message].insref = insref_dst;

		message_dst->messages[message_dst->current_message].mref = message_src->messages[i].mref;
		message_dst->messages[message_dst->current_message].fields = message_src->messages[i].fields;
		message_dst->messages[message_dst->current_message].delay = message_src->messages[i].delay;

		if (message_dst->fields_num + message_src->messages[i].fields >= message_dst->fields_max) {
			message_dst->fields_max = message_dst->fields_num + message_src->messages[i].fields;
			message_dst->fields_max += FIELDS_CHUNK_SIZE - (message_dst->fields_max % FIELDS_CHUNK_SIZE);
			message_dst->fields = (struct field *) realloc (message_dst->fields, sizeof *message_dst->fields * message_dst->fields_max);
		}

		for (int j = 0, k = current_field; j < message_src->messages[i].fields; j++, k++) {
			message_dst->fields[message_dst->fields_num].offset = message_dst->data_used;
			message_dst->fields[message_dst->fields_num].tag = message_src->fields[k].tag;
			message_dst->fields[message_dst->fields_num].len = message_src->fields[k].len;

			if (message_dst->data_used + message_src->fields[k].len > message_dst->data_size) {
				message_dst->data_size = message_dst->data_used + message_src->fields[k].len;
				message_dst->data_size += BUFFER_CHUNK_SIZE - (message_dst->data_size % BUFFER_CHUNK_SIZE);
				message_dst->data = (uint8_t *) realloc (message_dst->data, message_dst->data_size);
			}

			memcpy (message_dst->data + message_dst->data_used, message_src->data + message_src->fields[k].offset, message_src->fields[k].len);
			message_dst->data_used += message_src->fields[k].len;

			message_dst->fields_num++;
		}

		memmove (message_src->fields + current_field, message_src->fields + current_field + message_src->messages[i].fields, (message_src->fields_num - (current_field + message_src->messages[i].fields)) * sizeof (struct field));
		message_src->fields_num -= message_src->messages[i].fields;
		memmove (message_src->messages + i, message_src->messages + i + 1, (message_src->current_message - i) * sizeof (struct message));
		message_src->current_message--;

		if (message_src->current_message == -1)
			message_src->data_used = 0;

		i--;
	}

	return 1;
}

static void add_tag (struct mdf_message_s * const message, const uint32_t tag, const size_t offset, const int len)
{
	message->messages[message->current_message].fields++;

	if (message->fields_num == message->fields_max) {
		message->fields_max += FIELDS_CHUNK_SIZE;
		message->fields = (struct field *) realloc (message->fields, sizeof *message->fields * message->fields_max);
	}

	message->fields[message->fields_num].offset = offset;
	message->fields[message->fields_num].tag = tag;
	message->fields[message->fields_num++].len = len;
}

static int add_null (struct mdf_message_s * const message, const uint32_t tag)
{
	add_tag (message, tag, message->data_used, 1);
	grow_buffer (message, 1);
	message->data[message->data_used++] = 240;

	return 1;
}

static int add_as_bcd (struct mdf_message_s * const message, const uint32_t tag, const char *value, const int neg)
{
	size_t oldpos = message->data_used;
	size_t len = strlen (value);

	grow_buffer (message, ((len + 1) / 2) + 2);
	message->data[message->data_used++] = 0xFF;

	int even = 0;
	if (neg == 1) {
		message->data[message->data_used] = 0xC0;
		even = 1;
	}

	for (; *value != '\0'; value++) {
		int code;

		if (*value == '.')
			code = 0x0A;
		else
			code = *value - '0';

		if (code > 10 || code < 0)
			continue;

		if (even == 0) {
			even = 1;
			message->data[message->data_used] = (uint8_t) (code << 4);
		} else {
			even = 0;
			message->data[message->data_used++] |= code;
		}
	}

	if (even == 0)
		message->data[message->data_used++] = 0xDD;
	else
		message->data[message->data_used++] |= 0x0D;

	add_tag (message, tag, oldpos, message->data_used - oldpos);
	return 1;
}

/* buf must be at least 5 bytes for uint32_t and 10 bytes for uint64_t */
static int add_serialised_integer (uint8_t *buf, uint64_t value)
{
	uint64_t tmp = value;
	uint64_t offset = 0;
	int c = 0;

	do {
		++c;
		offset = offset * 128 + 1;
		tmp = (tmp - 1) / 128;
	} while (tmp != 0);

	value -= offset;

	int d = c - 1;

	uint8_t *p = buf;
	do {
		p[--c] = (uint8_t) (value % 128);
		value /= 128;
	} while (c != 0);

	p += d;
	*p++ |= 128;

	return p - buf;
}

LIBMDF_API int mdf_message_add_list (mdf_message_t message, const uint32_t tag, const char *value)
{
	if (message == NULL || message->current_message == -1)
		return 0;

	if (value == NULL)
		return add_null (message, tag);

	uint32_t num = 0;
	for (const char *p = value; *p != '\0'; p++) {
		if (isdigit ((unsigned char) *p) != 0) {
			num++;

			while (isdigit ((unsigned char) p[1]) != 0)
				p++;
		}
	}

	/* sanity limit */
	if (num > 1000000)
		return 0;

	if (num == 0)
		return add_null (message, tag);

	const size_t oldpos = message->data_used;

	grow_buffer (message, num * 10 + 6);

	if (isdigit ((unsigned char) *value) != 0) {
		message->data[message->data_used++] = 250;
	} else if (*value == '=') {
		message->data[message->data_used++] = 250;
		value++;
	} else if (*value == '-') {
		message->data[message->data_used++] = 249;
		value++;
	} else if (*value == '+') {
		message->data[message->data_used++] = 248;
		value++;
	} else {
		return 0;
	}

	message->data_used += add_serialised_integer (message->data + message->data_used, (uint64_t) num);

	while (*value != '\0') {
		char *endp;
		const uint64_t v = strtoull (value, &endp, 10);

		if (v == 0 && value == endp) {
			value++;
			continue;
		}

		message->data_used += add_serialised_integer (message->data + message->data_used, v + 1);
		value = endp;
	}

	add_tag (message, tag, oldpos, message->data_used - oldpos);
	return 1;
}

LIBMDF_API int mdf_message_add_numeric (mdf_message_t message, const uint32_t tag, const char *value)
{
	if (message == NULL || message->current_message == -1)
		return 0;

	if (value == NULL)
		return add_null (message, tag);

	while (isspace ((unsigned char) *value) != 0)
		++value;

	int neg = 0;
	if (*value == '-') {
		++value;
		neg = 1;
	} else if (*value == '+') {
		++value;
	} else if (*value == '=') {
		++value;
	}

	if (*value == '\0')
		return add_null (message, tag);

	char *endptr;
	errno = 0;
	uint64_t num = strtoull (value, &endptr, 10);

	if (errno != 0)
		return add_as_bcd (message, tag, value, neg);

	uint64_t dec = 0;
	unsigned int decimals = 0;
	if (*endptr == '.') {
		const uint64_t cutoff = __UINT64_C (1844674407370955161);
		char *ptr = endptr + strlen (endptr);

		--ptr;

		while (*ptr == '0' || isspace ((unsigned char) *ptr) != 0)
			--ptr;

		++endptr;
		for (unsigned char c = *endptr; endptr <= ptr; c = *++endptr) {
			if (c < '0' || c > '9')
				return 0;

			c -= '0';

			/* do we overflow num? */
			if (num > cutoff || (num == cutoff && c > 5))
				return add_as_bcd (message, tag, value, neg);

			dec = dec * 10 + c;
			num = num * 10 + c;
			decimals++;
		}
	} else if (*endptr != '\0')
		return 0;

	if (decimals > 15)
		return add_as_bcd (message, tag, value, neg);

	if (neg == 0)
		return mdf_message_add_uint (message, tag, num, decimals);

	/* add -integer with up to 15 decimals */
	uint64_t tmpval = num, offset = 0;
	int bytes = 0;
	do {
		++bytes;
		offset = offset * 128UL + 1UL;
		tmpval = (tmpval - 1UL) / 128UL;
	} while (tmpval != 0UL);

	num -= offset;

	add_tag (message, tag, message->data_used, bytes + 1);
	grow_buffer (message, bytes + 1);

	message->data[message->data_used++] = (uint8_t) (0xC0 + decimals);

	uint8_t *p = message->data + message->data_used;
	message->data_used += bytes;

	do {
		p[--bytes] = (uint8_t) (num % 128);

		num /= 128UL;
	} while (bytes != 0);

	message->data[message->data_used - 1] |= 128;
	return 1;
}

LIBMDF_API int mdf_message_add_uint (mdf_message_t message, const uint32_t tag, uint64_t value, int decimals)
{
	static const uint64_t divisor[20] = {
		1,
		10,
		100,
		1000,
		10000,
		100000,
		1000000,
		10000000,
		100000000,
		1000000000,
    __UINT64_C (10000000000),
    __UINT64_C (100000000000),
    __UINT64_C (1000000000000),
    __UINT64_C (10000000000000),
    __UINT64_C (100000000000000),
    __UINT64_C (1000000000000000),
    __UINT64_C (10000000000000000),
    __UINT64_C (100000000000000000),
    __UINT64_C (1000000000000000000),
    __UINT64_C (10000000000000000000)
	};

	if (message == NULL || message->current_message == -1 || decimals > 19 || decimals < 0)
		return 0;

	/* we must normalize the scaled value */
	while (decimals > 0 && value % 10 == 0) {
		value /= 10;
		decimals--;
	}

	uint64_t integer = value / divisor[decimals];

	if (decimals > 15) {
		char string[30];
		sprintf (string, "%" PRIu64 ".%0*" PRIu64, integer, decimals, value % divisor[decimals]);
		return add_as_bcd (message, tag, string, 0);
	}

	if (decimals == 0) {
		int scale = 0;

		if (value < 28) {
			add_tag (message, tag, message->data_used, 1);
			grow_buffer (message, 1);
			message->data[message->data_used++] = (uint8_t) (value + 100);
			return 1;
		}

		if (value < 284) {
			add_tag (message, tag, message->data_used, 2);
			grow_buffer (message, 2);
			message->data[message->data_used++] = 0;
			message->data[message->data_used++] = (uint8_t) (value - 28);
			return 1;
		}

		while (integer % 10 == 0) {
			integer /= 10;
			++scale;

			if (scale == 16)
				break;
		}

		if (scale > 0) {
			uint64_t tmpval = integer;
			uint64_t offset = 0;
			int bytes = 0;

			do {
				++bytes;
				offset = offset * 128 + 1;
				tmpval = (tmpval - 1) / 128;
			} while (tmpval != 0);

			integer -= offset;

			add_tag (message, tag, message->data_used, bytes + 1);
			grow_buffer (message, bytes + 1);

			message->data[message->data_used++] = (uint8_t) (0xA0 + (scale - 1));
			uint8_t *p = message->data + message->data_used;
			message->data_used += bytes;

			do {
				p[--bytes] = (uint8_t) (integer % 128);
				
				integer /= 128;
			} while (bytes != 0);

			message->data[message->data_used - 1] |= 128;
			return 1;
		}
	}

	uint64_t dec = value % divisor[decimals];

	if (decimals < 3 && integer < 256) {
		/* decimals == 0 for this price range is taken care of by the previous code */
		if (decimals == 1)
			dec *= 10;

		add_tag (message, tag, message->data_used, 2);
		grow_buffer (message, 2);
		message->data[message->data_used++] = (uint8_t) dec;
		message->data[message->data_used++] = (uint8_t) integer;
		return 1;
	}

	if ((decimals < 5 && (integer < 104 || (integer == 104 && ((decimals == 3 && dec < 858) || (decimals == 4 && dec < 8577)))))) {
		/* decimals < 3 for this price range is taken care of by the previous code */
		if (decimals == 3)
			value *= 10;

		--value;

		add_tag (message, tag, message->data_used, 3);
		grow_buffer (message, 3);
		message->data[message->data_used++] = (uint8_t) (0x80 + (value >> 16));
		message->data[message->data_used++] = (uint8_t) (value % 256);
		message->data[message->data_used++] = (uint8_t) ((value & 0x0000FFFF) / 256);
		return 1;
	}

	if (decimals < 3 && (integer < 10741 || (integer == 10741 && dec < 77 && !(decimals == 1 && dec > 7)))) {
		if (decimals == 0)
			value *= 100;
		else if (decimals == 1)
			value *= 10;

		value -= 25601;

		add_tag (message, tag, message->data_used, 3);
		grow_buffer (message, 3);
		message->data[message->data_used++] = (uint8_t) (0x90 + (value >> 16));
		message->data[message->data_used++] = (uint8_t) (value % 256);
		message->data[message->data_used++] = (uint8_t) ((value & 0x0000FFFF) / 256);
		return 1;
	}

	/* add +/- integer with up to 15 decimals */
	uint64_t tmpval = value;
	uint64_t offset = 0;
	int bytes = 0;

	do {
		++bytes;
		offset = offset * 128UL + 1UL;
		tmpval = (tmpval - 1UL) / 128UL;
	} while (tmpval != 0UL);

	value -= offset;

	add_tag (message, tag, message->data_used, bytes + 1);
	grow_buffer (message, bytes + 1);

	message->data[message->data_used++] = (uint8_t) (0xB0 + decimals);
	uint8_t *p = message->data + message->data_used;
	message->data_used += bytes;

	do {
		p[--bytes] = (uint8_t) (value % 128);
				
		value /= 128UL;
	} while (bytes != 0);

	message->data[message->data_used - 1] |= 128;
	return 1;
}

LIBMDF_API int mdf_message_add_int (mdf_message_t message, const uint32_t tag, int64_t value, int decimals)
{
	static const uint64_t divisor[20] = {
		1,
		10,
		100,
		1000,
		10000,
		100000,
		1000000,
		10000000,
		100000000,
		1000000000,
    __UINT64_C (10000000000),
    __UINT64_C (100000000000),
    __UINT64_C (1000000000000),
    __UINT64_C (10000000000000),
    __UINT64_C (100000000000000),
    __UINT64_C (1000000000000000),
    __UINT64_C (10000000000000000),
    __UINT64_C (100000000000000000),
    __UINT64_C (1000000000000000000),
    __UINT64_C (10000000000000000000)
	};

	if (message == NULL || message->current_message == -1 || decimals > 19 || decimals < 0)
		return 0;

	int neg = 0;
	if (value < 0) {
		neg = 1;
		value = -value;
	}

	/* we must normalize the scaled value */
	while (decimals > 0 && value % 10 == 0) {
		value /= 10;
		decimals--;
	}

	uint64_t integer = value / divisor[decimals];

	if (decimals > 15) {
		char string[30];
		sprintf (string, "%" PRIu64 ".%0*" PRIu64, integer, decimals, value % divisor[decimals]);
		return add_as_bcd (message, tag, string, neg);
	}

	if (neg == 0) {
		if (decimals == 0) {
			int scale = 0;

			if (value < 28) {
				add_tag (message, tag, message->data_used, 1);
				grow_buffer (message, 1);
				message->data[message->data_used++] = (uint8_t) (value + 100);
				return 1;
			}

			if (value < 284) {
				add_tag (message, tag, message->data_used, 2);
				grow_buffer (message, 2);
				message->data[message->data_used++] = 0;
				message->data[message->data_used++] = (uint8_t) (value - 28);
				return 1;
			}

			while (integer % 10 == 0) {
				integer /= 10;
				++scale;

				if (scale == 16)
					break;
			}

			if (scale > 0) {
				uint64_t tmpval = integer;
				uint64_t offset = 0;
				int bytes = 0;

				do {
					++bytes;
					offset = offset * 128 + 1;
					tmpval = (tmpval - 1) / 128;
				} while (tmpval != 0);

				integer -= offset;

				add_tag (message, tag, message->data_used, bytes + 1);
				grow_buffer (message, bytes + 1);

				message->data[message->data_used++] = (uint8_t) (0xA0 + (scale - 1));
				uint8_t *p = message->data + message->data_used;
				message->data_used += bytes;

				do {
					p[--bytes] = (uint8_t) (integer % 128);
				
					integer /= 128;
				} while (bytes != 0);

				message->data[message->data_used - 1] |= 128;
				return 1;
			}
		}

		uint64_t dec = value % divisor[decimals];

		if (decimals < 3 && integer < 256) {
			/* decimals == 0 for this price range is taken care of by the previous code */
			if (decimals == 1)
				dec *= 10;

			add_tag (message, tag, message->data_used, 2);
			grow_buffer (message, 2);
			message->data[message->data_used++] = (uint8_t) dec;
			message->data[message->data_used++] = (uint8_t) integer;
			return 1;
		}

		if ((decimals < 5 && (integer < 104 || (integer == 104 && ((decimals == 3 && dec < 858) || (decimals == 4 && dec < 8577)))))) {
			/* decimals < 3 for this price range is taken care of by the previous code */
			if (decimals == 3)
				value *= 10;

			--value;

			add_tag (message, tag, message->data_used, 3);
			grow_buffer (message, 3);
			message->data[message->data_used++] = (uint8_t) (0x80 + (value >> 16));
			message->data[message->data_used++] = (uint8_t) (value % 256);
			message->data[message->data_used++] = (uint8_t) ((value & 0x0000FFFF) / 256);
			return 1;
		}

		if (decimals < 3 && (integer < 10741 || (integer == 10741 && dec < 77 && !(decimals == 1 && dec > 7)))) {
			if (decimals == 0)
				value *= 100;
			else if (decimals == 1)
				value *= 10;

			value -= 25601;

			add_tag (message, tag, message->data_used, 3);
			grow_buffer (message, 3);
			message->data[message->data_used++] = (uint8_t) (0x90 + (value >> 16));
			message->data[message->data_used++] = (uint8_t) (value % 256);
			message->data[message->data_used++] = (uint8_t) ((value & 0x0000FFFF) / 256);
			return 1;
		}
	}

	/* add +/- integer with up to 15 decimals */
	uint64_t tmpval = value;
	uint64_t offset = 0;
	int bytes = 0;

	do {
		++bytes;
		offset = offset * 128UL + 1UL;
		tmpval = (tmpval - 1UL) / 128UL;
	} while (tmpval != 0UL);

	value -= offset;

	add_tag (message, tag, message->data_used, bytes + 1);
	grow_buffer (message, bytes + 1);

	if (neg == 0)
		message->data[message->data_used++] = (uint8_t) (0xB0 + decimals);
	else
		message->data[message->data_used++] = (uint8_t) (0xC0 + decimals);

	uint8_t *p = message->data + message->data_used;
	message->data_used += bytes;

	do {
		p[--bytes] = (uint8_t) (value % 128);
				
		value /= 128UL;
	} while (bytes != 0);

	message->data[message->data_used - 1] |= 128;
	return 1;
}

/* Unicode functions modified from files located here: <http://unicode.org/Public/PROGRAMS/CVTUTF/>
 *
 * Copyright 2001-2004 Unicode, Inc.
 * 
 * Disclaimer
 * 
 * This source code is provided as is by Unicode, Inc. No claims are
 * made as to fitness for any particular purpose. No warranties of any
 * kind are expressed or implied. The recipient agrees to determine
 * applicability of information provided. If this file has been
 * purchased on magnetic or optical media from Unicode, Inc., the
 * sole remedy for any claim will be exchange of defective media
 * within 90 days of receipt.
 * 
 * Limitations on Rights to Redistribute This Code
 * 
 * Unicode, Inc. hereby grants the right to freely use the information
 * supplied in this file in the creation of products supporting the
 * Unicode Standard, and to make copies of this file in any form
 * for internal or external distribution as long as this notice
 * remains attached.
 *
 * The above copyright is for the isLegalUTF8() function, the
 * string verification part of mdf_add_string() and the 
 * accompanying tables.
*/

#define UNI_SUR_HIGH_START   (uint32_t) 0xD800
#define UNI_SUR_LOW_END      (uint32_t) 0xDFFF
#define UNI_REPLACEMENT_CHAR (uint32_t) 0x0000FFFD
#define UNI_MAX_LEGAL_UTF32  (uint32_t) 0x0010FFFF

static const uint8_t trailingBytesForUTF8[256] = {
    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5
};

static const uint32_t offsetsFromUTF8[6] = {
	0x00000000, 0x00003080, 0x000E2080,
	0x03C82080, 0xFA082080, 0x82082080
};

static int isLegalUTF8(const uint8_t *source, const int length)
{
    uint8_t a;
    const uint8_t *srcptr = source+length;
    switch (length) {
    default: return 0;
        /* Everything else falls through when "true"... */
/*  case 6: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return 0; RFC3629 makes 5 & 6 bytes UTF-8 illegal
    case 5: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return 0; */
    case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return 0;	/* FALLTHRU */
    case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return 0;	/* FALLTHRU */
    case 2: if ((a = (*--srcptr)) > 0xBF) return 0;	/* FALLTHRU */

        switch (*source) {
            /* no fall-through in this inner switch */
            case 0xE0: if (a < 0xA0) return 0; break;
            case 0xED: if (a > 0x9F) return 0; break;
            case 0xF0: if (a < 0x90) return 0; break;
            case 0xF4: if (a > 0x8F) return 0; break;
            default:   if (a < 0x80) return 0; break;
        }

	/* FALLTHRU */
    case 1: if (*source >= 0x80 && *source < 0xC2) return 0;
    }
    if (*source > 0xF4) return 0;
    return 1;
}

LIBMDF_API int mdf_message_add_string2 (mdf_message_t message, const uint32_t tag, const char * value, size_t len)
{
	if (message == NULL || message->current_message == -1)
		return 0;

	if (value == NULL)
		return add_null (message, tag);

	/* remove any initial BOM */
	if (len > 2 && (unsigned char)value[0] == 0xEF && (unsigned char)value[1] == 0xBB && (unsigned char)value[2] == 0xBF) {
		len -= 3;
		value += 3;
	}

	/* is the string valid UTF-8? */
	if (message->utf8_validate == 1) {
		for (size_t i = 0; i < len; i++) {
			uint32_t ch = 0;
			uint8_t extrabytes = trailingBytesForUTF8[(uint8_t) value[i]];

			if (extrabytes + i > len)
				return 0;

			if (isLegalUTF8 ((const uint8_t *) (value + i), extrabytes + 1) == 0)
				return 0;

			switch (extrabytes) {
				case 5 : ch += (uint8_t) value[i++]; ch <<= 6;	/* FALLTHRU */
				case 4 : ch += (uint8_t) value[i++]; ch <<= 6;	/* FALLTHRU */
				case 3 : ch += (uint8_t) value[i++]; ch <<= 6;	/* FALLTHRU */
				case 2 : ch += (uint8_t) value[i++]; ch <<= 6;	/* FALLTHRU */
				case 1 : ch += (uint8_t) value[i++]; ch <<= 6;	/* FALLTHRU */
				case 0 : ch += (uint8_t) value[i];
			}

			ch -= offsetsFromUTF8[extrabytes];

			if (ch <= UNI_MAX_LEGAL_UTF32) {
				if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END)
					return 0;
			} else {
				return 0;
			}
		}
	}

	/* we need this location if compression fails */
	const size_t oldpos = message->data_used;

	/* trying to compress too small strings is just a big waste of time!
	 * news headlines might be worth it though */
	if (len <= 100 && tag != MDF_F_HEADLINE) {
		grow_buffer (message, len + 2);
		message->data[message->data_used++] = 241;

		if (tag == MDF_F_MMT) {
			uint8_t num_blanks = 0;
			for (size_t i = 0; i < len; i++) {
				if (value[i] == '-') {
					num_blanks++;
				} else {
					if (num_blanks != 0) {
						message->data[message->data_used++] = num_blanks + 0x60;
						num_blanks = 0;
					}

					message->data[message->data_used++] = value[i];
				}
			}
		} else {
			memcpy (message->data + message->data_used, value, len);
			message->data_used += len;
		}

		message->data[message->data_used++] = '\0';
	} else {
		int ret;
		uLongf complen = compressBound (len);

		grow_buffer (message, complen + 6);
		message->data[message->data_used++] = 242;
		message->data_used += add_serialised_integer (message->data + message->data_used, (uint64_t) len);
		ret = compress2 ((Bytef *) (message->data + message->data_used), &complen, (const Bytef *) value, len, message->compression_level);

		if (ret == Z_OK)
			message->data_used += complen;

		/* compression either failed or increased the size; add the string uncompressed */
		if (message->data_used - oldpos - 1 >= len + 1 || ret != Z_OK) {
			message->data_used = oldpos;
			grow_buffer (message, len + 2);
			message->data[message->data_used++] = 241;
			memcpy (message->data + message->data_used, value, len);
			message->data_used += len;
			message->data[message->data_used++] = '\0';
		}

		/* compress with ZSTD, disbled for now */
/*		size_t complen = ZSTD_compressBound (len);

		grow_buffer (message, complen + 2);
		message->data[message->data_used++] = 242;
		message->data[message->data_used++] = 128;

		size_t ret = ZSTD_compress (message->data + message->data_used, complen, value, len, message->compression_level);

		if (!ZSTD_isError (ret))
			message->data_used += ret;
*/
		/* compression either failed or increased the size; add the string uncompressed */
/*		if (message->data_used - oldpos - 1 >= len + 1 || ZSTD_isError (ret)) {
			message->data_used = oldpos;
			grow_buffer (message, len + 2);
			message->data[message->data_used++] = 241;
			memcpy (message->data + message->data_used, value, len);
			message->data_used += len;
			message->data[message->data_used++] = '\0';
		}
*/
	}

	/* obfuscate the password length if the length is < 50 characters */
	if (tag == MDF_F_PASSWORD) {
		size_t added_len = message->data_used - oldpos;

		if (added_len < 44) { /* 50-7 == 43 */
			added_len = 45 - added_len; /* 50 - added_len - 5 */

			add_tag (message, tag, oldpos, message->data_used - oldpos);
			add_tag (message, 0xffffffff, message->data_used, added_len);

			grow_buffer (message, added_len);

			message->data[message->data_used++] = 241;
			added_len--;

			memset (message->data + message->data_used, ' ', added_len - 1);
			message->data_used += added_len;
			message->data[message->data_used - 1] = '\0';
			return 1;
		}
	}

	add_tag (message, tag, oldpos, message->data_used - oldpos);
	return 1;
}

LIBMDF_API int mdf_message_add_string (mdf_message_t message, const uint32_t tag, const char * value)
{
	if (message == NULL || message->current_message == -1)
		return 0;

	if (value == NULL)
		return add_null (message, tag);

	return mdf_message_add_string2 (message, tag, value, strlen (value));
}


LIBMDF_API int mdf_message_add_date2 (mdf_message_t message, const uint32_t tag, const int year, const int mon, const int day)
{
	if (message == NULL || message->current_message == -1)
		return 0;

	const int num = year * 448 + mon * 32 + day;

	if (mon > 13 || mon < 0 || day > 31 || day < 0)
		return 0;

	if (num > 1048575)
		return 0;

	add_tag (message, tag, message->data_used, 3);
	grow_buffer (message, 3);

	message->data[message->data_used++] = (0xD0 + (num >> 16)) & 0xFF;
	message->data[message->data_used++] = num & 0xFF;
	message->data[message->data_used++] = (num >> 8) & 0xFF;

	return 1;
}

LIBMDF_API int mdf_message_add_date (mdf_message_t message, const uint32_t tag, const char *value)
{
	if (message == NULL || message->current_message == -1)
		return 0;

	if (value == NULL)
		return add_null (message, tag);

	char *endp;
	int year = strtol (value, &endp, 10);

	if (year <= 0)
		return 0;

	if (*endp == '\0') {
		if (year > 2339)
			return mdf_message_add_string (message, tag, value);

		return mdf_message_add_date2 (message, tag, year, 0, 0);
	} else if (*endp != '-') {
		return 0;
	}

	endp++;

	switch (*endp) {
		case 'H' : {
			int day = strtol (endp + 1, NULL, 10);

			if (day < 1 || day > 2)
				return 0;

			if (year > 2339)
				return mdf_message_add_string (message, tag, value);

			return mdf_message_add_date2 (message, tag, year, 0, day);
		}

		case 'T' : {
			int day = strtol (endp + 1, NULL, 10);

			if (day < 1 || day > 3)
				return 0;

			if (year > 2339)
				return mdf_message_add_string (message, tag, value);

			return mdf_message_add_date2 (message, tag, year, 0, day + 2);
		}

		case 'Q' : {
			int day = strtol (endp + 1, NULL, 10);

			if (day < 1 || day > 4)
				return 0;

			if (year > 2339)
				return mdf_message_add_string (message, tag, value);

			return mdf_message_add_date2 (message, tag, year, 0, day + 5);
		}

		case 'W' : {
			int day = strtol (endp + 1, NULL, 10);

			if (day < 1 || day > 53)
				return 0;

			if (year > 2339)
				return mdf_message_add_string (message, tag, value);

			if (day < 23)
				return mdf_message_add_date2 (message, tag, year, 0, day + 9);

			return mdf_message_add_date2 (message, tag, year, 13, day - 22);
		}
	}

	int mon = strtol (endp, &endp, 10);
	int day;

	if (*endp == '\0')
		day = 0;
	else if (*endp == '-')
		day = strtol (endp + 1, NULL, 10);
	else if (*endp == 'M')
		return mdf_message_add_string (message, tag, value);
	else
		return 0;

	if (year > 2339)
		return mdf_message_add_string (message, tag, value);

	return mdf_message_add_date2 (message, tag, year, mon, day);
}

LIBMDF_API int mdf_message_add_time3 (mdf_message_t message, const uint32_t tag, const int hour, const int min, const int sec, int nsec)
{
	if (message == NULL || message->current_message == -1)
		return 0;

	if (hour > 23 || hour < 0 || min > 59 || min < 0 || sec > 59 || sec < 0 || nsec < 0)
		return 0;

	/* scale value down to nanoseconds */
	while (nsec > 999999999)
		nsec /= 10;

	if (nsec != 0) {
		if (nsec % 1000000 == 0) { /* so timestamp was in milliseconds apparantly (mmm000000) */
			add_tag (message, tag, message->data_used, 5);
			grow_buffer (message, 5);
			uint32_t num = cpu_to_le32 (hour * 3600000 + min * 60000 + sec * 1000 + nsec / 1000000); /* 0 - 86399999 */

			message->data[message->data_used++] = 0xF4;
			memcpy (message->data + message->data_used, &num, sizeof num);
			message->data_used += 4;
			return 1;
		}

		add_tag (message, tag, message->data_used, 7);
		grow_buffer (message, 7);

		const uint64_t num2 = hour * __UINT64_C(3600000000000) + min * __UINT64_C(60000000000) + sec * __UINT64_C(1000000000) + nsec + __UINT64_C(0x800000000000);

		message->data[message->data_used++] = 0xF4;
		message->data[message->data_used++] = (num2 >> 32) & 0xFF;
		message->data[message->data_used++] = (num2 >> 24) & 0xFF;
		message->data[message->data_used++] = (num2 >> 16) & 0xFF;
		message->data[message->data_used++] = (num2 >> 40) & 0xFF;
		message->data[message->data_used++] = (num2 >> 8) & 0xFF;
		message->data[message->data_used++] = num2 & 0xFF;

		return 1;
	}

	add_tag (message, tag, message->data_used, 3);
	grow_buffer (message, 3);

	uint32_t num = hour * 3600 + min * 60 + sec;

	if (num > 65535) {
		message->data[message->data_used++] = 0xF7;
		num -= 65535;
	} else {
		message->data[message->data_used++] = 0xF6;
	}

	uint16_t num2 = cpu_to_le16 (num);
	memcpy (message->data + message->data_used, &num2, sizeof num2);
	message->data_used += 2;

	return 1;
}

LIBMDF_API int mdf_message_add_time2 (mdf_message_t message, const uint32_t tag, const int hour, const int min, const int sec, int msec)
{
	if (message == NULL || message->current_message == -1)
		return 0;

	if (hour > 23 || hour < 0 || min > 59 || min < 0 || sec > 59 || sec < 0 || msec < 0)
		return 0;

	while (msec > 999)
		msec /= 10;

	if (msec != 0) {
		add_tag (message, tag, message->data_used, 5);
		grow_buffer (message, 5);
		uint32_t num = cpu_to_le32 (hour * 3600000 + min * 60000 + sec * 1000 + msec); /* 0 - 86399999 */

		message->data[message->data_used++] = 0xF4;
		memcpy (message->data + message->data_used, &num, sizeof num);
		message->data_used += 4;
		return 1;
	}

	add_tag (message, tag, message->data_used, 3);
	grow_buffer (message, 3);

	uint32_t num = hour * 3600 + min * 60 + sec;

	if (num > 65535) {
		message->data[message->data_used++] = 0xF7;
		num -= 65535;
	} else {
		message->data[message->data_used++] = 0xF6;
	}

	uint16_t num2 = cpu_to_le16 (num);
	memcpy (message->data + message->data_used, &num2, sizeof num2);
	message->data_used += 2;

	return 1;
}

LIBMDF_API int mdf_message_add_time (mdf_message_t message, const uint32_t tag, const char *value)
{
	if (value == NULL)
		return 0;

	char *endp;
	int hour = strtol (value, &endp, 10);
	if (*endp != ':')
		return 0;

	int min = strtol (endp + 1, &endp, 10);
	if (*endp != ':')
		return 0;

	int sec = strtol (endp + 1, &endp, 10);
	int msec = 0;

	if (*endp == '.') {
		int num_digits = 0;
		++endp;
		for (unsigned char c = *endp; c != '\0'; c = *++endp) {
			if (c < '0' || c > '9')
				return 0;

			c -= '0';

			msec = msec * 10 + c;
			num_digits++;
		}

		for (int min_digits = 9 - num_digits; min_digits > 0; min_digits--)
			msec *= 10;
	}

	return mdf_message_add_time3 (message, tag, hour, min, sec, msec);
}

