#include <memory.h>
#include <stdio.h>
#include <stdlib.h>

#define	BUFFER_CAPACITY		1024 * 1024 * 25

#define INDICES_EXPECTED	32
#define INDICES_MAXIMUM		40
#define INDICES_WRITE		16

#define	PRINT_FREQUENCY		16

int		main (int argc, char* argv[])
{
	unsigned char*	bufferInput;
	unsigned char*	bufferLast;
	unsigned char*	bufferPrev;
	unsigned long	bufferSize;
	unsigned char*	bufferSwap;
	unsigned long	indicesArray[INDICES_MAXIMUM + 1];
	unsigned long	indicesCount;
	unsigned long	indicesWrite[INDICES_WRITE] = {0, 1, 14, 7, 8, 16, 10, 12, 13, 11, 9, 25, 26, 27, 17, 18};
	FILE*			streamWriter;
	FILE*			streamReader;
	unsigned long	block;
	unsigned long	dupes;
	unsigned long	lines;
	unsigned long	size;
	unsigned long	i;
	unsigned long	j;

	if (argc < 3)
	{
		printf ("usage: %s <input> <output>\n", argv[0]);

		return 0;
	}

	streamReader = fopen (argv[1], "rb");

	if (!streamReader)
	{
		fprintf (stderr, "cannot open input file \"%s\" for reading\n", argv[1]);

		return 1;
	}

	streamWriter = fopen (argv[2], "wb");

	if (!streamWriter)
	{
		fprintf (stderr, "cannot open output file \"%s\" for writing\n", argv[2]);
		fclose (streamReader);

		return 1;
	}

	bufferInput = malloc (sizeof (*bufferInput) * BUFFER_CAPACITY * 3);
	bufferLast = bufferInput + BUFFER_CAPACITY;
	bufferPrev = bufferLast + BUFFER_CAPACITY;
	bufferSize = 0;
	indicesArray[0] = 0;
	indicesCount = 1;
	dupes = 0;
	lines = 0;

	printf ("starting process...\n");

	for (i = 0; i < bufferSize || !feof (streamReader); ++i)
	{
		// End of buffer reached: content must be shifted to the left before
		// buffer is populated with incoming data from input stream
		if (i == bufferSize)
		{
			i = bufferSize - indicesArray[0];

			memmove (bufferInput, bufferInput + indicesArray[0], i * sizeof (*bufferInput));

			bufferSize = i + fread (bufferInput + i, sizeof (*bufferInput), BUFFER_CAPACITY - i, streamReader);

			for (j = indicesCount; j--; )
				indicesArray[j] -= indicesArray[0];

			if (i >= bufferSize)
				break;
		}

		// End of item found: save starting index of the next one
		if (bufferInput[i] == '|')
		{
			if (indicesCount < INDICES_MAXIMUM)
				indicesArray[indicesCount++] = i + 1;
		}

		// End of line found: write required indices and flush list
		else if (bufferInput[i] < ' ')
		{
			if (indicesCount == INDICES_EXPECTED)
			{
				indicesArray[indicesCount] = i + 1;
				size = 0;

				for (j = 0; j + 1 < INDICES_WRITE; ++j)
				{
					block = indicesArray[indicesWrite[j] + 1] - indicesArray[indicesWrite[j]] - 1;

					memcpy (bufferLast + size, bufferInput + indicesArray[indicesWrite[j]], block * sizeof (*bufferInput));
					size += block;

					bufferLast[size++] = '|';
				}

				if (j < INDICES_WRITE)
				{
					block = indicesArray[indicesWrite[j] + 1] - indicesArray[indicesWrite[j]] - 1;

					memcpy (bufferLast + size, bufferInput + indicesArray[indicesWrite[j]], block * sizeof (*bufferInput));
					size += block;

					bufferLast[size++] = 0;
				}

				if (size > 0)
				{
					if (memcmp (bufferLast, bufferPrev, size * sizeof (*bufferLast)) != 0)
					{
						if ((++lines & (((unsigned)1 << PRINT_FREQUENCY) - 1)) == 0)
							printf ("writing line %lu (%lu duplicate(s))...\n", lines, dupes);

						fwrite (bufferLast, sizeof (*bufferLast), size - 1, streamWriter);

						bufferSwap = bufferLast;
						bufferLast = bufferPrev;
						bufferPrev = bufferSwap;
					}
					else
						++dupes;
				}
			}

			fwrite (bufferInput + i, sizeof (*bufferInput), 1, streamWriter);

			indicesArray[0] = i + 1;
			indicesCount = 1;
		}
	}

	if ((lines & (((unsigned)1 << PRINT_FREQUENCY) - 1)) != 0)
		printf ("writing line %lu (%lu duplicate(s))...\n", lines, dupes);

	printf ("done.\n");

	fclose (streamWriter);
	fclose (streamReader);
	free (bufferInput);

	return 0;
}