This challenge spawns a listening socket that takes 4 bytes. Then; it ANDs this with 0xFF (so effectively only uses one byte) and feeds it to a function that decodes a piece of code in the executable (hence the expression ‘greek to me’; there’s code in there that is unreadable).

I extracted the hash function and encoded data from the binary to be able to easely brute-force it:

#include <stdint.h>
#include <string.h>
#include <stdio.h>

const char magic[] = {
	0x33, 0xe1, 0xc4, 0x99, 0x11, 0x06, 0x81, 0x16, 0xf0, 0x32, 0x9f, 0xc4, 0x91, 0x17, 0x06, 0x81,
	0x14, 0xf0, 0x06, 0x81, 0x15, 0xf1, 0xc4, 0x91, 0x1a, 0x06, 0x81, 0x1b, 0xe2, 0x06, 0x81, 0x18,
	0xf2, 0x06, 0x81, 0x19, 0xf1, 0x06, 0x81, 0x1e, 0xf0, 0xc4, 0x99, 0x1f, 0xc4, 0x91, 0x1c, 0x06,
	0x81, 0x1d, 0xe6, 0x06, 0x81, 0x62, 0xef, 0x06, 0x81, 0x63, 0xf2, 0x06, 0x81, 0x60, 0xe3, 0xc4,
	0x99, 0x61, 0x06, 0x81, 0x66, 0xbc, 0x06, 0x81, 0x67, 0xe6, 0x06, 0x81, 0x64, 0xe8, 0x06, 0x81,
	0x65, 0x9d, 0x06, 0x81, 0x6a, 0xf2, 0xc4, 0x99, 0x6b, 0x06, 0x81, 0x68, 0xa9, 0x06, 0x81, 0x69,
	0xef, 0x06, 0x81, 0x6e, 0xee, 0x06, 0x81, 0x6f, 0xae, 0x06, 0x81, 0x6c, 0xe3, 0x06, 0x81, 0x6d,
	0xef, 0x06, 0x81, 0x72, 0xe9, 0x06, 0x81, 0x73, 0x7c
};

const int len = 121;

int32_t __declspec(naked) hashfunc(char * a1, int32_t a2) {
	__asm {
		push ebp
		mov ebp, esp
		push ecx
		mov edx, dword ptr ss : [ebp + 0x0C]
		mov ecx, 0xFF
		mov dword ptr ss : [ebp - 4], ecx
		test edx, edx
		je J124A
		push ebx
		mov ebx, dword ptr ss : [ebp + 8]
		push esi
		push edi
		push 14
		pop eax
		J1202:
		mov di, word ptr ss : [ebp - 4]
		cmp edx, eax
		mov esi, edx
		cmova esi, eax
		sub edx, esi
		J120F:
		movzx eax, byte ptr ds : [ebx]
		add di, ax
		mov word ptr ss : [ebp - 4], di
		add ecx, dword ptr ss : [ebp - 4]
		inc ebx
		sub esi, 1
		jne J120F
		movzx eax, byte ptr ss : [ebp - 4]
		shr di, 8
		add ax, di
		movzx eax, ax
		J124A:
		mov dword ptr ss : [ebp - 4], eax
		movzx eax, cl
		shr cx, 8
		add ax, cx
		movzx ecx, ax
		push 14
		pop eax
		test edx, edx
		jne J1202
		pop edi
		pop esi
		pop ebx
		movzx edx, byte ptr ss : [ebp - 4]
		mov eax, ecx
		shl ecx, 8
		and eax, 0xFF00
		add eax, ecx
		mov cx, word ptr ss : [ebp - 4]
		shr cx, 8
		add dx, cx
		or ax, dx
		mov esp, ebp
		pop ebp
		ret }
}

uint16_t guess(uint8_t inp) {
	uint16_t hash;
	char b[121];
	memcpy(b, magic, len);

	for (int x = 0; x < len; x++) {
		b[x] = (b[x] ^ inp) + 34;
	}

	hash = hashfunc(b, len);	
	return hash;
}

int main(void) {
	for (uint8_t i = 0; i < 255; i++) {
		uint16_t hash = guess(i);
		printf("%02X = %04X\n", i, hash);
		if (hash == 0xFB5E) {
			printf("Done!\n");
			break;
		}
	}
	system("pause");
	return 0;
}

The key turns out to be A2 00 00 00. On sending this to the socket, we get the following message:

Congratulations! But wait, where's my flag?

The debugger provides the answer: