New hamcorebuilder implementation, independent from Cedar and Mayaqua

This new implementation can be easily compiled and executed without the need for other components to be present.

It relies on standard C functions, aside from stat() which is part of POSIX but available on Windows as well.

There's only one third-party dependency, which is tinydir: a single-file header-only library for traversing directories.

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "src/Mayaqua/3rdparty/cpu_features"]
 	path = src/Mayaqua/3rdparty/cpu_features
 	url = https://github.com/google/cpu_features.git
+[submodule "3rdparty/tinydir"]
+	path = 3rdparty/tinydir
+	url = https://github.com/cxong/tinydir.git

3rdparty/tinydir

@@ -0,0 +1 @@
+Subproject commit ec6bff2043eaac3ad25423705e63a781762a0dfd

src/CMakeLists.txt

@@ -144,7 +144,7 @@ add_subdirectory(vpntest)
 # hamcore.se2 archive file
 add_custom_target(hamcore-archive-build
   ALL
-  COMMAND hamcorebuilder "${TOP_DIRECTORY}/src/bin/hamcore/" "${BUILD_DIRECTORY}/hamcore.se2"
+  COMMAND hamcorebuilder "${BUILD_DIRECTORY}/hamcore.se2" "${TOP_DIRECTORY}/src/bin/hamcore"
   DEPENDS hamcorebuilder
   COMMENT "Building hamcore.se2 archive file..."
   VERBATIM

src/GlobalConst.h

@@ -64,7 +64,9 @@
 
 #define	GC_UI_APPID_CM				L"SoftEther.SoftEther VPN Client Developer Edition"
 
+//// Hamcore
 
+#define HAMCORE_HEADER_DATA			"HamCore"
+#define HAMCORE_HEADER_SIZE			7
 
 #endif	// GLOBAL_CONST_H
-

src/Mayaqua/FileIO.h

@@ -13,8 +13,6 @@
 #define	HAMCORE_FILE_NAME			"hamcore.se2"
 #define	HAMCORE_FILE_NAME_2			"_hamcore.se2"
 #define	HAMCORE_TEXT_NAME			"hamcore.txt"
-#define	HAMCORE_HEADER_DATA			"HamCore"
-#define	HAMCORE_HEADER_SIZE			7
 #define	HAMCORE_CACHE_EXPIRES		(5 * 60 * 1000)
 
 // IO structure

src/hamcorebuilder/CMakeLists.txt

@@ -1,3 +1,21 @@
-add_executable(hamcorebuilder hamcorebuilder.c)
+include(TestBigEndian)
 
-target_link_libraries(hamcorebuilder cedar mayaqua)
+add_executable(hamcorebuilder
+  main.c
+  FileSystem.c
+  FileSystem.h
+)
+
+if(WIN32)
+  target_compile_definitions(hamcorebuilder PRIVATE "OS_WINDOWS")
+endif()
+
+test_big_endian(BIG_ENDIAN)
+if(BIG_ENDIAN)
+  target_compile_definitions(hamcorebuilder PRIVATE "BYTE_ORDER_BIG_ENDIAN")
+endif()
+
+target_include_directories(hamcorebuilder PRIVATE "${TOP_DIRECTORY}/3rdparty/tinydir")
+
+find_package(ZLIB REQUIRED)
+target_link_libraries(hamcorebuilder PRIVATE ZLIB::ZLIB)

src/hamcorebuilder/FileSystem.c

@@ -0,0 +1,220 @@
+#include "FileSystem.h"
+
+#include <string.h>
+
+#include <sys/stat.h>
+
+ENTRIES *EnumEntries(const char *path)
+{
+	if (!path)
+	{
+		return NULL;
+	}
+
+	tinydir_dir dir;
+	if (tinydir_open_sorted(&dir, path) == -1)
+	{
+		printf("tinydir_open_sorted() failed!\n");
+		return NULL;
+	}
+
+	ENTRIES *entries = calloc(1, sizeof(ENTRIES));
+
+	for (size_t i = 0; i < dir.n_files; ++i)
+	{
+		tinydir_file file;
+		if (tinydir_readfile_n(&dir, &file, i) == -1)
+		{
+			printf("tinydir_readfile_n() failed at index %zu!\n", i);
+			FreeEntries(entries);
+			return NULL;
+		}
+
+		if (file.is_dir)
+		{
+			if (strcmp(file.name, ".") == 0 || strcmp(file.name, "..") == 0)
+			{
+				continue;
+			}
+		}
+#ifndef OS_WINDOWS
+		if (IsWindowsExtension(file.extension))
+		{
+			continue;
+		}
+#endif
+		++entries->Num;
+		entries->List = realloc(entries->List, sizeof(ENTRY) * entries->Num);
+
+		ENTRY *entry = &entries->List[entries->Num - 1];
+		entry->IsDir = file.is_dir;
+		strcpy(entry->Path, file.path);
+	}
+
+	tinydir_close(&dir);
+	return entries;
+}
+
+ENTRIES *EnumEntriesRecursively(const char *path, const bool files_only)
+{
+	if (!path)
+	{
+		return NULL;
+	}
+
+	ENTRIES *tmp = EnumEntries(path);
+	if (!tmp)
+	{
+		return NULL;
+	}
+
+	ENTRIES *entries = calloc(1, sizeof(ENTRIES));
+
+	for (size_t i = 0; i < tmp->Num; ++i)
+	{
+		ENTRY *entry = &tmp->List[i];
+		if (!files_only || !entry->IsDir)
+		{
+			++entries->Num;
+			entries->List = realloc(entries->List, sizeof(ENTRY) * entries->Num);
+			memcpy(&entries->List[entries->Num - 1], entry, sizeof(ENTRY));
+		}
+
+		if (!entry->IsDir)
+		{
+			continue;
+		}
+
+		ENTRIES *tmp_2 = EnumEntries(entry->Path);
+		if (!tmp_2)
+		{
+			continue;
+		}
+
+		const size_t offset = tmp->Num;
+
+		tmp->Num += tmp_2->Num;
+		tmp->List = realloc(tmp->List, sizeof(ENTRY) * tmp->Num);
+
+		memcpy(&tmp->List[offset], tmp_2->List, sizeof(ENTRY) * tmp_2->Num);
+
+		FreeEntries(tmp_2);
+	}
+
+	FreeEntries(tmp);
+
+	return entries;
+}
+
+void FreeEntries(ENTRIES *entries)
+{
+	if (!entries)
+	{
+		return;
+	}
+
+	if (entries->List)
+	{
+		free(entries->List);
+	}
+
+	free(entries);
+}
+
+FILE *FileOpen(const char *path, const bool write)
+{
+	if (!path)
+	{
+		return NULL;
+	}
+
+	return fopen(path, write ? "wb" : "rb");
+}
+
+bool FileClose(FILE *file)
+{
+	if (!file)
+	{
+		return false;
+	}
+
+	return fclose(file) == 0;
+}
+
+bool FileRead(FILE *file, void *dst, const size_t size)
+{
+	if (!file || !dst || size == 0)
+	{
+		return false;
+	}
+
+	return fread(dst, 1, size, file) == size;
+}
+
+bool FileWrite(FILE *file, const void *src, const size_t size)
+{
+	if (!file || !src || size == 0)
+	{
+		return false;
+	}
+
+	return fwrite(src, 1, size, file) == size;
+}
+
+size_t FileSize(const char *path)
+{
+	if (!path)
+	{
+		return 0;
+	}
+
+	struct stat st;
+	if (stat(path, &st) == -1)
+	{
+		return 0;
+	}
+
+	return st.st_size;
+}
+
+char *PathRelativeToBase(char *full, const char *base)
+{
+	if (!full || !base)
+	{
+		return NULL;
+	}
+
+	if (strstr(full, base) != &full[0])
+	{
+		return NULL;
+	}
+
+	full += strlen(base);
+	if (full[0] == '/')
+	{
+		++full;
+	}
+
+	return full;
+}
+
+#ifndef OS_WINDOWS
+bool IsWindowsExtension(const char *extension)
+{
+	if (!extension)
+	{
+		return false;
+	}
+
+	if (strcmp(extension, "cat") == 0 ||
+		strcmp(extension, "dll") == 0 ||
+		strcmp(extension, "exe") == 0 ||
+		strcmp(extension, "inf") == 0 ||
+		strcmp(extension, "sys") == 0)
+	{
+		return true;
+	}
+
+	return false;
+}
+#endif

src/hamcorebuilder/FileSystem.h

@@ -0,0 +1,39 @@
+#ifndef FILESYSTEM_H
+#define FILESYSTEM_H
+
+#include <stdbool.h>
+#include <stdio.h>
+
+#include <tinydir.h>
+
+#define MAX_PATH_LENGTH _TINYDIR_PATH_MAX
+
+typedef struct ENTRY
+{
+	bool IsDir;
+	char Path[MAX_PATH_LENGTH];
+} ENTRY;
+
+typedef struct ENTRIES
+{
+	size_t Num;
+	ENTRY *List;
+} ENTRIES;
+
+ENTRIES *EnumEntries(const char *path);
+ENTRIES *EnumEntriesRecursively(const char *path, const bool files_only);
+void FreeEntries(ENTRIES *entries);
+
+FILE *FileOpen(const char *path, const bool write);
+bool FileClose(FILE *file);
+bool FileRead(FILE *file, void *dst, const size_t size);
+bool FileWrite(FILE *file, const void *src, const size_t size);
+size_t FileSize(const char *path);
+
+char *PathRelativeToBase(char *full, const char *base);
+
+#ifndef OS_WINDOWS
+bool IsWindowsExtension(const char *extension);
+#endif
+
+#endif

src/hamcorebuilder/hamcorebuilder.c

@@ -1,75 +0,0 @@
-// SoftEther VPN Source Code - Developer Edition Master Branch
-// Cedar Communication Module
-
-
-// hamcorebuilder.c
-// hamcore.se2 Build Utility
-
-#include <GlobalConst.h>
-
-#ifdef	WIN32
-#include <winsock2.h>
-#include <windows.h>
-#include <wincrypt.h>
-#include <wininet.h>
-#include <shlobj.h>
-#include <commctrl.h>
-#include <Dbghelp.h>
-#endif	// WIN32
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <wchar.h>
-#include <stdarg.h>
-#include <time.h>
-#include <Mayaqua/Mayaqua.h>
-#include <Cedar/Cedar.h>
-
-
-// main function
-int main(int argc, char *argv[])
-{
-	MayaquaMinimalMode();
-
-#if defined(_DEBUG) || defined(DEBUG)	// In VC++ compilers, the macro is "_DEBUG", not "DEBUG".
-	// If set memcheck = true, the program will be vitally slow since it will log all malloc() / realloc() / free() calls to find the cause of memory leak.
-	// For normal debug we set memcheck = false.
-	// Please set memcheck = true if you want to test the cause of memory leaks.
-	InitMayaqua(false, true, argc, argv);
-#else
-	InitMayaqua(false, false, argc, argv);
-#endif
-	InitCedar();
-
-	Print("hamcore.se2 Build Utility\n");
-	Print("Copyright (c) SoftEther VPN Project. All Rights Reserved.\n\n");
-
-	if (argc < 3)
-	{
-		Print("Usage: hamcorebuilder <src_dir> <dest_hamcore_filename>\n\n");
-	}
-	else
-	{
-		char *src_dir = argv[1];
-		char *dst_filename = argv[2];
-
-		Print("Src Dir: '%s'\n", src_dir);
-		Print("Dest Filename: '%s'\n", dst_filename);
-
-		Print("\nProcessing...\n");
-
-#ifdef	WIN32
-		BuildHamcore(dst_filename, src_dir, false);
-#else
-		BuildHamcore(dst_filename, src_dir, true);
-#endif
-
-		Print("\nDone.\n");
-	}
-
-	FreeCedar();
-	FreeMayaqua();
-
-	return 0;
-}
-

src/hamcorebuilder/main.c

@@ -0,0 +1,258 @@
+#include "GlobalConst.h"
+
+#include "FileSystem.h"
+
+#include <stdint.h>
+
+#include <zlib.h>
+
+#ifdef BYTE_ORDER_BIG_ENDIAN
+#	define BigEndian32
+#else
+#	define BigEndian32 Swap32
+#endif
+
+typedef struct CompressedFile
+{
+	char *Path;
+	uint8_t *Data;
+	size_t Size;
+	size_t OriginalSize;
+	size_t Offset;
+} CompressedFile;
+
+size_t CompressionBufferSize(const size_t original_size)
+{
+	return original_size * 2 + 256;
+}
+
+uint32_t Swap32(const uint32_t value)
+{
+	uint32_t swapped;
+	((uint8_t *)&swapped)[0] = ((uint8_t *)&value)[3];
+	((uint8_t *)&swapped)[1] = ((uint8_t *)&value)[2];
+	((uint8_t *)&swapped)[2] = ((uint8_t *)&value)[1];
+	((uint8_t *)&swapped)[3] = ((uint8_t *)&value)[0];
+	return swapped;
+}
+
+void WriteAndSeek(uint8_t **dst, const void *src, const size_t size)
+{
+	if (!dst || !*dst)
+	{
+		return;
+	}
+
+	memcpy(*dst, src, size);
+	*dst += size;
+}
+
+bool BuildHamcore(const char *dst, const char *src)
+{
+	ENTRIES *entries = EnumEntriesRecursively(src, true);
+	if (!entries)
+	{
+		return false;
+	}
+
+	uint8_t *buffer = NULL;
+	size_t buffer_size = 0;
+	const size_t num = entries->Num;
+	CompressedFile *files = calloc(num, sizeof(CompressedFile));
+
+	for (size_t i = 0; i < num; ++i)
+	{
+		CompressedFile *file = &files[i];
+		char *path = entries->List[i].Path;
+
+		file->OriginalSize = FileSize(path);
+		if (file->OriginalSize == 0)
+		{
+			printf("Skipping \"%s\" because empty...\n", path);
+			continue;
+		}
+
+		FILE *handle = FileOpen(path, false);
+		if (!handle)
+		{
+			printf("Failed to open \"%s\", skipping...\n", path);
+			continue;
+		}
+
+		uint8_t *content = malloc(file->OriginalSize);
+		if (!FileRead(handle, content, file->OriginalSize))
+		{
+			printf("FileRead() failed for \"%s\", skipping...\n", path);
+			free(content);
+			continue;
+		}
+
+		FileClose(handle);
+
+		const size_t wanted_size = CompressionBufferSize(file->OriginalSize);
+		if (buffer_size < wanted_size)
+		{
+			const size_t prev_size = buffer_size;
+			buffer_size = wanted_size;
+			buffer = realloc(buffer, buffer_size);
+			memset(buffer + prev_size, 0, buffer_size - prev_size);
+		}
+
+		file->Size = buffer_size;
+		const int ret = compress(buffer, (uLongf *)&file->Size, content, (uLong)file->OriginalSize);
+		free(content);
+
+		if (ret != Z_OK)
+		{
+			printf("Failed to compress \"%s\" with error %d, skipping...\n", path, ret);
+			file->Size = 0;
+			continue;
+		}
+
+		char *relative_path = PathRelativeToBase(path, src);
+		if (!relative_path)
+		{
+			printf("Failed to get relative path for \"%s\", skipping...\n", path);
+			file->Size = 0;
+			continue;
+		}
+
+		const size_t path_size = strlen(relative_path) + 1;
+		file->Path = malloc(path_size);
+		memcpy(file->Path, relative_path, path_size);
+
+		file->Data = malloc(file->Size);
+		memcpy(file->Data, buffer, file->Size);
+
+		printf("\"%s\": %zu bytes -> %zu bytes\n", file->Path, file->OriginalSize, file->Size);
+	}
+
+	FreeEntries(entries);
+
+	size_t offset = HAMCORE_HEADER_SIZE;
+	// Number of files
+	offset += sizeof(uint32_t);
+	// File table
+	for (size_t i = 0; i < num; ++i)
+	{
+		CompressedFile *file = &files[i];
+		if (file->Size == 0)
+		{
+			continue;
+		}
+
+		// Path (length + string)
+		offset += sizeof(uint32_t) + strlen(file->Path);
+		// Original size
+		offset += sizeof(uint32_t);
+		// Size
+		offset += sizeof(uint32_t);
+		// Offset
+		offset += sizeof(uint32_t);
+	}
+
+	for (size_t i = 0; i < num; ++i)
+	{
+		CompressedFile *file = &files[i];
+		if (file->Size == 0)
+		{
+			continue;
+		}
+
+		file->Offset = offset;
+		printf("Offset for \"%s\": %zu\n", file->Path, file->Offset);
+		offset += file->Size;
+	}
+
+	if (buffer_size < offset)
+	{
+		buffer_size = offset;
+		buffer = realloc(buffer, buffer_size);
+	}
+
+	uint8_t *ptr = buffer;
+	WriteAndSeek(&ptr, HAMCORE_HEADER_DATA, HAMCORE_HEADER_SIZE);
+	uint32_t tmp = BigEndian32((uint32_t)num);
+	WriteAndSeek(&ptr, &tmp, sizeof(tmp));
+
+	for (size_t i = 0; i < num; ++i)
+	{
+		CompressedFile *file = &files[i];
+		if (file->Size == 0)
+		{
+			continue;
+		}
+
+		const size_t path_length = strlen(file->Path);
+		tmp = BigEndian32((uint32_t)path_length + 1);
+		WriteAndSeek(&ptr, &tmp, sizeof(tmp));
+		WriteAndSeek(&ptr, file->Path, path_length);
+		free(file->Path);
+
+		tmp = BigEndian32((uint32_t)file->OriginalSize);
+		WriteAndSeek(&ptr, &tmp, sizeof(tmp));
+
+		tmp = BigEndian32((uint32_t)file->Size);
+		WriteAndSeek(&ptr, &tmp, sizeof(tmp));
+
+		tmp = BigEndian32((uint32_t)file->Offset);
+		WriteAndSeek(&ptr, &tmp, sizeof(tmp));
+	}
+
+	for (size_t i = 0; i < num; ++i)
+	{
+		CompressedFile *file = &files[i];
+		WriteAndSeek(&ptr, file->Data, file->Size);
+		free(file->Data);
+	}
+
+	free(files);
+
+	bool ok = false;
+
+	FILE *handle = FileOpen(dst, true);
+	if (!handle)
+	{
+		printf("FileOpen() failed!\n");
+		goto FINAL;
+	}
+
+	printf("\nWriting to \"%s\"...\n", dst);
+
+	if (!FileWrite(handle, buffer, buffer_size))
+	{
+		printf("FileWrite() failed!\n");
+		goto FINAL;
+	}
+
+	ok = true;
+FINAL:
+	FileClose(handle);
+	free(buffer);
+	return ok;
+}
+
+int main(const int argc, const char *argv[])
+{
+	printf("hamcore.se2 builder\n\n");
+
+	if (argc < 3)
+	{
+		printf("Usage: hamcorebuilder <dest_file> <src_dir>\n\n");
+		return 0;
+	}
+
+	const char *dst = argv[1];
+	const char *src = argv[2];
+
+	printf("Destination: \"%s\"\n", dst);
+	printf("Source: \"%s\"\n\n", src);
+
+	if (!BuildHamcore(dst, src))
+	{
+		return 1;
+	}
+
+	printf("\nDone!\n");
+	return 0;
+}