From 1301dc93c682dd1d3586953064ca4f36c52bf6ec Mon Sep 17 00:00:00 2001 From: Davide Beatrici Date: Fri, 26 Feb 2021 07:06:26 +0100 Subject: [PATCH] New hamcorebuilder implementation, independent from Cedar and Mayaqua This new implementation can be easily compiled and executed without the need for other components to be present. It relies on standard C functions, aside from stat() which is part of POSIX but available on Windows as well. There's only one third-party dependency, which is tinydir: a single-file header-only library for traversing directories. --- .gitmodules | 3 + 3rdparty/tinydir | 1 + src/CMakeLists.txt | 2 +- src/GlobalConst.h | 4 +- src/Mayaqua/FileIO.h | 2 - src/hamcorebuilder/CMakeLists.txt | 22 ++- src/hamcorebuilder/FileSystem.c | 220 ++++++++++++++++++++++++ src/hamcorebuilder/FileSystem.h | 39 +++++ src/hamcorebuilder/hamcorebuilder.c | 75 -------- src/hamcorebuilder/main.c | 258 ++++++++++++++++++++++++++++ 10 files changed, 545 insertions(+), 81 deletions(-) create mode 160000 3rdparty/tinydir mode change 100644 => 100755 src/hamcorebuilder/CMakeLists.txt create mode 100755 src/hamcorebuilder/FileSystem.c create mode 100755 src/hamcorebuilder/FileSystem.h delete mode 100644 src/hamcorebuilder/hamcorebuilder.c create mode 100755 src/hamcorebuilder/main.c diff --git a/.gitmodules b/.gitmodules index 5f3f129a..e1dee0ef 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "src/Mayaqua/3rdparty/cpu_features"] path = src/Mayaqua/3rdparty/cpu_features url = https://github.com/google/cpu_features.git +[submodule "3rdparty/tinydir"] + path = 3rdparty/tinydir + url = https://github.com/cxong/tinydir.git diff --git a/3rdparty/tinydir b/3rdparty/tinydir new file mode 160000 index 00000000..ec6bff20 --- /dev/null +++ b/3rdparty/tinydir @@ -0,0 +1 @@ +Subproject commit ec6bff2043eaac3ad25423705e63a781762a0dfd diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index aa357290..ff3da41d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -144,7 +144,7 @@ add_subdirectory(vpntest) # hamcore.se2 archive file add_custom_target(hamcore-archive-build ALL - COMMAND hamcorebuilder "${TOP_DIRECTORY}/src/bin/hamcore/" "${BUILD_DIRECTORY}/hamcore.se2" + COMMAND hamcorebuilder "${BUILD_DIRECTORY}/hamcore.se2" "${TOP_DIRECTORY}/src/bin/hamcore" DEPENDS hamcorebuilder COMMENT "Building hamcore.se2 archive file..." VERBATIM diff --git a/src/GlobalConst.h b/src/GlobalConst.h index b7662d3b..7a6db194 100644 --- a/src/GlobalConst.h +++ b/src/GlobalConst.h @@ -64,7 +64,9 @@ #define GC_UI_APPID_CM L"SoftEther.SoftEther VPN Client Developer Edition" +//// Hamcore +#define HAMCORE_HEADER_DATA "HamCore" +#define HAMCORE_HEADER_SIZE 7 #endif // GLOBAL_CONST_H - diff --git a/src/Mayaqua/FileIO.h b/src/Mayaqua/FileIO.h index 3233ce00..6dc7cea5 100644 --- a/src/Mayaqua/FileIO.h +++ b/src/Mayaqua/FileIO.h @@ -13,8 +13,6 @@ #define HAMCORE_FILE_NAME "hamcore.se2" #define HAMCORE_FILE_NAME_2 "_hamcore.se2" #define HAMCORE_TEXT_NAME "hamcore.txt" -#define HAMCORE_HEADER_DATA "HamCore" -#define HAMCORE_HEADER_SIZE 7 #define HAMCORE_CACHE_EXPIRES (5 * 60 * 1000) // IO structure diff --git a/src/hamcorebuilder/CMakeLists.txt b/src/hamcorebuilder/CMakeLists.txt old mode 100644 new mode 100755 index 41be69ad..dc52ebbb --- a/src/hamcorebuilder/CMakeLists.txt +++ b/src/hamcorebuilder/CMakeLists.txt @@ -1,3 +1,21 @@ -add_executable(hamcorebuilder hamcorebuilder.c) +include(TestBigEndian) -target_link_libraries(hamcorebuilder cedar mayaqua) +add_executable(hamcorebuilder + main.c + FileSystem.c + FileSystem.h +) + +if(WIN32) + target_compile_definitions(hamcorebuilder PRIVATE "OS_WINDOWS") +endif() + +test_big_endian(BIG_ENDIAN) +if(BIG_ENDIAN) + target_compile_definitions(hamcorebuilder PRIVATE "BYTE_ORDER_BIG_ENDIAN") +endif() + +target_include_directories(hamcorebuilder PRIVATE "${TOP_DIRECTORY}/3rdparty/tinydir") + +find_package(ZLIB REQUIRED) +target_link_libraries(hamcorebuilder PRIVATE ZLIB::ZLIB) diff --git a/src/hamcorebuilder/FileSystem.c b/src/hamcorebuilder/FileSystem.c new file mode 100755 index 00000000..93f0b5af --- /dev/null +++ b/src/hamcorebuilder/FileSystem.c @@ -0,0 +1,220 @@ +#include "FileSystem.h" + +#include + +#include + +ENTRIES *EnumEntries(const char *path) +{ + if (!path) + { + return NULL; + } + + tinydir_dir dir; + if (tinydir_open_sorted(&dir, path) == -1) + { + printf("tinydir_open_sorted() failed!\n"); + return NULL; + } + + ENTRIES *entries = calloc(1, sizeof(ENTRIES)); + + for (size_t i = 0; i < dir.n_files; ++i) + { + tinydir_file file; + if (tinydir_readfile_n(&dir, &file, i) == -1) + { + printf("tinydir_readfile_n() failed at index %zu!\n", i); + FreeEntries(entries); + return NULL; + } + + if (file.is_dir) + { + if (strcmp(file.name, ".") == 0 || strcmp(file.name, "..") == 0) + { + continue; + } + } +#ifndef OS_WINDOWS + if (IsWindowsExtension(file.extension)) + { + continue; + } +#endif + ++entries->Num; + entries->List = realloc(entries->List, sizeof(ENTRY) * entries->Num); + + ENTRY *entry = &entries->List[entries->Num - 1]; + entry->IsDir = file.is_dir; + strcpy(entry->Path, file.path); + } + + tinydir_close(&dir); + return entries; +} + +ENTRIES *EnumEntriesRecursively(const char *path, const bool files_only) +{ + if (!path) + { + return NULL; + } + + ENTRIES *tmp = EnumEntries(path); + if (!tmp) + { + return NULL; + } + + ENTRIES *entries = calloc(1, sizeof(ENTRIES)); + + for (size_t i = 0; i < tmp->Num; ++i) + { + ENTRY *entry = &tmp->List[i]; + if (!files_only || !entry->IsDir) + { + ++entries->Num; + entries->List = realloc(entries->List, sizeof(ENTRY) * entries->Num); + memcpy(&entries->List[entries->Num - 1], entry, sizeof(ENTRY)); + } + + if (!entry->IsDir) + { + continue; + } + + ENTRIES *tmp_2 = EnumEntries(entry->Path); + if (!tmp_2) + { + continue; + } + + const size_t offset = tmp->Num; + + tmp->Num += tmp_2->Num; + tmp->List = realloc(tmp->List, sizeof(ENTRY) * tmp->Num); + + memcpy(&tmp->List[offset], tmp_2->List, sizeof(ENTRY) * tmp_2->Num); + + FreeEntries(tmp_2); + } + + FreeEntries(tmp); + + return entries; +} + +void FreeEntries(ENTRIES *entries) +{ + if (!entries) + { + return; + } + + if (entries->List) + { + free(entries->List); + } + + free(entries); +} + +FILE *FileOpen(const char *path, const bool write) +{ + if (!path) + { + return NULL; + } + + return fopen(path, write ? "wb" : "rb"); +} + +bool FileClose(FILE *file) +{ + if (!file) + { + return false; + } + + return fclose(file) == 0; +} + +bool FileRead(FILE *file, void *dst, const size_t size) +{ + if (!file || !dst || size == 0) + { + return false; + } + + return fread(dst, 1, size, file) == size; +} + +bool FileWrite(FILE *file, const void *src, const size_t size) +{ + if (!file || !src || size == 0) + { + return false; + } + + return fwrite(src, 1, size, file) == size; +} + +size_t FileSize(const char *path) +{ + if (!path) + { + return 0; + } + + struct stat st; + if (stat(path, &st) == -1) + { + return 0; + } + + return st.st_size; +} + +char *PathRelativeToBase(char *full, const char *base) +{ + if (!full || !base) + { + return NULL; + } + + if (strstr(full, base) != &full[0]) + { + return NULL; + } + + full += strlen(base); + if (full[0] == '/') + { + ++full; + } + + return full; +} + +#ifndef OS_WINDOWS +bool IsWindowsExtension(const char *extension) +{ + if (!extension) + { + return false; + } + + if (strcmp(extension, "cat") == 0 || + strcmp(extension, "dll") == 0 || + strcmp(extension, "exe") == 0 || + strcmp(extension, "inf") == 0 || + strcmp(extension, "sys") == 0) + { + return true; + } + + return false; +} +#endif diff --git a/src/hamcorebuilder/FileSystem.h b/src/hamcorebuilder/FileSystem.h new file mode 100755 index 00000000..303df228 --- /dev/null +++ b/src/hamcorebuilder/FileSystem.h @@ -0,0 +1,39 @@ +#ifndef FILESYSTEM_H +#define FILESYSTEM_H + +#include +#include + +#include + +#define MAX_PATH_LENGTH _TINYDIR_PATH_MAX + +typedef struct ENTRY +{ + bool IsDir; + char Path[MAX_PATH_LENGTH]; +} ENTRY; + +typedef struct ENTRIES +{ + size_t Num; + ENTRY *List; +} ENTRIES; + +ENTRIES *EnumEntries(const char *path); +ENTRIES *EnumEntriesRecursively(const char *path, const bool files_only); +void FreeEntries(ENTRIES *entries); + +FILE *FileOpen(const char *path, const bool write); +bool FileClose(FILE *file); +bool FileRead(FILE *file, void *dst, const size_t size); +bool FileWrite(FILE *file, const void *src, const size_t size); +size_t FileSize(const char *path); + +char *PathRelativeToBase(char *full, const char *base); + +#ifndef OS_WINDOWS +bool IsWindowsExtension(const char *extension); +#endif + +#endif diff --git a/src/hamcorebuilder/hamcorebuilder.c b/src/hamcorebuilder/hamcorebuilder.c deleted file mode 100644 index a9a498d9..00000000 --- a/src/hamcorebuilder/hamcorebuilder.c +++ /dev/null @@ -1,75 +0,0 @@ -// SoftEther VPN Source Code - Developer Edition Master Branch -// Cedar Communication Module - - -// hamcorebuilder.c -// hamcore.se2 Build Utility - -#include - -#ifdef WIN32 -#include -#include -#include -#include -#include -#include -#include -#endif // WIN32 -#include -#include -#include -#include -#include -#include -#include -#include - - -// main function -int main(int argc, char *argv[]) -{ - MayaquaMinimalMode(); - -#if defined(_DEBUG) || defined(DEBUG) // In VC++ compilers, the macro is "_DEBUG", not "DEBUG". - // If set memcheck = true, the program will be vitally slow since it will log all malloc() / realloc() / free() calls to find the cause of memory leak. - // For normal debug we set memcheck = false. - // Please set memcheck = true if you want to test the cause of memory leaks. - InitMayaqua(false, true, argc, argv); -#else - InitMayaqua(false, false, argc, argv); -#endif - InitCedar(); - - Print("hamcore.se2 Build Utility\n"); - Print("Copyright (c) SoftEther VPN Project. All Rights Reserved.\n\n"); - - if (argc < 3) - { - Print("Usage: hamcorebuilder \n\n"); - } - else - { - char *src_dir = argv[1]; - char *dst_filename = argv[2]; - - Print("Src Dir: '%s'\n", src_dir); - Print("Dest Filename: '%s'\n", dst_filename); - - Print("\nProcessing...\n"); - -#ifdef WIN32 - BuildHamcore(dst_filename, src_dir, false); -#else - BuildHamcore(dst_filename, src_dir, true); -#endif - - Print("\nDone.\n"); - } - - FreeCedar(); - FreeMayaqua(); - - return 0; -} - diff --git a/src/hamcorebuilder/main.c b/src/hamcorebuilder/main.c new file mode 100755 index 00000000..8a9a6b25 --- /dev/null +++ b/src/hamcorebuilder/main.c @@ -0,0 +1,258 @@ +#include "GlobalConst.h" + +#include "FileSystem.h" + +#include + +#include + +#ifdef BYTE_ORDER_BIG_ENDIAN +# define BigEndian32 +#else +# define BigEndian32 Swap32 +#endif + +typedef struct CompressedFile +{ + char *Path; + uint8_t *Data; + size_t Size; + size_t OriginalSize; + size_t Offset; +} CompressedFile; + +size_t CompressionBufferSize(const size_t original_size) +{ + return original_size * 2 + 256; +} + +uint32_t Swap32(const uint32_t value) +{ + uint32_t swapped; + ((uint8_t *)&swapped)[0] = ((uint8_t *)&value)[3]; + ((uint8_t *)&swapped)[1] = ((uint8_t *)&value)[2]; + ((uint8_t *)&swapped)[2] = ((uint8_t *)&value)[1]; + ((uint8_t *)&swapped)[3] = ((uint8_t *)&value)[0]; + return swapped; +} + +void WriteAndSeek(uint8_t **dst, const void *src, const size_t size) +{ + if (!dst || !*dst) + { + return; + } + + memcpy(*dst, src, size); + *dst += size; +} + +bool BuildHamcore(const char *dst, const char *src) +{ + ENTRIES *entries = EnumEntriesRecursively(src, true); + if (!entries) + { + return false; + } + + uint8_t *buffer = NULL; + size_t buffer_size = 0; + const size_t num = entries->Num; + CompressedFile *files = calloc(num, sizeof(CompressedFile)); + + for (size_t i = 0; i < num; ++i) + { + CompressedFile *file = &files[i]; + char *path = entries->List[i].Path; + + file->OriginalSize = FileSize(path); + if (file->OriginalSize == 0) + { + printf("Skipping \"%s\" because empty...\n", path); + continue; + } + + FILE *handle = FileOpen(path, false); + if (!handle) + { + printf("Failed to open \"%s\", skipping...\n", path); + continue; + } + + uint8_t *content = malloc(file->OriginalSize); + if (!FileRead(handle, content, file->OriginalSize)) + { + printf("FileRead() failed for \"%s\", skipping...\n", path); + free(content); + continue; + } + + FileClose(handle); + + const size_t wanted_size = CompressionBufferSize(file->OriginalSize); + if (buffer_size < wanted_size) + { + const size_t prev_size = buffer_size; + buffer_size = wanted_size; + buffer = realloc(buffer, buffer_size); + memset(buffer + prev_size, 0, buffer_size - prev_size); + } + + file->Size = buffer_size; + const int ret = compress(buffer, (uLongf *)&file->Size, content, (uLong)file->OriginalSize); + free(content); + + if (ret != Z_OK) + { + printf("Failed to compress \"%s\" with error %d, skipping...\n", path, ret); + file->Size = 0; + continue; + } + + char *relative_path = PathRelativeToBase(path, src); + if (!relative_path) + { + printf("Failed to get relative path for \"%s\", skipping...\n", path); + file->Size = 0; + continue; + } + + const size_t path_size = strlen(relative_path) + 1; + file->Path = malloc(path_size); + memcpy(file->Path, relative_path, path_size); + + file->Data = malloc(file->Size); + memcpy(file->Data, buffer, file->Size); + + printf("\"%s\": %zu bytes -> %zu bytes\n", file->Path, file->OriginalSize, file->Size); + } + + FreeEntries(entries); + + size_t offset = HAMCORE_HEADER_SIZE; + // Number of files + offset += sizeof(uint32_t); + // File table + for (size_t i = 0; i < num; ++i) + { + CompressedFile *file = &files[i]; + if (file->Size == 0) + { + continue; + } + + // Path (length + string) + offset += sizeof(uint32_t) + strlen(file->Path); + // Original size + offset += sizeof(uint32_t); + // Size + offset += sizeof(uint32_t); + // Offset + offset += sizeof(uint32_t); + } + + for (size_t i = 0; i < num; ++i) + { + CompressedFile *file = &files[i]; + if (file->Size == 0) + { + continue; + } + + file->Offset = offset; + printf("Offset for \"%s\": %zu\n", file->Path, file->Offset); + offset += file->Size; + } + + if (buffer_size < offset) + { + buffer_size = offset; + buffer = realloc(buffer, buffer_size); + } + + uint8_t *ptr = buffer; + WriteAndSeek(&ptr, HAMCORE_HEADER_DATA, HAMCORE_HEADER_SIZE); + uint32_t tmp = BigEndian32((uint32_t)num); + WriteAndSeek(&ptr, &tmp, sizeof(tmp)); + + for (size_t i = 0; i < num; ++i) + { + CompressedFile *file = &files[i]; + if (file->Size == 0) + { + continue; + } + + const size_t path_length = strlen(file->Path); + tmp = BigEndian32((uint32_t)path_length + 1); + WriteAndSeek(&ptr, &tmp, sizeof(tmp)); + WriteAndSeek(&ptr, file->Path, path_length); + free(file->Path); + + tmp = BigEndian32((uint32_t)file->OriginalSize); + WriteAndSeek(&ptr, &tmp, sizeof(tmp)); + + tmp = BigEndian32((uint32_t)file->Size); + WriteAndSeek(&ptr, &tmp, sizeof(tmp)); + + tmp = BigEndian32((uint32_t)file->Offset); + WriteAndSeek(&ptr, &tmp, sizeof(tmp)); + } + + for (size_t i = 0; i < num; ++i) + { + CompressedFile *file = &files[i]; + WriteAndSeek(&ptr, file->Data, file->Size); + free(file->Data); + } + + free(files); + + bool ok = false; + + FILE *handle = FileOpen(dst, true); + if (!handle) + { + printf("FileOpen() failed!\n"); + goto FINAL; + } + + printf("\nWriting to \"%s\"...\n", dst); + + if (!FileWrite(handle, buffer, buffer_size)) + { + printf("FileWrite() failed!\n"); + goto FINAL; + } + + ok = true; +FINAL: + FileClose(handle); + free(buffer); + return ok; +} + +int main(const int argc, const char *argv[]) +{ + printf("hamcore.se2 builder\n\n"); + + if (argc < 3) + { + printf("Usage: hamcorebuilder \n\n"); + return 0; + } + + const char *dst = argv[1]; + const char *src = argv[2]; + + printf("Destination: \"%s\"\n", dst); + printf("Source: \"%s\"\n\n", src); + + if (!BuildHamcore(dst, src)) + { + return 1; + } + + printf("\nDone!\n"); + return 0; +}