This Program just goes through all the itemdata files and then textdata files
Then it searches for the names of the items in the textdata files and writes them in two files
I had to split the output up, because my browser always crashed when i hadonly one file(looks like a 3MB unicode file with one long table is not comfortable for a browser) Don't blame it on FF, same happens on IE(didn't expect anything else lol)
Oh and make sure that you have more than 135MB RAM available! The program allocates more memory than needed
The program doesn't check if the memory has been allocated, so if you lack of memory the program will crash!
Also make sure that you don't touch the input files, or if you really need to make sure that their encoding is UNICODE!!
Here is the whole project(compiled programs are in the bin folder; there's a x64 executable as well):
main.c(pretty empty^^):
Code:
#include "parse_itemdata.h"
int main(int argc, char** argv) {
UNREFERENCED_PARAMETER(argc);
UNREFERENCED_PARAMETER(argv);
clock_t beginning = 0;
atexit(&destroy);
beginning = clock();
start();
printf("It took %d ms to parse all Files and create the HTML File.\n", (int)(clock() - beginning));
printf("Press any Key to close the Program...\n");
getch();
return 0;
}
Code:
#include "parse_itemdata.h"
#define NO_OF_FILES 8
#ifdef WIN64
#define POINTER_SIZE 8
#else
#define POINTER_SIZE 4
#endif
#define null NULL
//Input Names of the Files.If you want to add some, change NO_OF_FILES and write the name into this Array
static char* itemdata_input[NO_OF_FILES] = {"itemdata_5000.txt", "itemdata_10000.txt", "itemdata_15000.txt", "itemdata_20000.txt",
"itemdata_25000.txt", "ItemData_30000.txt", "ItemData_35000.txt", "itemdata_40000.txt"
};
static char* textdataEaS_input = "textdata_equip&skill.txt";
static char* textdataO_input = "textdata_object.txt";
//Memory with the content of every File
static wchar_t* itemdata[NO_OF_FILES];
static wchar_t* textdata;
//This is a 2D Array. [FILE_INDEX][INDEX_IN_THIS_FILE]
static wchar_t*** itemID;
static wchar_t*** itemname;
static wchar_t*** itemname_in_textdata;
static wchar_t*** itemlevel;
//Contains the Info from the Textdata Files
static wchar_t** itemname_textdata;
static wchar_t** ingamename_textdata;
static int count[NO_OF_FILES];
static boolean finished_normally;
static int textdata_count;
//Contains the Index the Item has in the Textdata Files
static int itemID_in_textdata[NO_OF_FILES][5000];
void initialize();
void writeToFile();
void parse_itemdata(int index);
void parse_textdata();
void findNameInTextdata();
/*
Allocates all the Memory needed and writes the Content of every File in the Memory
*/
void initialize() {
FILE* input = null;
long filesizeEaS = 0, filesize;
int i = 0, j = 0;
finished_normally = false;
//Writing the Content of every File into the Memory(makes everything a lot faster)
for(i = 0; i < NO_OF_FILES; i++) {
input = fopen(itemdata_input[i], "rb, ccs=UNICODE");
filesize = _filelength(_fileno(input));
itemdata[i] = malloc(filesize + 1);
fread(itemdata[i], filesize, 1, input);
fclose(input);
}
//Writing the Content of the File textdataEaS_input into the Memory
input = fopen(textdataEaS_input, "rb, ccs=UNICODE");
filesizeEaS = _filelength(_fileno(input));
textdata = malloc(11000000);
fread(textdata, filesizeEaS, 1, input);
fclose(input);
//Writing the Content of the File textdataO_input at the end of the first File into the Memory
input = fopen(textdataO_input, "rb, ccs=UNICODE");
filesize = _filelength(_fileno(input));
fread(textdata + filesizeEaS/2, filesize, 1, input);
fclose(input);
//Allocating the Memory for the Array Index of every File
itemID = malloc(NO_OF_FILES * POINTER_SIZE);
itemname = malloc(NO_OF_FILES * POINTER_SIZE);
itemname_in_textdata = malloc(NO_OF_FILES * POINTER_SIZE);
itemlevel = malloc(NO_OF_FILES * POINTER_SIZE);
for(j = 0; j < NO_OF_FILES; j++) {
itemID[j] = malloc(5000 * POINTER_SIZE);
itemname[j] = malloc(5000 * POINTER_SIZE);
itemname_in_textdata[j] = malloc(5000 * POINTER_SIZE);
itemlevel[j] = malloc(5000 * POINTER_SIZE);
}
itemname_textdata = malloc(30000 * POINTER_SIZE);
ingamename_textdata = malloc(30000 * POINTER_SIZE);
ZeroMemory(itemname_textdata, 30000 * POINTER_SIZE);
ZeroMemory(ingamename_textdata, 30000 * POINTER_SIZE);
ZeroMemory(&itemID_in_textdata, sizeof(itemID_in_textdata));
}
/*
Calls every needed Function
There's no need to call any other Function
*/
void start() {
int i = 0;
initialize();
for(i = 0; i < NO_OF_FILES; i++) {
parse_itemdata(i);
printf("Finished parsing the %d. File\n", i + 1);
}
parse_textdata();
printf("Finished parsing textdata_equip&skill.txt and textdata_object.txt\n");
printf("Finished Parsing all Files!\n");
findNameInTextdata();
printf("Finished finding the right Names.\n");
writeToFile();
destroy();
}
/*
Parsing a itemdata File
Index defines which itemdata File is parsed
*/
void parse_itemdata(int index) {
wchar_t* line, *source;
//Setting source to the start of the itemdata File
source = itemdata[index];
//Searching for the end of the first Line
line = wcstok(source, L"\n");
do {
//Setting source to the beginning of the next Line
source = source + wcslen(line) + 1;
//Allocating all the needed Memory
itemID[index][count[index]] = malloc(50);
itemname[index][count[index]] = malloc(1500);
itemname_in_textdata[index][count[index]] = malloc(1500);
itemlevel[index][count[index]] = malloc(50);
wcstok(line, L"\t");
wcscpy(itemID[index][count[index]], wcstok(null, L"\t")); //ID
wcscpy(itemname[index][count[index]], wcstok(null, L"\t")); //Name
wcstok(null, L"\t");
wcstok(null, L"\t");
wcscpy(itemname_in_textdata[index][count[index]], wcstok(null, L"\t")); //The Name, the Item has in the textdata Files
wcstok(null, L"\t");
wcstok(null, L"\t");
wcstok(null, L"\t");
wcstok(null, L"\t");
wcstok(null, L"\t");
wcstok(null, L"\t");
wcstok(null, L"\t");
wcstok(null, L"\t");
wcstok(null, L"\t");
wcstok(null, L"\t");
wcstok(null, L"\t");
wcstok(null, L"\t");
wcstok(null, L"\t");
wcstok(null, L"\t");
wcstok(null, L"\t");
wcstok(null, L"\t");
wcstok(null, L"\t");
wcstok(null, L"\t");
wcstok(null, L"\t");
wcstok(null, L"\t");
wcstok(null, L"\t");
wcstok(null, L"\t");
wcstok(null, L"\t");
wcstok(null, L"\t");
wcstok(null, L"\t");
wcstok(null, L"\t");
wcstok(null, L"\t");
wcscpy(itemlevel[index][count[index]], wcstok(null, L"\t")); //Level
count[index]++;
} while((line = wcstok(source, L"\n")));
free(itemdata[index]);
}
/*
Parses the Textdata File
All two Files are parsed at one Time, because they are written in the same Memory block
*/
void parse_textdata() {
wchar_t* source, *line, *name, *ingame;
//Copying the Start of the File into another Variable(Original one is needed to free the Memory again)
source = textdata;
//Searching for the End of the first Line
line = wcstok(source, L"\r");
do {
//Setting Source to the beginning to the next Line
source += wcslen(line) + 2;
wcstok(line, L"\t");
if(!(name = wcstok(null, L"\t"))) {
continue;
}
wcstok(null, L"\t");
wcstok(null, L"\t");
wcstok(null, L"\t");
wcstok(null, L"\t");
wcstok(null, L"\t");
wcstok(null, L"\t");
if(!(ingame = wcstok(null, L"\t"))) {
continue;
}
//Allocating the needed Memory
itemname_textdata[textdata_count] = malloc(300);
ingamename_textdata[textdata_count] = malloc(3000);
//Copying the Strings into the previously allocated Memory
wcscpy(itemname_textdata[textdata_count], name);
wcscpy(ingamename_textdata[textdata_count], ingame);
//Increasing the Variable by one(it is used as Index)
textdata_count++;
} while((line = wcstok(source, L"\r")));
free(textdata);
//Allocating Memory
itemname_textdata[textdata_count] = malloc(300);
ingamename_textdata[textdata_count] = malloc(3000);
//Copying the Message into the two variables(These are written into the File, when there's no Name for an Item)
wcscpy(itemname_textdata[textdata_count], L"No Name");
wcscpy(ingamename_textdata[textdata_count], L"No Name");
}
/*
Searches for the Name in the textdata File
*/
void findNameInTextdata() {
int i = 0, j = 0, k = 0;
for(i = 0; i < NO_OF_FILES; i++) { //Loops through every File
for(j = 0; j < count[i]; j++) { //Loops through every Line of the current File
for(k = 0; k < textdata_count; k++) { //Loops through every Line of the Textdata File
if(!wcscmp(itemname_in_textdata[i][j], itemname_textdata[k])) { //If the Names match the loop breaks
break;
}
}
itemID_in_textdata[i][j] = k; //The ID is saved in a 2D Array of Integers
}
}
}
/*
Writes everything into 2 Files
Browser may crash on low-end Systems!
*/
void writeToFile() {
clock_t beginning = clock();
FILE *output;
wchar_t *puffer = malloc(20000000), *line = malloc(2000);
int i = 0, j = 0;
printf("Creating File. This might take some Time.\nPlease wait...\n");
//Opening the first File and writing half of the Content to it
output = fopen("Items 1st Half.html", "wb, ccs=UNICODE");
memset(puffer, 0, 20000000);
wcscpy(puffer, L"<HTML><Head><title>Items</title></Head><Body><table border=\"1\"><tr><th>ID</th><th>PK2 Name</th><th>Ingame Name</th><th>Level</th></tr>");
for(j = 0; j < NO_OF_FILES/2; j++) { //Loops through half of the Input Files
for(i = 0; i < count[j]; i++) { //Loops through every Line of the current Input files
//Copying Line per Line into the String Buffer
swprintf(line, L"<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>", itemID[j][i], itemname[j][i], ingamename_textdata[itemID_in_textdata[j][i]], itemlevel[j][i]);
wcscat(puffer, line);
}
}
wcscat(puffer, L"</table></Body></HTML>");
//Finally writing into the File
fwprintf(output, puffer);
fclose(output);
//Opening the second File and writing the rest of the Content to it
output = fopen("Items 2nd Half.html", "wb, ccs=UNICODE");
memset(puffer, 0, 20000000);
wcscpy(puffer, L"<HTML><Head><title>Items</title></Head><Body><table border=\"1\"><tr><th>ID</th><th>PK2 Name</th><th>Ingame Name</th><th>Level</th></tr>");
//Setting j to the next File to progress
j = NO_OF_FILES%2 ? NO_OF_FILES/2 + 1 : NO_OF_FILES/2;
for(; j < NO_OF_FILES; j++) { //Loops through the rest of the Input Files
for(i = 0; i < count[j]; i++) { //Loops through every Line of the current Input files
//Copying Line per Line into the String Buffer
swprintf(line, L"<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>", itemID[j][i], itemname[j][i], ingamename_textdata[itemID_in_textdata[j][i]], itemlevel[j][i]);
wcscat(puffer, line);
}
}
wcscat(puffer, L"</table></Body></HTML>");
//Finally writing into the File
fwprintf(output, puffer);
fclose(output);
printf("It took %d ms to create the Files!\n", (int)(clock() - beginning));
printf("Finished. The Files are located in the Program's Folder.\n");
free(puffer);
free(line);
}
/*
Frees all the allocated Memory and closes all Files
*/
void destroy() {
int i = 0, j = 0;
if(finished_normally) {
return;
}
_fcloseall();
for(j = 0; j < NO_OF_FILES; j++) {
for(i = 0; i < count[j]; i++) {
free(itemID[j][i]);
free(itemname[j][i]);
free(itemname_in_textdata[j][i]);
free(itemlevel[j][i]);
}
free(itemID[j]);
free(itemname[j]);
free(itemname_in_textdata[j]);
free(itemlevel[j]);
}
free(itemID);
free(itemname);
free(itemname_in_textdata);
free(itemlevel);
for(i = 0; i < textdata_count; i++) {
free(itemname_textdata[i]);
free(ingamename_textdata[i]);
}
free(itemname_textdata);
free(ingamename_textdata);
finished_normally = true;
}
Code:
#ifndef PARSE_ITEMDATA_H_INCLUDED #define PARSE_ITEMDATA_H_INCLUDED #include <stdlib.h> #include <stdio.h> #include <windows.h> #include <io.h> #include <time.h> #define null NULL #define boolean unsigned char #define true 1 #define false 0 void start(); void destroy(); #endif // PARSE_ITEMDATA_H_INCLUDED
It took the x86 file ~15 seconds and the x64 program ~10 seconds to parse all the files(at my pc)
I only added the most important things to the output files
you can add some yourself, or ask me if you want
The project has been created with Codeblocks(you can download it here:
)
