Program for parsing Items

Schickl · 11/04/2011, 17:07

Another small program related to Silkroad i made.
This Program just goes through all the itemdata files and then textdata files
Then it searches for the names of the items in the textdata files and writes them in two files
I had to split the output up, because my browser always crashed when i hadonly one file(looks like a 3MB unicode file with one long table is not comfortable for a browser) Don't blame it on FF, same happens on IE(didn't expect anything else lol)

Oh and make sure that you have more than 135MB RAM available! The program allocates more memory than needed
The program doesn't check if the memory has been allocated, so if you lack of memory the program will crash!
Also make sure that you don't touch the input files, or if you really need to make sure that their encoding is UNICODE!!

Here is the whole project(compiled programs are in the bin folder; there's a x64 executable as well):

main.c(pretty empty^^):

Code:

#include "parse_itemdata.h"

int main(int argc, char** argv) {
    UNREFERENCED_PARAMETER(argc);
    UNREFERENCED_PARAMETER(argv);

    clock_t beginning = 0;
    atexit(&destroy);

    beginning = clock();
    start();

    printf("It took %d ms to parse all Files and create the HTML File.\n", (int)(clock() - beginning));
    printf("Press any Key to close the Program...\n");
    getch();

    return 0;
}

parse_main.c:

Code:

#include "parse_itemdata.h"

#define NO_OF_FILES 8

#ifdef WIN64
#define POINTER_SIZE 8
#else
#define POINTER_SIZE 4
#endif

#define null NULL

//Input Names of the Files.If you want to add some, change NO_OF_FILES and write the name into this Array
static char* itemdata_input[NO_OF_FILES] = {"itemdata_5000.txt", "itemdata_10000.txt", "itemdata_15000.txt", "itemdata_20000.txt",
                                            "itemdata_25000.txt", "ItemData_30000.txt", "ItemData_35000.txt", "itemdata_40000.txt"
                                           };
static char* textdataEaS_input = "textdata_equip&skill.txt";
static char* textdataO_input = "textdata_object.txt";


//Memory with the content of every File
static wchar_t* itemdata[NO_OF_FILES];
static wchar_t* textdata;

//This is a 2D Array. [FILE_INDEX][INDEX_IN_THIS_FILE]
static wchar_t*** itemID;
static wchar_t*** itemname;
static wchar_t*** itemname_in_textdata;
static wchar_t*** itemlevel;

//Contains the Info from the Textdata Files
static wchar_t** itemname_textdata;
static wchar_t** ingamename_textdata;

static int count[NO_OF_FILES];
static boolean finished_normally;

static int textdata_count;

//Contains the Index the Item has in the Textdata Files
static int itemID_in_textdata[NO_OF_FILES][5000];


void initialize();
void writeToFile();

void parse_itemdata(int index);

void parse_textdata();

void findNameInTextdata();


/*
Allocates all the Memory needed and writes the Content of every File in the Memory
*/
void initialize() {
    FILE* input = null;
    long filesizeEaS = 0, filesize;
    int i = 0, j = 0;

    finished_normally = false;

    //Writing the Content of every File into the Memory(makes everything a lot faster)
    for(i = 0; i < NO_OF_FILES; i++) {
        input = fopen(itemdata_input[i], "rb, ccs=UNICODE");

        filesize = _filelength(_fileno(input));

        itemdata[i] = malloc(filesize + 1);

        fread(itemdata[i], filesize, 1, input);

        fclose(input);
    }

    //Writing the Content of the File textdataEaS_input into the Memory
    input = fopen(textdataEaS_input, "rb, ccs=UNICODE");

    filesizeEaS = _filelength(_fileno(input));

    textdata = malloc(11000000);

    fread(textdata, filesizeEaS, 1, input);

    fclose(input);


    //Writing the Content of the File textdataO_input at the end of the first File into the Memory
    input = fopen(textdataO_input, "rb, ccs=UNICODE");

    filesize = _filelength(_fileno(input));

    fread(textdata + filesizeEaS/2, filesize, 1, input);

    fclose(input);


    //Allocating the Memory for the Array Index of every File
    itemID = malloc(NO_OF_FILES * POINTER_SIZE);
    itemname = malloc(NO_OF_FILES * POINTER_SIZE);
    itemname_in_textdata = malloc(NO_OF_FILES * POINTER_SIZE);
    itemlevel = malloc(NO_OF_FILES * POINTER_SIZE);

    for(j = 0; j < NO_OF_FILES; j++) {
        itemID[j] = malloc(5000 * POINTER_SIZE);
        itemname[j] = malloc(5000 * POINTER_SIZE);
        itemname_in_textdata[j] = malloc(5000 * POINTER_SIZE);
        itemlevel[j] = malloc(5000 * POINTER_SIZE);
    }


    itemname_textdata = malloc(30000 * POINTER_SIZE);
    ingamename_textdata = malloc(30000 * POINTER_SIZE);

    ZeroMemory(itemname_textdata, 30000 * POINTER_SIZE);
    ZeroMemory(ingamename_textdata, 30000 * POINTER_SIZE);


    ZeroMemory(&itemID_in_textdata, sizeof(itemID_in_textdata));
}

/*
Calls every needed Function
There's no need to call any other Function
*/
void start() {
    int i = 0;

    initialize();

    for(i = 0; i < NO_OF_FILES; i++) {
        parse_itemdata(i);
        printf("Finished parsing the %d. File\n", i + 1);
    }

    parse_textdata();
    printf("Finished parsing textdata_equip&skill.txt and textdata_object.txt\n");

    printf("Finished Parsing all Files!\n");


    findNameInTextdata();
    printf("Finished finding the right Names.\n");

    writeToFile();

    destroy();
}

/*
Parsing a itemdata File
Index defines which itemdata File is parsed
*/
void parse_itemdata(int index) {
    wchar_t* line, *source;

    //Setting source to the start of the itemdata File
    source = itemdata[index];

    //Searching for the end of the first Line
    line = wcstok(source, L"\n");
    do {
        //Setting source to the beginning of the next Line
        source = source + wcslen(line) + 1;

        //Allocating all the needed Memory
        itemID[index][count[index]] = malloc(50);
        itemname[index][count[index]] = malloc(1500);
        itemname_in_textdata[index][count[index]] = malloc(1500);
        itemlevel[index][count[index]] = malloc(50);

        wcstok(line, L"\t");
        wcscpy(itemID[index][count[index]], wcstok(null, L"\t"));               //ID
        wcscpy(itemname[index][count[index]], wcstok(null, L"\t"));             //Name
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcscpy(itemname_in_textdata[index][count[index]], wcstok(null, L"\t")); //The Name, the Item has in the textdata Files
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcscpy(itemlevel[index][count[index]], wcstok(null, L"\t"));            //Level
        count[index]++;
    } while((line = wcstok(source, L"\n")));

    free(itemdata[index]);
}

/*
Parses the Textdata File
All two Files are parsed at one Time, because they are written in the same Memory block
*/
void parse_textdata() {
    wchar_t* source, *line, *name, *ingame;

    //Copying the Start of the File into another Variable(Original one is needed to free the Memory again)
    source = textdata;

    //Searching for the End of the first Line
    line = wcstok(source, L"\r");

    do {
        //Setting Source to the beginning to the next Line
        source += wcslen(line) + 2;

        wcstok(line, L"\t");
        if(!(name = wcstok(null, L"\t"))) {
            continue;
        }
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        wcstok(null, L"\t");
        if(!(ingame = wcstok(null, L"\t"))) {
            continue;
        }

        //Allocating the needed Memory
        itemname_textdata[textdata_count] = malloc(300);
        ingamename_textdata[textdata_count] = malloc(3000);

        //Copying the Strings into the previously allocated Memory
        wcscpy(itemname_textdata[textdata_count], name);
        wcscpy(ingamename_textdata[textdata_count], ingame);

        //Increasing the Variable by one(it is used as Index)
        textdata_count++;

    } while((line = wcstok(source, L"\r")));
    free(textdata);

    //Allocating Memory
    itemname_textdata[textdata_count] = malloc(300);
    ingamename_textdata[textdata_count] = malloc(3000);

    //Copying the Message into the two variables(These are written into the File, when there's no Name for an Item)
    wcscpy(itemname_textdata[textdata_count], L"No Name");
    wcscpy(ingamename_textdata[textdata_count], L"No Name");
}

/*
Searches for the Name in the textdata File
*/
void findNameInTextdata() {
    int i = 0, j = 0, k = 0;

    for(i = 0; i < NO_OF_FILES; i++) {              //Loops through every File
        for(j = 0; j < count[i]; j++) {             //Loops through every Line of the current File
            for(k = 0; k < textdata_count; k++) {   //Loops through every Line of the Textdata File
                if(!wcscmp(itemname_in_textdata[i][j], itemname_textdata[k])) { //If the Names match the loop breaks
                    break;
                }
            }
            itemID_in_textdata[i][j] = k;           //The ID is saved in a 2D Array of Integers
        }
    }
}

/*
    Writes everything into 2 Files
    Browser may crash on low-end Systems!
*/
void writeToFile() {
    clock_t beginning = clock();
    FILE *output;
    wchar_t *puffer = malloc(20000000), *line = malloc(2000);
    int i = 0, j = 0;

    printf("Creating File. This might take some Time.\nPlease wait...\n");

    //Opening the first File and writing half of the Content to it
    output = fopen("Items 1st Half.html", "wb, ccs=UNICODE");

    memset(puffer, 0, 20000000);

    wcscpy(puffer, L"<HTML><Head><title>Items</title></Head><Body><table border=\"1\"><tr><th>ID</th><th>PK2 Name</th><th>Ingame Name</th><th>Level</th></tr>");

    for(j = 0; j < NO_OF_FILES/2; j++) {          //Loops through half of the Input Files
        for(i = 0; i < count[j]; i++) {           //Loops through every Line of the current Input files
            //Copying Line per Line into the String Buffer
            swprintf(line, L"<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>", itemID[j][i], itemname[j][i], ingamename_textdata[itemID_in_textdata[j][i]],  itemlevel[j][i]);
            wcscat(puffer, line);
        }
    }

    wcscat(puffer, L"</table></Body></HTML>");

    //Finally writing into the File
    fwprintf(output, puffer);

    fclose(output);

    //Opening the second File and writing the rest of the Content to it
    output = fopen("Items 2nd Half.html", "wb, ccs=UNICODE");

    memset(puffer, 0, 20000000);

    wcscpy(puffer, L"<HTML><Head><title>Items</title></Head><Body><table border=\"1\"><tr><th>ID</th><th>PK2 Name</th><th>Ingame Name</th><th>Level</th></tr>");

    //Setting j to the next File to progress
    j = NO_OF_FILES%2 ? NO_OF_FILES/2 + 1 : NO_OF_FILES/2;

    for(; j < NO_OF_FILES; j++) {                 //Loops through the rest of the Input Files
        for(i = 0; i < count[j]; i++) {           //Loops through every Line of the current Input files
            //Copying Line per Line into the String Buffer
            swprintf(line, L"<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>", itemID[j][i], itemname[j][i], ingamename_textdata[itemID_in_textdata[j][i]],  itemlevel[j][i]);
            wcscat(puffer, line);
        }
    }

    wcscat(puffer, L"</table></Body></HTML>");

    //Finally writing into the File
    fwprintf(output, puffer);

    fclose(output);


    printf("It took %d ms to create the Files!\n", (int)(clock() - beginning));
    printf("Finished. The Files are located in the Program's Folder.\n");

    free(puffer);
    free(line);
}

/*
Frees all the allocated Memory and closes all Files
*/
void destroy() {
    int i = 0, j = 0;

    if(finished_normally) {
        return;
    }

    _fcloseall();

    for(j = 0; j < NO_OF_FILES; j++) {
        for(i = 0; i < count[j]; i++) {
            free(itemID[j][i]);
            free(itemname[j][i]);
            free(itemname_in_textdata[j][i]);
            free(itemlevel[j][i]);
        }
        free(itemID[j]);
        free(itemname[j]);
        free(itemname_in_textdata[j]);
        free(itemlevel[j]);
    }

    free(itemID);
    free(itemname);
    free(itemname_in_textdata);
    free(itemlevel);


    for(i = 0; i < textdata_count; i++) {
        free(itemname_textdata[i]);
        free(ingamename_textdata[i]);
    }

    free(itemname_textdata);
    free(ingamename_textdata);


    finished_normally = true;
}

parse_itemdata.h:

Code:

#ifndef PARSE_ITEMDATA_H_INCLUDED
#define PARSE_ITEMDATA_H_INCLUDED

#include <stdlib.h>
#include <stdio.h>
#include <windows.h>
#include <io.h>
#include <time.h>

#define null NULL
#define boolean unsigned char
#define true 1
#define false 0

void start();
void destroy();

#endif // PARSE_ITEMDATA_H_INCLUDED

You need to copy the binary file out of the bin folder and into the main project folder to run it

It took the x86 file ~15 seconds and the x64 program ~10 seconds to parse all the files(at my pc)

I only added the most important things to the output files
you can add some yourself, or ask me if you want

The project has been created with Codeblocks(you can download it here:

)

bootdisk · 11/05/2011, 01:50

That's awesome!
I can't believe you did it on C, my respects & congratulations.

The only thing I would change is the ZeroMemory calls, use memset.

Schickl · 11/05/2011, 10:23

Quote:

Originally Posted by bootdisk

That's awesome!
I can't believe you did it on C, my respects & congratulations.

The only thing I would change is the ZeroMemory calls, use memset.

Thanks

I do everything in C or sometimes C++

and ZeroMemory is defined as RtlZeroMemory and RtlZeroMemory is defined as
RtlZeroMemory(d,l) RtlFillMemory((d),(l),0)
and RtlFillMemory as
RtlFillMemory(d,l,f) memset((d), (f), (l))

so it does the same after all
But i use memset now(the project is a few months old and i used ZeroMemory back then)

oh and I've done the same for mobs
I can release it if someone's interested(Would require some documentation; that's why I'll only release it if someone wants it)