Logo Search packages:      
Sourcecode: teeworlds version File versions  Download package

e_datafile.c

/* copyright (c) 2007 magnus auvinen, see licence.txt for more info */
#include <base/system.h>
#include "e_datafile.h"
#include "e_engine.h"
#include <zlib.h>

static const int DEBUG=0;

typedef struct 
{
      int type;
      int start;
      int num;
} DATAFILE_ITEM_TYPE;

typedef struct 
{
      int type_and_id;
      int size;
} DATAFILE_ITEM;

typedef struct
{
      char id[4];
      int version;
      int size;
      int swaplen;
      int num_item_types;
      int num_items;
      int num_raw_data;
      int item_size;
      int data_size;
} DATAFILE_HEADER;

typedef struct
{
      int num_item_types;
      int num_items;
      int num_raw_data;
      int item_size;
      int data_size;
      char start[4];
} DATAFILE_DATA;

typedef struct
{
      DATAFILE_ITEM_TYPE *item_types;
      int *item_offsets;
      int *data_offsets;
      int *data_sizes;

      char *item_start;
      char *data_start;
} DATAFILE_INFO;

struct DATAFILE_t
{
      IOHANDLE file;
      DATAFILE_INFO info;
      DATAFILE_HEADER header;
      int data_start_offset;
      char **data_ptrs;
      char *data;
};

DATAFILE *datafile_load(const char *filename)
{
      DATAFILE *df;
      IOHANDLE file;
      DATAFILE_HEADER header;
      unsigned readsize;
      
      unsigned *dst;
      unsigned char *src;
      unsigned j;
      int size = 0;
      int allocsize = 0;
      
      (void)dst;
      (void)src;
      (void)j;
      
      
      dbg_msg("datafile", "datafile loading. filename='%s'", filename);

      file = engine_openfile(filename, IOFLAG_READ);
      if(!file)
            return 0;
      
      /* TODO: change this header */
      io_read(file, &header, sizeof(header));
      if(header.id[0] != 'A' || header.id[1] != 'T' || header.id[2] != 'A' || header.id[3] != 'D')
      {
            if(header.id[0] != 'D' || header.id[1] != 'A' || header.id[2] != 'T' || header.id[3] != 'A')
            {
                  dbg_msg("datafile", "wrong signature. %x %x %x %x", header.id[0], header.id[1], header.id[2], header.id[3]);
                  return 0;
            }
      }

#if defined(CONF_ARCH_ENDIAN_BIG)
      swap_endian(&header, sizeof(int), sizeof(header)/sizeof(int));    
#endif
      if(header.version != 3 && header.version != 4)
      {
            dbg_msg("datafile", "wrong version. version=%x", header.version);
            return 0;
      }
      
      /* read in the rest except the data */
      size = 0;
      size += header.num_item_types*sizeof(DATAFILE_ITEM_TYPE);
      size += (header.num_items+header.num_raw_data)*sizeof(int);
      if(header.version == 4)
            size += header.num_raw_data*sizeof(int); /* v4 has uncompressed data sizes aswell */
      size += header.item_size;
      
      allocsize = size;
      allocsize += sizeof(DATAFILE); /* add space for info structure */
      allocsize += header.num_raw_data*sizeof(void*); /* add space for data pointers */

      df = (DATAFILE*)mem_alloc(allocsize, 1);
      df->header = header;
      df->data_start_offset = sizeof(DATAFILE_HEADER) + size;
      df->data_ptrs = (char**)(df+1);
      df->data = (char *)(df+1)+header.num_raw_data*sizeof(char *);
      df->file = file;
      
      /* clear the data pointers */
      mem_zero(df->data_ptrs, header.num_raw_data*sizeof(void*));
      
      /* read types, offsets, sizes and item data */
      readsize = io_read(file, df->data, size);
      if(readsize != size)
      {
            dbg_msg("datafile", "couldn't load the whole thing, wanted=%d got=%d", size, readsize);
            return 0;
      }

#if defined(CONF_ARCH_ENDIAN_BIG)
      swap_endian(df->data, sizeof(int), header.swaplen / sizeof(int));
#endif

      if(DEBUG)
      {
            dbg_msg("datafile", "allocsize=%d", allocsize);
            dbg_msg("datafile", "readsize=%d", readsize);
            dbg_msg("datafile", "swaplen=%d", header.swaplen);
            dbg_msg("datafile", "item_size=%d", df->header.item_size);
      }
      
      df->info.item_types = (DATAFILE_ITEM_TYPE *)df->data;
      df->info.item_offsets = (int *)&df->info.item_types[df->header.num_item_types];
      df->info.data_offsets = (int *)&df->info.item_offsets[df->header.num_items];
      df->info.data_sizes = (int *)&df->info.data_offsets[df->header.num_raw_data];
      
      if(header.version == 4)
            df->info.item_start = (char *)&df->info.data_sizes[df->header.num_raw_data];
      else
            df->info.item_start = (char *)&df->info.data_offsets[df->header.num_raw_data];
      df->info.data_start = df->info.item_start + df->header.item_size;

      if(DEBUG)
            dbg_msg("datafile", "datafile loading done. datafile='%s'", filename);

      if(DEBUG)
      {
            /*
            for(int i = 0; i < df->data.num_raw_data; i++)
            {
                  void *p = datafile_get_data(df, i);
                  dbg_msg("datafile", "%d %d", (int)((char*)p - (char*)(&df->data)), size);
            }
                  
            for(int i = 0; i < datafile_num_items(df); i++)
            {
                  int type, id;
                  void *data = datafile_get_item(df, i, &type, &id);
                  dbg_msg("map", "\t%d: type=%x id=%x p=%p offset=%d", i, type, id, data, df->info.item_offsets[i]);
                  int *idata = (int*)data;
                  for(int k = 0; k < 3; k++)
                        dbg_msg("datafile", "\t\t%d=%d (%x)", k, idata[k], idata[k]);
            }

            for(int i = 0; i < df->data.num_item_types; i++)
            {
                  dbg_msg("map", "\t%d: type=%x start=%d num=%d", i,
                        df->info.item_types[i].type,
                        df->info.item_types[i].start,
                        df->info.item_types[i].num);
                  for(int k = 0; k < df->info.item_types[i].num; k++)
                  {
                        int type, id;
                        datafile_get_item(df, df->info.item_types[i].start+k, &type, &id);
                        if(type != df->info.item_types[i].type)
                              dbg_msg("map", "\tERROR");
                  }
            }
            */
      }
            
      return df;
}

int datafile_num_data(DATAFILE *df)
{
      return df->header.num_raw_data;
}

/* always returns the size in the file */
int datafile_get_datasize(DATAFILE *df, int index)
{
      if(index == df->header.num_raw_data-1)
            return df->header.data_size-df->info.data_offsets[index];
      return  df->info.data_offsets[index+1]-df->info.data_offsets[index];
}

static void *datafile_get_data_impl(DATAFILE *df, int index, int swap)
{
      /* load it if needed */
      if(!df->data_ptrs[index])
      {
            /* fetch the data size */
            int datasize = datafile_get_datasize(df, index);
            int swapsize = datasize;
            
            if(df->header.version == 4)
            {
                  /* v4 has compressed data */
                  void *temp = (char *)mem_alloc(datasize, 1);
                  unsigned long uncompressed_size = df->info.data_sizes[index];
                  unsigned long s;

                  dbg_msg("datafile", "loading data index=%d size=%d uncompressed=%d", index, datasize, uncompressed_size);
                  df->data_ptrs[index] = (char *)mem_alloc(uncompressed_size, 1);
                  
                  /* read the compressed data */
                  io_seek(df->file, df->data_start_offset+df->info.data_offsets[index], IOSEEK_START);
                  io_read(df->file, temp, datasize);

                  /* decompress the data, TODO: check for errors */
                  s = uncompressed_size;
                  uncompress((Bytef*)df->data_ptrs[index], &s, (Bytef*)temp, datasize);
                  swapsize = s;

                  /* clean up the temporary buffers */
                  mem_free(temp);
            }
            else
            {
                  /* load the data */
                  dbg_msg("datafile", "loading data index=%d size=%d", index, datasize);
                  df->data_ptrs[index] = (char *)mem_alloc(datasize, 1);
                  io_seek(df->file, df->data_start_offset+df->info.data_offsets[index], IOSEEK_START);
                  io_read(df->file, df->data_ptrs[index], datasize);
            }

#if defined(CONF_ARCH_ENDIAN_BIG)
            if(swap && swapsize)
                  swap_endian(df->data_ptrs[index], sizeof(int), swapsize/sizeof(int));
#endif
      }
      
      return df->data_ptrs[index];
}

void *datafile_get_data(DATAFILE *df, int index)
{
      return datafile_get_data_impl(df, index, 0);
}

void *datafile_get_data_swapped(DATAFILE *df, int index)
{
      return datafile_get_data_impl(df, index, 1);
}

void datafile_unload_data(DATAFILE *df, int index)
{
      if(index < 0)
            return;
            
      /* */
      mem_free(df->data_ptrs[index]);
      df->data_ptrs[index] = 0x0;
}

int datafile_get_itemsize(DATAFILE *df, int index)
{
      if(index == df->header.num_items-1)
            return df->header.item_size-df->info.item_offsets[index];
      return  df->info.item_offsets[index+1]-df->info.item_offsets[index];
}

void *datafile_get_item(DATAFILE *df, int index, int *type, int *id)
{
      DATAFILE_ITEM *i = (DATAFILE_ITEM *)(df->info.item_start+df->info.item_offsets[index]);
      if(type)
            *type = (i->type_and_id>>16)&0xffff; /* remove sign extention */
      if(id)
            *id = i->type_and_id&0xffff;
      return (void *)(i+1);
}

void datafile_get_type(DATAFILE *df, int type, int *start, int *num)
{
      int i;
      for(i = 0; i < df->header.num_item_types; i++)
      {
            if(df->info.item_types[i].type == type)
            {
                  *start = df->info.item_types[i].start;
                  *num = df->info.item_types[i].num;
                  return;
            }
      }
      
      *start = 0;
      *num = 0;
}

void *datafile_find_item(DATAFILE *df, int type, int id)
{
      int start, num,i ;
      int item_id;
      void *item;
      
      datafile_get_type(df, type, &start, &num);
      for(i = 0; i < num; i++)
      {
            item = datafile_get_item(df, start+i,0, &item_id);
            if(id == item_id)
                  return item;
      }
      return 0;
}

int datafile_num_items(DATAFILE *df)
{
      return df->header.num_items;
}

void datafile_unload(DATAFILE *df)
{
      if(df)
      {
            /* free the data that is loaded */
            int i;
            for(i = 0; i < df->header.num_raw_data; i++)
                  mem_free(df->data_ptrs[i]);
            
            io_close(df->file);
            mem_free(df);
      }
}

/* DATAFILE output */
typedef struct
{
      int uncompressed_size;
      int compressed_size;
      void *compressed_data;
} DATA_INFO;

typedef struct
{
      int type;
      int id;
      int size;
      int next;
      int prev;
      void *data;
} ITEM_INFO;

typedef struct
{
      int num;
      int first;
      int last;
} ITEMTYPE_INFO;

/* */
struct DATAFILE_OUT_t
{
      IOHANDLE file;
      int num_items;
      int num_datas;
      int num_item_types;
      ITEMTYPE_INFO item_types[0xffff];
      ITEM_INFO items[1024];
      DATA_INFO datas[1024];
};

DATAFILE_OUT *datafile_create(const char *filename)
{
      int i;
      DATAFILE_OUT *df = (DATAFILE_OUT*)mem_alloc(sizeof(DATAFILE_OUT), 1);
      df->file = engine_openfile(filename, IOFLAG_WRITE);
      if(!df->file)
      {
            mem_free(df);
            return 0;
      }
      
      df->num_items = 0;
      df->num_datas = 0;
      df->num_item_types = 0;
      mem_zero(&df->item_types, sizeof(df->item_types));

      for(i = 0; i < 0xffff; i++)
      {
            df->item_types[i].first = -1;
            df->item_types[i].last = -1;
      }
      
      return df;
}

int datafile_add_item(DATAFILE_OUT *df, int type, int id, int size, void *data)
{
      df->items[df->num_items].type = type;
      df->items[df->num_items].id = id;
      df->items[df->num_items].size = size;
      
      /*
      dbg_msg("datafile", "added item type=%d id=%d size=%d", type, id, size);
      int i;
      for(i = 0; i < size/4; i++)
            dbg_msg("datafile", "\t%d: %08x %d", i, ((int*)data)[i], ((int*)data)[i]);
      */
      
      /* copy data */
      df->items[df->num_items].data = mem_alloc(size, 1);
      mem_copy(df->items[df->num_items].data, data, size);

      if(!df->item_types[type].num) /* count item types */
            df->num_item_types++;

      /* link */
      df->items[df->num_items].prev = df->item_types[type].last;
      df->items[df->num_items].next = -1;
      
      if(df->item_types[type].last != -1)
            df->items[df->item_types[type].last].next = df->num_items;
      df->item_types[type].last = df->num_items;
      
      if(df->item_types[type].first == -1)
            df->item_types[type].first = df->num_items;
      
      df->item_types[type].num++;
            
      df->num_items++;
      return df->num_items-1;
}

int datafile_add_data(DATAFILE_OUT *df, int size, void *data)
{
      DATA_INFO *info = &df->datas[df->num_datas];
      unsigned long s = compressBound(size);
      void *compdata = mem_alloc(s, 1); /* temporary buffer that we use duing compression */

      int result = compress((Bytef*)compdata, &s, (Bytef*)data, size);
      if(result != Z_OK)
      {
            dbg_msg("datafile", "compression error %d", result);
            dbg_assert(0, "zlib error");
      }
            
      info->uncompressed_size = size;
      info->compressed_size = (int)s;
      info->compressed_data = mem_alloc(info->compressed_size, 1);
      mem_copy(info->compressed_data, compdata, info->compressed_size);
      mem_free(compdata);

      df->num_datas++;
      return df->num_datas-1;
}

int datafile_add_data_swapped(DATAFILE_OUT *df, int size, void *data)
{
#if defined(CONF_ARCH_ENDIAN_BIG)
      void *swapped = mem_alloc(size, 1); /* temporary buffer that we use duing compression */
      int index;
      mem_copy(swapped, data, size);
      swap_endian(&swapped, sizeof(int), size/sizeof(int));
      index = datafile_add_data(df, size, swapped);
      mem_free(swapped);
      return index;
#else
      return datafile_add_data(df, size, data);
#endif
}


int datafile_finish(DATAFILE_OUT *df)
{
      int itemsize = 0;
      int i, count, offset;
      int typessize, headersize, offsetsize, filesize, swapsize;
      int datasize = 0;
      DATAFILE_ITEM_TYPE info;
      DATAFILE_ITEM itm;
      DATAFILE_HEADER header;

      /* we should now write this file! */
      if(DEBUG)
            dbg_msg("datafile", "writing");

      /* calculate sizes */
      for(i = 0; i < df->num_items; i++)
      {
            if(DEBUG)
                  dbg_msg("datafile", "item=%d size=%d (%d)", i, df->items[i].size, df->items[i].size+sizeof(DATAFILE_ITEM));
            itemsize += df->items[i].size + sizeof(DATAFILE_ITEM);
      }
      
      
      for(i = 0; i < df->num_datas; i++)
            datasize += df->datas[i].compressed_size;
      
      /* calculate the complete size */
      typessize = df->num_item_types*sizeof(DATAFILE_ITEM_TYPE);
      headersize = sizeof(DATAFILE_HEADER);
      offsetsize = df->num_items*sizeof(int) + df->num_datas*sizeof(int);
      filesize = headersize + typessize + offsetsize + itemsize + datasize;
      swapsize = filesize - datasize;
      
      (void)swapsize;
      
      if(DEBUG)
            dbg_msg("datafile", "num_item_types=%d typessize=%d itemsize=%d datasize=%d", df->num_item_types, typessize, itemsize, datasize);
      
      /* construct header */
      {
            header.id[0] = 'D';
            header.id[1] = 'A';
            header.id[2] = 'T';
            header.id[3] = 'A';
            header.version = 4;
            header.size = filesize - 16;
            header.swaplen = swapsize - 16;
            header.num_item_types = df->num_item_types;
            header.num_items = df->num_items;
            header.num_raw_data = df->num_datas;
            header.item_size = itemsize;
            header.data_size = datasize;
            
            /* TODO: apply swapping */
            /* write header */
            if(DEBUG)
                  dbg_msg("datafile", "headersize=%d", sizeof(header));
            io_write(df->file, &header, sizeof(header));
      }
      
      /* write types */
      for(i = 0, count = 0; i < 0xffff; i++)
      {
            if(df->item_types[i].num)
            {
                  /* write info */
                  info.type = i;
                  info.start = count;
                  info.num = df->item_types[i].num;
                  if(DEBUG)
                        dbg_msg("datafile", "writing type=%x start=%d num=%d", info.type, info.start, info.num);
                  io_write(df->file, &info, sizeof(info));
                  count += df->item_types[i].num;
            }
      }
      
      /* write item offsets */
      for(i = 0, offset = 0; i < 0xffff; i++)
      {
            if(df->item_types[i].num)
            {
                  /* write all items in of this type */
                  int k = df->item_types[i].first;
                  while(k != -1)
                  {
                        if(DEBUG)
                              dbg_msg("datafile", "writing item offset num=%d offset=%d", k, offset);
                        io_write(df->file, &offset, sizeof(offset));
                        offset += df->items[k].size + sizeof(DATAFILE_ITEM);
                        
                        /* next */
                        k = df->items[k].next;
                  }
            }
      }
      
      /* write data offsets */
      for(i = 0, offset = 0; i < df->num_datas; i++)
      {
            if(DEBUG)
                  dbg_msg("datafile", "writing data offset num=%d offset=%d", i, offset);
            io_write(df->file, &offset, sizeof(offset));
            offset += df->datas[i].compressed_size;
      }

      /* write data uncompressed sizes */
      for(i = 0, offset = 0; i < df->num_datas; i++)
      {
            /*
            if(DEBUG)
                  dbg_msg("datafile", "writing data offset num=%d offset=%d", i, offset);
            */
            io_write(df->file, &df->datas[i].uncompressed_size, sizeof(int));
      }
      
      /* write items */
      for(i = 0; i < 0xffff; i++)
      {
            if(df->item_types[i].num)
            {
                  /* write all items in of this type */
                  int k = df->item_types[i].first;
                  while(k != -1)
                  {
                        itm.type_and_id = (i<<16)|df->items[k].id;
                        itm.size = df->items[k].size;
                        if(DEBUG)
                              dbg_msg("datafile", "writing item type=%x idx=%d id=%d size=%d", i, k, df->items[k].id, df->items[k].size);
                        
                        io_write(df->file, &itm, sizeof(itm));
                        io_write(df->file, df->items[k].data, df->items[k].size);
                        
                        /* next */
                        k = df->items[k].next;
                  }
            }
      }
      
      /* write data */
      for(i = 0; i < df->num_datas; i++)
      {
            if(DEBUG)
                  dbg_msg("datafile", "writing data id=%d size=%d", i, df->datas[i].compressed_size);
            io_write(df->file, df->datas[i].compressed_data, df->datas[i].compressed_size);
      }

      /* free data */
      for(i = 0; i < df->num_items; i++)
            mem_free(df->items[i].data);

      
      io_close(df->file);
      mem_free(df);

      if(DEBUG)
            dbg_msg("datafile", "done");
      return 0;
}

#define BUFFER_SIZE 64*1024

int datafile_crc(const char *filename)
{
      unsigned char buffer[BUFFER_SIZE];
      IOHANDLE file;
      int crc = 0;
      unsigned bytes = 0;
      
      file = engine_openfile(filename, IOFLAG_READ);
      if(!file)
            return 0;
            
      while(1)
      {
            bytes = io_read(file, buffer, BUFFER_SIZE);
            if(bytes <= 0)
                  break;
            crc = crc32(crc, buffer, bytes);
      }
      
      io_close(file);

      return crc; 
}

Generated by  Doxygen 1.6.0   Back to index