Hide navigation sidebar
Hide table of contents sidebar
Back to top
Toggle table of contents sidebar

# C API

The GSD C API consists of a single header and source file. Developers can drop the implementation into any package that needs it.

struct gsd_byte_buffer
  #include <gsd.h>

  Byte buffer.

  Used to buffer of small data chunks held for a buffered write at the end of a frame. Also used to hold the names.

  Public Members

  char *data

    Data.

  size_t size

    Number of bytes in the buffer.

  size_t reserved

    Number of bytes available in the buffer.

struct gsd_handle
  #include <gsd.h>

  File handle.

  A handle to an open GSD file.

  This handle is obtained when opening a GSD file and is passed into every method that operates on the file.

  Warning

  All members are read-only to the caller.

  Public Members

  int fd

    File descriptor.

  struct gsd_header header

    The file header.

  struct gsd_index_buffer file_index

    Mapped data chunk index.

  struct gsd_index_buffer frame_index

    Index entries to append to the current frame.

  struct gsd_index_buffer buffer_index

    Buffered index entries to append to the current frame.

  struct gsd_byte_buffer write_buffer

    Buffered write data.

  struct gsd_name_buffer file_names

    List of names stored in the file.

  struct gsd_name_buffer frame_names

    List of names added in the current frame.

  uint64_t cur_frame

    The index of the last frame in the file.

  int64_t file_size

    Size of the file (in bytes)

  enum gsd_open_flag open_flags

    Flags passed to gsd_open() when opening this handle.

  struct gsd_name_id_map name_map

    Access the names in the namelist.

  uint64_t pending_index_entries

    Number of index entries pending in the current frame.

  uint64_t maximum_write_buffer_size

    Maximum write buffer size (bytes).

  uint64_t index_entries_to_buffer

    Number of index entries to buffer before flushing.

struct gsd_header
  #include <gsd.h>

  GSD file header.

  The in-memory and on-disk storage of the GSD file header. Stored in the first 256 bytes of the file.

  Warning

  All members are read-only to the caller.

  Public Members

  uint64_t magic

    Magic number marking that this is a GSD file.

  uint64_t index_location

    Location of the chunk index in the file.

  uint64_t index_allocated_entries

    Number of index entries that will fit in the space allocated.

  uint64_t namelist_location

    Location of the name list in the file.

  uint64_t namelist_allocated_entries

    Number of bytes in the namelist divided by GSD_NAME_SIZE.

  uint32_t schema_version

    Schema version: from gsd_make_version().

  uint32_t gsd_version

    GSD file format version from gsd_make_version().

  char application[GSD_NAME_SIZE]

    Name of the application that generated this file.

  char schema[GSD_NAME_SIZE]

    Name of data schema.

  char reserved[GSD_RESERVED_BYTES]

    Reserved for future use.

struct gsd_index_buffer
  #include <gsd.h>

  Array of index entries.

  May point to a mapped location of index entries in the file or an in-memory buffer.

  Public Members

  struct gsd_index_entry *data

    Indices in the buffer.

  size_t size

    Number of entries in the buffer.

  size_t reserved

    Number of entries available in the buffer.

  void *mapped_data

    Pointer to mapped data (NULL if not mapped)

  size_t mapped_len

    Number of bytes mapped.

struct gsd_index_entry
  #include <gsd.h>

  Index entry.

  An index entry for a single chunk of data.

  Warning

  All members are read-only to the caller.

  Public Members

  uint64_t frame

    Frame index of the chunk.

  uint64_t N

    Number of rows in the chunk.

  int64_t location

    Location of the chunk in the file.

  uint32_t M

    Number of columns in the chunk.

  uint16_t id

    Index of the chunk name in the name list.

  uint8_t type

    Data type of the chunk: one of gsd_type.

  uint8_t flags

    Flags (for internal use).

struct gsd_name_buffer
  #include <gsd.h>

  Name buffer.

  Holds a list of string names in order separated by NULL terminators. In v1 files, each name is 64 bytes. In v2 files, only one NULL terminator is placed between each name.

  Public Members

  struct gsd_byte_buffer data

    Data.

  size_t n_names

    Number of names in the list.

struct gsd_name_id_map
  #include <gsd.h>

  Name/id hash map.

  A hash map of string names to integer identifiers.

  Public Members

  struct gsd_name_id_pair *v

    Name/id mappings.

  size_t size

    Number of entries in the mapping.

struct gsd_name_id_pair
  #include <gsd.h>

  Name/id mapping.

  A string name paired with an ID. Used for storing sorted name/id mappings in a hash map.

  Public Members

  char *name

    Pointer to name (actual name storage is allocated in gsd_handle)

  struct gsd_name_id_pair *next

    Next name/id pair with the same hash.

  uint16_t id

    Entry id.

file gsd.h
  #include <stdbool.h>
  #include <stdint.h>
  #include <string.h>

  Declare GSD data types and C API.

  Enums

  enum gsd_type

    Identifiers for the gsd data chunk element types.

    Values:

    enumerator GSD_TYPE_UINT8

      Unsigned 8-bit integer.

    enumerator GSD_TYPE_UINT16

      Unsigned 16-bit integer.

    enumerator GSD_TYPE_UINT32

      Unsigned 32-bit integer.

    enumerator GSD_TYPE_UINT64

      Unsigned 53-bit integer.

    enumerator GSD_TYPE_INT8

      Signed 8-bit integer.

    enumerator GSD_TYPE_INT16

      Signed 16-bit integer.

    enumerator GSD_TYPE_INT32

      Signed 32-bit integer.

    enumerator GSD_TYPE_INT64

      Signed 64-bit integer.

    enumerator GSD_TYPE_FLOAT

      32-bit floating point number.

    enumerator GSD_TYPE_DOUBLE

      64-bit floating point number.

  enum gsd_open_flag

    Flag for GSD file open options.

    Values:

    enumerator GSD_OPEN_READWRITE

      Open for both reading and writing.

    enumerator GSD_OPEN_READONLY

      Open only for reading.

    enumerator GSD_OPEN_APPEND

      Open only for writing.

  enum gsd_error

    Error return values.

    Values:

    enumerator GSD_SUCCESS

      Success.

    enumerator GSD_ERROR_IO

      IO error. Check `errno` for details.

    enumerator GSD_ERROR_INVALID_ARGUMENT

      Invalid argument passed to function.

    enumerator GSD_ERROR_NOT_A_GSD_FILE

      The file is not a GSD file.

    enumerator GSD_ERROR_INVALID_GSD_FILE_VERSION

      The GSD file version cannot be read.

    enumerator GSD_ERROR_FILE_CORRUPT

      The GSD file is corrupt.

    enumerator GSD_ERROR_MEMORY_ALLOCATION_FAILED

      GSD failed to allocated memory.

    enumerator GSD_ERROR_NAMELIST_FULL

      The GSD file cannot store any additional unique data chunk names.

    enumerator GSD_ERROR_FILE_MUST_BE_WRITABLE

      This API call requires that the GSD file opened in with the mode GSD_OPEN_APPEND or GSD_OPEN_READWRITE.

    enumerator GSD_ERROR_FILE_MUST_BE_READABLE

      This API call requires that the GSD file opened the mode GSD_OPEN_READ or GSD_OPEN_READWRITE.

    Values:

    enumerator GSD_NAME_SIZE

      v1 file: Size of a GSD name in memory.

      v2 file: The name buffer size is a multiple of GSD_NAME_SIZE.

    Values:

    enumerator GSD_RESERVED_BYTES

      Reserved bytes in the header structure.

  Functions

  uint32_t gsd_make_version(unsigned int major, unsigned int minor)

    Specify a version.

    Parameters:

      - major – major version

      - minor – minor version

    Returns:

      a packed version number aaaa.bbbb suitable for storing in a gsd file version entry.

  int gsd_create(const char *fname, const char *application, const char *schema, uint32_t schema_version)

    Create a GSD file.

    The generated gsd file is not opened. Call gsd_open() to open it for writing.

    Parameters:

      - fname – File name (UTF-8 encoded).

      - application – Generating application name (truncated to 63 chars).

      - schema – Schema name for data to be written in this GSD file (truncated to 63 chars).

      - schema_version – Version of the scheme data to be written (make with gsd_make_version()).

    Post:

      Create an empty gsd file in a file of the given name. Overwrite any existing file at that location.

    Returns:

      - GSD_SUCCESS (0) on success. Negative value on failure:

      - GSD_ERROR_IO: IO error (check errno).

  int gsd_create_and_open(struct gsd_handle *handle, const char *fname, const char *application, const char *schema, uint32_t schema_version, enum gsd_open_flag flags, int exclusive_create)

    Create and open a GSD file.

    Open the generated gsd file in handle.

    The file descriptor is closed if there when an error opening the file.

    Parameters:

      - handle – Handle to open.

      - fname – File name (UTF-8 encoded).

      - application – Generating application name (truncated to 63 chars).

      - schema – Schema name for data to be written in this GSD file (truncated to 63 chars).

      - schema_version – Version of the scheme data to be written (make with gsd_make_version()).

      - flags – Either GSD_OPEN_READWRITE, or GSD_OPEN_APPEND.

      - exclusive_create – Set to non-zero to force exclusive creation of the file.

    Post:

      Create an empty gsd file with the given name. Overwrite any existing file at that location.

    Returns:

      - GSD_SUCCESS (0) on success. Negative value on failure:

      - GSD_ERROR_IO: IO error (check errno).

      - GSD_ERROR_NOT_A_GSD_FILE: Not a GSD file.

      - GSD_ERROR_INVALID_GSD_FILE_VERSION: Invalid GSD file version.

      - GSD_ERROR_FILE_CORRUPT: Corrupt file.

      - GSD_ERROR_MEMORY_ALLOCATION_FAILED: Unable to allocate memory.

  int gsd_open(struct gsd_handle *handle, const char *fname, enum gsd_open_flag flags)

    Open a GSD file.

    The file descriptor is closed if there is an error opening the file.

    Parameters:

      - handle – Handle to open.

      - fname – File name to open (UTF-8 encoded).

      - flags – Either GSD_OPEN_READWRITE, GSD_OPEN_READONLY, or GSD_OPEN_APPEND.

    Pre:

      The file name fname is a GSD file.

    Post:

      Open a GSD file and populates the handle for use by API calls.

    Returns:

      - GSD_SUCCESS (0) on success. Negative value on failure:

      - GSD_ERROR_IO: IO error (check errno).

      - GSD_ERROR_NOT_A_GSD_FILE: Not a GSD file.

      - GSD_ERROR_INVALID_GSD_FILE_VERSION: Invalid GSD file version.

      - GSD_ERROR_FILE_CORRUPT: Corrupt file.

      - GSD_ERROR_MEMORY_ALLOCATION_FAILED: Unable to allocate memory.

  int gsd_truncate(struct gsd_handle *handle)

    Truncate a GSD file.

    After truncating, a file will have no frames and no data chunks. The file size will be that of a newly created gsd file. The application, schema, and schema version metadata will be kept. Truncate does not close and reopen the file, so it is suitable for writing restart files on Lustre file systems without any metadata access.

    Parameters:

      - handle – Open GSD file to truncate.

    Returns:

      - GSD_SUCCESS (0) on success. Negative value on failure:

      - GSD_ERROR_IO: IO error (check errno).

      - GSD_ERROR_NOT_A_GSD_FILE: Not a GSD file.

      - GSD_ERROR_INVALID_GSD_FILE_VERSION: Invalid GSD file version.

      - GSD_ERROR_FILE_CORRUPT: Corrupt file.

      - GSD_ERROR_MEMORY_ALLOCATION_FAILED: Unable to allocate memory.

  int gsd_close(struct gsd_handle *handle)

    Close a GSD file.

    Warning

    Ensure that all gsd_write_chunk() calls are completed with gsd_end_frame() before closing the file.

    Parameters:

      - handle – GSD file to close.

    Pre:

      handle was opened by gsd_open().

    Post:

      Writable files: All data and index entries buffered before the previous call to gsd_end_frame() is written to the file (see gsd_flush()).

    Post:

      The file is closed.

    Post:

      handle is freed and can no longer be used.

    Returns:

      - GSD_SUCCESS (0) on success. Negative value on failure:

      - GSD_ERROR_IO: IO error (check errno).

      - GSD_ERROR_INVALID_ARGUMENT: handle is NULL.

  int gsd_end_frame(struct gsd_handle *handle)

    Complete the current frame.

    Parameters:

      - handle – Handle to an open GSD file

    Pre:

      handle was opened by gsd_open().

    Post:

      The current frame counter is increased by 1.

    Post:

      Flush the write buffer if it has overflowed. See gsd_flush().

    Returns:

      - GSD_SUCCESS (0) on success. Negative value on failure:

      - GSD_ERROR_IO: IO error (check errno).

      - GSD_ERROR_INVALID_ARGUMENT: handle is NULL.

      - GSD_ERROR_FILE_MUST_BE_WRITABLE: The file was opened read-only.

      - GSD_ERROR_MEMORY_ALLOCATION_FAILED: Unable to allocate memory.

  int gsd_flush(struct gsd_handle *handle)

    Flush the write buffer.

    Parameters:

      - handle – Handle to an open GSD file

    Pre:

      handle was opened by gsd_open().

    Post:

      All data buffered by gsd_write_chunk() are present in the file.

    Post:

      All index entries buffered by gsd_write_chunk() prior to the last call to gsd_end_frame() are present in the file.

    Returns:

      - GSD_SUCCESS (0) on success. Negative value on failure:

      - GSD_ERROR_IO: IO error (check errno).

      - GSD_ERROR_INVALID_ARGUMENT: handle is NULL.

      - GSD_ERROR_FILE_MUST_BE_WRITABLE: The file was opened read-only.

      - GSD_ERROR_MEMORY_ALLOCATION_FAILED: Unable to allocate memory.

  int gsd_write_chunk(struct gsd_handle *handle, const char *name, enum gsd_type type, uint64_t N, uint32_t M, uint8_t flags, const void *data)

    Add a data chunk to the current frame.

    Note

    If the GSD file is version 1.0, the chunk name is truncated to 63 bytes. GSD version 2.0 files support arbitrarily long names.

    Note

    N == 0 is allowed. When N is 0, data may be NULL.

    Parameters:

      - handle – Handle to an open GSD file.

      - name – Name of the data chunk.

      - type – type ID that identifies the type of data in data.

      - N – Number of rows in the data.

      - M – Number of columns in the data.

      - flags – set to 0, non-zero values reserved for future use.

      - data – Data buffer.

    Pre:

      handle was opened by gsd_open().

    Pre:

      name is a unique name for data chunks in the given frame.

    Pre:

      data is allocated and contains at least `N * M * gsd_sizeof_type(type)` bytes.

    Post:

      When there is space in the buffer: The given data is present in the write buffer. Otherwise, the data is present at the end of the file.

    Post:

      The index is present in the buffer.

    Returns:

      - GSD_SUCCESS (0) on success. Negative value on failure:

      - GSD_ERROR_IO: IO error (check errno).

      - GSD_ERROR_INVALID_ARGUMENT: handle is NULL, N == 0, M == 0, type is invalid, or flags* != 0.

      - GSD_ERROR_FILE_MUST_BE_WRITABLE: The file was opened read-only.

      - GSD_ERROR_NAMELIST_FULL: The file cannot store any additional unique chunk names.

      - GSD_ERROR_MEMORY_ALLOCATION_FAILED: failed to allocate memory.

  const struct gsd_index_entry *gsd_find_chunk(struct gsd_handle *handle, uint64_t frame, const char *name)

    Find a chunk in the GSD file.

    The found entry contains size and type metadata and can be passed to gsd_read_chunk() to read the data.

    Note

    gsd_find_chunk() calls gsd_flush() when the file is writable.

    Parameters:

      - handle – Handle to an open GSD file

      - frame – Frame to look for chunk

      - name – Name of the chunk to find

    Pre:

      handle was opened by gsd_open() in read or readwrite mode.

    Returns:

      A pointer to the found chunk, or NULL if not found.

  int gsd_read_chunk(struct gsd_handle *handle, void *data, const struct gsd_index_entry *chunk)

    Read a chunk from the GSD file.

    Note

    gsd_read_chunk() calls gsd_flush() when the file is writable.

    Parameters:

      - handle – Handle to an open GSD file.

      - data – Data buffer to read into.

      - chunk – Chunk to read.

    Pre:

      handle was opened in read or readwrite mode.

    Pre:

      chunk was found by gsd_find_chunk().

    Pre:

      data points to an allocated buffer with at least `N * M * gsd_sizeof_type(type)` bytes.

    Returns:

      - GSD_SUCCESS (0) on success. Negative value on failure:

      - GSD_ERROR_IO: IO error (check errno).

      - GSD_ERROR_INVALID_ARGUMENT: handle is NULL, data is NULL, or chunk is NULL.

      - GSD_ERROR_FILE_MUST_BE_READABLE: The file was opened in append mode.

      - GSD_ERROR_FILE_CORRUPT: The GSD file is corrupt.

  uint64_t gsd_get_nframes(struct gsd_handle *handle)

    Get the number of frames in the GSD file.

    Parameters:

      - handle – Handle to an open GSD file

    Pre:

      handle was opened by gsd_open().

    Returns:

      The number of frames in the file, or 0 on error.

  size_t gsd_sizeof_type(enum gsd_type type)

    Query size of a GSD type ID.

    Parameters:

      - type – Type ID to query.

    Returns:

      Size of the given type in bytes, or 0 for an unknown type ID.

  const char *gsd_find_matching_chunk_name(struct gsd_handle *handle, const char *match, const char *prev)

    Search for chunk names in a gsd file.

    To find the first matching chunk name, pass NULL for prev. Pass in the previous found string to find the next after that, and so on. Chunk names match if they begin with the string in match*. Chunk names returned by this function may be present in at least one frame.

    Note

    gsd_find_matching_chunk_name() calls gsd_flush() when the file is writable.

    Parameters:

      - handle – Handle to an open GSD file.

      - match – String to match.

      - prev – Search starting point.

    Pre:

      handle was opened by gsd_open()

    Pre:

      prev was returned by a previous call to gsd_find_matching_chunk_name()

    Returns:

      Pointer to a string, NULL if no more matching chunks are found found, or NULL if prev* is invalid

  int gsd_upgrade(struct gsd_handle *handle)

    Upgrade a GSD file to the latest specification.

    Parameters:

      - handle – Handle to an open GSD file

    Pre:

      handle was opened by gsd_open() with a writable mode.

    Pre:

      There are no pending data to write to the file in gsd_end_frame()

    Returns:

      - GSD_SUCCESS (0) on success. Negative value on failure:

      - GSD_ERROR_IO: IO error (check errno).

      - GSD_ERROR_INVALID_ARGUMENT: handle is NULL

      - GSD_ERROR_FILE_MUST_BE_WRITABLE: The file was opened in read-only mode.

  uint64_t gsd_get_maximum_write_buffer_size(struct gsd_handle *handle)

    Get the maximum write buffer size.

    Parameters:

      - handle – Handle to an open GSD file

    Pre:

      handle was opened by gsd_open().

    Returns:

      The maximum write buffer size in bytes, or 0 on error.

  int gsd_set_maximum_write_buffer_size(struct gsd_handle *handle, uint64_t size)

    Set the maximum write buffer size.

    Parameters:

      - handle – Handle to an open GSD file

      - size – Maximum number of bytes to allocate in the write buffer (must be greater than 0).

    Pre:

      handle was opened by gsd_open().

    Returns:

      - GSD_SUCCESS (0) on success. Negative value on failure:

      - GSD_ERROR_INVALID_ARGUMENT: handle is NULL

      - GSD_ERROR_INVALID_ARGUMENT: size == 0

  uint64_t gsd_get_index_entries_to_buffer(struct gsd_handle *handle)

    Get the number of index entries to buffer.

    Parameters:

      - handle – Handle to an open GSD file

    Pre:

      handle was opened by gsd_open().

    Returns:

      The number of index entries to buffer, or 0 on error.

  int gsd_set_index_entries_to_buffer(struct gsd_handle *handle, uint64_t number)

    Set the number of index entries to buffer.

    Note

    GSD may allocate more than this number of entries in the buffer, as needed to store all index entries for the already buffered frames and the current frame.

    Parameters:

      - handle – Handle to an open GSD file

      - number – Number of index entries to buffer before automatically flushing in `gsd_end_frame()` (must be greater than 0).

    Pre:

      handle was opened by gsd_open().

    Returns:

      - GSD_SUCCESS (0) on success. Negative value on failure:

      - GSD_ERROR_INVALID_ARGUMENT: handle is NULL

      - GSD_ERROR_INVALID_ARGUMENT: number == 0

dir gsd

type uint8_t

  8-bit unsigned integer (defined by C compiler).

type uint16_t

  16-bit unsigned integer (defined by C compiler).

type uint32_t

  32-bit unsigned integer (defined by C compiler).

type uint64_t

  64-bit unsigned integer (defined by C compiler).

type int64_t

  64-bit signed integer (defined by C compiler).

type size_t

  unsigned integer (defined by C compiler).