Loading...
Searching...
No Matches
filter_example.c
Go to the documentation of this file.
/* Copyright 2018, UCAR/Unidata.
See COPYRIGHT file for conditions of use. */
#include "config.h"
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <hdf5.h>
#include "netcdf.h"
#include "netcdf_filter.h"
/* The HDF assigned id for bzip compression */
#define BZIP2_ID 307
/* The compression level used in this example */
#define BZIP2_LEVEL 9
#define TESTFILE "bzip2.nc"
/* Point at which we give up */
#define MAXERRS 8
#define NDIMS 4
#define DIMSIZE 4
#define CHUNKSIZE 4 /* Note: not the total size of the chunk, but size wrt a dim*/
static size_t dimsize = DIMSIZE;
static size_t chunksize = CHUNKSIZE;
static size_t actualdims = NDIMS;
static size_t actualproduct = 1; /* x-product over dim sizes */
static size_t chunkproduct = 1; /* x-product over chunksizes */
static size_t dims[NDIMS];
static size_t chunks[NDIMS];
static int nerrs = 0;
static int ncid, varid;
static int dimids[NDIMS];
static float* array = NULL;
static float* expected = NULL;
static unsigned int filterid = 0;
static unsigned int* params = NULL;
/* Forward */
static void init(int argc, char** argv);
static int test_bzip2(void);
static int verifychunks(void);
#define ERRR do { \
fflush(stdout); /* Make sure our stdout is synced with stderr. */ \
fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \
__FILE__, __LINE__); \
nerrs++;\
} while (0)
static int
check(int err,int line)
{
if(err != NC_NOERR) {
fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err));
fflush(stderr);
exit(1);
}
return NC_NOERR;
}
#define CHECK(x) check(x,__LINE__)
/*
Read the chunking information about the variable
and verify that it is as expected.
*/
static int
verifychunks(void)
{
int i;
int store = -1;
size_t chunksizes[NDIMS];
memset(chunksizes,0,sizeof(chunksizes));
CHECK(nc_inq_var_chunking(ncid, varid, &store, chunksizes));
/* Storage must be chunked, not contiguous */
if(store != NC_CHUNKED) {
fprintf(stderr,"bad chunk store\n");
return NC_ESTORAGE;
}
/* Chunk sizes must match our predefined set */
for(i=0;i<actualdims;i++) {
if(chunksizes[i] != chunks[i]) {
fprintf(stderr,"bad chunk size: %d\n",i);
return NC_EBADCHUNK;
}
}
return 1;
}
/*
Compare the data we wrote against the data we read.
*/
static int
compare(void)
{
int errs = 0;
int i;
printf("data comparison: |array|=%ld\n",(unsigned long)actualproduct);
for(i=0;i<actualproduct;i++) {
if(expected[i] != array[i]) {
printf("mismatch: array[%d]=%f expected[%d]=%f\n",
i,array[i],i,expected[i]);
errs++;
if(errs >= MAXERRS)
break;
}
}
if(errs == 0)
printf("no data errors\n");
if(actualproduct <= 1)
return NC_EBADDIM;
return (errs == 0 ? NC_NOERR: NC_EINVAL);
}
/*
Create the file, write it, then re-read for comparison.
*/
static int
test_bzip2(void)
{
int i;
unsigned int level = BZIP2_LEVEL;
size_t nparams = 0;
printf("\n*** Testing API: bzip2 compression.\n");
/* Clear the data array */
memset(array,0,sizeof(float)*actualproduct);
/* Create a file */
CHECK(nc_create(TESTFILE, NC_NETCDF4|NC_CLOBBER, &ncid));
/* Do not use fill for this file */
CHECK(nc_set_fill(ncid, NC_NOFILL, NULL));
/* Define the dimensions */
for(i=0;i<actualdims;i++) {
char dimname[1024];
snprintf(dimname,sizeof(dimname),"dim%d",i);
CHECK(nc_def_dim(ncid, dimname, dims[i], &dimids[i]));
}
/* Define the variable */
CHECK(nc_def_var(ncid, "var", NC_FLOAT, actualdims, dimids, &varid));
/* Set chunking on the variable */
CHECK(nc_def_var_chunking(ncid,varid,NC_CHUNKED,chunks));
/* Verify that chunking succeeded */
if(!verifychunks())
return NC_EINVAL;
/* Set bzip2 compression for the variable: takes one parameter == level */
CHECK(nc_def_var_filter(ncid,varid,BZIP2_ID,1,&level));
/* Read back the compression info and verify it */
level = 0;
CHECK(nc_inq_var_filter_info(ncid,varid,BZIP2_ID,&nparams,&level));
if(nparams != 1 || level != BZIP2_LEVEL) {
printf("test_filter: filter def/inq mismatch\n");
return NC_EFILTER;
}
/* Show the level */
printf("show parameters for bzip2: level=%u\n",level);
/* Show chunking */
printf("show chunks:");
for(i=0;i<actualdims;i++)
printf("%s%ld",(i==0?" chunks=":","),(unsigned long)chunks[i]);
printf("\n");
/* prepare to write */
CHECK(nc_enddef(ncid));
/* Fill in the array */
for(i=0;i<actualproduct;i++)
expected[i] = (float)i;
/* write array */
CHECK(nc_put_var(ncid,varid,expected));
/* Close file */
CHECK(nc_close(ncid));
/* Now re-open and verify */
printf("\n*** Testing API: bzip2 decompression.\n");
/* Clear the data array */
memset(array,0,sizeof(float)*actualproduct);
/* Open the file */
CHECK(nc_open(TESTFILE, NC_NOWRITE, &ncid));
/* Get the variable id */
CHECK(nc_inq_varid(ncid, "var", &varid));
/* Check the compression algorithm */
filterid = 0;
nparams = 0;
params = NULL;
CHECK(nc_inq_var_filter(ncid,varid,&filterid,&nparams,NULL));
if(filterid == 0) CHECK(NC_ENOFILTER); /* Not defined */
if(nparams > 0) {
params = (unsigned int*)malloc(sizeof(unsigned int)*nparams);
if(params == NULL)
return NC_ENOMEM;
CHECK(nc_inq_var_filter(ncid,varid,&filterid,&nparams,params));
}
if(filterid != BZIP2_ID) {
printf("Bzip2 id mismatch: %d\n",filterid);
return NC_EFILTER;
}
if(nparams != 1 && params != NULL && params[0] != BZIP2_LEVEL) {
printf("Compression parameter mismatch\n");
return NC_EFILTER;
}
/* Verify chunking */
if(!verifychunks())
return 0;
/* Read the data */
CHECK(nc_get_var_float(ncid, varid, array));
/* Close the file */
CHECK(nc_close(ncid));
return (compare() == NC_NOERR ? 0 : 1);
}
/**************************************************/
/* Utilities */
static void
init(int argc, char** argv)
{
int i;
/* Setup various variables */
actualproduct = 1;
chunkproduct = 1;
for(i=0;i<NDIMS;i++) {
dims[i] = dimsize;
chunks[i] = chunksize;
if(i < actualdims) {
actualproduct *= dims[i];
chunkproduct *= chunks[i];
}
}
/* Allocate max size */
array = (float*)calloc(1,sizeof(float)*actualproduct);
expected = (float*)calloc(1,sizeof(float)*actualproduct);
}
/**************************************************/
int
main(int argc, char **argv)
{
H5Eprint1(stderr);
init(argc,argv);
if(test_bzip2() != NC_NOERR) ERRR;
exit(nerrs > 0?1:0);
}
BZIP2_ID
#define BZIP2_ID
Definition filter_example.c:51
nc_close
EXTERNL int nc_close(int ncid)
Close an open netCDF dataset.
Definition dfile.c:1330
nc_create
EXTERNL int nc_create(const char *path, int cmode, int *ncidp)
Create a new netCDF file.
Definition dfile.c:428
nc_set_fill
EXTERNL int nc_set_fill(int ncid, int fillmode, int *old_modep)
Change the fill-value mode to improve write performance.
Definition dfile.c:1504
nc_enddef
EXTERNL int nc_enddef(int ncid)
Leave define mode.
Definition dfile.c:1057
nc_open
EXTERNL int nc_open(const char *path, int mode, int *ncidp)
Open an existing netCDF file.
Definition dfile.c:694
nc_def_dim
EXTERNL int nc_def_dim(int ncid, const char *name, size_t len, int *idp)
Define a new dimension.
Definition ddim.c:121
nc_strerror
EXTERNL const char * nc_strerror(int ncerr)
Given an error number, return an error message.
Definition derror.c:87
nc_inq_varid
EXTERNL int nc_inq_varid(int ncid, const char *name, int *varidp)
Find the ID of a variable, from the name.
Definition dvarinq.c:60
nc_put_var
EXTERNL int nc_put_var(int ncid, int varid, const void *op)
Write an entire variable with one call.
Definition dvarput.c:922
nc_inq_var_filter_info
EXTERNL int nc_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t *nparams, unsigned int *params)
Find the param info about filter (if any) associated with a variable and with specified id.
Definition dfilter.c:98
nc_inq_var_filter
EXTERNL int nc_inq_var_filter(int ncid, int varid, unsigned int *idp, size_t *nparams, unsigned int *params)
Find the first filter (if any) associated with a variable.
Definition dfilter.c:170
nc_inq_var_chunking
EXTERNL int nc_inq_var_chunking(int ncid, int varid, int *storagep, size_t *chunksizesp)
Get the storage and (for chunked variables) the chunksizes of a variable.
Definition dvarinq.c:466
nc_def_var
EXTERNL int nc_def_var(int ncid, const char *name, nc_type xtype, int ndims, const int *dimidsp, int *varidp)
Define a new variable.
Definition dvar.c:216
nc_def_var_chunking
EXTERNL int nc_def_var_chunking(int ncid, int varid, int storage, const size_t *chunksizesp)
Define storage and, if chunked storage is used, chunking parameters for a variable.
Definition dvar.c:732
netcdf.h
Main header file for the C API.
NC_NETCDF4
#define NC_NETCDF4
Use netCDF-4/HDF5 format.
Definition netcdf.h:179
NC_EFILTER
#define NC_EFILTER
Filter operation failed.
Definition netcdf.h:562
NC_CLOBBER
#define NC_CLOBBER
Destroy existing file.
Definition netcdf.h:138
NC_FLOAT
#define NC_FLOAT
single precision floating point number
Definition netcdf.h:40
NC_NOWRITE
#define NC_NOWRITE
Set read-only access for nc_open().
Definition netcdf.h:135
NC_ENOMEM
#define NC_ENOMEM
Memory allocation (malloc) failure.
Definition netcdf.h:497
NC_CHUNKED
#define NC_CHUNKED
In HDF5 files you can set storage for each variable to be either contiguous or chunked,...
Definition netcdf.h:353
NC_ESTORAGE
#define NC_ESTORAGE
Can't specify both contiguous and chunking.
Definition netcdf.h:555
NC_NOFILL
#define NC_NOFILL
Argument to nc_set_fill() to turn off filling of data.
Definition netcdf.h:123
NC_EINVAL
#define NC_EINVAL
Invalid Argument.
Definition netcdf.h:427
NC_NOERR
#define NC_NOERR
No Error.
Definition netcdf.h:417
nc_def_var_filter
EXTERNL int nc_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const unsigned int *parms)
Define a new variable filter Assumes HDF5 format using unsigned ints.
Definition dfilter.c:129
NC_EBADCHUNK
#define NC_EBADCHUNK
Bad chunksize.
Definition netcdf.h:556
NC_ENOFILTER
#define NC_ENOFILTER
Filter not defined on variable.
Definition netcdf.h:566
NC_EBADDIM
#define NC_EBADDIM
Invalid dimension id or name.
Definition netcdf.h:460
netcdf_filter.h