Give nccopy control of output filter suppression

Makefile.am

@@ -7,6 +7,9 @@
 
 # Ed Hartnett, Ward Fisher
 
+
+
+
 # This directory stores libtool macros, put there by aclocal.
 ACLOCAL_AMFLAGS = -I m4
 

cf

@@ -1,6 +1,6 @@
 #!/bin/bash
 #NB=1
-#DB=1
+DB=1
 #X=-x
 FAST=1
 
@@ -124,7 +124,8 @@ FLAGS="$FLAGS --disable-parallel4"
 fi
 
 if test "x${DB}" = x1 ; then
-FLAGS="$FLAGS --disable-shared --enable-static"
+#FLAGS="$FLAGS --disable-shared --enable-static"
+FLAGS="$FLAGS --enable-static"
 else
 FLAGS="$FLAGS --enable-shared"
 fi

docs/Doxyfile.in

@@ -750,7 +750,6 @@ INPUT = \
     @abs_top_srcdir@/docs/install-fortran.md \
     @abs_top_srcdir@/docs/types.dox \
     @abs_top_srcdir@/docs/internal.dox \
-    @abs_top_srcdir@/docs/indexing.dox \
     @abs_top_srcdir@/docs/windows-binaries.md \
     @abs_top_srcdir@/docs/guide.dox \
     @abs_top_srcdir@/docs/OPeNDAP.dox \

docs/filters.md

@@ -142,10 +142,34 @@ The "-F" option can be used repeatedly as long as the variable name
 part is different. A different filter id and parameters can be
 specified for each occurrence.
 
-Note that if the input file has compressed variables, that fact
-will be invisble to nccopy because it is handled within the
-netcdf-c/hdf5 library code. This is true for any program that calls
-the netcdf-c library.
+As a rule, any input filter on an input variable will be applied
+to the equivalent output variable -- assuming the output file type
+is netcdf-4. It is, however, sometimes convenient to suppress
+output compression either totally or on a per-variable basis.
+Total suppression of output filters can be accomplished by specifying
+a special case of "-F", namely this.
+````
+nccopy -F "none" input.nc output.nc
+````
+Suppression of output filtering for a specific variable can be accomplished
+using this format.
+````
+nccopy -F "var,none" input.nc output.nc
+````
+where "var" is the fully qualified name of the variable.
+
+The rules for all possible cases of the "-F" flag are defined
+by this table.
+
+<table>
+<tr><th>-F none<th>-Fvar,...<th>Input Filter<th>Applied Output Filter
+<tr><td>true<td>unspecified<td>NA<td>unfiltered
+<tr><td>true<td>-Fvar,none<td>NA<td>unfiltered
+<tr><td>true<td>-Fvar,...<td>NA<td>use output filter
+<tr><td>false<td>unspecified<td>defined<td>use input filter
+<tr><td>false<td>-Fvar,none<td>NA<td>unfiltered
+<tr><td>false<td>-Fvar,...<td>NA<td>use output filter
+</table> 
 
 Parameter Encoding {#ParamEncode}
 ==========
@@ -416,5 +440,5 @@ References {#References}
 
 1. https://support.hdfgroup.org/HDF5/doc/Advanced/DynamicallyLoadedFilters/HDF5DynamicallyLoadedFilters.pdf
 2. https://support.hdfgroup.org/HDF5/doc/TechNotes/TechNote-HDF5-CompressionTroubleshooting.pdf
-3. https://support.hdfgroup.org/services/filters.html
+3. https://portal.hdfgroup.org/display/support/Contributions#Contributions-filters
 4. https://support.hdfgroup.org/services/contributions.html#filters

examples/C/hdf5plugins/Makefile.am

@@ -21,7 +21,7 @@ DLLSRC=${PLUGINSRC} ${BZIP2SRC}
 lib_LTLIBRARIES = libbzip2.la
 
 libbzip2_la_SOURCES = ${DLLSRC}
-libbzip2_la_LDFLAGS = -module -avoid-version -shared -export-dynamic -no-undefined
+libbzip2_la_LDFLAGS = -module -avoid-version -shared -export-dynamic -no-undefined -rpath ${abs_builddir}
 
 endif #ENABLE_FILTER_TESTING
 EXTRA_DIST = CMakeLists.txt H5Zbzip2.c Makefile.am blocksort.c bzlib.c bzlib.h bzlib_private.h compress.c \

nc_test4/hdf5plugins/Makefile.am

@@ -18,9 +18,9 @@ DLLSRC=${PLUGINSRC} ${BZIP2SRC} ${PLUGINHDRS} ${BZIP2HDRS}
 lib_LTLIBRARIES = libbzip2.la libmisc.la
 
 libbzip2_la_SOURCES = ${DLLSRC}
-libbzip2_la_LDFLAGS = -module -avoid-version -shared -export-dynamic -no-undefined
+libbzip2_la_LDFLAGS = -module -avoid-version -shared -export-dynamic -no-undefined -rpath ${abs_builddir}
 
 libmisc_la_SOURCES = H5Zmisc.c h5misc.h
-libmisc_la_LDFLAGS = -module -avoid-version -shared -export-dynamic -no-undefined
+libmisc_la_LDFLAGS = -module -avoid-version -shared -export-dynamic -no-undefined -rpath ${abs_builddir}
 
 endif #ENABLE_FILTER_TESTING

nc_test4/tst_filter.sh

@@ -55,73 +55,100 @@ if ! test -f ${MISCPATH} ; then echo "Unable to locate ${MISCPATH}"; exit 1; fi
 
 if test "x$API" = x1 ; then
 echo "*** Testing dynamic filters using API"
-rm -f ./bzip2.nc ./bzip2.dump ./tmp_tst_filter
+rm -f ./bzip2.nc ./bzip2.dump ./tst_filter.txt
 ${execdir}/test_filter
-${NCDUMP} -s bzip2.nc > ./tmp_tst_filter
+${NCDUMP} -s bzip2.nc > ./tst_filter.txt
 # Remove irrelevant -s output
-sclean ./tmp_tst_filter ./bzip2.dump
+sclean ./tst_filter.txt ./bzip2.dump
 diff -b -w ${srcdir}/bzip2.cdl ./bzip2.dump
 echo "*** Pass: API dynamic filter"
 fi
 
 if test "x$MISC" = x1 ; then
 echo
 echo "*** Testing dynamic filters parameter passing"
-rm -f ./testmisc.nc tmp_tst_filter tmp_tst_filter2
+rm -f ./testmisc.nc tst_filter.txt tst_filter2.txt
 ${execdir}/test_filter_misc
 # Verify the parameters via ncdump
-${NCDUMP} -s testmisc.nc > ./tmp_tst_filter
+${NCDUMP} -s testmisc.nc > ./tst_filter.txt
 # Extract the parameters
-getfilterattr ./tmp_tst_filter ./tmp_tst_filter2
-rm -f ./tmp_tst_filter
-trimleft ./tmp_tst_filter2 ./tmp_tst_filter
-rm -f ./tmp_tst_filter2
-cat >./tmp_tst_filter2 <<EOF
+getfilterattr ./tst_filter.txt ./tst_filter2.txt
+rm -f ./tst_filter.txt
+trimleft ./tst_filter2.txt ./tst_filter.txt
+rm -f ./tst_filter2.txt
+cat >./tst_filter2.txt <<EOF
 var:_Filter = "32768,1,239,23,65511,27,77,93,1145389056,3287505826,1097305129,1,2147483648,4294967295,4294967295" ;
 EOF
-diff -b -w ./tmp_tst_filter ./tmp_tst_filter2
+diff -b -w ./tst_filter.txt ./tst_filter2.txt
 echo "*** Pass: parameter passing"
 fi
 
 if test "x$NG" = x1 ; then
 echo "*** Testing dynamic filters using ncgen"
-rm -f ./bzip2.nc ./bzip2.dump ./tmp_tst_filter
+rm -f ./bzip2.nc ./bzip2.dump ./tst_filter.txt
 ${NCGEN} -lb -4 -o bzip2.nc ${srcdir}/bzip2.cdl
-${NCDUMP} -s bzip2.nc > ./tmp_tst_filter
+${NCDUMP} -s bzip2.nc > ./tst_filter.txt
 # Remove irrelevant -s output
-sclean ./tmp_tst_filter ./bzip2.dump
+sclean ./tst_filter.txt ./bzip2.dump
 diff -b -w ${srcdir}/bzip2.cdl ./bzip2.dump
 echo "*** Pass: ncgen dynamic filter"
 fi
 
 if test "x$NCP" = x1 ; then
 echo "*** Testing dynamic filters using nccopy"
-rm -f ./unfiltered.nc ./filtered.nc ./filtered.dump ./tmp_tst_filter
+rm -f ./unfiltered.nc ./filtered.nc ./tmp.nc ./filtered.dump ./tst_filter.txt
 ${NCGEN} -4 -lb -o unfiltered.nc ${srcdir}/unfiltered.cdl
+echo "	*** Testing simple filter application"
 ${NCCOPY} -F "/g/var,307,9,4" unfiltered.nc filtered.nc
-${NCDUMP} -s filtered.nc > ./tmp_tst_filter
+${NCDUMP} -s filtered.nc > ./tst_filter.txt
 # Remove irrelevant -s output
-sclean ./tmp_tst_filter ./filtered.dump
+sclean ./tst_filter.txt ./filtered.dump
 diff -b -w ${srcdir}/filtered.cdl ./filtered.dump
-echo "*** Pass: nccopy dynamic filter"
+echo "	*** Pass: nccopy simple filter"
+
+echo "	*** Testing pass-thru of filters"
+rm -f ./tst_filter.txt tst_filter2.txt ./tst_filter2.nc
+${NCCOPY} ./filtered.nc ./tst_filter2.nc
+${NCDUMP} -s tst_filter2.nc > ./tst_filter.txt
+sed -e '/_Filter/p' -e d < ./tst_filter.txt >tst_filter2.txt
+test -s tst_filter2.txt
+echo "	*** Pass: pass-thru of filters"
+
+echo "	*** Testing -F none"
+rm -f ./tst_filter.txt ./tst_filter2.txt ./tst_filter.nc
+${NCCOPY} -F none ./filtered.nc ./tst_filter.nc
+${NCDUMP} -s tst_filter.nc > ./tst_filter.txt
+sed -e '/_Filter/p' -e d < ./tst_filter.txt >./tst_filter2.txt
+test ! -s tst_filter2.txt
+echo "	*** Pass: -F none"
+
+echo "	*** Testing -F var,none "
+rm -f ./tst_filter.txt ./tst_filter.nc
+${NCCOPY} -F "/g/var,none" ./filtered.nc ./tst_filter.nc
+${NCDUMP} -s tst_filter.nc > ./tst_filter.txt
+sed -e '/_Filter/p' -e d < ./tst_filter.txt >tst_filter2.txt
+test ! -s tst_filter2.txt
+echo "	*** Pass: -F var,none"
+
+echo "*** Pass: all nccopy filter tests"
 fi
 
 if test "x$UNK" = x1 ; then
 echo "*** Testing access to filter info when filter dll is not available"
-rm -f bzip2.nc ./tmp_tst_filter
+rm -f bzip2.nc ./tst_filter.txt
 # build bzip2.nc
 ${NCGEN} -lb -4 -o bzip2.nc ${srcdir}/bzip2.cdl
 # dump and clean bzip2.nc header only when filter is avail
-${NCDUMP} -hs bzip2.nc > ./tmp_tst_filter
+${NCDUMP} -hs bzip2.nc > ./tst_filter.txt
 # Remove irrelevant -s output
-sclean ./tmp_tst_filter bzip2.dump
+sclean ./tst_filter.txt bzip2.dump
 # Now hide the filter code
 mv ${BZIP2PATH} ${BZIP2PATH}.save
 # dump and clean bzip2.nc header only when filter is not avail
-rm -f ./tmp_tst_filter
-${NCDUMP} -hs bzip2.nc > ./tmp_tst_filter
+rm -f ./tst_filter.txt
+${NCDUMP} -hs bzip2.nc > ./tst_filter.txt
 # Remove irrelevant -s output
-sclean ./tmp_tst_filter bzip2x.dump
+sclean ./tst_filter.txt bzip2x.dump
 # Restore the filter code
 mv ${BZIP2PATH}.save ${BZIP2PATH}
 diff -b -w ./bzip2.dump ./bzip2x.dump
@@ -137,10 +164,10 @@ echo "*** Pass: ncgen dynamic filter"
 fi
 
 #cleanup
-rm -f ./bzip*.nc ./unfiltered.nc ./filtered.nc ./tmp_tst_filter ./tmp_tst_filter2 *.dump bzip*hdr.*
-rm -fr ./test_bzip2.c
-rm -fr ./testmisc.nc
-
+rm -f ./bzip*.nc ./unfiltered.nc ./filtered.nc ./tst_filter.txt ./tst_filter2.txt *.dump bzip*hdr.*
+rm -f ./test_bzip2.c
+rm -f ./testmisc.nc
+rm -f ./tst_filter2.nc
 echo "*** Pass: all selected tests passed"
 
 exit 0

nc_test4/tst_rename.c

@@ -178,7 +178,6 @@ main(int argc, char **argv)
       fprintf(stderr,"*** Test Charlie's test for renaming with one enddef...");
       {
          int ncid, dimid, varid;
-         nc_set_log_level(5);
 
          /* Create a nice, simple file. This file will contain one
           * dataset, "lon", which is a dimscale. */

ncdump/Makefile.am

@@ -4,6 +4,10 @@
 
 # Ed Hartnett, Dennis Heimbigner, Ward Fisher
 
+#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
+#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
+#LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
+
 # Put together AM_CPPFLAGS and AM_LDFLAGS.
 include $(top_srcdir)/lib_flags.am
 LDADD = ${top_builddir}/liblib/libnetcdf.la