Provide ability to control URL encoding for DAP2

RELEASE_NOTES.md

@@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release
 
 ## 4.8.0 - TBD
 
+* [Enhancement] Give the client control over what parts of a DAP2 URL are URL encoded (i.e. %xx). This is to support the different decoding rules that servers apply to incoming URLS.
 * [Bug Fix] Fix incorrect time offsets from `ncdump -t`, in some cases when the time `units` attribute contains both a **non-zero** time-of-day, and a time zone suffix containing the letter "T", such as "UTC".  See [Github #1866](https://github.com/Unidata/netcdf-c/pull/1866) for more information.
 * [Bug Fix] Cleanup the NCZarr S3 build options. See [Github #1869](https://github.com/Unidata/netcdf-c/pull/1869) for more information.
 * [Bug Fix] Support aligned access for selected ARM processors.  See [Github #1871](https://github.com/Unidata/netcdf-c/pull/1871) for more information.

configure.ac

@@ -498,11 +498,11 @@ CFLAGS="$SAVECFLAGS"
 # --enable-dap => enable-dap4
 enable_dap4=$enable_dap
 # Default is to do the short remote tests.
-# Temporary: Change default to npt do these tests
-AC_MSG_CHECKING([whether dap remote testing should be enabled (default on)])
+# Temporary: Change default to not do these tests
+AC_MSG_CHECKING([whether dap remote testing should be enabled (default off)])
 AC_ARG_ENABLE([dap-remote-tests],
-              [AS_HELP_STRING([--disable-dap-remote-tests],
-                                 [disable dap remote tests])])
+              [AS_HELP_STRING([--enable-dap-remote-tests],
+                                 [enable dap remote tests])])
 test "x$enable_dap_remote_tests" = xyes || enable_dap_remote_tests=no
 if test "x$enable_dap" = "xno" ; then
   enable_dap_remote_tests=no

include/ncdap.h

@@ -36,6 +36,9 @@ typedef unsigned int NCFLAGS;
 #define NCF_PREFETCH_ALL    (0x0800) /* Prefetch all variables */
 /* Allow _FillValue/Variable type mismatch */
 #define NCF_FILLMISMATCH    (0x1000)
+/* Hack to control URL encoding */
+#define NCF_ENCODE_PATH     (0x2000) 
+#define NCF_ENCODE_QUERY    (0x4000) 
 /*COLUMBIA_HACK*/
 #define NCF_COLUMBIA        (0x80000000) /* Hack for columbia server */
 

include/ncuri.h

@@ -9,15 +9,16 @@
 #include "ncexternl.h"
 
 /* Define flags to control what is included by ncuribuild*/
-#define NCURIPATH      1
-#define NCURIPWD       2
-#define NCURIQUERY     4
-#define NCURIFRAG      8
-#define NCURIENCODE    16 /* If output should be encoded */
-#define NCURIBASE    (NCURIPWD|NCURIPATH)
-#define NCURISVC     (NCURIQUERY|NCURIBASE) /* for sending to server  */
-#define NCURIALL     (NCURIPATH|NCURIPWD|NCURIQUERY|NCURIFRAG) /* for rebuilding after changes */
-
+#define NCURIPATH	    1
+#define NCURIPWD	    2
+#define NCURIQUERY	    4
+#define NCURIFRAG	    8
+#define NCURIENCODEPATH	    16 /* If output url path should be encoded */
+#define NCURIENCODEQUERY    32 /* If output url query should be encoded */
+#define NCURIENCODE	    (NCURIENCODEPATH|NCURIENCODEQUERY)
+#define NCURIBASE	    (NCURIPWD|NCURIPATH)
+#define NCURISVC	    (NCURIQUERY|NCURIBASE) /* for sending to server  */
+#define NCURIALL	    (NCURIPATH|NCURIPWD|NCURIQUERY|NCURIFRAG) /* for rebuilding after changes */
 
 /*! This is an open structure meaning
 	it is ok to directly access its fields

libdap2/daputil.c

@@ -654,8 +654,8 @@ dap_fetch(NCDAPCOMMON* nccomm, OClink conn, const char* ce,
     NCerror ncstat = NC_NOERR;
     OCerror ocstat = OC_NOERR;
     char* ext = NULL;
-    OCflags flags = 0;
     int httpcode = 0;
+    OCflags ocflags = 0;
 #ifdef HAVE_GETTIMEOFDAY
     struct timeval time0;
     struct timeval time1;
@@ -668,13 +668,14 @@ dap_fetch(NCDAPCOMMON* nccomm, OClink conn, const char* ce,
     if(ce != NULL && strlen(ce) == 0)
 	ce = NULL;
 
-    if(FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE)) {
+    if(FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE))
 	ce = NULL;
-    }
-
-    if(FLAGSET(nccomm->controls,NCF_ONDISK)) {
-	flags |= OCONDISK;
-    }
+    if(FLAGSET(nccomm->controls,NCF_ONDISK))
+	ocflags |= OCONDISK;
+    if(FLAGSET(nccomm->controls,NCF_ENCODE_PATH))
+	ocflags |= OCENCODEPATH;
+    if(FLAGSET(nccomm->controls,NCF_ENCODE_QUERY))
+	ocflags |= OCENCODEQUERY;
 
     if(SHOWFETCH) {
 	/* Build uri string minus the constraint and #tag */
@@ -688,7 +689,7 @@ dap_fetch(NCDAPCOMMON* nccomm, OClink conn, const char* ce,
 	gettimeofday(&time0,NULL);
 #endif
     }
-    ocstat = oc_fetch(conn,ce,dxd,flags,rootp);
+    ocstat = oc_fetch(conn,ce,dxd,ocflags,rootp);
     if(FLAGSET(nccomm->controls,NCF_SHOWFETCH)) {
 #ifdef HAVE_GETTIMEOFDAY
         double secs;
@@ -810,3 +811,19 @@ nccpadding(unsigned long offset, int alignment)
     return pad;
 }
 
+int
+dapparamparselist(const char* s0, int delim, NClist* list)
+{
+    int stat = NC_NOERR;
+    char* s = strdup(s0);
+    char* p;
+    int i,count = 1;
+    if(s0 == NULL || strlen(s) == 0) goto done;
+    for(p=s;*p;p++) {if(*p == delim) {*p = '\0'; count++;}}
+    for(i=0,p=s;i<count;i++,p+=(strlen(p)+1)) {
+	if(strlen(p)>0)
+	    nclistpush(list,strdup(p));
+    }
+done:
+    return stat;
+}

libdap2/daputil.h

@@ -72,4 +72,6 @@ extern int dap_badname(char* name);
 extern char* dap_repairname(char* name);
 extern char* dap_getselection(NCURI* uri);
 
+extern int dapparamparselist(const char* s0, int delim, NClist* list);
+
 #endif /*DAPUTIL_H*/

libdap2/ncd2dispatch.c

@@ -2207,13 +2207,17 @@ fixzerodims(NCDAPCOMMON* dapcomm)
 static void
 applyclientparamcontrols(NCDAPCOMMON* dapcomm)
 {
+    const char* value = NULL;
+
     /* clear the flags */
     CLRFLAG(dapcomm->controls,NCF_CACHE);
     CLRFLAG(dapcomm->controls,NCF_SHOWFETCH);
     CLRFLAG(dapcomm->controls,NCF_NC3);
     CLRFLAG(dapcomm->controls,NCF_NCDAP);
     CLRFLAG(dapcomm->controls,NCF_PREFETCH);
     CLRFLAG(dapcomm->controls,NCF_PREFETCH_EAGER);
+    CLRFLAG(dapcomm->controls,NCF_ENCODE_PATH);
+    CLRFLAG(dapcomm->controls,NCF_ENCODE_QUERY);
 
     /* Turn on any default on flags */
     SETFLAG(dapcomm->controls,DFALT_ON_FLAGS);
@@ -2248,6 +2252,30 @@ applyclientparamcontrols(NCDAPCOMMON* dapcomm)
     else if(dapparamcheck(dapcomm,"nofillmismatch",NULL))
 	CLRFLAG(dapcomm->controls,NCF_FILLMISMATCH);
 
+    if((value=dapparamvalue(dapcomm,"encode")) != NULL) {
+	int i;
+	NClist* encode = nclistnew();
+	if(dapparamparselist(value,',',encode)) 
+            nclog(NCLOGERR,"Malformed encode parameter: %s",value);
+	else {
+	    /* First, turn off all the encode flags */
+            CLRFLAG(dapcomm->controls,NCF_ENCODE_PATH|NCF_ENCODE_QUERY);
+	    for(i=0;i<nclistlength(encode);i++) {
+	        char* s = nclistremove(encode,i);
+	        if(strcmp(s,"path")==0)
+	            SETFLAG(dapcomm->controls,NCF_ENCODE_PATH);
+	        else if(strcmp(s,"query")==0)
+	            SETFLAG(dapcomm->controls,NCF_ENCODE_QUERY);
+	        else if(strcmp(s,"all")==0)
+	            SETFLAG(dapcomm->controls,NCF_ENCODE_PATH|NCF_ENCODE_QUERY);
+	        else if(strcmp(s,"none")==0)
+	            CLRFLAG(dapcomm->controls,NCF_ENCODE_PATH|NCF_ENCODE_QUERY);
+	    }
+	}
+        nclistfree(encode);
+    } else { /* Set defaults */
+	SETFLAG(dapcomm->controls,NCF_ENCODE_QUERY);
+    }
     nclog(NCLOGNOTE,"Caching=%d",FLAGSET(dapcomm->controls,NCF_CACHE));
 
 }

libdispatch/ncuri.c

@@ -629,7 +629,8 @@ ncuribuild(NCURI* duri, const char* prefix, const char* suffix, int flags)
 {
     char* newuri = NULL;
     NCbytes* buf = ncbytesnew();
-    const int encode = (flags&NCURIENCODE ? 1 : 0);
+    const int encodepath = (flags&NCURIENCODEPATH ? 1 : 0);
+    const int encodequery = (flags&NCURIENCODEQUERY ? 1 : 0);
 
     if(prefix != NULL)
 	ncbytescat(buf,prefix);
@@ -656,7 +657,7 @@ ncuribuild(NCURI* duri, const char* prefix, const char* suffix, int flags)
     if((flags & NCURIPATH)) {
 	if(duri->path == NULL)
 	    ncbytescat(buf,"/");
-	else if(encode) {
+	else if(encodepath) {
 	    char* encoded = ncuriencodeonly(duri->path,pathallow);
 	    ncbytescat(buf,encoded);
 	    nullfree(encoded);
@@ -675,8 +676,13 @@ ncuribuild(NCURI* duri, const char* prefix, const char* suffix, int flags)
  	ensurequerylist(duri);
         if(duri->query != NULL) {
             ncbytescat(buf,"?");
-	    ncbytescat(buf,duri->query);
-        }
+	    if(encodequery) {
+	        char* encoded = ncuriencodeonly(duri->query,queryallow);
+	        ncbytescat(buf,encoded);
+	        nullfree(encoded);
+	    } else
+	        ncbytescat(buf,duri->query);
+	}
     }
     if(flags & NCURIFRAG) {
  	ensurefraglist(duri);

libsrc/putget.m4

@@ -644,6 +644,8 @@ putNCvx_$1_$2(NC3_INFO* ncp, const NC_var *varp,
 	void *xp;
         void *fillp=NULL;
 
+	NC_UNUSED(fillp);
+
 	if(nelems == 0)
 		return NC_NOERR;
 
@@ -847,7 +849,9 @@ getNCvx_$1_$2(const NC3_INFO* ncp, const NC_var *varp,
 }
 ')dnl
 
+#if 0 /*unused*/
 GETNCVX(char, char)
+#endif
 
 GETNCVX(schar, schar)
 GETNCVX(schar, short)

ncdap_test/CMakeLists.txt

@@ -40,6 +40,7 @@ IF(ENABLE_TESTS)
       add_sh_test(ncdap tst_ber)
       add_sh_test(ncdap tst_remote3)
       add_sh_test(ncdap tst_zero_len_var)
+      add_sh_test(ncdap tst_encode)
 # not yet      add_sh_test(ncdap tst_hyrax)
       add_sh_test(ncdap tst_fillmismatch)
       IF(ENABLE_DAP_LONG_TESTS)

ncdap_test/Makefile.am

@@ -45,7 +45,7 @@ endif
 
 if ENABLE_DAP_REMOTE_TESTS
 if BUILD_UTILITIES
-TESTS += tst_ber.sh tst_remote3.sh tst_formatx.sh testurl.sh tst_fillmismatch.sh tst_zero_len_var.sh
+TESTS += tst_ber.sh tst_remote3.sh tst_formatx.sh testurl.sh tst_fillmismatch.sh tst_zero_len_var.sh tst_encode.sh
 endif
 
 TESTS += test_partvar

ncdap_test/tst_encode.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+# Dennis Heimbigner
+
+if test "x$srcdir" = x ; then srcdir=`pwd`; fi
+. ../test_common.sh
+
+set -e
+echo ""
+echo "*** Testing #encode=" mechanism
+
+${NCDUMP} -h 'http://opendap2.oceanbrowser.net/thredds/dodsC/data/emodnet1-domains/tmp%20test.nc?lon[0:8]#encode=none'
+${NCDUMP} -h 'http://iridl.ldeo.columbia.edu/SOURCES/.Indices/.soi/.c8110/.anomaly/T/%28Jan%201979%29/VALUE/dods?anomaly[0]'
+
+

oc2/oc.c

@@ -1804,12 +1804,12 @@ oc_raw_xdrsize(OCobject link, OCobject ddsroot, off_t* xdrsizep)
 
 /* Resend a url as a head request to check the Last-Modified time */
 OCerror
-oc_update_lastmodified_data(OCobject link)
+oc_update_lastmodified_data(OCobject link, OCflags flags)
 {
     OCstate* state;
     OCVERIFY(OC_State,link);
     OCDEREF(OCstate*,state,link);
-    return OCTHROW(ocupdatelastmodifieddata(state));
+    return OCTHROW(ocupdatelastmodifieddata(state,flags));
 }
 
 long

oc2/oc.h

@@ -51,9 +51,18 @@ typedef int OCflags;
 /*!\def OCONDISK
 Cause oc_fetch to store the retrieved data on disk.
 */
-
 #define OCONDISK 1
 
+/*!\def OCENCODEPATH
+Cause oc_fetch to encode path part of a URL
+*/
+#define OCENCODEPATH 2
+
+/*!\def OCENCODEQUERY
+Cause oc_fetch to encode query part of a URL
+*/
+#define OCENCODEQUERY 4
+
 /**************************************************/
 /* OCtype */
 
@@ -583,7 +592,7 @@ EXTERNL OCerror oc_set_curlopt(OClink link, const char* option, void* value);
 EXTERNL OCerror oc_get_connection(OCobject ocnode, OCobject* linkp);
 
 /* Resend a url as a head request to check the Last-Modified time */
-EXTERNL OCerror oc_update_lastmodified_data(OClink);
+EXTERNL OCerror oc_update_lastmodified_data(OClink,OCflags);
 
 /* Get last known modification time; -1 => data unknown */
 EXTERNL long oc_get_lastmodified_data(OClink);

oc2/ocinternal.c

@@ -180,14 +180,14 @@ ocfetch(OCstate* state, const char* constraint, OCdxd kind, OCflags flags,
 
     switch (kind) {
     case OCDAS:
-        stat = readDAS(state,tree);
+        stat = readDAS(state,tree,flags);
 	if(stat == OC_NOERR) {
             tree->text = ncbytesdup(state->packet);
 	    if(tree->text == NULL) stat = OC_EDAS;
 	}
 	break;
     case OCDDS:
-        stat = readDDS(state,tree);
+        stat = readDDS(state,tree,flags);
 	if(stat == OC_NOERR) {
             tree->text = ncbytesdup(state->packet);
 	    if(tree->text == NULL) stat = OC_EDDS;
@@ -460,12 +460,15 @@ fprintf(stderr,"missing bod: ddslen=%lu bod=%lu\n",
 }
 
 OCerror
-ocupdatelastmodifieddata(OCstate* state)
+ocupdatelastmodifieddata(OCstate* state, OCflags ocflags)
 {
     OCerror status = OC_NOERR;
     long lastmodified;
     char* base = NULL;
-    base = ncuribuild(state->uri,NULL,NULL,NCURIENCODE);
+    int flags = 0;
+    if(ocflags & OCENCODEPATH) flags |= NCURIENCODEPATH;
+    if(ocflags & OCENCODEQUERY) flags |= NCURIENCODEQUERY;
+    base = ncuribuild(state->uri,NULL,NULL,flags);
     status = ocfetchlastmodified(state->curl, base, &lastmodified);
     free(base);
     if(status == OC_NOERR) {

oc2/ocinternal.h

@@ -229,7 +229,7 @@ extern int oc_network_order;
 extern int oc_invert_xdr_double;
 extern OCerror ocinternalinitialize(void);
 
-extern OCerror ocupdatelastmodifieddata(OCstate* state);
+extern OCerror ocupdatelastmodifieddata(OCstate* state, OCflags);
 
 extern OCerror ocset_useragent(OCstate* state, const char* agent);
 extern OCerror ocset_netrc(OCstate* state, const char* path);