From 0b7a5382e7b9535781906642690d2a7bb2ee7efc Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Thu, 4 Mar 2021 13:41:31 -0700 Subject: [PATCH] Codify cross-platform file paths The netcdf-c code has to deal with a variety of platforms: Windows, OSX, Linux, Cygwin, MSYS, etc. These platforms differ significantly in the kind of file paths that they accept. So in order to handle this, I have created a set of replacements for the most common file system operations such as _open_ or _fopen_ or _access_ to manage the file path differences correctly. A more limited version of this idea was already implemented via the ncwinpath.h and dwinpath.c code. So this can be viewed as a replacement for that code. And in path in many cases, the only change that was required was to replace '#include ' with '#include ' and then replace file operation calls with the NCxxx equivalent from ncpathmgr.h Note that recently, the ncwinpath.h was renamed ncpathmgmt.h, so this pull request should not require dealing with winpath. The heart of the change is include/ncpathmgmt.h, which provides alternate operations such as NCfopen or NCaccess and which properly parse and rebuild path arguments to work for the platform on which the code is executing. This mostly matters for Windows because of the way that it uses backslash and drive letters, as compared to *nix*. One important feature is that the user can do string manipulations on a file path without having to worry too much about the platform because the path management code will properly handle most mixed cases. So one can for example concatenate a path suffix that uses forward slashes to a Windows path and have it work correctly. The conversion code is in libdispatch/dpathmgr.c, and the important function there is NCpathcvt which does the proper conversions to the local path format. As a rule, most code should just replace their file operations with the corresponding NCxxx ones defined in include/ncpathmgmt.h. These NCxxx functions all call NCpathcvt on their path arguments before executing the actual file operation. In some rare cases, the client may need to directly use NCpathcvt, but this should be avoided as much as possible. If there is a need for supporting a new file operation not already in ncpathmgmt.h, then use the code in dpathmgr.c as a template. Also please notify Unidata so we can include it as a formal part or our supported operations. Also, if you see an operation in the library that is not using the NCxxx form, then please submit an issue so we can fix it. Misc. Changes: * Clean up the utf8 testing code; it is impossible to get some tests to work under windows using shell scripts; the args do not pass as utf8 but as some other encoding. * Added an extra utf8 test case: test_unicode_path.sh * Add a true test for HDF5 1.10.6 or later because as noted in PR https://github.com/Unidata/netcdf-c/pull/1794, HDF5 changed its Windows file path handling. --- CMakeLists.txt | 11 +- configure.ac | 4 +- debug/cf.cmake | 2 +- include/Makefile.am | 2 +- include/hdf5internal.h | 20 +- include/nchttp.h | 1 + include/ncpathmgr.h | 97 +++- include/ncrc.h | 2 - include/ncuri.h | 3 + lib_flags.am | 2 +- libdispatch/ddispatch.c | 45 +- libdispatch/dfile.c | 10 - libdispatch/dhttp.c | 32 +- libdispatch/dinfermodel.c | 28 - libdispatch/dpathmgr.c | 933 +++++++++++++++++++++++++------ libdispatch/drc.c | 2 +- libdispatch/dutil.c | 5 +- libdispatch/ncuri.c | 9 + libhdf5/hdf5create.c | 39 +- libhdf5/hdf5internal.c | 3 +- libhdf5/hdf5open.c | 39 +- libhdf5/nc4memcb.c | 5 +- libnczarr/zmap.h | 1 - libsrc/ncFile.c | 6 +- libsrc/posixio.c | 9 +- libsrc/winceio.c | 9 +- nc_test/CMakeLists.txt | 6 +- nc_test/Makefile.am | 2 + ncdump/CMakeLists.txt | 26 +- ncdump/Makefile.am | 3 +- ncdump/nc4print.c | 1 + ncdump/nccopy.c | 8 +- ncdump/ncdump.c | 64 +-- ncdump/ocprint.c | 1 + ncdump/test_unicode_directory.sh | 26 +- ncdump/test_unicode_path.sh | 53 ++ ncdump/tst_nccopy4.sh | 22 +- ncdump/tst_netcdf4.sh | 9 - ncdump/tst_netcdf4_4.sh | 9 +- ncdump/tst_unicode.c | 142 +++-- ncgen/main.c | 8 +- ncgen/semantics.c | 1 + ncgen3/Makefile.am | 8 +- ncgen3/main.c | 11 +- ncgen3/ncgen.y | 2 - ncgen3/ncgeny.c | 2 - nczarr_test/itmeta.c | 724 ------------------------ nczarr_test/ncdumpchunks.c | 2 +- nczarr_test/s3util.c | 2 +- nczarr_test/tst_utils.c | 2 +- nczarr_test/ut_test.c | 24 +- nczarr_test/zmapio.c | 2 +- nczarr_test/zs3parse.c | 6 +- test_common.in | 5 + unit_test/test_pathcvt.c | 126 +++-- 55 files changed, 1331 insertions(+), 1285 deletions(-) create mode 100755 ncdump/test_unicode_path.sh delete mode 100644 nczarr_test/itmeta.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 5c3d929842..804be33f0d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -662,7 +662,6 @@ IF(USE_HDF5) IF(HDF5_VERSION_STRING AND NOT HDF5_VERSION) SET(HDF5_VERSION ${HDF5_VERSION_STRING}) ENDIF() - IF("${HDF5_VERSION}" STREQUAL "") MESSAGE(STATUS "Unable to determine hdf5 version. NetCDF requires at least version ${HDF5_VERSION_REQUIRED}") ELSE() @@ -803,6 +802,15 @@ IF(USE_HDF5) SET(HAS_PAR_FILTERS no CACHE STRING "") ENDIF() + # Check to see if HDF5 library is 1.10.6 or greater. + # Used to control path name conversion + IF(${HDF5_VERSION} VERSION_LESS "1.10.6") + SET(HDF5_UTF8_PATHS FALSE) + ELSE() + SET(HDF5_UTF8_PATHS TRUE) + ENDIF() + MESSAGE("-- Checking for HDF5 version 1.10.6 or later: ${HDF5_UTF8_PATHS}") + SET(H5_USE_16_API 1) OPTION(NC_ENABLE_HDF_16_API "Enable HDF5 1.6.x Compatibility(Required)" ON) IF(NOT NC_ENABLE_HDF_16_API) @@ -2204,6 +2212,7 @@ configure_file( SET(EXTRA_DIST ${EXTRA_DIST} ${CMAKE_CURRENT_SOURCE_DIR}/test_common.in) SET(TOPSRCDIR "${CMAKE_CURRENT_SOURCE_DIR}") SET(TOPBUILDDIR "${CMAKE_CURRENT_BINARY_DIR}") +SET(ISMSVC "${MSVC}") configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test_common.in ${CMAKE_CURRENT_BINARY_DIR}/test_common.sh @ONLY NEWLINE_STYLE LF) diff --git a/configure.ac b/configure.ac index 5680c7b187..1ff05a88c5 100644 --- a/configure.ac +++ b/configure.ac @@ -79,9 +79,9 @@ AC_MSG_NOTICE([checking supported formats]) # An explicit disable of netcdf-4 | netcdf4 is treated as if it was disable-hdf5 AC_MSG_CHECKING([whether we should build with netcdf4 (alias for HDF5)]) AC_ARG_ENABLE([netcdf4], [AS_HELP_STRING([--disable-netcdf4], - [(synonym for --enable-hdf5)])]) + [(deprecated synonym for --enable-hdf5)])]) test "x$enable_netcdf4" = xno || enable_netcdf4=yes -AC_MSG_RESULT([$enable_netcdf4]) +AC_MSG_RESULT([$enable_netcdf4 (deprecated; Please use with --disable-hdf5)]) AC_MSG_CHECKING([whether we should build with netcdf-4 (alias for HDF5)]) AC_ARG_ENABLE([netcdf-4], [AS_HELP_STRING([--disable-netcdf-4], [(synonym for --disable-netcdf4)])]) diff --git a/debug/cf.cmake b/debug/cf.cmake index f6acbd5b74..2303c8cbb7 100644 --- a/debug/cf.cmake +++ b/debug/cf.cmake @@ -15,7 +15,7 @@ S3TEST=1 #CDF5=1 #HDF4=1 -#TR=--trace +H518=1 for arg in "$@" ; do case "$arg" in diff --git a/include/Makefile.am b/include/Makefile.am index 060e32c127..cf7723b67c 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -19,7 +19,7 @@ ncbytes.h nchashmap.h ceconstraints.h rnd.h nclog.h ncconfigure.h \ nc4internal.h nctime.h nc3internal.h onstack.h ncrc.h ncauth.h \ ncoffsets.h nctestserver.h nc4dispatch.h nc3dispatch.h ncexternl.h \ ncpathmgr.h ncindex.h hdf4dispatch.h hdf5internal.h nc_provenance.h \ -hdf5dispatch.h ncmodel.h isnan.h nccrc.h ncexhash.h ncxcache.h +hdf5dispatch.h ncmodel.h isnan.h nccrc.h ncexhash.h ncxcache.h ncfilter.h if USE_DAP noinst_HEADERS += ncdap.h diff --git a/include/hdf5internal.h b/include/hdf5internal.h index 964f6ec8a1..e3bcca3e3d 100644 --- a/include/hdf5internal.h +++ b/include/hdf5internal.h @@ -201,23 +201,9 @@ extern int NC4_hdf5get_libversion(unsigned*,unsigned*,unsigned*);/*libsrc4/nc4hd extern int NC4_hdf5get_superblock(struct NC_FILE_INFO*, int*);/*libsrc4/nc4hdf.c*/ extern int NC4_isnetcdf4(struct NC_FILE_INFO*); /*libsrc4/nc4hdf.c*/ -#ifdef _WIN32 +extern int nc4_find_default_chunksizes2(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var); -/* Maxinum length of a typical path in UTF-8. - * When converting from ANSI to UTF-8, the length will be up to 3 times, - * so round up 260*3 to 1024. (260=MAX_PATH) */ -#define MAX_PATHBUF_SIZE 1024 - -/* Struct for converting ANSI to UTF-8. */ -typedef struct pathbuf -{ - void *ptr; - char buffer[MAX_PATHBUF_SIZE]; -} pathbuf_t; - -const char *nc4_ndf5_ansi_to_utf8(pathbuf_t *pb, const char *path); -void nc4_hdf5_free_pathbuf(pathbuf_t *pb); - -#endif /* _WIN32 */ +EXTERNL hid_t nc4_H5Fopen(const char *filename, unsigned flags, hid_t fapl_id); +EXTERNL hid_t nc4_H5Fcreate(const char *filename, unsigned flags, hid_t fcpl_id, hid_t fapl_id); #endif /* _HDF5INTERNAL_ */ diff --git a/include/nchttp.h b/include/nchttp.h index 2e7aab948d..e239ae4b99 100644 --- a/include/nchttp.h +++ b/include/nchttp.h @@ -20,6 +20,7 @@ typedef struct NC_HTTP_STATE { const char** headset; /* which headers to capture */ NClist* headers; NCbytes* buf; + char errbuf[1024]; /* assert(CURL_ERROR_SIZE <= 1024) */ } NC_HTTP_STATE; extern int nc_http_open(const char* objecturl, NC_HTTP_STATE** state, size64_t* lenp); diff --git a/include/ncpathmgr.h b/include/ncpathmgr.h index a158bd46a7..8a92f647de 100644 --- a/include/ncpathmgr.h +++ b/include/ncpathmgr.h @@ -2,8 +2,8 @@ * Copyright 2018, University Corporation for Atmospheric Research * See netcdf/COPYRIGHT file for copying and redistribution conditions. */ -#ifndef _NCWINIO_H_ -#define _NCWINIO_H_ +#ifndef _NCPATHMGR_H_ +#define _NCPATHMGR_H_ #include "config.h" #include @@ -28,6 +28,7 @@ #endif #endif +/* Define wrapper constants for use with NCaccess */ /* Define wrapper constants for use with NCaccess */ #ifdef _WIN32 #define ACCESS_MODE_EXISTS 0 @@ -49,16 +50,63 @@ #define ACCESS_MODE_RW (R_OK|W_OK) #endif -/* Path Converter */ +#ifdef _WIN32 +#ifndef S_IFDIR +#define S_IFDIR _S_IFDIR +#define S_IFREG _S_IFREG +#endif +#ifndef S_ISDIR +#define S_ISDIR(mode) ((mode) & _S_IFDIR) +#define S_ISREG(mode) ((mode) & _S_IFREG) +#endif +#endif /*_WIN32*/ + +/* +WARNING: you should never need to explictly call this function; +rather it is invoked as part of the wrappers for e.g. NCfopen, etc. + +This function attempts to take an arbitrary path and convert +it to a canonical form. +Assumptions about Input path: +1. It is a relative or absolute path +2. It is not a URL +3. It conforms to the format expected by one of the following: + Linux (/x/y/...), Cygwin (/cygdrive/D/...), + Windows (D:/...), or MSYS (/D/...), or relative (x/y...) +4. It is encoded in the local platform character set. + Note that for most systems, this is utf-8. But for Windows, + the encoding is most likely some form of ANSI code page, probably + the windows 1252 encoding. + Note that in any case, the path must be representable in the + local Code Page. + +On output it produces a re-written path that has the following +properties: +1. The path is normalized to match the platform on which the code + is running (e.g. cygwin, windows, msys, linux). So for example + using a cygwin path under visual studio will convert e.g. + /cygdrive/d/x/y to d:\x\y. See ../unit_test/test_pathcvt.c + for example conversions. +It returns the converted path. + +Note that this function is intended to be Idempotent: f(f(x) == f(x). +This means it is ok to call it repeatedly with no harm. +*/ EXTERNL char* NCpathcvt(const char* path); -/* path -> URL Path converter */ -EXTERNL char* NCurlpath(const char* path); +/* Canonicalize and make absolute by prefixing the current working directory */ +EXTERNL char* NCpathabsolute(const char* name); -/* Fix path in case it was escaped by shell */ -EXTERNL char* NCdeescape(const char* name); +/* Convert from the local coding (e.g. ANSI) to utf-8; + note that this can produce unexpected results for Windows + because it first converts to wide character and then to utf8. */ +EXTERNL int NCpath2utf8(const char* path, char** u8p); -#ifdef WINPATH +/* Wrap various stdio and unistd IO functions. +It is especially important to use for windows so that +NCpathcvt (above) is invoked on the path. +*/ +#if defined(WINPATH) /* path converter wrappers*/ EXTERNL FILE* NCfopen(const char* path, const char* flags); EXTERNL int NCopen3(const char* path, int flags, int mode); @@ -68,6 +116,10 @@ EXTERNL int NCremove(const char* path); EXTERNL int NCmkdir(const char* path, int mode); EXTERNL int NCrmdir(const char* path); EXTERNL char* NCcwd(char* cwdbuf, size_t len); +EXTERNL char* NCcwd(char* cwdbuf, size_t len); +#ifdef HAVE_SYS_STAT_H +EXTERNL int NCstat(char* path, struct stat* buf); +#endif #ifdef HAVE_DIRENT_H EXTERNL DIR* NCopendir(const char* path); EXTERNL int NCclosedir(DIR* ent); @@ -78,8 +130,12 @@ EXTERNL int NCclosedir(DIR* ent); #define NCopen2(path,flags) open((path),(flags)) #define NCremove(path) remove(path) #define NCaccess(path,mode) access(path,mode) +#define NCmkdir(path,mode) mkdir(path,mode) +#define NCgetcwd(buf,len) getcwd(buf,len) +#ifdef HAVE_SYS_STAT_H +#define NCstat(path,buf) stat(path,buf) +#endif #define NCcwd(buf, len) getcwd(buf,len) -#define NCmkdir(path, mode) mkdir(path,mode) #define NCrmdir(path) rmdir(path) #ifdef HAVE_DIRENT_H #define NCopendir(path) opendir(path) @@ -89,7 +145,26 @@ EXTERNL int NCclosedir(DIR* ent); /* Platform independent */ #define NCclose(fd) close(fd) +#define NCfstat(fd,buf) fstat(fd,buf) + +/**************************************************/ +/* Following definitions are for testing only */ + +/* Possible Kinds Of Output */ +#define NCPD_UNKNOWN 0 +#define NCPD_NIX 1 +#define NCPD_MSYS 2 +#define NCPD_CYGWIN 3 +#define NCPD_WIN 4 +#define NCPD_REL 5 /* actual kind is unknown */ + +EXTERNL char* NCpathcvt_test(const char* path, int ukind, int udrive); + +EXTERNL void printutf8hex(const char* s, char* sx); -EXTERNL int NChasdriveletter(const char* path); +/**************************************************/ +/* From dutil.c */ +EXTERNL char* NC_backslashEscape(const char* s); +EXTERNL char* NC_backslashUnescape(const char* esc); -#endif /* _NCWINIO_H_ */ +#endif /* _NCPATHMGR_H_ */ diff --git a/include/ncrc.h b/include/ncrc.h index 22fe448caf..b983743d29 100644 --- a/include/ncrc.h +++ b/include/ncrc.h @@ -55,8 +55,6 @@ extern NCTriple* NC_rcfile_ith(NCRCinfo*,size_t); /* From dutil.c (Might later move to e.g. nc.h */ extern int NC__testurl(const char* path, char** basenamep); extern int NC_isLittleEndian(void); -extern char* NC_backslashEscape(const char* s); -extern char* NC_backslashUnescape(const char* esc); extern char* NC_entityescape(const char* s); extern int NC_readfile(const char* filename, NCbytes* content); extern int NC_writefile(const char* filename, size_t size, void* content); diff --git a/include/ncuri.h b/include/ncuri.h index d2cd5dbfa5..b68ed46f8f 100644 --- a/include/ncuri.h +++ b/include/ncuri.h @@ -67,6 +67,9 @@ EXTERNL void ncurifree(NCURI* ncuri); /* Replace the protocol */ EXTERNL int ncurisetprotocol(NCURI*,const char* newprotocol); +/* Replace the path */ +EXTERNL int ncurisetpath(NCURI*,const char* newpath); + /* Replace the constraints */ EXTERNL int ncurisetquery(NCURI*,const char* query); diff --git a/lib_flags.am b/lib_flags.am index 5f46e7e04c..5dd1c36397 100644 --- a/lib_flags.am +++ b/lib_flags.am @@ -13,7 +13,7 @@ if USE_DAP AM_CPPFLAGS += -I${top_srcdir}/oc2 endif -if HAVE_AWS +if ENABLE_S3_SDK AM_LDFLAGS += -lstdc++ endif diff --git a/libdispatch/ddispatch.c b/libdispatch/ddispatch.c index b8bf733644..5b32691d83 100644 --- a/libdispatch/ddispatch.c +++ b/libdispatch/ddispatch.c @@ -10,6 +10,7 @@ See LICENSE.txt for license information. #include "ncbytes.h" #include "ncrc.h" #include "ncoffsets.h" +#include "ncpathmgr.h" /* Required for getcwd, other functions. */ #ifdef HAVE_UNISTD_H @@ -19,7 +20,6 @@ See LICENSE.txt for license information. /* Required for getcwd, other functions. */ #ifdef _WIN32 #include -#define getcwd _getcwd #endif #if defined(ENABLE_BYTERANGE) || defined(ENABLE_DAP) || defined(ENABLE_DAP4) @@ -57,61 +57,28 @@ NCDISPATCH_initialize(void) /* Capture temp dir*/ { - char* tempdir; - char* p; - char* q; - char cwd[4096]; -#ifdef _WIN32 + char* tempdir = NULL; +#if defined _WIN32 || defined __MSYS__ tempdir = getenv("TEMP"); #else tempdir = "/tmp"; #endif if(tempdir == NULL) { fprintf(stderr,"Cannot find a temp dir; using ./\n"); - tempdir = getcwd(cwd,sizeof(cwd)); - if(tempdir == NULL || *tempdir == '\0') tempdir = "."; - } - globalstate->tempdir= (char*)malloc(strlen(tempdir) + 1); - for(p=tempdir,q=globalstate->tempdir;*p;p++,q++) { - if((*p == '/' && *(p+1) == '/') - || (*p == '\\' && *(p+1) == '\\')) {p++;} - *q = *p; - } - *q = '\0'; -#ifdef _WIN32 -#else - /* Canonicalize */ - for(p=globalstate->tempdir;*p;p++) { - if(*p == '\\') {*p = '/'; }; + tempdir = "."; } - *q = '\0'; -#endif + globalstate->tempdir= strdup(tempdir); } /* Capture $HOME */ { - char* p; - char* q; char* home = getenv("HOME"); if(home == NULL) { /* use tempdir */ home = globalstate->tempdir; } - globalstate->home = (char*)malloc(strlen(home) + 1); - for(p=home,q=globalstate->home;*p;p++,q++) { - if((*p == '/' && *(p+1) == '/') - || (*p == '\\' && *(p+1) == '\\')) {p++;} - *q = *p; - } - *q = '\0'; -#ifdef _WIN32 -#else - /* Canonicalize */ - for(p=home;*p;p++) { - if(*p == '\\') {*p = '/'; }; - } -#endif + globalstate->home = strdup(home); } /* Now load RC File */ diff --git a/libdispatch/dfile.c b/libdispatch/dfile.c index f3471cf596..ffc4275397 100644 --- a/libdispatch/dfile.c +++ b/libdispatch/dfile.c @@ -1853,12 +1853,7 @@ NC_create(const char *path0, int cmode, size_t initialsz, /* Skip past any leading whitespace in path */ const unsigned char* p; for(p=(const unsigned char*)path0;*p;p++) {if(*p > ' ') break;} -#ifdef WINPATH - /* Need to do path conversion */ - path = NCpathcvt((const char*)p); -#else path = nulldup((const char*)p); -#endif } memset(&model,0,sizeof(model)); @@ -2009,12 +2004,7 @@ NC_open(const char *path0, int omode, int basepe, size_t *chunksizehintp, /* Skip past any leading whitespace in path */ const char* p; for(p=(const char*)path0;*p;p++) {if(*p < 0 || *p > ' ') break;} -#ifdef WINPATH - /* Need to do path conversion */ - path = NCpathcvt(p); -#else path = nulldup(p); -#endif } memset(&model,0,sizeof(model)); diff --git a/libdispatch/dhttp.c b/libdispatch/dhttp.c index aee1fbd465..815e1c8959 100644 --- a/libdispatch/dhttp.c +++ b/libdispatch/dhttp.c @@ -24,9 +24,10 @@ #include "nclist.h" #include "nchttp.h" +#undef VERBOSE #undef TRACE -#define CURLERR(e) (e) +#define CURLERR(e) reporterror(state,(e)) /* Mnemonics */ #define GETCMD 0 @@ -40,6 +41,8 @@ static int setupconn(NC_HTTP_STATE* state, const char* objecturl, NCbytes* buf); static int execute(NC_HTTP_STATE* state, int headcmd); static int headerson(NC_HTTP_STATE* state, const char** which); static void headersoff(NC_HTTP_STATE* state); +static void showerrors(NC_HTTP_STATE* state); +static int reporterror(NC_HTTP_STATE* state, CURLcode cstat); #ifdef TRACE static void @@ -81,6 +84,15 @@ nc_http_open(const char* objecturl, NC_HTTP_STATE** statep, size64_t* filelenp) /* initialize curl*/ state->curl = curl_easy_init(); if (state->curl == NULL) {stat = NC_ECURL; goto done;} + showerrors(state); +#ifdef VERBOSE + {long onoff = 1; + CURLcode cstat = CURLE_OK; + cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_VERBOSE, onoff)); + if(cstat != CURLE_OK) + {stat = NC_ECURL; goto done;} + } +#endif if(filelenp) { *filelenp = -1; /* Attempt to get the file length using HEAD */ @@ -324,6 +336,9 @@ setupconn(NC_HTTP_STATE* state, const char* objecturl, NCbytes* buf) if(objecturl != NULL) { /* Set the URL */ +#ifdef TRACE + fprintf(stderr,"curl.setup: url |%s|\n",objecturl); +#endif cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_URL, (void*)objecturl)); if (cstat != CURLE_OK) goto fail; } @@ -419,3 +434,18 @@ headersoff(NC_HTTP_STATE* state) (void)CURLERR(curl_easy_setopt(state->curl, CURLOPT_HEADERFUNCTION, NULL)); (void)CURLERR(curl_easy_setopt(state->curl, CURLOPT_HEADERDATA, NULL)); } + +static void +showerrors(NC_HTTP_STATE* state) +{ + (void)curl_easy_setopt(state->curl, CURLOPT_ERRORBUFFER, state->errbuf); +} + +static int +reporterror(NC_HTTP_STATE* state, CURLcode cstat) +{ + if(cstat != CURLE_OK) + fprintf(stderr,"curlcode: (%d)%s : %s\n", + cstat,curl_easy_strerror(cstat),state->errbuf); + return cstat; +} diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index a7a31177f1..2b7ff21853 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -860,34 +860,6 @@ nullify(const char* s) #endif /**************************************************/ -#if 0 -/* return 1 if path looks like a url; 0 otherwise */ -int -NC_testurl(const char* path) -{ - int isurl = 0; - NCURI* tmpurl = NULL; - - if(path == NULL) return 0; - - /* Ok, try to parse as a url */ - if(ncuriparse(path,&tmpurl)==NCU_OK) { - /* Do some extra testing to make sure this really is a url */ - /* Look for a known/accepted protocol */ - struct NCPROTOCOLLIST* protolist; - for(protolist=ncprotolist;protolist->protocol;protolist++) { - if(strcmp(tmpurl->protocol,protolist->protocol) == 0) { - isurl=1; - break; - } - } - ncurifree(tmpurl); - return isurl; - } - return 0; -} -#endif - /**************************************************/ /** * Provide a hidden interface to allow utilities diff --git a/libdispatch/dpathmgr.c b/libdispatch/dpathmgr.c index a453a26f21..0a3e312eb7 100644 --- a/libdispatch/dpathmgr.c +++ b/libdispatch/dpathmgr.c @@ -12,6 +12,9 @@ #ifdef HAVE_FCNTL_H #include #endif +#ifdef HAVE_SYS_STAT_H +#include +#endif #ifdef HAVE_UNISTD_H #include #endif @@ -19,14 +22,23 @@ #include #endif #ifdef _WIN32 +#include #include +#include +#include #include #endif - -#include "ncexternl.h" +#ifdef __hpux +#include +#endif + +#include "netcdf.h" #include "ncpathmgr.h" - -extern char *realpath(const char *path, char *resolved_path); +#include "nclog.h" +#include "nclist.h" +#include "ncbytes.h" +#include "ncuri.h" +#include "ncutf8.h" #undef PATHFORMAT @@ -47,7 +59,6 @@ for Windows. Other cases will be added as needed. 2. a leading '/cygdrive/X' will be converted to a drive letter X if X is alpha-char. 3. a leading D:/... is treated as a windows drive letter -4. a relative path will be converted to an absolute path. 5. If any of the above is encountered, then forward slashes will be converted to backslashes. All other cases are passed thru unchanged @@ -58,203 +69,246 @@ static const char* windrive = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVW static const size_t cdlen = 10; /* strlen("/cygdrive/") */ +static int pathinitialized = 0; + static int pathdebug = -1; -static char* makeabsolute(const char* relpath); +static const struct Path { + int kind; + int drive; + char* path; +} empty = {NCPD_UNKNOWN,0,NULL}; + +/* Keep the working directory kind and drive */ +static struct Path wdpath = {NCPD_UNKNOWN,0,NULL}; +static char wdstaticpath[8192]; + +static int parsepath(const char* inpath, struct Path* path); +static int unparsepath(struct Path* p, char** pathp); +static int getwdpath(struct Path* wd); +static char* printPATH(struct Path* p); +static int getlocalpathkind(void); +static void clearPath(struct Path* path); +static void pathinit(void); +static int iscygwinspecial(const char* path); +static int testurl(const char* path); + +#ifdef WINPATH +static int ansi2utf8(const char* local, char** u8p); +static int ansi2wide(const char* local, wchar_t** u16p); +static int utf82wide(const char* utf8, wchar_t** u16p); +static int wide2utf8(const wchar_t* u16, char** u8p); +#endif EXTERNL char* /* caller frees */ -NCpathcvt(const char* path) +NCpathcvt(const char* inpath) { - char* outpath = NULL; - char* p; - char* q; - size_t pathlen; - int forwardslash; + int stat = NC_NOERR; + char* tmp1 = NULL; + struct Path canon = empty; - if(path == NULL) goto done; /* defensive driving */ + if(inpath == NULL) goto done; /* defensive driving */ -#ifdef _WIN32 - forwardslash = 0; -#else - forwardslash = 1; -#endif + if(!pathinitialized) pathinit(); - /* Check for path debug env vars */ - if(pathdebug < 0) { - const char* s = getenv("NCPATHDEBUG"); - pathdebug = (s == NULL ? 0 : 1); + if(testurl(inpath)) { /* Pass thru URLs */ + if((tmp1 = strdup(inpath))==NULL) stat = NC_ENOMEM; + goto done; } - pathlen = strlen(path); + if((stat = parsepath(inpath,&canon))) {goto done;} - /* 1. look for MSYS path /D/... */ - if(pathlen >= 2 - && (path[0] == '/' || path[0] == '\\') - && strchr(windrive,path[1]) != NULL - && (path[2] == '/' || path[2] == '\\' || path[2] == '\0')) { - /* Assume this is a mingw path */ - outpath = (char*)malloc(pathlen+3); /* conservative */ - if(outpath == NULL) goto done; - q = outpath; - *q++ = path[1]; - *q++ = ':'; - strncpy(q,&path[2],pathlen); - if(strlen(outpath) == 2) - strcat(outpath,"/"); - goto slashtrans; - } + /* Special check for special cygwin paths: /tmp,usr etc */ + if(getlocalpathkind() == NCPD_CYGWIN + && iscygwinspecial(canon.path) + && canon.kind == NCPD_NIX) + canon.kind = NCPD_CYGWIN; - /* 2. Look for leading /cygdrive/D where D is a single-char drive letter */ - if(pathlen >= (cdlen+1) - && memcmp(path,"/cygdrive/",cdlen)==0 - && strchr(windrive,path[cdlen]) != NULL - && (path[cdlen+1] == '/' - || path[cdlen+1] == '\\' - || path[cdlen+1] == '\0')) { - /* Assume this is a cygwin path */ - outpath = (char*)malloc(pathlen+1); /* conservative */ - if(outpath == NULL) goto done; - outpath[0] = path[cdlen]; /* drive letter */ - outpath[1] = ':'; - strcpy(&outpath[2],&path[cdlen+1]); - if(strlen(outpath) == 2) - strcat(outpath,"/"); - goto slashtrans; + if(canon.kind != NCPD_REL && wdpath.kind != canon.kind) { + nclog(NCLOGWARN,"NCpathcvt: path mismatch: platform=%d inpath=%d\n", + wdpath.kind,canon.kind); + canon.kind = wdpath.kind; /* override */ } - /* 3. Look for leading D: where D is a single-char drive letter */ - /* This could be cygwin or Windows or mingw */ - if(pathlen >= 2 - && strchr(windrive,path[0]) != NULL - && path[1] == ':' - && (path[2] == '\0' || path[2] == '/' || path[2] == '\\')) { - outpath = strdup(path); - goto slashtrans; + if((stat = unparsepath(&canon,&tmp1))) {goto done;} +done: + if(pathdebug) { + fprintf(stderr,"xxx: inpath=|%s| outpath=|%s|\n", + inpath?inpath:"NULL",tmp1?tmp1:"NULL"); + fflush(stderr); } - - /* 4. Look for relative path */ - if(pathlen > 1 && path[0] == '.') { - outpath = makeabsolute(path); - goto slashtrans; + if(stat) { + nullfree(tmp1); tmp1 = NULL; + nclog(NCLOGERR,"NCpathcvt: stat=%d (%s)", + stat,nc_strerror(stat)); } + clearPath(&canon); + return tmp1; +} - /* Other: just pass thru */ - outpath = strdup(path); - goto done; - -slashtrans: - /* In order to help debugging, and if not using MSC_VER or MINGW or CYGWIN, - convert back slashes to forward, else convert forward to back - */ - if(!forwardslash) { - p = outpath; - /* In all #1 or #2 cases, translate '/' -> '\\' */ - for(;*p;p++) { - if(*p == '/') {*p = '\\';} - } - } -#ifdef PATHFORMAT - if(forwardslash) { - p = outpath; - /* Convert '\' back to '/' */ - for(;*p;p++) { - if(*p == '\\') {*p = '/';} - } +EXTERNL +char* /* caller frees */ +NCpathabsolute(const char* relpath) +{ + int stat = NC_NOERR; + struct Path canon = empty; + char* tmp1 = NULL; + char* result = NULL; + size_t len; + + if(relpath == NULL) goto done; /* defensive driving */ + + if(!pathinitialized) pathinit(); + + /* Canonicalize relpath */ + if((stat = parsepath(relpath,&canon))) {goto done;} + + /* See if relative */ + if(canon.kind == NCPD_REL) { + /* prepend the wd path to the inpath, including drive letter, if any */ + len = strlen(wdpath.path)+strlen(canon.path)+1+1; + if((tmp1 = (char*)malloc(len))==NULL) + {stat = NC_ENOMEM; {goto done;}} + tmp1[0] = '\0'; + strlcat(tmp1,wdpath.path,len); + strlcat(tmp1,"/",len); + strlcat(tmp1,canon.path,len); + nullfree(canon.path); + canon.path = tmp1; tmp1 = NULL; + canon.drive = wdpath.drive; + canon.kind = wdpath.kind; } -#endif /*PATHFORMAT*/ - + /* rebuild */ + if((stat=unparsepath(&canon,&result))) goto done; done: if(pathdebug) { - fprintf(stderr,"XXXX: inpath=|%s| outpath=|%s|\n", - path?path:"NULL",outpath?outpath:"NULL"); + fprintf(stderr,"xxx: relpath=|%s| result=|%s|\n", + relpath?relpath:"NULL",result?result:"NULL"); fflush(stderr); } - return outpath; + if(stat) { + nullfree(tmp1); tmp1 = NULL; + nclog(NCLOGERR,"NCpathcvt: stat=%d (%s)", + stat,nc_strerror(stat)); + } + clearPath(&canon); + nullfree(tmp1); + return result; } -/* Make path suitable for inclusion in url */ + +/* Testing support */ +/* Force drive and wd before invoking NCpathcvt + and then revert */ EXTERNL char* /* caller frees */ -NCurlpath(const char* path) +NCpathcvt_test(const char* inpath, int ukind, int udrive) { - char* upath = NCpathcvt(path); - char* p = upath; - for(;*p;p++) { - if(*p == '\\') {*p = '/';} - } - return upath; + char* result = NULL; + struct Path oldwd = wdpath; + + if(!pathinitialized) pathinit(); + /* Override */ + wdpath.kind = ukind; + wdpath.drive = udrive; + wdpath.path = strdup("/"); + if(pathdebug) + fprintf(stderr,"xxx: wd=|%s|",printPATH(&wdpath)); + result = NCpathcvt(inpath); + clearPath(&wdpath); + wdpath = oldwd; + return result; } -static char* -makeabsolute(const char* relpath) +static void +pathinit(void) { - char* path = NULL; -#ifdef _WIN32 - path = _fullpath(NULL,relpath,8192); -#else - path = realpath(relpath, NULL); -#endif - if(path == NULL) - path = strdup(relpath); - return path; + if(pathinitialized) return; + + /* Check for path debug env vars */ + if(pathdebug < 0) { + const char* s = getenv("NCPATHDEBUG"); + pathdebug = (s == NULL ? 0 : 1); + } + + (void)getwdpath(&wdpath); + /* make the path static but remember to never free it (Ugh!) */ + wdstaticpath[0] = '\0'; + strlcat(wdstaticpath,wdpath.path,sizeof(wdstaticpath)); + clearPath(&wdpath); + wdpath.path = wdstaticpath; + + pathinitialized = 1; } -/* Fix up a path in case extra escapes were added by shell */ -EXTERNL -char* -NCdeescape(const char* name) +static void +clearPath(struct Path* path) { - char* ename = NULL; - const char* p; - char* q; - - if(name == NULL) return NULL; - ename = strdup(name); - if(ename == NULL) return NULL; - for(p=name,q=ename;*p;) { - switch (*p) { - case '\0': break; - case '\\': - if(p[1] == '#') { - p++; - break; - } - /* fall thru */ - default: *q++ = *p++; break; - } - } - *q++ = '\0'; - return ename; + nullfree(path->path); + path->path = NULL; } +static const char* cygwinspecial[] = + {"/bin/","/dev/","/etc/","/home/", + "/lib/","/proc/","/sbin/","/tmp/", + "/usr/","/var/",NULL}; -int -NChasdriveletter(const char* path) +/* Unfortunately, not all cygwin paths start with /cygdrive. + So see if the path starts with one of the special paths. +*/ +static int +iscygwinspecial(const char* path) { - /* Check for windows drive letter */ - if(path == NULL || strlen(path) < 2) - return 0; - if(strchr(windrive,path[0]) != NULL && path[1] == ':') - return 1; /* windows path with drive letter */ + const char** p; + if(path == NULL) return 0; + for(p=cygwinspecial;*p;p++) { + if(strncmp(*p,path,strlen(*p))==0) return 1; + } return 0; } +/* return 1 if path looks like a url; 0 otherwise */ +static int +testurl(const char* path) +{ + int isurl = 0; + NCURI* tmpurl = NULL; + + if(path == NULL) return 0; + + /* Ok, try to parse as a url */ + ncuriparse(path,&tmpurl); + isurl = (tmpurl == NULL?0:1); + ncurifree(tmpurl); + return isurl; +} + #ifdef WINPATH /* -Provide wrappers for open and fopen. +Provide wrappers for Path-related functions */ EXTERNL FILE* NCfopen(const char* path, const char* flags) { + int stat = NC_NOERR; FILE* f = NULL; - char* cvtname = NCpathcvt(path); - if(cvtname == NULL) return NULL; - f = fopen(cvtname,flags); - free(cvtname); + char* cvtpath = NULL; + wchar_t* wpath = NULL; + wchar_t* wflags = NULL; + cvtpath = NCpathcvt(path); + if(cvtpath == NULL) return NULL; + /* Convert from local to wide */ + if((stat = utf82wide(cvtpath,&wpath))) goto done; + if((stat = ansi2wide(flags,&wflags))) goto done; + f = _wfopen(wpath,wflags); +done: + nullfree(cvtpath); + nullfree(wpath); + nullfree(wflags); return f; } @@ -262,11 +316,18 @@ EXTERNL int NCopen3(const char* path, int flags, int mode) { + int stat = NC_NOERR; int fd = -1; - char* cvtname = NCpathcvt(path); - if(cvtname == NULL) return -1; - fd = open(cvtname,flags,mode); - free(cvtname); + char* cvtpath = NULL; + wchar_t* wpath = NULL; + cvtpath = NCpathcvt(path); + if(cvtpath == NULL) goto done; + /* Convert from utf8 to wide */ + if((stat = utf82wide(cvtpath,&wpath))) goto done; + fd = _wopen(wpath,flags,mode); +done: + nullfree(cvtpath); + nullfree(wpath); return fd; } @@ -313,11 +374,17 @@ int NCaccess(const char* path, int mode) { int status = 0; - char* cvtname = NCpathcvt(path); - if(cvtname == NULL) return -1; - status = access(cvtname,mode); - free(cvtname); - return status; + char* cvtpath = NULL; + wchar_t* wpath = NULL; + if((cvtpath = NCpathcvt(path)) == NULL) + {status = EINVAL; goto done;} + if((status = utf82wide(cvtpath,&wpath))) {status = ENOENT; goto done;} + if(_waccess(wpath,mode) < 0) {status = errno; goto done;} +done: + free(cvtpath); + free(wpath); + errno = status; + return (errno?-1:0); } EXTERNL @@ -325,11 +392,16 @@ int NCremove(const char* path) { int status = 0; - char* cvtname = NCpathcvt(path); - if(cvtname == NULL) {errno = ENOENT; return -1;} - status = remove(cvtname); - free(cvtname); - return status; + char* cvtpath = NULL; + wchar_t* wpath = NULL; + if((cvtpath = NCpathcvt(path)) == NULL) {status=ENOMEM; goto done;} + if((status = utf82wide(cvtpath,&wpath))) {status = ENOENT; goto done;} + if(_wremove(wpath) < 0) {status = errno; goto done;} +done: + free(cvtpath); + free(wpath); + errno = status; + return (errno?-1:0); } EXTERNL @@ -337,11 +409,16 @@ int NCmkdir(const char* path, int mode) { int status = 0; - char* cvtname = NCpathcvt(path); - if(cvtname == NULL) {errno = ENOENT; return -1;} - status = mkdir(cvtname,mode); - free(cvtname); - return status; + char* cvtpath = NULL; + wchar_t* wpath = NULL; + if((cvtpath = NCpathcvt(path)) == NULL) {status=ENOMEM; goto done;} + if((status = utf82wide(cvtpath,&wpath))) {status = ENOENT; goto done;} + if(_wmkdir(wpath) < 0) {status = errno; goto done;} +done: + free(cvtpath); + free(wpath); + errno = status; + return (errno?-1:0); } EXTERNL @@ -358,21 +435,509 @@ NCrmdir(const char* path) EXTERNL char* -NCcwd(char* cwdbuf, size_t len) -{ - int ret = 0; - char* cvtname = NULL; - - if(cwdbuf == NULL || len == 0) {errno = ENAMETOOLONG; goto done;} - if(getcwd(cwdbuf,len) == NULL) {goto done;} - cvtname = NCpathcvt(cwdbuf); - if(cvtname == NULL || strlen(cvtname)+1 > len) - {errno = ENAMETOOLONG; goto done;} - cwdbuf[0] = '\0'; - strlcat(cwdbuf,cvtname,len); +NCgetcwd(char* cwdbuf, size_t cwdlen) +{ + int status = NC_NOERR; + struct Path wd = empty; + char* path = NULL; + size_t len; + + errno = 0; + if(cwdlen == 0) {status = ENAMETOOLONG; goto done;} + if(!pathinitialized) pathinit(); + if((status = getwdpath(&wd))) {status = ENOENT; goto done;} + if((status = unparsepath(&wd,&path))) {status = EINVAL; goto done;} + len = strlen(path); + if(len >= cwdlen) {status = ENAMETOOLONG; goto done;} + if(cwdbuf == NULL) { + if((cwdbuf = malloc(cwdlen))==NULL) {status = ENOMEM; goto done;} + } + memcpy(cwdbuf,path,len+1); done: - nullfree(cvtname); - if(errno) return NULL; + clearPath(&wd); + nullfree(path); + errno = status; return cwdbuf; } + +#ifdef HAVE_SYS_STAT_H +EXTERNL +int +NCstat(char* path, struct stat* buf) +{ + int status = 0; + char* cvtpath = NULL; + wchar_t* wpath = NULL; + if((cvtpath = NCpathcvt(path)) == NULL) {status=ENOMEM; goto done;} + if((status = utf82wide(cvtpath,&wpath))) {status = ENOENT; goto done;} + if(_wstat(wpath,buf) < 0) {status = errno; goto done;} +done: + free(cvtpath); + free(wpath); + errno = status; + return (errno?-1:0); +} +#endif /*HAVE_SYS_STAT_H*/ + +int +NCpath2utf8(const char* s, char** u8p) +{ + return ansi2utf8(s,u8p); +} + +#else /*!WINPATH*/ + +int +NCpath2utf8(const char* path, char** u8p) +{ + int stat = NC_NOERR; + char* u8 = NULL; + if(path != NULL) { + u8 = strdup(path); + if(u8 == NULL) {stat = NC_ENOMEM; goto done;} + } + if(u8p) {*u8p = u8; u8 = NULL;} +done: + return stat; +} +#endif /*!WINPATH*/ + +EXTERNL int +NChasdriveletter(const char* path) +{ + int stat = NC_NOERR; + int hasdl = 0; + struct Path canon = empty; + + if(!pathinitialized) pathinit(); + + if((stat = parsepath(path,&canon))) goto done; + if(canon.kind == NCPD_REL) { + clearPath(&canon); + /* Get the drive letter (if any) from the local wd */ + canon.drive = wdpath.drive; + } + hasdl = (canon.drive != 0); +done: + clearPath(&canon); + return hasdl; +} + +/**************************************************/ +/* Utilities */ + +/* Parse a path */ +static int +parsepath(const char* inpath, struct Path* path) +{ + int stat = NC_NOERR; + char* tmp1 = NULL; + size_t len; + char* p; + + assert(path); + memset(path,0,sizeof(struct Path)); + + if(inpath == NULL) goto done; /* defensive driving */ + + /* Convert to UTF8 */ +#if 0 + if((stat = NCpath2utf8(inpath,&tmp1))) goto done; +#else + tmp1 = strdup(inpath); +#endif + /* Convert to forward slash */ + for(p=tmp1;*p;p++) {if(*p == '\\') *p = '/';} + + /* parse all paths to 2-parts: + 1. drive letter (optional) + 2. path after drive letter + */ + + len = strlen(tmp1); + + /* 1. look for MSYS path /D/... */ + if(len >= 2 + && (tmp1[0] == '/') + && strchr(windrive,tmp1[1]) != NULL + && (tmp1[2] == '/' || tmp1[2] == '\0')) { + /* Assume this is a mingw path */ + path->drive = tmp1[1]; + /* Remainder */ + if(tmp1[2] == '\0') + path->path = NULL; + else + path->path = strdup(tmp1+2); + if(path == NULL) + {stat = NC_ENOMEM; goto done;} + path->kind = NCPD_MSYS; + } + /* 2. Look for leading /cygdrive/D where D is a single-char drive letter */ + else if(len >= (cdlen+1) + && memcmp(tmp1,"/cygdrive/",cdlen)==0 + && strchr(windrive,tmp1[cdlen]) != NULL + && (tmp1[cdlen+1] == '/' + || tmp1[cdlen+1] == '\0')) { + /* Assume this is a cygwin path */ + path->drive = tmp1[cdlen]; + /* Remainder */ + if(tmp1[cdlen+1] == '\0') + path->path = NULL; + else + path->path = strdup(tmp1+cdlen+1); + if(path == NULL) + {stat = NC_ENOMEM; goto done;} + path->kind = NCPD_CYGWIN; + } + /* 3. Look for windows path: D:/... where D is a single-char + drive letter */ + else if(len >= 2 + && strchr(windrive,tmp1[0]) != NULL + && tmp1[1] == ':' + && (tmp1[2] == '\0' || tmp1[2] == '/')) { + /* Assume this is a windows path */ + path->drive = tmp1[0]; + /* Remainder */ + if(tmp1[2] == '\0') + path->path = NULL; + else + path->path = strdup(tmp1+2); + if(path == NULL) + {stat = NC_ENOMEM; goto done;} + path->kind = NCPD_WIN; + } + /* look for *nix path */ + else if(len >= 1 && tmp1[0] == '/') { + /* Assume this is a *nix path */ + path->drive = 0; /* no drive letter */ + /* Remainder */ + path->path = tmp1; tmp1 = NULL; + path->kind = NCPD_NIX; + } else {/* Relative path of unknown type */ + path->kind = NCPD_REL; + path->path = tmp1; tmp1 = NULL; + } + +done: + nullfree(tmp1); + if(stat) {clearPath(path);} + return stat; +} + +static int +unparsepath(struct Path* xp, char** pathp) +{ + int stat = NC_NOERR; + size_t len; + char* path = NULL; + char sdrive[2] = {'\0','\0'}; + char* p = NULL; + int kind = xp->kind; + int cygspecial = 0; + + switch (kind) { + case NCPD_NIX: + len = nulllen(xp->path); + if(xp->drive != 0) { + len += 2; + sdrive[0] = xp->drive; + } + len++; /* nul terminate */ + if((path = (char*)malloc(len))==NULL) + {stat = NC_ENOMEM; goto done;} + path[0] = '\0'; + if(xp->drive != 0) { + strlcat(path,"/",len); + strlcat(path,sdrive,len); + } + if(xp->path != NULL) + strlcat(path,xp->path,len); + break; + case NCPD_CYGWIN: + /* Is this one of the special cygwin paths? */ + cygspecial = iscygwinspecial(xp->path); + if(xp->drive == 0) {xp->drive = wdpath.drive;} /*may require a drive */ + len = nulllen(xp->path)+cdlen+1+1; + if((path = (char*)malloc(len))==NULL) + {stat = NC_ENOMEM; goto done;} + path[0] = '\0'; + if(!cygspecial) { + strlcat(path,"/cygdrive/",len); + sdrive[0] = xp->drive; + strlcat(path,sdrive,len); + } + if(xp->path) + strlcat(path,xp->path,len); + break; + case NCPD_WIN: + if(xp->drive == 0) {xp->drive = wdpath.drive;} /*requires a drive */ + len = nulllen(xp->path)+2+1; + if((path = (char*)malloc(len))==NULL) + {stat = NC_ENOMEM; goto done;} + path[0] = '\0'; + sdrive[0] = xp->drive; + strlcat(path,sdrive,len); + strlcat(path,":",len); + if(xp->path) + strlcat(path,xp->path,len); + /* Convert forward to back */ + for(p=path;*p;p++) {if(*p == '/') *p = '\\';} + break; + case NCPD_MSYS: + if(xp->drive == 0) {xp->drive = wdpath.drive;} /*requires a drive */ + len = nulllen(xp->path)+2+1; + if((path = (char*)malloc(len))==NULL) + {stat = NC_ENOMEM; goto done;} + path[0] = '\0'; + sdrive[0] = xp->drive; + strlcat(path,"/",len); + strlcat(path,sdrive,len); + if(xp->path) + strlcat(path,xp->path,len); + break; + case NCPD_REL: + path = strdup(xp->path); + /* Use wdpath to decide slashing */ + if(wdpath.kind == NCPD_WIN) { + /* Convert forward to back */ + for(p=path;*p;p++) {if(*p == '/') *p = '\\';} + } + break; + default: stat = NC_EINTERNAL; goto done; + } + if(pathp) {*pathp = path; path = NULL;} +done: + nullfree(path); + return stat; +} + +static int +getwdpath(struct Path* wd) +{ + int stat = NC_NOERR; + char* path = NULL; + if(wd->path != NULL) return stat; + memset(wd,0,sizeof(struct Path)); + { +#ifdef _WIN32 + wchar_t* wpath = NULL; + wpath = _wgetcwd(NULL,8192); + if((stat = wide2utf8(wpath,&path))) + {nullfree(wpath); wpath = NULL; return stat;} +#else + path = getcwd(NULL,8192); +#endif + } + stat = parsepath(path,wd); + /* Force the kind */ + wd->kind = getlocalpathkind(); + nullfree(path); path = NULL; + return stat; +} + +static int +getlocalpathkind(void) +{ + int kind = NCPD_UNKNOWN; +#ifdef __CYGWIN__ + kind = NCPD_CYGWIN; +#elif __MSYS__ + kind = NCPD_MSYS; +#elif _MSC_VER /* not _WIN32 */ + kind = NCPD_WIN; +#else + kind = NCPD_NIX; +#endif + return kind; +} + +#ifdef WINPATH +/** + * Converts the filename from Locale character set (presumably some + * ANSI character set like ISO-Latin-1 or UTF-8 to UTF-8 + * @param local Pointer to a nul-terminated string in locale char set. + * @param u8p Pointer for returning the output utf8 string + * + * @return NC_NOERR return converted filename + * @return NC_EINVAL if conversion fails + * @return NC_ENOMEM if no memory available + * + */ +static int +ansi2utf8(const char* local, char** u8p) +{ + int stat=NC_NOERR; + char* u8 = NULL; + int n; + wchar_t* u16 = NULL; + + { + /* Get length of the converted string */ + n = MultiByteToWideChar(CP_ACP, 0, local, -1, NULL, 0); + if (!n) {stat = NC_EINVAL; goto done;} + if((u16 = malloc(sizeof(wchar_t) * n))==NULL) + {stat = NC_ENOMEM; goto done;} + /* do the conversion */ + if (!MultiByteToWideChar(CP_ACP, 0, local, -1, u16, n)) + {stat = NC_EINVAL; goto done;} + /* Now reverse the process to produce utf8 */ + n = WideCharToMultiByte(CP_UTF8, 0, u16, -1, NULL, 0, NULL, NULL); + if (!n) {stat = NC_EINVAL; goto done;} + if((u8 = malloc(sizeof(char) * n))==NULL) + {stat = NC_ENOMEM; goto done;} + if (!WideCharToMultiByte(CP_UTF8, 0, u16, -1, u8, n, NULL, NULL)) + {stat = NC_EINVAL; goto done;} + } + if(u8p) {*u8p = u8; u8 = NULL;} +done: + nullfree(u8); + return stat; +} + +static int +ansi2wide(const char* local, wchar_t** u16p) +{ + int stat=NC_NOERR; + wchar_t* u16 = NULL; + int n; + + /* Get length of the converted string */ + n = MultiByteToWideChar(CP_ACP, 0, local, -1, NULL, 0); + if (!n) {stat = NC_EINVAL; goto done;} + if((u16 = malloc(sizeof(wchar_t) * n))==NULL) + {stat = NC_ENOMEM; goto done;} + /* do the conversion */ + if (!MultiByteToWideChar(CP_ACP, 0, local, -1, u16, n)) + {stat = NC_EINVAL; goto done;} + if(u16p) {*u16p = u16; u16 = NULL;} +done: + nullfree(u16); + return stat; +} + +static int +utf82wide(const char* utf8, wchar_t** u16p) +{ + int stat=NC_NOERR; + wchar_t* u16 = NULL; + int n; + + /* Get length of the converted string */ + n = MultiByteToWideChar(CP_UTF8, 0, utf8, -1, NULL, 0); + if (!n) {stat = NC_EINVAL; goto done;} + if((u16 = malloc(sizeof(wchar_t) * n))==NULL) + {stat = NC_ENOMEM; goto done;} + /* do the conversion */ + if (!MultiByteToWideChar(CP_UTF8, 0, utf8, -1, u16, n)) + {stat = NC_EINVAL; goto done;} + if(u16p) {*u16p = u16; u16 = NULL;} +done: + nullfree(u16); + return stat; +} + +static int +wide2utf8(const wchar_t* u16, char** u8p) +{ + int stat=NC_NOERR; + char* u8 = NULL; + int n; + + /* Get length of the converted string */ + n = WideCharToMultiByte(CP_UTF8, 0, u16, -1, NULL, 0, NULL, NULL); + if (!n) {stat = NC_EINVAL; goto done;} + if((u8 = malloc(sizeof(char) * n))==NULL) + {stat = NC_ENOMEM; goto done;} + /* do the conversion */ + if (!WideCharToMultiByte(CP_UTF8, 0, u16, -1, u8, n, NULL, NULL)) + {stat = NC_EINVAL; goto done;} + if(u8p) {*u8p = u8; u8 = NULL;} +done: + nullfree(u8); + return stat; +} + +/* Set locale */ +void +nc_setlocale_utf8(void) +{ +#ifdef __hpux + setlocale(LC_CTYPE,""); +#else + setlocale(LC_ALL,"C.UTF8"); +#endif +} + #endif /*WINPATH*/ + +static char* +printPATH(struct Path* p) +{ + static char buf[4096]; + buf[0] = '\0'; + snprintf(buf,sizeof(buf),"Path{kind=%d drive='%c' path=|%s|}", + p->kind,(p->drive > 0?p->drive:'0'),p->path); + return buf; +} + +#if 0 +static int +hexfor(int c) +{ + if(c >= '0' && c <= '9') return c - '0'; + if(c >= 'a' && c <= 'f') return (c - 'a')+10; + if(c >= 'A' && c <= 'F') return (c - 'A')+10; + return -1; +} +#endif + +static char hexdigit[] = "0123456789abcdef"; + +EXTERNL void +printutf8hex(const char* s, char* sx) +{ + const char* p; + char* q; + for(q=sx,p=s;*p;p++) { + unsigned int c = (unsigned char)*p; + if(c >= ' ' && c <= 127) + *q++ = (char)c; + else { + *q++ = '\\'; + *q++ = 'x'; + *q++ = hexdigit[(c>>4)&0xf]; + *q++ = hexdigit[(c)&0xf]; + } + } + *q = '\0'; +} + +/**************************************************/ +#if 0 +#ifdef HAVE_DIRENT_H +EXTERNL +DIR* +NCopendir(const char* path) +{ + DIR* ent = NULL; + char* cvtpath = NCpathcvt(path); + if(cvtpath == NULL) return -1; + ent = opendir(cvtpath); + free(cvtpath); + return ent; +} + +EXTERNL +int +NCclosedir(DIR* ent) +{ + int stat = 0; + char* cvtpath = NCpathcvt(path); + if(cvtpath == NULL) return -1; + stat = closedir(cvtpath); + free(cvtpath); + return stat; +} +#endif +#endif /*0*/ + diff --git a/libdispatch/drc.c b/libdispatch/drc.c index 4e7646edeb..0283e237a0 100644 --- a/libdispatch/drc.c +++ b/libdispatch/drc.c @@ -433,7 +433,7 @@ rcsearch(const char* prefix, const char* rcname, char** pathp) strncat(path,"/",pathlen); strncat(path,rcname,pathlen); /* see if file is readable */ - f = fopen(path,"r"); + f = NCfopen(path,"r"); if(f != NULL) nclog(NCLOGDBG, "Found rc file=%s",path); done: diff --git a/libdispatch/dutil.c b/libdispatch/dutil.c index 26cd386870..d3163bddfd 100644 --- a/libdispatch/dutil.c +++ b/libdispatch/dutil.c @@ -182,7 +182,8 @@ NC_mktmp(const char* base) mode_t mask; #endif - /* Make sure that this path conversion has been applied */ + /* Make sure that this path conversion has been applied + since we do not wrap mkstemp */ cvtpath = NCpathcvt(base); strncpy(tmp,cvtpath,sizeof(tmp)); nullfree(cvtpath); @@ -214,7 +215,7 @@ NC_mktmp(const char* base) strncat(tmp,spid,sizeof(tmp) - strlen(tmp) - 1); } #endif /* HAVE_MKTEMP */ -#ifdef _MSC_VER +#ifdef _WIN32 fd=NCopen3(tmp,O_RDWR|O_BINARY|O_CREAT, _S_IREAD|_S_IWRITE); #else fd=NCopen3(tmp,O_RDWR|O_CREAT|O_EXCL, S_IRWXU); diff --git a/libdispatch/ncuri.c b/libdispatch/ncuri.c index 6541423836..1a20447019 100644 --- a/libdispatch/ncuri.c +++ b/libdispatch/ncuri.c @@ -485,6 +485,15 @@ ncurisetprotocol(NCURI* duri,const char* protocol) return (NC_NOERR); } +/* Replace the path */ +int +ncurisetpath(NCURI* duri,const char* newpath) +{ + nullfree(duri->path); + duri->path = strdup(newpath); + return (NC_NOERR); +} + /* Replace the query */ int ncurisetquery(NCURI* duri,const char* query) diff --git a/libhdf5/hdf5create.c b/libhdf5/hdf5create.c index 27a99405c9..f043193610 100644 --- a/libhdf5/hdf5create.c +++ b/libhdf5/hdf5create.c @@ -9,6 +9,9 @@ */ #include "config.h" +#include "netcdf.h" +#include "ncpathmgr.h" +#include "ncpathmgr.h" #include "hdf5internal.h" /* From hdf5file.c. */ @@ -22,12 +25,6 @@ static const int ILLEGAL_CREATE_FLAGS = (NC_NOWRITE|NC_MMAP|NC_64BIT_OFFSET|NC_C /* From nc4mem.c */ extern int NC4_create_image_file(NC_FILE_INFO_T* h5, size_t); -#ifdef _WIN32 -static hid_t nc4_H5Fcreate(const char *filename, unsigned flags, hid_t fcpl_id, hid_t fapl_id); -#else -#define nc4_H5Fcreate H5Fcreate -#endif - /** * @internal Create a netCDF-4/HDF5 file. * @@ -114,7 +111,7 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, /* If this file already exists, and NC_NOCLOBBER is specified, return an error (unless diskless|inmemory) */ if (!nc4_info->mem.diskless && !nc4_info->mem.inmemory) { - if ((cmode & NC_NOCLOBBER) && (fp = fopen(path, "r"))) { + if ((cmode & NC_NOCLOBBER) && (fp = NCfopen(path, "r"))) { fclose(fp); BAIL(NC_EEXIST); } @@ -313,8 +310,6 @@ NC4_create(const char* path, int cmode, size_t initialsz, int basepe, return res; } -#ifdef _WIN32 - /** * Wrapper function for H5Fcreate. * Converts the filename from ANSI to UTF-8 as needed before calling H5Fcreate. @@ -325,18 +320,24 @@ NC4_create(const char* path, int cmode, size_t initialsz, int basepe, * @param fapl_id File access property list identifier. * @return A file identifier if succeeded. A negative value if failed. */ -static hid_t -nc4_H5Fcreate(const char *filename, unsigned flags, hid_t fcpl_id, hid_t fapl_id) +hid_t +nc4_H5Fcreate(const char *filename0, unsigned flags, hid_t fcpl_id, hid_t fapl_id) { - pathbuf_t pb; hid_t hid; + char* localname = NULL; + char* filename = NULL; - filename = nc4_ndf5_ansi_to_utf8(&pb, filename); - if (!filename) - return H5I_INVALID_HID; - hid = H5Fcreate(filename, flags, fcpl_id, fapl_id); - nc4_hdf5_free_pathbuf(&pb); +#ifdef HDF5_UTF8_PATHS + NCpath2utf8(filename0,&filename); +#else + filename = strdup(filename0); +#endif + /* Canonicalize it since we are not opening the file ourselves */ + if((localname = NCpathcvt(filename))==NULL) + {hid = H5I_INVALID_HID; goto done;} + hid = H5Fcreate(localname, flags, fcpl_id, fapl_id); +done: + nullfree(filename); + nullfree(localname); return hid; } - -#endif /* _WIN32 */ diff --git a/libhdf5/hdf5internal.c b/libhdf5/hdf5internal.c index 15cc0e335d..7817bed6e1 100644 --- a/libhdf5/hdf5internal.c +++ b/libhdf5/hdf5internal.c @@ -1026,7 +1026,7 @@ nc_log_hdf5(void) #endif /* LOGGING */ - +#if 0 #ifdef _WIN32 /** @@ -1112,3 +1112,4 @@ nc4_hdf5_free_pathbuf(pathbuf_t *pb) } #endif /* _WIN32 */ +#endif /*0*/ diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 0b68e96786..846d09652a 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -15,6 +15,8 @@ #include "ncrc.h" #include "ncauth.h" #include "ncmodel.h" +#include "ncfilter.h" +#include "ncpathmgr.h" #ifdef ENABLE_BYTERANGE #include "H5FDhttp.h" @@ -67,15 +69,11 @@ extern int NC4_open_image_file(NC_FILE_INFO_T* h5); /* Defined later in this file. */ static int rec_read_metadata(NC_GRP_INFO_T *grp); -#ifdef _WIN32 -static hid_t nc4_H5Fopen(const char *filename, unsigned flags, hid_t fapl_id); -#else -#define nc4_H5Fopen H5Fopen -#endif - +#ifdef ENABLE_BYTERANGE #ifdef ENABLE_HDF5_ROS3 static int ros3info(NCauth** auth, NCURI* uri, char** hostportp, char** regionp); #endif +#endif /** * @internal Struct to track HDF5 object info, for @@ -2759,6 +2757,7 @@ rec_read_metadata(NC_GRP_INFO_T *grp) return retval; } +#ifdef ENABLE_BYTERANGE #ifdef ENABLE_HDF5_ROS3 static int ros3info(NCauth** authp, NCURI* uri, char** hostportp, char** regionp) @@ -2806,8 +2805,7 @@ ros3info(NCauth** authp, NCURI* uri, char** hostportp, char** regionp) return stat; } #endif /*ENABLE_HDF5_ROS3*/ - -#ifdef _WIN32 +#endif /*ENABLE_BYTERANGE*/ /** * Wrapper function for H5Fopen. @@ -2818,18 +2816,23 @@ ros3info(NCauth** authp, NCURI* uri, char** hostportp, char** regionp) * @param fapl_id File access property list identifier. * @return A file identifier if succeeded. A negative value if failed. */ -static hid_t -nc4_H5Fopen(const char *filename, unsigned flags, hid_t fapl_id) +hid_t +nc4_H5Fopen(const char *filename0, unsigned flags, hid_t fapl_id) { - pathbuf_t pb; hid_t hid; + char* localname = NULL; + char* filename = NULL; - filename = nc4_ndf5_ansi_to_utf8(&pb, filename); - if (!filename) - return H5I_INVALID_HID; - hid = H5Fopen(filename, flags, fapl_id); - nc4_hdf5_free_pathbuf(&pb); +#ifdef HDF5_UTF8_PATHS + NCpath2utf8(filename0,&filename); +#else + filename = strdup(filename0); +#endif + if((localname = NCpathcvt(filename))==NULL) + {hid = H5I_INVALID_HID; goto done;} + hid = H5Fopen(localname, flags, fapl_id); +done: + nullfree(filename); + nullfree(localname); return hid; } - -#endif /* _WIN32 */ diff --git a/libhdf5/nc4memcb.c b/libhdf5/nc4memcb.c index 0c70bfbcdb..3228d2bceb 100644 --- a/libhdf5/nc4memcb.c +++ b/libhdf5/nc4memcb.c @@ -83,6 +83,7 @@ #include #include "nc4internal.h" +#include "hdf5internal.h" #ifndef HDrealloc #define HDrealloc(x,y) realloc(x,y) @@ -806,10 +807,10 @@ NC4_image_init(NC_FILE_INFO_T* h5) /* Assign file image in FAPL to the core file driver */ if(create) { - if ((file_id = H5Fcreate(file_name, file_open_flags, H5P_DEFAULT, fapl)) < 0) + if ((file_id = nc4_H5Fcreate(file_name, file_open_flags, H5P_DEFAULT, fapl)) < 0) goto out; } else { - if ((file_id = H5Fopen(file_name, file_open_flags, fapl)) < 0) + if ((file_id = nc4_H5Fopen(file_name, file_open_flags, fapl)) < 0) goto out; } diff --git a/libnczarr/zmap.h b/libnczarr/zmap.h index c75e041755..63040c297e 100644 --- a/libnczarr/zmap.h +++ b/libnczarr/zmap.h @@ -356,7 +356,6 @@ EXTERNL void nczm_sortlist(struct NClist* l); EXTERNL void nczm_sortenvv(int n, char** envv); EXTERNL void NCZ_freeenvv(int n, char** envv); - #ifdef __cplusplus } #endif diff --git a/libsrc/ncFile.c b/libsrc/ncFile.c index 92c1500033..e448d32e94 100644 --- a/libsrc/ncFile.c +++ b/libsrc/ncFile.c @@ -7,6 +7,8 @@ #include #include +#include "ncpathmgr.h" + /* Implement the ncstdio.h interface using unix stdio.h */ @@ -84,9 +86,9 @@ ncFile_open(const char *path, int ioflags, ncstdio** filepp) File* f; if(fIsSet(ioflags,NC_NOCLOBBER)) - f = fopen(path,"r"); + f = NCfopen(path,"r"); else - f = fopen(path,"w+"); + f = NCfopen(path,"w+"); if(f == NULL) return errno; diff --git a/libsrc/posixio.c b/libsrc/posixio.c index bb68ea91e7..0e1c185003 100644 --- a/libsrc/posixio.c +++ b/libsrc/posixio.c @@ -47,6 +47,7 @@ #define SEEK_END 2 #endif +#include "ncpathmgr.h" #include "ncio.h" #include "fbits.h" #include "rnd.h" @@ -1622,10 +1623,10 @@ posixio_create(const char *path, int ioflags, fSet(oflags, O_BINARY); #endif #ifdef vms - fd = open(path, oflags, NC_DEFAULT_CREAT_MODE, "ctx=stm"); + fd = NCopen3(path, oflags, NC_DEFAULT_CREAT_MODE, "ctx=stm"); #else /* Should we mess with the mode based on NC_SHARE ?? */ - fd = open(path, oflags, NC_DEFAULT_CREAT_MODE); + fd = NCopen3(path, oflags, NC_DEFAULT_CREAT_MODE); #endif #if 0 (void) fprintf(stderr, "ncio_create(): path=\"%s\"\n", path); @@ -1760,9 +1761,9 @@ posixio_open(const char *path, #endif #ifdef vms - fd = open(path, oflags, 0, "ctx=stm"); + fd = NCopen3(path, oflags, 0, "ctx=stm"); #else - fd = open(path, oflags, 0); + fd = NCopen3(path, oflags, 0); #endif if(fd < 0) { diff --git a/libsrc/winceio.c b/libsrc/winceio.c index f30daba420..5c9652eec3 100644 --- a/libsrc/winceio.c +++ b/libsrc/winceio.c @@ -31,6 +31,7 @@ #include #include +#include "ncpathmgr.h" #include "ncio.h" #include "fbits.h" #include "rnd.h" @@ -508,9 +509,9 @@ ncio_create(const char *path, int ioflags, /* Since we do not have use of the O_EXCL flag, we need to fake it */ #ifdef WINCE - f = fopen(path,"rb"); + f = NCfopen(path,"rb"); #else - f = fopen(path,"r"); + f = NCfopen(path,"r"); #endif if(f != NULL) { /* do not overwrite */ (void)fclose(f); @@ -518,7 +519,7 @@ ncio_create(const char *path, int ioflags, } } - f = fopen(path, oflags); + f = NCfopen(path, oflags); if(f == NULL) { status = errno; @@ -602,7 +603,7 @@ ncio_open(const char *path, if(nciop == NULL) return ENOMEM; - f = fopen(path, oflags); + f = NCfopen(path, oflags); if(f == NULL) { diff --git a/nc_test/CMakeLists.txt b/nc_test/CMakeLists.txt index 0a6a0bbd27..02a9a9d33a 100644 --- a/nc_test/CMakeLists.txt +++ b/nc_test/CMakeLists.txt @@ -36,7 +36,11 @@ TARGET_LINK_LIBRARIES(nc_test ) # Some extra stand-alone tests -SET(TESTS t_nc tst_small tst_misc tst_norm tst_names tst_nofill tst_nofill2 tst_nofill3 tst_meta tst_inq_type tst_utf8_validate tst_utf8_phrases tst_global_fillval tst_max_var_dims tst_formats tst_def_var_fill tst_err_enddef tst_default_format) +SET(TESTS t_nc tst_small tst_misc tst_norm tst_names tst_nofill tst_nofill2 tst_nofill3 tst_meta tst_inq_type tst_utf8_phrases tst_global_fillval tst_max_var_dims tst_formats tst_def_var_fill tst_err_enddef tst_default_format) + +IF(NOT MSVC) +SET(TESTS ${TESTS} tst_utf8_validate) +ENDIF() IF(NOT HAVE_BASH) SET(TESTS ${TESTS} tst_atts3) diff --git a/nc_test/Makefile.am b/nc_test/Makefile.am index 3e2d33ea72..7885411330 100644 --- a/nc_test/Makefile.am +++ b/nc_test/Makefile.am @@ -8,7 +8,9 @@ # Un comment to use a more verbose test driver #SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose #LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#TESTS_ENVIRONMENT = export SETX=1; # Put together AM_CPPFLAGS and AM_LDFLAGS. include $(top_srcdir)/lib_flags.am diff --git a/ncdump/CMakeLists.txt b/ncdump/CMakeLists.txt index e71031341a..828aa18d25 100644 --- a/ncdump/CMakeLists.txt +++ b/ncdump/CMakeLists.txt @@ -94,7 +94,6 @@ IF(MSVC) ENDIF() - IF(ENABLE_TESTS) ADD_EXECUTABLE(rewrite-scalar rewrite-scalar.c) ADD_EXECUTABLE(bom bom.c) @@ -146,8 +145,8 @@ IF(ENABLE_TESTS) SET_TARGET_PROPERTIES(tst_fileinfo PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR}) - ENDIF(USE_HDF5) -ENDIF(MSVC) + ENDIF(USE_HDF5) + ENDIF(MSVC) # Base tests # The tests are set up as a combination of shell scripts and executables that @@ -169,7 +168,6 @@ ENDIF(MSVC) add_sh_test(ncdump tst_fileinfo) add_sh_test(ncdump tst_hdf5_offset) ENDIF(USE_HDF5) - add_sh_test(ncdump test_unicode_directory) add_sh_test(ncdump tst_null_byte_padding) IF(USE_STRICT_NULL_BYTE_HEADER_PADDING) @@ -240,8 +238,6 @@ ENDIF(MSVC) build_bin_test_no_prefix(tst_h_rdc0) ENDIF() - build_bin_test_no_prefix(tst_unicode) - build_bin_test_no_prefix(tst_fillbug) add_sh_test(ncdump_sh tst_fillbug) @@ -277,6 +273,8 @@ ENDIF(MSVC) add_sh_test(ncdump tst_ncgen4) ENDIF(USE_HDF5) + ENDIF(USE_HDF5) + add_sh_test(ncdump tst_inttags) IF(USE_HDF5) add_sh_test(ncdump tst_inttags4) @@ -291,13 +289,23 @@ ENDIF(MSVC) PROPERTIES ENVIRONMENT NC_VLEN_NOTEST=1) ENDIF() + # The unicode tests are complicated + IF(USE_HDF5) + IF(HAVE_BASH) + build_bin_test_no_prefix(tst_unicode) + IF(NOT MSVC) + # These tests do not work under windows + add_sh_test(ncdump test_unicode_directory) + add_sh_test(ncdump test_unicode_path) + ENDIF(NOT MSVC) + ENDIF(HAVE_BASH) + ENDIF(USE_HDF5) + IF(USE_CDF5) add_sh_test(ncdump test_keywords) ENDIF() -ENDIF() - -ENDIF() +ENDIF(ENABLE_TESTS) #IF(MSVC) # SET_TARGET_PROPERTIES(ncdump diff --git a/ncdump/Makefile.am b/ncdump/Makefile.am index bdcb053d24..996c16e9bd 100644 --- a/ncdump/Makefile.am +++ b/ncdump/Makefile.am @@ -162,7 +162,8 @@ ref_provenance_v1.nc ref_tst_radix.cdl tst_radix.cdl test_radix.sh \ ref_nccopy_w.cdl tst_nccopy_w3.sh tst_nccopy_w4.sh \ ref_no_ncproperty.nc test_unicode_directory.sh \ ref_roman_szip_simple.cdl ref_roman_szip_unlim.cdl ref_tst_perdimspecs.cdl \ -test_keywords.sh ref_keyword1.cdl ref_keyword2.cdl ref_tst_nofilters.cdl +test_keywords.sh ref_keyword1.cdl ref_keyword2.cdl ref_tst_nofilters.cdl \ +test_unicode_path.sh # The L512.bin file is file containing exactly 512 bytes each of value 0. # It is used for creating hdf5 files with varying offsets for testing. diff --git a/ncdump/nc4print.c b/ncdump/nc4print.c index 0dab9cd9bb..b33024e975 100644 --- a/ncdump/nc4print.c +++ b/ncdump/nc4print.c @@ -16,6 +16,7 @@ create other tools. #include #include "netcdf.h" #include "ncbytes.h" +#include "ncpathmgr.h" extern int NC4print(NCbytes* buf, int ncid); diff --git a/ncdump/nccopy.c b/ncdump/nccopy.c index 4d909772f6..3fcf9b837a 100644 --- a/ncdump/nccopy.c +++ b/ncdump/nccopy.c @@ -32,6 +32,7 @@ #include "nccomps.h" #include "list.h" #include "ncpathmgr.h" +#include "ncfilter.h" #undef DEBUGFILTER #undef DEBUGCHUNK @@ -2410,10 +2411,9 @@ main(int argc, char**argv) if (argc != 2) { error("one input file and one output file required"); } - /* Escape the file names in case bash escaped them */ - inputfile = NCdeescape(argv[0]); - outputfile = NCdeescape(argv[1]); - + /* Canonicalize the input and output files names */ + inputfile = NC_backslashUnescape(argv[0]); /* Remove shell added escapes */ + outputfile = NC_backslashUnescape(argv[1]); if(strcmp(inputfile, outputfile) == 0) { error("output would overwrite input"); } diff --git a/ncdump/ncdump.c b/ncdump/ncdump.c index df640ba492..d333cc37b7 100644 --- a/ncdump/ncdump.c +++ b/ncdump/ncdump.c @@ -25,9 +25,6 @@ Research/Unidata. See \ref copyright file for more info. */ #include #include #include -#ifdef HAVE_LOCALE_H -#include -#endif /* HAVE_LOCALE_H */ #include "netcdf.h" #include "netcdf_mem.h" @@ -47,6 +44,7 @@ Research/Unidata. See \ref copyright file for more info. */ #include "nclist.h" #include "ncuri.h" #include "nc_provenance.h" +#include "ncpathmgr.h" #ifdef USE_NETCDF4 #include "nc4internal.h" /* to get name of the special properties file */ @@ -145,39 +143,35 @@ usage(void) static char * name_path(const char *path) { - const char *cp; - char *new; - char *sp; + char* cvtpath = NULL; + const char *cp = NULL; + char *sp = NULL; + size_t cplen = 0; + char* base = NULL; -#ifdef vms -#define FILE_DELIMITER ']' -#endif -#if defined(_WIN32) || defined(msdos) -#define FILE_DELIMITER '\\' -#endif -#ifndef FILE_DELIMITER /* default to unix */ -#define FILE_DELIMITER '/' -#endif + if((cvtpath = NCpathcvt(path))==NULL) + return NULL; /* See if this is a url */ - { - char* base; - if(nc__testurl(path,&base)) { - return base; /* Looks like a url */ - } - /* else fall thru and treat like a file path */ - } + if(nc__testurl(cvtpath,&base)) + goto done; /* Looks like a url */ + /* else fall thru and treat like a file path */ - cp = strrchr(path, FILE_DELIMITER); - if (cp == 0) /* no delimiter */ - cp = path; + cp = strrchr(cvtpath, '/'); + if (cp == NULL) /* no delimiter */ + cp = cvtpath; else /* skip delimiter */ cp++; - new = (char *) emalloc((unsigned) (strlen(cp)+1)); - (void) strncpy(new, cp, strlen(cp) + 1); /* copy last component of path */ - if ((sp = strrchr(new, '.')) != NULL) + cplen = strlen(cp); + base = (char *) emalloc((unsigned) (cplen+1)); + base[0] = '\0'; + strlcat(base,cp,cplen+1); + if ((sp = strrchr(base, '.')) != NULL) *sp = '\0'; /* strip off any extension */ - return new; + +done: + nullfree(cvtpath); + return base; } /* Return primitive type name */ @@ -333,7 +327,7 @@ fileopen(const char* path, void** memp, size_t* sizep) #endif oflags |= O_EXCL; #ifdef vms - fd = open(path, oflags, 0, "ctx=stm"); + fd = NCopen3(path, oflags, 0, "ctx=stm"); #else fd = NCopen2(path, oflags); #endif @@ -2182,9 +2176,6 @@ main(int argc, char *argv[]) putenv("PRINTF_EXPONENT_DIGITS=2"); /* Enforce unix/linux style exponent formatting. */ #endif -#ifdef HAVE_LOCALE_H - setlocale(LC_ALL, "C"); /* CDL may be ambiguous with other locales */ -#endif /* HAVE_LOCALE_H */ progname = argv[0]; set_formats(FLT_DIGITS, DBL_DIGITS); /* default for float, double data */ @@ -2342,10 +2333,11 @@ main(int argc, char *argv[]) init_epsilons(); - /* Deescape the user name */ - path = NCdeescape(argv[i]); + /* We need to look for escape characters because the argument + may have come in via a shell script */ + path = NC_backslashUnescape(argv[i]); if(path == NULL) { - snprintf(errmsg,sizeof(errmsg),"out of memory copying argument %s", argv[i]); + snprintf(errmsg,sizeof(errmsg),"out of memory un-escaping argument %s", argv[i]); goto fail; } diff --git a/ncdump/ocprint.c b/ncdump/ocprint.c index f386b8db13..c005a13ac4 100755 --- a/ncdump/ocprint.c +++ b/ncdump/ocprint.c @@ -26,6 +26,7 @@ #include "ncuri.h" #include "ncbytes.h" #include "nclog.h" +#include "ncpathmgr.h" #include "oc.h" #include "ocx.h" diff --git a/ncdump/test_unicode_directory.sh b/ncdump/test_unicode_directory.sh index d5b5966d1b..f83d8f6fbf 100755 --- a/ncdump/test_unicode_directory.sh +++ b/ncdump/test_unicode_directory.sh @@ -8,7 +8,6 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh - ERR() { RES=$? if [ $RES -ne 0 ]; then @@ -17,21 +16,28 @@ ERR() { fi } -LC_ALL="C.UTF-8" -export LC_ALL - -#UNISTRING='海' -UNISTRING=$(echo '\xe6\xb5\xb7') +#UNISTRING=$(echo '\xe6\xb5\xb7') +UNISTRING='海' echo "" echo "Creating Unicode String Directory ${UNISTRING}" -mkdir -p ${UNISTRING}; ERR +mkdir -p "${UNISTRING}"; ERR +ls -ld "${UNISTRING}" + +# Do test for netcdf-3 and (optionally) netcdf-4 -echo "*** Generating binary file ${UNISTRING}/tst_utf.nc..." +echo "*** Generating netcdf-3 binary file ${UNISTRING}/tst_utf.nc..." ${NCGEN} -b -o "${UNISTRING}/tst_utf.nc" "${srcdir}/ref_tst_utf8.cdl"; ERR echo "*** Accessing binary file ${UNISTRING}/tst_utf.nc..." ${NCDUMP} -h "${UNISTRING}/tst_utf.nc"; ERR +if test "x$FEATURE_HDF5" = xyes ; then +echo "*** Generating netcdf-4 binary file ${UNISTRING}/tst_utf.nc..." +rm -f "${UNISTRING}/tst_utf.nc" +${NCGEN} -4 -b -o "${UNISTRING}/tst_utf.nc" "${srcdir}/ref_tst_utf8.cdl"; ERR +echo "*** Accessing binary file ${UNISTRING}/tst_utf.nc..." +${NCDUMP} -h "${UNISTRING}/tst_utf.nc"; ERR +fi + echo "Test Passed. Cleaning up." -rm "${UNISTRING}/tst_utf.nc"; ERR -rmdir "${UNISTRING}"; ERR +rm -fr "${UNISTRING}"; ERR diff --git a/ncdump/test_unicode_path.sh b/ncdump/test_unicode_path.sh new file mode 100755 index 0000000000..b9e08b3048 --- /dev/null +++ b/ncdump/test_unicode_path.sh @@ -0,0 +1,53 @@ +#!/bin/sh +# +# Test to make sure ncdump works with a subdirectory which starts +# with a unicode character. +# See https://github.com/Unidata/netcdf-c/issues/1666 for more information. +# Ward Fisher + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +set -e + +LC_ALL="C.UTF-8" +export LC_ALL + +# Passing a utf8 name using either \x or actual characters +# to Visual Studio does not work well. +if test "x$FP_ISMSVC" = x ; then +#UNISTRING='\xe6\xb5\xb7' +UNISTRING='海' +else +UNISTRING='海' +fi + +UNIFILE="tst_utf8_${UNISTRING}" + +echo "" + +echo "*** Generating netcdf-3 binary file ${UNIFILE}.nc ..." +${NCGEN} -3 -b -o "${UNIFILE}.nc" "${srcdir}/ref_tst_utf8.cdl" +echo "*** Accessing binary file ${UNIFILE}.nc ..." +${NCDUMP} -h "${UNIFILE}.nc" + +if test "x$FEATURE_HDF5" = xyes ; then +echo "*** Generating netcdf-4 binary file ${UNIFILE}.nc ..." +rm -f "${UNIFILE}.nc" +${NCGEN} -4 -b -o "${UNIFILE}.nc" "${srcdir}/ref_tst_utf8.cdl" +echo "*** Accessing binary file ${UNIFILE}.nc ..." +${NCDUMP} -h "${UNIFILE}.nc" +fi + +# This test was moved here from tst_nccopy4.sh +# to unify all the unicode path tests +echo "*** Test nccopy ${UNIFILe} copy_of_${UNIFILE} ..." +${NCCOPY} ${UNIFILE}.nc copy_of_${UNIFILE}.nc +${NCDUMP} -n copy_of_${UNIFILE} ${UNIFILE}.nc > tmp_${UNIFILE}.cdl +${NCDUMP} copy_of_${UNIFILE}.nc > copy_of_${UNIFILE}.cdl +echo "*** compare " with copy_of_${UNIFILE}.cdl +diff copy_of_${UNIFILE}.cdl tmp_${UNIFILE}.cdl +rm copy_of_${UNIFILE}.nc copy_of_${UNIFILE}.cdl tmp_${UNIFILE}.cdl + +echo "Test Passed. Cleaning up." +rm ${UNIFILE}.nc diff --git a/ncdump/tst_nccopy4.sh b/ncdump/tst_nccopy4.sh index 92e2801da9..ac81cd0f25 100755 --- a/ncdump/tst_nccopy4.sh +++ b/ncdump/tst_nccopy4.sh @@ -7,7 +7,6 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi if test -f tst_comp2${ext} ; then ${execdir}/tst_comp2 ; fi - set -e echo "" @@ -16,22 +15,27 @@ echo "" # ref_tst_compounds2 ref_tst_compounds3 ref_tst_compounds4 TESTFILES='tst_comp tst_comp2 tst_enum_data tst_fillbug tst_group_data tst_nans tst_opaque_data tst_solar_1 tst_solar_2 - tst_solar_cmp tst_special_atts tst_string_data tst_unicode' + tst_solar_cmp tst_special_atts tst_string_data' if test "x$NC_VLEN_NOTEST" = x ; then -TESFILES="$TESTFILES tst_vlen_data" +TESTFILES="$TESTFILES tst_vlen_data" fi echo "*** Testing netCDF-4 features of nccopy on ncdump/*.nc files" for i in $TESTFILES ; do echo "*** Test nccopy $i.nc copy_of_$i.nc ..." + if test "x$i" = xtst_vlen_data ; then + ls -l tst_vlen_data* + ls -l *.nc + fi ${NCCOPY} $i.nc copy_of_$i.nc -${NCDUMP} -n copy_of_$i $i.nc > tmp.cdl -${NCDUMP} copy_of_$i.nc > copy_of_$i.cdl -# echo "*** compare " with copy_of_$i.cdl - diff copy_of_$i.cdl tmp.cdl - rm copy_of_$i.nc copy_of_$i.cdl tmp.cdl + ${NCDUMP} -n copy_of_$i $i.nc > tmp_$i.cdl + ${NCDUMP} copy_of_$i.nc > copy_of_$i.cdl + echo "*** compare " with copy_of_$i.cdl + diff copy_of_$i.cdl tmp_$i.cdl + rm copy_of_$i.nc copy_of_$i.cdl tmp_$i.cdl done + # echo "*** Testing compression of deflatable files ..." ./tst_compress echo "*** Test nccopy -d1 can compress a classic format file ..." @@ -94,6 +98,8 @@ ${NCCOPY} -k nc7 -c"lat/2,lon/2" tst_bug321.nc tmp.nc ${NCDUMP} -n tst_bug321 tmp.nc > tmp.cdl diff -b $srcdir/tst_bug321.cdl tmp.cdl +rm tst_chunking.nc tmp.nc tmp.cdl tmp-chunked.nc tmp-chunked.cdl tmp-unchunked.nc tmp-unchunked.cdl + echo "*** Test that nccopy -c dim/n works as intended " ${NCGEN} -4 -b -o tst_perdimspecs.nc $srcdir/ref_tst_perdimspecs.cdl ${NCCOPY} -M0 -4 -c "time/10,lat/15,lon/20" tst_perdimspecs.nc tmppds.nc diff --git a/ncdump/tst_netcdf4.sh b/ncdump/tst_netcdf4.sh index d3737e3592..fecf1db156 100755 --- a/ncdump/tst_netcdf4.sh +++ b/ncdump/tst_netcdf4.sh @@ -97,15 +97,6 @@ ${execdir}/tst_nans ; ERR ${NCDUMP} tst_nans.nc | sed 's/e+0/e+/g' > tst_nans.cdl ; ERR diff -b tst_nans.cdl $srcdir/ref_tst_nans.cdl ; ERR -# Do unicode test only if it exists => BUILD_UTF8 is true -if test -f ./tst_unicode -o -f ./tst_unicode.exe ; then - echo "*** dumping tst_unicode.nc to tst_unicode.cdl..." - ${execdir}/tst_unicode ; ERR -${NCDUMP} tst_unicode.nc | sed 's/e+0/e+/g' > tst_unicode.cdl ; ERR - #echo "*** comparing tst_unicode.cdl with ref_tst_unicode.cdl..." - #diff -b tst_unicode.cdl $srcdir/ref_tst_unicode.cdl -fi - echo "*** Running tst_special_atts.c to create test files." ${execdir}/tst_special_atts ; ERR ${NCDUMP} -c -s tst_special_atts.nc > tst_special_atts.cdl ; ERR diff --git a/ncdump/tst_netcdf4_4.sh b/ncdump/tst_netcdf4_4.sh index dd9fb88362..52d7f6c965 100755 --- a/ncdump/tst_netcdf4_4.sh +++ b/ncdump/tst_netcdf4_4.sh @@ -72,13 +72,8 @@ ${NCDUMP} -s -h tst_ncf213.nc > tst_ncf213.cdl cleanncprops tst_ncf213.cdl tst_ncf213.tmp cleanncprops ${srcdir}/ref_tst_ncf213.cdl ref_tst_ncf213.tmp # Now compare -ok=1; -if diff -b tst_ncf213.tmp ref_tst_ncf213.tmp ; then ok=1; else ok=0; fi - -if test $ok = 0 ; then - echo "*** FAIL: NCF-213 Bug Fix test" - exit 1 -fi +diff -b tst_ncf213.tmp ref_tst_ncf213.tmp echo "*** All ncgen and ncdump extra test output for netCDF-4 format passed!" exit 0 + diff --git a/ncdump/tst_unicode.c b/ncdump/tst_unicode.c index 5711e554ec..e742260db5 100644 --- a/ncdump/tst_unicode.c +++ b/ncdump/tst_unicode.c @@ -11,21 +11,33 @@ #include "err_macros.h" #include #include -#include +#include "netcdf.h" +#include "ncpathmgr.h" + +#ifdef _WIN32 +#include +#include +#endif + +#ifdef HAVE_LOCALE_H +#include +#endif + +#define DEBUG /* The data file we will create. */ -#define FILE7_NAME "tst_unicode.nc" +static const unsigned char prefix[] = { +'t','s','t','_','u','t','f','8','_', +'\xe6', '\xb5', '\xb7', +'\0' +}; + +/* Other meta-data */ #define UNITS "units" #define NDIMS 1 #define UTF8_BYTES 18 -int -main(int argc, char **argv) -{ - int ncid, dimid, varid; - int dimids[NDIMS]; - - unsigned char name_utf8[] = { +static unsigned char name_utf8[] = { 0xCE, 0x9A, /* GREEK CAPITAL LETTER KAPPA : 2-bytes utf8 */ 0xCE, 0xB1, /* GREEK SMALL LETTER LAMBDA : 2-bytes utf8 */ 0xCE, 0xBB, /* GREEK SMALL LETTER ALPHA : 2-bytes utf8 */ @@ -41,65 +53,103 @@ main(int argc, char **argv) #define UNAME ((char *) name_utf8) #define UNAMELEN (sizeof name_utf8) + /* Note, name was normalized before storing, so retrieved name + won't match original unnormalized name. Check that we get + normalized version, instead. */ + + /* NFC normalized UTF-8 for Unicode 8-character "Hello" in Greek */ +static unsigned char norm_utf8[] = { + 0xCE, 0x9A, /* GREEK CAPITAL LETTER KAPPA : 2-bytes utf8 */ + 0xCE, 0xB1, /* GREEK SMALL LETTER LAMBDA : 2-bytes utf8 */ + 0xCE, 0xBB, /* GREEK SMALL LETTER ALPHA : 2-bytes utf8 */ + 0xCE, 0xB7, /* GREEK SMALL LETTER ETA : 2-bytes utf8 */ + 0xCE, 0xBC, /* GREEK SMALL LETTER MU : 2-bytes utf8 */ + 0xCE, 0xAD, /* GREEK SMALL LETTER EPSILON WITH TONOS + : 2-bytes utf8 */ + 0xCF, 0x81, /* GREEK SMALL LETTER RHO : 2-bytes utf8 */ + 0xCE, 0xB1, /* GREEK SMALL LETTER ALPHA : 2-bytes utf8 */ + 0x00 + }; +#define NNAME ((char *) norm_utf8) +#define NNAMELEN (sizeof norm_utf8) + +static int +check(int err, int line, const char* file) +{ + if(err != 0) { + fprintf(stderr,"ERR %s.%d (%d) %s\n",file,line,err,nc_strerror(err)); + fflush(stderr); + } + return err; +} + +#define CHECK(err) {if((ret=check(err,__LINE__,__FILE__))) goto done;} + +static int +test(int flags, const char* model) +{ + int ret = NC_NOERR; + int ncid, dimid, varid; + int dimids[NDIMS]; char name_in[UNAMELEN + 1], strings_in[UNAMELEN + 1]; nc_type att_type; size_t att_len; + char filename[4096]; + + /* Construct the file name */ + snprintf(filename,sizeof(filename),"%s_%s.nc",prefix,model); + + printf("\n*** Testing UTF-8: %s model\n",model); + printf("*** creating UTF-8 test file |%s|...", filename); - printf("\n*** Testing UTF-8.\n"); - printf("*** creating UTF-8 test file %s...", FILE7_NAME); - if (nc_create(FILE7_NAME, NC_NETCDF4, &ncid)) ERR; + CHECK(nc_create(filename, flags, &ncid)); /* Define dimension with Unicode UTF-8 encoded name */ - if (nc_def_dim(ncid, UNAME, UTF8_BYTES, &dimid)) ERR; + CHECK(nc_def_dim(ncid, UNAME, UTF8_BYTES, &dimid)); dimids[0] = dimid; /* Define variable with same name */ - if (nc_def_var(ncid, UNAME, NC_CHAR, NDIMS, dimids, &varid)) ERR; + CHECK(nc_def_var(ncid, UNAME, NC_CHAR, NDIMS, dimids, &varid)); /* Create string attribute with same value */ - if (nc_put_att_text(ncid, varid, UNITS, UNAMELEN, UNAME)) ERR; + CHECK(nc_put_att_text(ncid, varid, UNITS, UNAMELEN, UNAME)); - if (nc_enddef(ncid)) ERR; + CHECK(nc_enddef(ncid)); /* Write string data, UTF-8 encoded, to the file */ - if (nc_put_var_text(ncid, varid, UNAME)) ERR; + CHECK(nc_put_var_text(ncid, varid, UNAME)); - if (nc_close(ncid)) ERR; + CHECK(nc_close(ncid)); /* Check it out. */ /* Reopen the file. */ - if (nc_open(FILE7_NAME, NC_NOWRITE, &ncid)) ERR; - if (nc_inq_varid(ncid, UNAME, &varid)) ERR; - if (nc_inq_varname(ncid, varid, name_in)) ERR; + CHECK(nc_open(filename, NC_NOWRITE, &ncid)); + CHECK(nc_inq_varid(ncid, UNAME, &varid)); + CHECK(nc_inq_varname(ncid, varid, name_in)); { - /* Note, name was normalized before storing, so retrieved name - won't match original unnormalized name. Check that we get - normalized version, instead. */ - - /* NFC normalized UTF-8 for Unicode 8-character "Hello" in Greek */ - unsigned char norm_utf8[] = { - 0xCE, 0x9A, /* GREEK CAPITAL LETTER KAPPA : 2-bytes utf8 */ - 0xCE, 0xB1, /* GREEK SMALL LETTER LAMBDA : 2-bytes utf8 */ - 0xCE, 0xBB, /* GREEK SMALL LETTER ALPHA : 2-bytes utf8 */ - 0xCE, 0xB7, /* GREEK SMALL LETTER ETA : 2-bytes utf8 */ - 0xCE, 0xBC, /* GREEK SMALL LETTER MU : 2-bytes utf8 */ - 0xCE, 0xAD, /* GREEK SMALL LETTER EPSILON WITH TONOS - : 2-bytes utf8 */ - 0xCF, 0x81, /* GREEK SMALL LETTER RHO : 2-bytes utf8 */ - 0xCE, 0xB1, /* GREEK SMALL LETTER ALPHA : 2-bytes utf8 */ - 0x00 - }; -#define NNAME ((char *) norm_utf8) -#define NNAMELEN (sizeof norm_utf8) - if (strncmp(NNAME, name_in, NNAMELEN) != 0) ERR; + if (strncmp(NNAME, name_in, NNAMELEN) != 0) + {CHECK(NC_EBADNAME);} } - if (nc_inq_att(ncid, varid, UNITS, &att_type, &att_len)) ERR; - if (att_type != NC_CHAR || att_len != UNAMELEN) ERR; - if (nc_get_att_text(ncid, varid, UNITS, strings_in)) ERR; + CHECK(nc_inq_att(ncid, varid, UNITS, &att_type, &att_len)); + CHECK(att_type != NC_CHAR || att_len != UNAMELEN); + CHECK(nc_get_att_text(ncid, varid, UNITS, strings_in)); strings_in[att_len] = '\0'; /* null terminate, because nc_get_att_text doesn't */ - if (strncmp(UNAME, strings_in, UNAMELEN) != 0) ERR; - if (nc_close(ncid)) ERR; + if (strncmp(UNAME, strings_in, UNAMELEN) != 0) + {CHECK(NC_EBADNAME);} + CHECK(nc_close(ncid)); +done: + return ret; +} + +int +main(int argc, char **argv) +{ + /* Run the utf8 test both for netcdf-4 and netcdf-3 */ + if(test(0,"classic")) ERR; +#ifdef USE_HDF5 + if(test(NC_NETCDF4,"enhanced")) ERR; +#endif SUMMARIZE_ERR; FINAL_RESULTS; diff --git a/ncgen/main.c b/ncgen/main.c index 01ed14e0df..b411501664 100644 --- a/ncgen/main.c +++ b/ncgen/main.c @@ -200,9 +200,7 @@ main( int c; FILE *fp; struct Languages* langs; -#ifdef __hpux - setlocale(LC_CTYPE,""); -#endif + init_netcdf(); opterr = 1; /* print error message if bad option */ @@ -353,7 +351,7 @@ main( break; case 'o': /* to explicitly specify output name */ if(netcdf_name) efree(netcdf_name); - netcdf_name = NCdeescape(optarg); + netcdf_name = NC_backslashUnescape(optarg); break; case 'P': /* diskless with persistence */ diskless = 1; @@ -462,7 +460,7 @@ main( } } - cdlname = nulldup(argv[0]); + cdlname = NC_backslashUnescape(argv[0]); if(cdlname != NULL) { if(strlen(cdlname) > NC_MAX_NAME) cdlname[NC_MAX_NAME] = '\0'; diff --git a/ncgen/semantics.c b/ncgen/semantics.c index cac21b246f..3f21cb2753 100644 --- a/ncgen/semantics.c +++ b/ncgen/semantics.c @@ -9,6 +9,7 @@ #include "dump.h" #include "ncoffsets.h" #include "netcdf_aux.h" +#include "ncpathmgr.h" #define floordiv(x,y) ((x) / (y)) #define ceildiv(x,y) (((x) % (y)) == 0 ? ((x) / (y)) : (((x) / (y)) + 1)) diff --git a/ncgen3/Makefile.am b/ncgen3/Makefile.am index ef7b4b3f3b..e4de6f27f1 100644 --- a/ncgen3/Makefile.am +++ b/ncgen3/Makefile.am @@ -37,6 +37,7 @@ CLEANFILES = c0.nc c0_64.nc c0_4.nc c0_4c.nc # This is used if someone wants to rebuild the parser and lexer # Otherwise never invoked, but records how to do it. makeparser:: + rm -f ncgenl.c lex.ncg.c flex -Pncg -8 ncgen.l rm -f ncgenl.c sed -e s/lex.ncg.c/ncgenl.c/g ncgenl.c @@ -45,10 +46,3 @@ makeparser:: sed -e s/ncgen.tab.c/ncgeny.c/g -e s/ncgen.tab.h/ncgeny.h/g ncgeny.c sed -e s/ncgen.tab.c/ncgeny.c/g -e s/ncgen.tab.h/ncgeny.h/g ncgeny.h rm -f ncgen.tab.c ncgen.tab.h - -makeparserold:: - flex -Pncg -8 ncgen.l - mv lex.ncg.c ncgenyy.c - bison -pncg -d ncgen.y - mv ncgen.tab.c ncgentab.c - mv ncgen.tab.h ncgentab.h diff --git a/ncgen3/main.c b/ncgen3/main.c index 4bf5cd9f05..9c148cc6a4 100644 --- a/ncgen3/main.c +++ b/ncgen3/main.c @@ -20,10 +20,6 @@ #define snprintf _snprintf #endif -#ifdef __hpux -#include -#endif - #include "netcdf.h" #include "ncpathmgr.h" @@ -91,10 +87,6 @@ main( int c; FILE *fp; -#ifdef __hpux - setlocale(LC_CTYPE,""); -#endif - #ifdef MDEBUG malloc_debug(2) ; /* helps find malloc/free errors on Sun */ #endif /* MDEBUG */ @@ -213,6 +205,9 @@ main( argv += optind; if (argc > 1) { +int i; +for(i=0;i use default file: format */ - url = ncbytesextract(buf); - ncbytesfree(buf); -} - -int -main(int argc, char** argv) -{ - int stat = NC_NOERR; - int c; - struct Test* t = NULL; - struct Test* test = NULL; - char* file = NULL; - - memset((void*)&options,0,sizeof(options)); - while ((c = getopt(argc, argv, "dc:o:")) != EOF) { - switch(c) { - case 'd': - options.debug = 1; - break; - case 'c': - if(options.cmd != NULL) { - fprintf(stderr,"error: multiple tests specified\n"); - stat = NC_EINVAL; - goto done; - } - options.cmd = strdup(optarg); - break; - case 'o': - if(file != NULL) { - fprintf(stderr,"error: multiple output files specified\n"); - stat = NC_EINVAL; - goto done; - } - file = strdup(optarg); - break; - case '?': - fprintf(stderr,"unknown option\n"); - stat = NC_EINVAL; - goto done; - } - } - if(options.cmd == NULL) { - fprintf(stderr,"no command specified\n"); - stat = NC_EINVAL; - goto done; - } - - if(file == NULL) - file = strdup(FILE); - makeurl(file); -#ifdef DEBUG - fprintf(stderr,"test: %s url=|%s|\n",options.cmd,url); - fflush(stderr); -#endif - - /* Find the corresponding test for command */ - for(t=tests;t->cmd;t++) { - if(strcmp(options.cmd,t->cmd)==0) {test = t; break;} - } - if(test == NULL) { - fprintf(stderr,"unknown command: %s\n",optarg); - stat = NC_EINVAL; - goto done; - } - - /* Execute */ - test->test(); - -done: - if(stat) - nc_strerror(stat); - return (stat ? 1 : 0); -} - -/* Create test netcdf4 file via netcdf.h API*/ -static int -testcreate(void) -{ - int stat = NC_NOERR; - int ncid; - - unlink(FILE); - - if((stat = nc_create(url, 0, &ncid))) - goto done; - - if((stat = nc_close(ncid))) - goto done; - -done: - return THROW(stat); -} - -/* Create file and add a dimension */ -static int -testdim1(void) -{ - int stat = NC_NOERR; - int ncid, dimid; - - unlink(FILE); - - if((stat = nc_create(url, 0, &ncid))) - goto done; - - if((stat = nc_def_dim(ncid, "dim1", (size_t)1, &dimid))) - goto done; - - if((stat = nc_close(ncid))) - goto done; - -done: - return THROW(stat); -} - -/* Create file and add a variable */ -static int -testvar1(void) -{ - int stat = NC_NOERR; - int ncid, varid; - - unlink(FILE); - - if((stat = nc_create(url, 0, &ncid))) - goto done; - - if((stat = nc_def_var(ncid, VAR1, NC_INT, 0, NULL, &varid))) - goto done; - - if((stat = nc_close(ncid))) - goto done; - -done: - return THROW(stat); -} - -/* Create file and add a rank 2 variable */ -static int -testvar2(void) -{ - int stat = NC_NOERR; - int ncid, varid1, varid2; - int dimids[2]; - int ndims; - int fillval; - - unlink(FILE); - - if((stat = nc_create(url, 0, &ncid))) - goto done; - - ndims = 0; - - if((stat = nc_def_dim(ncid, "dim2", 2, &dimids[0]))) - goto done; - ndims++; - - if((stat = nc_def_dim(ncid, "dim3", 3, &dimids[1]))) - goto done; - ndims++; - - if((stat = nc_def_var(ncid, VAR1, NC_INT, ndims, dimids, &varid1))) - goto done; - - /* make VAR1 no-fill */ - if((stat = nc_def_var_fill(ncid,varid1,NC_NOFILL,NULL))) - goto done; - - /* make VAR1 contiguous */ - if((stat = nc_def_var_chunking(ncid,varid1,NC_CONTIGUOUS,NULL))) - goto done; - - /* Define var2 as chunked, but use default chunking */ - if((stat = nc_def_var(ncid, VAR2, NC_INT, ndims, dimids, &varid2))) - goto done; - - if((stat = nc_def_var_chunking(ncid,varid2,NC_CHUNKED,NULL))) - goto done; - - /* make VAR1 use non-default fill */ - fillval = 137; - if((stat = nc_def_var_fill(ncid,varid2,NC_FILL,&fillval))) - goto done; - - if((stat = nc_close(ncid))) - goto done; - -done: - return THROW(stat); -} - -/* Create file and add global attributes of all supported types */ -static int -testattr1(void) -{ - int stat = NC_NOERR; - int ncid; - AtomicsW atoms = { - {-17}, - {-17}, - {-17}, - {17.0f}, - {17.0}, - {17u}, - {17u}, - {17u}, - {17ll}, - {17ull}, - {'7'}}; - - unlink(FILE); - - if((stat = nc_create(url, 0, &ncid))) - goto done; - - if((stat = nc_put_att(ncid,NC_GLOBAL,"byte_a",NC_BYTE,1,atoms.bytev))) goto done; - if((stat = nc_put_att(ncid,NC_GLOBAL,"short_a",NC_SHORT,1,atoms.shortv))) goto done; - if((stat = nc_put_att(ncid,NC_GLOBAL,"int_a",NC_INT,1,atoms.intv))) goto done; - if((stat = nc_put_att(ncid,NC_GLOBAL,"float_a",NC_FLOAT,1,atoms.floatv))) goto done; - if((stat = nc_put_att(ncid,NC_GLOBAL,"double_a",NC_DOUBLE,1,atoms.doublev))) goto done; - if((stat = nc_put_att(ncid,NC_GLOBAL,"ubyte_a",NC_UBYTE,1,atoms.ubytev))) goto done; - if((stat = nc_put_att(ncid,NC_GLOBAL,"ushort_a",NC_USHORT,1,atoms.ushortv))) goto done; - if((stat = nc_put_att(ncid,NC_GLOBAL,"uint_a",NC_UINT,1,atoms.uintv))) goto done; - if((stat = nc_put_att(ncid,NC_GLOBAL,"int64_a",NC_INT64,1,atoms.int64v))) goto done; - if((stat = nc_put_att(ncid,NC_GLOBAL,"uint64_a",NC_UINT64,1,atoms.uint64v))) goto done; - if((stat = nc_put_att(ncid,NC_GLOBAL,"char_a",NC_CHAR,1,atoms.charv))) goto done; - - if((stat = nc_close(ncid))) - goto done; - -done: - return THROW(stat); -} - -/* Create file and a var and add some attributes to the var */ -static int -testattr2(void) -{ - int stat = NC_NOERR; - int ncid, varid1; - - int intv[1] = {-17}; - float floatv[1] = {17.0f}; - long long int64v[1] = {17ll}; - char charv[1] = {'7'}; - - unlink(FILE); - - if((stat = nc_create(url, 0, &ncid))) - goto done; - - if((stat = nc_def_var(ncid, VAR1, NC_INT, 0, NULL, &varid1))) - goto done; - - if((stat = nc_put_att(ncid,varid1,"int_a",NC_INT,1,intv))) goto done; - if((stat = nc_put_att(ncid,varid1,"float_a",NC_FLOAT,1,floatv))) goto done; - if((stat = nc_put_att(ncid,varid1,"int64_a",NC_INT64,1,int64v))) goto done; - if((stat = nc_put_att(ncid,varid1,"char_a",NC_CHAR,1,charv))) goto done; - - if((stat = nc_close(ncid))) - goto done; - -done: - return THROW(stat); -} - -static int -fv2string(nc_type xtype, size_t xtypesize, void* fill_value, char** fillstringp) -{ - char sval[1024]; - switch (xtype) { - case NC_BYTE: - snprintf(sval,sizeof(sval),"%d",*((char*)fill_value)); - break; - case NC_CHAR: - snprintf(sval,sizeof(sval),"'%c'",*((char*)fill_value)); - break; - case NC_SHORT: - snprintf(sval,sizeof(sval),"%d",*((short*)fill_value)); - break; - case NC_INT: - snprintf(sval,sizeof(sval),"%d",*((int*)fill_value)); - break; - case NC_FLOAT: - snprintf(sval,sizeof(sval),"%g",*((float*)fill_value)); - break; - case NC_DOUBLE: - snprintf(sval,sizeof(sval),"%lg",*((double*)fill_value)); - break; - case NC_UBYTE: - snprintf(sval,sizeof(sval),"%u",*((unsigned char*)fill_value)); - break; - case NC_USHORT: - snprintf(sval,sizeof(sval),"%u",*((unsigned short*)fill_value)); - break; - case NC_UINT: - snprintf(sval,sizeof(sval),"%u",*((unsigned int*)fill_value)); - break; - case NC_INT64: - snprintf(sval,sizeof(sval),"%lld",*((long long*)fill_value)); - break; - case NC_UINT64: - snprintf(sval,sizeof(sval),"%llu",*((unsigned long long*)fill_value)); - break; - default: - strlcat(sval,"",sizeof(sval)); - break; - } - if(fillstringp) *fillstringp = strdup(sval); - return NC_NOERR; -} - -static int -reportvar(int ncid, int varid) -{ - int i,stat = NC_NOERR; - char varname[NC_MAX_NAME]; - nc_type xtype; - int ndims, natts; - char typename[NC_MAX_NAME]; - size_t typesize; - int dimids[NC_MAX_VAR_DIMS]; - char* dimname[NC_MAX_VAR_DIMS]; - size_t dimsize[NC_MAX_VAR_DIMS]; - int no_fill; - void* fill_value = NULL; - int shuffle, deflate, deflate_level; - int fletcher32; - int endian; - unsigned int* filterids - size_t nfilters; - int storage; - size_t chunksizes[NC_MAX_VAR_DIMS]; - const char* tmp = NULL; - char* fillstring = NULL; - - if((stat = nc_inq_var(ncid,varid,varname,&xtype,&ndims,dimids,&natts))) - goto done; - /* get basetype name and size*/ - if((stat = nc_inq_type(ncid, xtype, typename, &typesize))) goto done; - /* Get dimension sizes */ - for(i=0;i 0) - filterids = malloc(sizeof(unsigned int)*nfilters); - if((stat = nc_inq_var_filters(ncid, varid, &nfilters, filterids))) - goto done; - /* Allocate space for fill_value */ - if((fill_value = calloc(1,typesize)) == NULL) - {stat = NC_ENOMEM; goto done;} - if((stat = nc_inq_var_fill(ncid, varid, &no_fill, fill_value))) - goto done; - if((stat = nc_inq_var_chunking(ncid, varid, &storage, chunksizes))) - goto done; - - /* Now dump everything */ - printf("%s %s",typename, varname); - printf("("); - for(i=0;i 0) printf(","); - printf("%s=%lu",dimname[i],(unsigned long)dimsize[i]); - } - printf(");\n"); - - switch (endian) { - case NC_ENDIAN_NATIVE: tmp = "native"; break; - case NC_ENDIAN_LITTLE: tmp = "little"; break; - case NC_ENDIAN_BIG: tmp = "big"; break; - default: tmp = "unknown"; break; - } - printf("\t_Endianness=%s\n",tmp); - - printf("\t_Fletcher32=%s\n",(fletcher32?"true":"false")); - - printf("\t_Shuffle=%s\n",(shuffle?"true":"false")); - if(deflate) printf("\t_DeflateLevel=%d\n",deflate_level); - - if(filterids != NULL) { - printf("\t_Filters=("); - for(i=0;i0) printf(","); - printf("%u",filterids[i]); - } - printf(")\n"); - } - - if((stat=fv2string(xtype,typesize,fill_value,&fillstring))) goto done; - printf("\t_FillValue=%s\n",fillstring); - - if(storage == NC_CONTIGUOUS) { - printf("\t_Storage=contiguous\n"); - } else { /* chunked */ - printf("\t_Storage=chunked\n"); - printf("\t_ChunkSizes="); - for(i=0;i0) printf(","); - printf("%lu",(unsigned long)chunksizes[i]); - } - printf("\n"); - } - -done: - return stat; -} - -/* Report variables from testvar2 */ -static int -testread1(void) -{ - int stat = NC_NOERR; - int ncid, varid1, varid2; - int ndims,nvars,natts,unlimdimid; - - unlink(FILE); - -#ifdef SETUP - NCCHECK(testvar2()); -#endif - - if((stat = nc_open(url, 0, &ncid))) - goto done; - - /* Report file level info */ - if((stat = nc_inq(ncid, &ndims, &nvars, &natts, &unlimdimid))) - goto done; - printf("file info: ndims=%d nvars=%d natts=%d unlimdimid=%d\n", - ndims,nvars,natts,unlimdimid); - - if((stat = nc_inq_varid(ncid, VAR1, &varid1))) goto done; - - /* Report info about the var */ - if((stat = reportvar(ncid,varid1))) goto done; - - if((stat = nc_inq_varid(ncid, VAR2, &varid2))) goto done; - - /* Report info about the var */ - if((stat = reportvar(ncid,varid2))) goto done; - - if((stat = nc_close(ncid))) - goto done; - -done: - return THROW(stat); -} - -static int -reportatt(int ncid, int varid, int natts, int index) -{ - int i,stat = NC_NOERR; - void* data = NULL; - char aname[NC_MAX_NAME]; - nc_type atype; - size_t alen; - char typename[NC_MAX_NAME]; - size_t typesize; - char* p = NULL; /* char* so we can do arithmetic */ - - /* Get some meta info about the attribute */ - if((stat = nc_inq_attname(ncid,varid,index,aname))) - goto done; - if((stat = nc_inq_att(ncid, varid, aname, &atype, &alen))) goto done; - - /* Get type info about atype */ - if((stat = nc_inq_type(ncid, atype, typename, &typesize))) goto done; - - /* Allocate space to read the attribute */ - if((data = malloc(typesize * alen)) == NULL) - {stat = NC_ENOMEM; goto done;} - - /* Read the attribute */ - if((stat = nc_get_att(ncid,varid,aname,data))) goto done; - - /* Now report */ - printf("\t[%d] %s:%s(%lu) = ",index,typename,aname,(unsigned long)alen); - for(p=(char*)data,i=0;ibytev[0]); break; - case NC_SHORT: printf("%ds",atom->shortv[0]); break; - case NC_INT: printf("%d",atom->intv[0]); break; - case NC_FLOAT: printf("%gf",atom->floatv[0]); break; - case NC_DOUBLE: printf("%lg",atom->doublev[0]); break; - case NC_UBYTE: printf("%uub",atom->ubytev[0]); break; - case NC_USHORT: printf("%uus",atom->ushortv[0]); break; - case NC_UINT: printf("%uu",atom->intv[0]); break; - case NC_INT64: printf("%lldll",atom->int64v[0]); break; - case NC_UINT64: printf("%lluull",atom->uint64v[0]); break; - case NC_CHAR: printf("'%c'",atom->charv[0]); break; - } - } - printf("}\n"); - -done: - return stat; -} - -/* Print out selected reserved attributes by name */ -static int -reportreserved(int ncid) -{ - int stat = NC_NOERR; - char* ncpropdata = NULL; - size_t len; - int superblockversion; - int isnetcdf4; - nc_type xtype; - - /* Get and print _NCProperties */ - if((stat = nc_inq_attlen(ncid, NC_GLOBAL, "_NCProperties", &len))) - goto done; - if((ncpropdata = calloc(1,len+1)) == NULL) {stat = NC_ENOMEM; goto done;} - if((stat = nc_get_att(ncid,NC_GLOBAL, "_NCProperties", ncpropdata))) - goto done; - ncpropdata[len] = '\0'; /* nul term */ - printf("\t%s:%s(%lu) = |%s|\n", "char","_NCProperties", (unsigned long)len, ncpropdata); - - /* Get and print _SuperblockVersion */ - if((stat = nc_inq_att(ncid, NC_GLOBAL, "_SuperblockVersion", &xtype,&len))) - goto done; - if(len != 1 || xtype != NC_INT) {stat = NC_ENCZARR; goto done;} - if((stat = nc_get_att_int(ncid,NC_GLOBAL, "_SuperblockVersion", &superblockversion))) - goto done; - printf("\t%s:%s(%lu) = %d\n", "int","_SuperblockVersion", (unsigned long)len, superblockversion); - - /* Get and print _Isnetcdf4 */ - if((stat = nc_inq_att(ncid, NC_GLOBAL, "_IsNetcdf4", &xtype,&len))) - goto done; - if(len != 1 || xtype != NC_INT) {stat = NC_ENCZARR; goto done;} - if((stat = nc_get_att_int(ncid,NC_GLOBAL, "_IsNetcdf4", &isnetcdf4))) - goto done; - printf("\t%s:%s(%lu) = %d\n", "int","_IsNetcdf4", (unsigned long)len, isnetcdf4); - -done: - nullfree(ncpropdata); - return stat; -} - -/* Read the global attributes in testattr1 */ -static int -testread2(void) -{ - int i,stat = NC_NOERR; - int ncid; - int natts; - - unlink(FILE); - -#ifdef SETUP - NCCHECK(testattr1()); -#endif - - if((stat = nc_open(url, 0, &ncid))) - goto done; - - printf("Reserved Attributes:\n"); - if((stat = reportreserved(ncid))) goto done; - fflush(stdout); - - printf("\nGlobal Attributes:\n"); - /* Get and print the global attributes */ - if((stat = nc_inq(ncid, NULL, NULL, &natts, NULL))) goto done; - - for(i=0;i 0) { - char* p = NCdeescape((*argvp)[0]); + char* p = NC_backslashUnescape((*argvp)[0]); strcpy(options->file,filenamefor(p)); nullfree(p); } diff --git a/nczarr_test/ut_test.c b/nczarr_test/ut_test.c index 09e49d3db8..5bddcf76c6 100755 --- a/nczarr_test/ut_test.c +++ b/nczarr_test/ut_test.c @@ -111,7 +111,7 @@ static void getpathcwd(char** cwdp) { char buf[4096]; - (void)NCcwd(buf,sizeof(buf)); + (void)NCgetcwd(buf,sizeof(buf)); if(cwdp) *cwdp = strdup(buf); } @@ -132,10 +132,14 @@ canonicalfile(char** fp) f = *fp; len = strlen(f); if(len <= 1) return; - if(f[0] == '/' || f[0] == '\\' || hasdriveletter(f)) - return; /* its already absolute */ ncuriparse(f,&uri); if(uri != NULL) {ncurifree(uri); return;} /* its a url */ + +#if 1 + abspath = NCpathabsolute(f); +#else + if(f[0] == '/' || f[0] == '\\' || hasdriveletter(f)) + return; /* its already absolute */ #ifdef _WIN32 for(p=f;*p;p++) {if(*p == '\\') {*p = '/';}} #endif @@ -165,10 +169,12 @@ canonicalfile(char** fp) #ifdef _WIN32 if(fwin32) for(p=abspath;*p;p++) {if(*p == '/') {*p = '\\';}} +#endif + nullfree(cwd); #endif nullfree(f); +fprintf(stderr,"canonicalfile: %s\n",abspath); *fp = abspath; - nullfree(cwd); } void @@ -188,16 +194,19 @@ makeurl(const char* file, NCZM_IMPL impl) NCbytes* buf = ncbytesnew(); NCURI* uri = NULL; const char* kind = impl2kind(impl); - char* path = NULL; + char* urlpath = NULL; + char* p; if(file && strlen(file) > 0) { switch (impl) { case NCZM_FILE: case NCZM_ZIP: /* Massage file to make it usable as URL path */ - if((path = NCurlpath(file))==NULL) return NULL; + urlpath = strdup(file); + for(p=urlpath;*p;p++) {if(*p == '\\') *p = '/';} ncbytescat(buf,"file://"); - ncbytescat(buf,path); + ncbytescat(buf,urlpath); + nullfree(urlpath); urlpath = NULL; ncbytescat(buf,"#mode=nczarr"); /* => use default file: format */ ncbytescat(buf,","); ncbytescat(buf,kind); @@ -217,7 +226,6 @@ makeurl(const char* file, NCZM_IMPL impl) } ncurifree(uri); ncbytesfree(buf); - nullfree(path); fprintf(stderr,"url=|%s|\n",url); fflush(stderr); return url; diff --git a/nczarr_test/zmapio.c b/nczarr_test/zmapio.c index b60ef97149..4f935b13d3 100644 --- a/nczarr_test/zmapio.c +++ b/nczarr_test/zmapio.c @@ -177,7 +177,7 @@ main(int argc, char** argv) } { - char* p = NCdeescape(argv[0]); + char* p = NC_backslashUnescape(argv[0]); strcpy(dumpoptions.infile,filenamefor(p)); if(p) free(p); } diff --git a/nczarr_test/zs3parse.c b/nczarr_test/zs3parse.c index f0f777e3cd..9ec5476c38 100644 --- a/nczarr_test/zs3parse.c +++ b/nczarr_test/zs3parse.c @@ -87,7 +87,7 @@ main(int argc, char** argv) fprintf(stderr, "zs3parse: no url|file specified\n"); goto fail; } - s3options.url = NCdeescape(argv[0]); + s3options.url = strdup(argv[0]); stat = processurl(s3options.op, s3options.url, &piece); if(stat == NC_NOERR) { @@ -126,10 +126,6 @@ processurl(S3op op, const char* surl, char** piece) && strcmp(url->protocol,"http") != 0) {stat = NC_EURL; goto done;} - /* Path better look absolute */ - if(!nczm_isabsolutepath(url->path)) - {stat = NC_EURL; goto done;} - if(url->host == NULL || strlen(url->host) == 0) {stat = NC_EURL; goto done;} if((host = strdup(url->host))==NULL) diff --git a/test_common.in b/test_common.in index b143198e0f..539be6fdf5 100644 --- a/test_common.in +++ b/test_common.in @@ -8,6 +8,11 @@ # Define location of execution TOPSRCDIR='@abs_top_srcdir@' TOPBUILDDIR='@abs_top_builddir@' +FP_ISCMAKE=@ISCMAKE@ +FP_ISMSVC=@ISMSVC@ + +# Feature flags +FEATURE_HDF5=@HAS_HDF5@ # Define selected features of the build FEATURE_HDF5=@HAS_HDF5@ diff --git a/unit_test/test_pathcvt.c b/unit_test/test_pathcvt.c index 66e1155248..772e140c55 100644 --- a/unit_test/test_pathcvt.c +++ b/unit_test/test_pathcvt.c @@ -10,61 +10,113 @@ Test the NCpathcvt #include #include #include +#include "netcdf.h" #include "ncpathmgr.h" -#undef VERBOSE +#define DEBUG + +#define NKINDS 4 +static const int kinds[NKINDS] = {NCPD_NIX,NCPD_MSYS,NCPD_CYGWIN,NCPD_WIN}; typedef struct Test { - char* path; - char* expected; + char* test; + char* expected[NKINDS]; } Test; /* Path conversion tests */ static Test PATHTESTS[] = { -#ifdef _MSC_VER -{"/xxx/a/b","/xxx/a/b"}, -{"d:/x/y","d:\\x\\y"}, -{"/cygdrive/d/x/y","d:\\x\\y"}, -{"/d/x/y","d:\\x\\y"}, -{"/cygdrive/d","d:\\"}, -{"/d","d:\\"}, -{"/cygdrive/d/git/netcdf-c/dap4_test/daptestfiles/test_anon_dim.2.syn","d:\\git\\netcdf-c\\dap4_test\\daptestfiles\\test_anon_dim.2.syn"}, -{"[dap4]file:///cygdrive/d/git/netcdf-c/dap4_test/daptestfiles/test_anon_dim.2.syn","[dap4]file:///cygdrive/d/git/netcdf-c/dap4_test/daptestfiles/test_anon_dim.2.syn"}, -#else -{"/xxx/a/b","/xxx/a/b"}, -{"d:/x/y","d:/x/y"}, -{"/cygdrive/d/x/y","d:/x/y"}, -{"/d/x/y","d:/x/y"}, -{"/cygdrive/d","d:/"}, -{"/d","d:/"}, -{"/cygdrive/d/git/netcdf-c/dap4_test/daptestfiles/test_anon_dim.2.syn","d:/git/netcdf-c/dap4_test/daptestfiles/test_anon_dim.2.syn"}, -{"[dap4]file:///cygdrive/d/git/netcdf-c/dap4_test/daptestfiles/test_anon_dim.2.syn","[dap4]file:///cygdrive/d/git/netcdf-c/dap4_test/daptestfiles/test_anon_dim.2.syn"}, +{"/xxx/a/b",{"/xxx/a/b", "/c/xxx/a/b", "/cygdrive/c/xxx/a/b", "c:\\xxx\\a\\b"}}, +{"d:/x/y",{ "/d/x/y", "/d/x/y", "/cygdrive/d/x/y", "d:\\x\\y"}}, +{"/cygdrive/d/x/y",{ "/d/x/y", "/d/x/y", "/cygdrive/d/x/y", "d:\\x\\y"}}, +{"/d/x/y",{ "/d/x/y", "/d/x/y", "/cygdrive/d/x/y", "d:\\x\\y"}}, +{"/cygdrive/d",{ "/d", "/d", "/cygdrive/d", "d:"}}, +{"/d", {"/d", "/d", "/cygdrive/d", "d:"}}, +{"/cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn",{ + "/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn", + "/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn", + "/cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn", + "d:\\git\\netcdf-c\\dap4_test\\test_anon_dim.2.syn"}}, +/* Test relative path */ +{"x/y",{ "x/y", "x/y", "x/y", "x\\y"}}, +#ifndef _WIN32 +/* Test utf8 path */ +{"/海/海",{ "/海/海", "/c/海/海", "/cygdrive/c/海/海", "c:\\海\\海"}}, #endif -{NULL,NULL} +{NULL, {NULL, NULL, NULL, NULL}} }; +/*Forward */ +static const char* kind2string(int kind); + int main(int argc, char** argv) { Test* test; int failcount = 0; + char* cvt = NULL; + int k; + int drive = 'c'; - for(test=PATHTESTS;test->path;test++) { - char* cvt = NCpathcvt(test->path); - if(cvt == NULL) { - fprintf(stderr,"TEST returned NULL: %s\n",test->path); - exit(1); - } - if(strcmp(cvt,test->expected) != 0) { - fprintf(stderr,"NCpathcvt failed:: input: |%s| expected=|%s| actual=|%s|\n",test->path,test->expected,cvt); - failcount++; - } -#ifdef VERBOSE - fprintf(stderr,"NCpathcvt:: input: |%s| actual=|%s|\n",test->path,cvt); + nc_initialize(); + + /* Test localkind X path kind */ + for(k=0;ktest;test++) { + /* Compare output for the localkind */ + if(test->expected[k] == NULL) { +#ifdef DEBUG + fprintf(stderr,"TEST local=%s: %s ignored\n",kind2string(kind),test->test); +#endif + continue; + } + cvt = NCpathcvt_test(test->test,kind,drive); +#ifdef DEBUG + fprintf(stderr,"TEST local=%s: input: |%s| expected=|%s| actual=|%s|: ", + kind2string(kind),test->test,test->expected[k],cvt); +#endif + fflush(stderr); fflush(stdout); + if(cvt == NULL) { +#ifdef DEBUG + fprintf(stderr," ILLEGAL"); +#endif + failcount++; + } else if(strcmp(cvt,test->expected[k]) != 0) { +#ifdef DEBUG + fprintf(stderr," FAIL"); #endif - free(cvt); + failcount++; + } else { +#ifdef DEBUG + fprintf(stderr," PASS"); +#endif + } +#ifdef DEBUG + fprintf(stderr,"\n"); +#endif + nullfree( cvt); cvt = NULL; + } } - - fprintf(stderr,"%s test_ncuri\n",failcount > 0 ? "***FAIL":"***PASS"); + nullfree(cvt); + fprintf(stderr,"%s test_pathcvt\n",failcount > 0 ? "***FAIL":"***PASS"); + nc_finalize(); return (failcount > 0 ? 1 : 0); } + +static const char* +kind2string(int kind) +{ + switch(kind) { + case NCPD_NIX: + return "Linux"; + case NCPD_MSYS: + return "MSYS"; + case NCPD_CYGWIN: + return "Cygwin"; + case NCPD_WIN: + return "Windows"; + default: break; + } + return "unknown"; +}