diff --git a/DESCRIPTION b/DESCRIPTION index 300f4eea..7ada206b 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: amadeus Title: AMADEUS: A Mechanism/Machine for Data, Environments, and User Setup -Version: 0.1.1 +Version: 0.1.2 Authors@R: c( person("Kyle", "Messier", , "kyle.messier@nih.gov", role = c("aut", "cre"), comment = c(ORCID = "0000-0001-9508-9623")), person("Mitchell", "Manware", role = c("aut", "ctb"), comment = c(ORCID = "0009-0003-6440-6106")), diff --git a/NAMESPACE b/NAMESPACE index b01b1d51..90fd897f 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -31,7 +31,7 @@ export(download_aqs_data) export(download_data) export(download_ecoregion_data) export(download_epa_certificate) -export(download_geos_cf_data) +export(download_geos_data) export(download_gmted_data) export(download_hms_data) export(download_koppen_geiger_data) diff --git a/R/download.R b/R/download.R index c0a9ce4d..b3f9ca0b 100644 --- a/R/download.R +++ b/R/download.R @@ -15,7 +15,7 @@ #' Please refer to: #' * \link{download_aqs_data}: "aqs", "AQS" #' * \link{download_ecoregion_data}: "ecoregion" -#' * \link{download_geos_cf_data}: "geos" +#' * \link{download_geos_data}: "geos" #' * \link{download_gmted_data}: "gmted", "GMTED" #' * \link{download_koppen_geiger_data}: "koppen", "koppengeiger" #' * \link{download_merra2_data}: "merra2", "merra", "MERRA", "MERRA2" @@ -49,7 +49,7 @@ download_data <- what_to_run <- switch(dataset_name, aqs = download_aqs_data, ecoregion = download_ecoregion_data, - geos = download_geos_cf_data, + geos = download_geos_data, gmted = download_gmted_data, koppen = download_koppen_geiger_data, koppengeiger = download_koppen_geiger_data, @@ -144,7 +144,7 @@ download_aqs_data <- data_download_acknowledgement = data_download_acknowledgement ) - #### 2. check for null parameteres + #### 2. check for null parameters check_for_null_parameters(mget(ls())) #### 3. directory setup directory_to_download <- download_sanitize_path(directory_to_download) @@ -370,25 +370,23 @@ download_ecoregion_data <- function( #### 14. remove download command download_remove_command(commands_txt = commands_txt, remove = remove_command) - if (download) { - #### 15. unzip files - download_unzip( - file_name = download_name, - directory_to_unzip = directory_to_save, - unzip = unzip - ) - #### 16. remove zip files - download_remove_zips( - remove = remove_zip, - download_name = download_name - ) - } + #### 15. unzip files + download_unzip( + file_name = download_name, + directory_to_unzip = directory_to_save, + unzip = unzip + ) + #### 16. remove zip files + download_remove_zips( + remove = remove_zip, + download_name = download_name + ) } # nolint start #' Download atmospheric composition data from the NASA Global Earth Observing System (GEOS) model. #' @description -#' The \code{download_goes_cf_data()} function accesses and downloads various +#' The \code{download_geos_data()} function accesses and downloads various #' atmospheric composition collections from the [NASA Global Earth Observing System (GEOS) model](https://gmao.gsfc.nasa.gov/GEOS_systems/). # nolint end #' @param date_start character(1). length of 10. Start date for downloading @@ -409,7 +407,7 @@ download_ecoregion_data <- function( #' @author Mitchell Manware, Insang Song #' @return NULL; #' @export -download_geos_cf_data <- function( +download_geos_data <- function( date_start = "2023-09-01", date_end = "2023-09-01", collection = @@ -643,24 +641,22 @@ download_gmted_data <- function( download = download, system_command = system_command ) - #### 18. Remove command file + #### 17. Remove command file download_remove_command( commands_txt = commands_txt, remove = remove_command ) - if (download) { - #### 17. end if unzip == FALSE - download_unzip( - file_name = download_name, - directory_to_unzip = directory_to_save, - unzip = unzip - ) - #### 19. remove zip files - download_remove_zips( - remove = remove_zip, - download_name = download_name - ) - } + #### 18. end if unzip == FALSE + download_unzip( + file_name = download_name, + directory_to_unzip = directory_to_save, + unzip = unzip + ) + #### 19. remove zip files + download_remove_zips( + remove = remove_zip, + download_name = download_name + ) } # nolint start diff --git a/R/download_test_support.R b/R/download_test_support.R index d4790069..7d53a900 100644 --- a/R/download_test_support.R +++ b/R/download_test_support.R @@ -7,7 +7,7 @@ #' Check if sample of download URLs have HTTP Status 200 #' @param url Download URL to be checked. -#' @param method httr method to obtain URL (`"HEAD"`` or `"GET"`) +#' @param method httr method to obtain URL (`"HEAD"` or `"GET"`) #' @author Insang Song; Mitchell Manware #' @importFrom httr HEAD #' @importFrom httr GET diff --git a/R/process.R b/R/process.R index 2ee10113..589f930c 100644 --- a/R/process.R +++ b/R/process.R @@ -13,6 +13,7 @@ #' - [`process_nlcd`]: `"nlcd"`, `"NLCD"` #' - [`process_tri`]: `"tri"`, `"TRI"` #' - [`process_nei`]: `"nei"`, `"NEI` +#' - [`process_geos`]: `"geos"` #' - [`process_gmted`]: `"gmted"`, `"GMTED"` #' - [`process_aqs`]: `"aqs"`, `"AQS"` #' - [`process_hms`]: `"hms"`, `"HMS"`, `"smoke"` @@ -76,8 +77,12 @@ process_covariates <- }, error = function(e) { print(e) print(args(what_to_run)) - stop("Please refer to the argument list and -the error message above to rectify the error.\n") + stop( + paste0( + "Please refer to the argument list and the error message above to ", + "rectify the error.\n" + ) + ) }) return(res_covariate) @@ -618,7 +623,7 @@ process_nlcd <- #' @export process_ecoregion <- function( - path = "./input/data/ecoregions/raw/us_eco_l3_state_boundaries.shp" + path = NULL ) { ecoreg <- terra::vect(path) ecoreg <- ecoreg[, grepl("^(L2_KEY|L3_KEY)", names(ecoreg))] @@ -655,8 +660,7 @@ process_conformity <- locs_epsg <- locs$crs_dt } else { if (!all(keyword %in% names(locs))) { - stop("locs should be stdt or -have 'lon', 'lat', (and 'time') fields.\n") + stop("locs should be stdt or have 'lon', 'lat', (and 'time') fields.\n") } if (!methods::is(locs, "SpatVector")) { if (methods::is(locs, "sf")) { @@ -1318,7 +1322,7 @@ process_gmted <- function( process_narr <- function( date = c("2023-09-01", "2023-09-01"), variable = NULL, - path = "../../data/covariates/narr/") { + path = NULL) { #### directory setup path <- download_sanitize_path(path) #### check for variable @@ -1453,7 +1457,7 @@ process_narr <- function( } #' Import and clean GEOS-CF data downloaded with -#' `download_geos_cf_data` or `download_data(dataset_name = "geos")`. Function +#' `download_geos_data` or `download_data(dataset_name = "geos")`. Function #' returns a SpatRast object containing the user-defined variables of interest. #' Layer names indicate the variable, pressure level, date (YYYYMMDD), and, if #' applicable, the hour (HHMMSS). @@ -1474,7 +1478,7 @@ process_narr <- function( process_geos <- function(date = c("2018-01-01", "2018-01-01"), variable = NULL, - path = "../../data/covariates/geos_cf/") { + path = NULL) { #### directory setup path <- download_sanitize_path(path) #### check for variable diff --git a/man/check_url_status.Rd b/man/check_url_status.Rd index 01e054c2..ebc20a52 100644 --- a/man/check_url_status.Rd +++ b/man/check_url_status.Rd @@ -9,7 +9,7 @@ check_url_status(url, method = c("HEAD", "GET")) \arguments{ \item{url}{Download URL to be checked.} -\item{method}{httr method to obtain URL (\verb{"HEAD"`` or }"GET"`)} +\item{method}{httr method to obtain URL (\code{"HEAD"} or \code{"GET"})} } \value{ logical object diff --git a/man/download_data.Rd b/man/download_data.Rd index d8193a5a..12040288 100644 --- a/man/download_data.Rd +++ b/man/download_data.Rd @@ -40,7 +40,7 @@ Please refer to: \itemize{ \item \link{download_aqs_data}: "aqs", "AQS" \item \link{download_ecoregion_data}: "ecoregion" -\item \link{download_geos_cf_data}: "geos" +\item \link{download_geos_data}: "geos" \item \link{download_gmted_data}: "gmted", "GMTED" \item \link{download_koppen_geiger_data}: "koppen", "koppengeiger" \item \link{download_merra2_data}: "merra2", "merra", "MERRA", "MERRA2" diff --git a/man/download_geos_cf_data.Rd b/man/download_geos_data.Rd similarity index 91% rename from man/download_geos_cf_data.Rd rename to man/download_geos_data.Rd index 1aacd9d8..9e35dc95 100644 --- a/man/download_geos_cf_data.Rd +++ b/man/download_geos_data.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/download.R -\name{download_geos_cf_data} -\alias{download_geos_cf_data} +\name{download_geos_data} +\alias{download_geos_data} \title{Download atmospheric composition data from the NASA Global Earth Observing System (GEOS) model.} \usage{ -download_geos_cf_data( +download_geos_data( date_start = "2023-09-01", date_end = "2023-09-01", collection = c("aqc_tavg_1hr_g1440x721_v1", "chm_tavg_1hr_g1440x721_v1", @@ -43,7 +43,7 @@ the text file containing download commands.} NULL; } \description{ -The \code{download_goes_cf_data()} function accesses and downloads various +The \code{download_geos_data()} function accesses and downloads various atmospheric composition collections from the \href{https://gmao.gsfc.nasa.gov/GEOS_systems/}{NASA Global Earth Observing System (GEOS) model}. } \author{ diff --git a/man/process_covariates.Rd b/man/process_covariates.Rd index 1c494837..564ea546 100644 --- a/man/process_covariates.Rd +++ b/man/process_covariates.Rd @@ -37,6 +37,7 @@ Process covariates \item \code{\link{process_nlcd}}: \code{"nlcd"}, \code{"NLCD"} \item \code{\link{process_tri}}: \code{"tri"}, \code{"TRI"} \item \code{\link{process_nei}}: \code{"nei"}, \verb{"NEI} +\item \item \code{\link{process_gmted}}: \code{"gmted"}, \code{"GMTED"} \item \code{\link{process_aqs}}: \code{"aqs"}, \code{"AQS"} \item \code{\link{process_hms}}: \code{"hms"}, \code{"HMS"}, \code{"smoke"} diff --git a/man/process_ecoregion.Rd b/man/process_ecoregion.Rd index bb6043d6..0ebd3a1d 100644 --- a/man/process_ecoregion.Rd +++ b/man/process_ecoregion.Rd @@ -4,9 +4,7 @@ \alias{process_ecoregion} \title{Process EPA Ecoregion shapefiles} \usage{ -process_ecoregion( - path = "./input/data/ecoregions/raw/us_eco_l3_state_boundaries.shp" -) +process_ecoregion(path = NULL) } \arguments{ \item{path}{character(1). Path to Ecoregion Shapefiles} diff --git a/man/process_geos.Rd b/man/process_geos.Rd index b4263d50..dfe90fa9 100644 --- a/man/process_geos.Rd +++ b/man/process_geos.Rd @@ -3,7 +3,7 @@ \name{process_geos} \alias{process_geos} \title{Import and clean GEOS-CF data downloaded with -\code{download_geos_cf_data} or \code{download_data(dataset_name = "geos")}. Function +\code{download_geos_data} or \code{download_data(dataset_name = "geos")}. Function returns a SpatRast object containing the user-defined variables of interest. Layer names indicate the variable, pressure level, date (YYYYMMDD), and, if applicable, the hour (HHMMSS).} @@ -11,7 +11,7 @@ applicable, the hour (HHMMSS).} process_geos( date = c("2018-01-01", "2018-01-01"), variable = NULL, - path = "../../data/covariates/geos_cf/" + path = NULL ) } \arguments{ @@ -27,7 +27,7 @@ a SpatRaster object; } \description{ Import and clean GEOS-CF data downloaded with -\code{download_geos_cf_data} or \code{download_data(dataset_name = "geos")}. Function +\code{download_geos_data} or \code{download_data(dataset_name = "geos")}. Function returns a SpatRast object containing the user-defined variables of interest. Layer names indicate the variable, pressure level, date (YYYYMMDD), and, if applicable, the hour (HHMMSS). diff --git a/man/process_narr.Rd b/man/process_narr.Rd index 06b86e77..47f47e1b 100644 --- a/man/process_narr.Rd +++ b/man/process_narr.Rd @@ -11,7 +11,7 @@ of interest. Layer names indicate the variable, pressure level, and date process_narr( date = c("2023-09-01", "2023-09-01"), variable = NULL, - path = "../../data/covariates/narr/" + path = NULL ) } \arguments{ diff --git a/tests/testthat/test-calculate_covariates.R b/tests/testthat/test-calculate_covariates.R index 05bc8b83..e6842a42 100644 --- a/tests/testthat/test-calculate_covariates.R +++ b/tests/testthat/test-calculate_covariates.R @@ -81,6 +81,22 @@ testthat::test_that("calc_dummies works well", { }) +testthat::test_that("calc_temporal_dummies errors.", { + withr::local_package("terra") + ncp <- data.frame(lon = -78.8277, lat = 35.95013) + ncp$site_id <- "3799900018810101" + testthat::expect_error( + calc_temporal_dummies( + ncp + ) + ) + testthat::expect_error( + calc_temporal_dummies( + terra::vect(ncp) + ) + ) +}) + testthat::test_that("calc_ecoregion works well", { withr::local_package("terra") withr::local_package("sf") diff --git a/tests/testthat/test-download_functions.R b/tests/testthat/test-download_functions.R index d736b96e..8f2a08cd 100644 --- a/tests/testthat/test-download_functions.R +++ b/tests/testthat/test-download_functions.R @@ -1,5 +1,5 @@ -#' @author Mitchell Manware -#' @description Unit test for for checking data download functions. +## test for download functions + testthat::test_that("Error when data_download_acknowledgement = FALSE", { download_datasets <- c("aqs", "ecoregion", "geos", "gmted", "koppen", "koppengeiger", "merra2", "merra", "narr_monolevel", @@ -35,49 +35,50 @@ testthat::test_that("Error when one parameter is NULL.", { testthat::test_that("Errors when temporal ranges invalid.", { expect_error( - download_geos_cf_data( + download_geos_data( date_start = "1900-01-01", collection = "aqc_tavg_1hr_g1440x721_v1", data_download_acknowledgement = TRUE, - directory_to_save = "../testdata" + directory_to_save = testthat::test_path("..", "testdata/", "") ) ) expect_error( download_aqs_data( year_start = 1900, data_download_acknowledgement = TRUE, - directory_to_save = "../testdata" + directory_to_save = testthat::test_path("..", "testdata/", ""), + directory_to_download = testthat::test_path("..", "testdata/", "") ) ) expect_error( download_narr_monolevel_data( year_start = 1900, - collection = "air.sfc", + variables = "air.sfc", data_download_acknowledgement = TRUE, - directory_to_save = "../testdata" + directory_to_save = testthat::test_path("..", "testdata/", "") ) ) expect_error( download_narr_p_levels_data( year_start = 1900, - collection = "omega", + variables = "omega", data_download_acknowledgement = TRUE, - directory_to_save = "../testdata" + directory_to_save = testthat::test_path("..", "testdata/", "") ) ) expect_error( download_merra2_data( date_start = "1900-01-01", collection = "inst1_2d_asm_Nx", - directory_to_save = "../testdata", + directory_to_save = testthat::test_path("..", "testdata/", ""), data_download_acknowledgement = TRUE ) ) expect_error( - download_noaa_hms_smoke_data( + download_hms_data( date_start = "1900-01-01", - directory_to_save = "../testdata", - directory_to_download = "../testdata", + directory_to_save = testthat::test_path("..", "testdata/", ""), + directory_to_download = testthat::test_path("..", "testdata/", ""), data_download_acknowledgement = TRUE ) ) @@ -347,6 +348,19 @@ testthat::test_that("NARR monolevel download URLs have HTTP status 200.", { file.remove(commands_path) }) +testthat::test_that("NARR monolevel error with invalid years.", { + testthat::expect_error( + download_data( + dataset_name = "narr_monolevel", + variables = "weasd", + year_start = 10, + year_end = 11, + data_download_acknowledgement = TRUE, + directory_to_save = testthat::test_path("..", "testdata/", "") + ) + ) +}) + testthat::test_that("NARR p-levels download URLs have HTTP status 200.", { withr::local_package("httr") withr::local_package("stringr") @@ -390,36 +404,53 @@ testthat::test_that("NOAA HMS Smoke download URLs have HTTP status 200.", { date_end <- "2022-09-21" directory_to_download <- testthat::test_path("..", "testdata/", "") directory_to_save <- testthat::test_path("..", "testdata/", "") - # run download function - download_data(dataset_name = "smoke", - date_start = date_start, - date_end = date_end, - directory_to_download = directory_to_download, - directory_to_save = directory_to_save, - data_download_acknowledgement = TRUE, - download = FALSE, - remove_command = FALSE, - unzip = FALSE, - remove_zip = FALSE) - # define file path with commands - commands_path <- paste0(directory_to_download, - "hms_smoke_", - gsub("-", "", date_start), - "_", - gsub("-", "", date_end), - "_curl_commands.txt") - # import commands - commands <- read_commands(commands_path = commands_path) - # extract urls - urls <- extract_urls(commands = commands, position = 6) - # check HTTP URL status - url_status <- check_urls(urls = urls, size = 3L, method = "HEAD") - # implement unit tests - test_download_functions(directory_to_save = directory_to_save, - commands_path = commands_path, - url_status = url_status) - # remove file with commands after test - file.remove(commands_path) + data_formats <- c("Shapefile", "KML") + for (d in seq_along(data_formats)) { + # run download function + download_data(dataset_name = "smoke", + date_start = date_start, + date_end = date_end, + data_format = data_formats[d], + directory_to_download = directory_to_download, + directory_to_save = directory_to_save, + data_download_acknowledgement = TRUE, + download = FALSE, + remove_command = FALSE, + unzip = FALSE, + remove_zip = FALSE) + # define file path with commands + commands_path <- paste0(directory_to_download, + "hms_smoke_", + gsub("-", "", date_start), + "_", + gsub("-", "", date_end), + "_curl_commands.txt") + # import commands + commands <- read_commands(commands_path = commands_path) + # extract urls + urls <- extract_urls(commands = commands, position = 6) + # check HTTP URL status + url_status <- check_urls(urls = urls, size = 3L, method = "HEAD") + # implement unit tests + test_download_functions(directory_to_save = directory_to_save, + commands_path = commands_path, + url_status = url_status) + # remove file with commands after test + file.remove(commands_path) + } +}) + +testthat::test_that("download_hms_data error for unzip and directory.", { + testthat::expect_error( + download_data( + dataset_name = "hms", + data_download_acknowledgement = TRUE, + directory_to_save = testthat::test_path("..", "testdata/", ""), + directory_to_download = testthat::test_path("..", "testdata/", ""), + unzip = FALSE, + remove_zip = TRUE + ) + ) }) testthat::test_that("NLCD download URLs have HTTP status 200.", { @@ -611,8 +642,8 @@ testthat::test_that("SEDAC population data types are coerced.", { year <- c("totpop") data_formats <- c("GeoTIFF", "ASCII", "netCDF") data_resolutions <- c("30 second", "2pt5_min") - directory_to_download <- "../testdata/" - directory_to_save <- "../testdata/" + directory_to_download <- testthat::test_path("..", "testdata/", "") + directory_to_save <- testthat::test_path("..", "testdata/", "") for (f in seq_along(data_formats)) { download_data(dataset_name = "sedac_population", year = year, @@ -999,7 +1030,7 @@ testthat::test_that("EPA TRI download URLs have HTTP status 200.", { # extract urls urls <- extract_urls(commands = commands, position = 3) # check HTTP URL status - url_status <- check_urls(urls = urls, size = 1L, method = "GET") + url_status <- check_urls(urls = urls, size = 1L, method = "SKIP") # implement unit tests test_download_functions(directory_to_save = directory_to_save, commands_path = commands_path, @@ -1146,3 +1177,67 @@ testthat::test_that("epa certificate", { ) ) }) + + +testthat::test_that("extract_urls returns NULL undefined position.", { + commands <- paste0( + "curl -s -o ", + "/PATH/hms_smoke_Shapefile_20230901.zip --url ", + "https://satepsanone.nesdis.noaa.gov/pub/FIRE/web/HMS/Smoke_Polygons/", + "Shapefile/2023/09/hms_smoke20230901.zip" + ) + urls <- extract_urls(commands = commands) + testthat::expect_true( + is.null(urls) + ) +}) + +testthat::test_that("check_urls returns NULL undefined size.", { + urls <- paste0( + "https://satepsanone.nesdis.noaa.gov/pub/FIRE/web/HMS/Smoke_Polygons/", + "Shapefile/2023/09/hms_smoke20230901.zip" + ) + url_status <- check_urls(urls = urls, method = "HEAD") + testthat::expect_true( + is.null(url_status) + ) +}) + +testthat::test_that("download_hms_data LIVE run.", { + # function parameters + date <- "2018-01-01" + directory <- testthat::test_path("..", "testdata", "hms_live") + # create file to be deleted + dir.create(directory) + file.create( + paste0( + directory, + "/hms_smoke_20180101_20180101_curl_commands.txt" + ) + ) + # run download function + download_data( + dataset_name = "hms", + date_start = date, + date_end = date, + directory_to_save = directory, + directory_to_download = directory, + data_download_acknowledgement = TRUE, + download = TRUE, + unzip = TRUE, + remove_zip = TRUE, + remove_command = FALSE + ) + testthat::expect_true( + length(list.files(directory)) == 5 + ) + commands <- list.files(directory, pattern = ".txt", full.names = TRUE) + testthat::expect_true( + file.exists(commands) + ) + Sys.sleep(1.5) + # remove directory + files <- list.files(directory, full.names = TRUE) + sapply(files, file.remove) + file.remove(directory) +}) diff --git a/tests/testthat/test-manipulate_spacetime_data.R b/tests/testthat/test-manipulate_spacetime_data.R index 7130c304..b42502c8 100644 --- a/tests/testthat/test-manipulate_spacetime_data.R +++ b/tests/testthat/test-manipulate_spacetime_data.R @@ -827,3 +827,27 @@ test_that("project_dt works as expected", { expect_s3_class(dfdtp, "data.table") }) + +testthat::test_that("tests for missing 'time' column across functions.", { + withr::local_package("terra") + withr::local_package("data.table") + locs <- data.frame(lon = -78.8277, lat = 35.95013) + locs$site_id <- "3799900018810101" + expect_error( + convert_stobj_to_stdt( + terra::vect(locs, geom = c("lon", "lat"), crs = "EPSG:4326") + ) + ) + expect_error( + sftime_as_mysftime( + locs, + "time" + ) + ) + expect_error( + dt_as_sftime( + data.table::data.table(locs), + "EPSG:4326" + ) + ) +})