From bacf5284396aef011286d689f78f96210cd91379 Mon Sep 17 00:00:00 2001 From: Mitchell Manware Date: Thu, 25 Jul 2024 15:08:25 -0400 Subject: [PATCH 1/6] cran debug 1 --- .Rbuildignore | 5 ++--- .gitignore | 4 ---- DESCRIPTION | 4 ++-- inst/REFERENCES.bib | 6 +++--- 4 files changed, 7 insertions(+), 12 deletions(-) diff --git a/.Rbuildignore b/.Rbuildignore index 043dd2aa..6beba603 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -1,4 +1,3 @@ -^vignettes/ ^input/ ^tools/ ^manuscript/ @@ -7,7 +6,6 @@ ^amadeus\.Rproj$ ^\.Rproj\.user$ ^\.github$ -^LICENSE\.md$ ^_pkgdown\.yml$ ^docs$ ^pkgdown$ @@ -19,4 +17,5 @@ pkgdown output manuscript input -tests \ No newline at end of file +tests +vignettes/epa_download.Rmd \ No newline at end of file diff --git a/.gitignore b/.gitignore index 3a3bd8a3..74a93ac0 100644 --- a/.gitignore +++ b/.gitignore @@ -21,10 +21,6 @@ # RStudio files .Rproj.user/ -# produced vignettes -vignettes/*.html -vignettes/*.pdf - # OAuth2 token, see https://github.com/hadley/httr/releases/tag/v0.3 .httr-oauth diff --git a/DESCRIPTION b/DESCRIPTION index de636379..97f8063b 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -9,10 +9,10 @@ Authors@R: c( person("Kyle", "Messier", role = c("aut", "cre"), email = "kyle.messier@nih.gov", comment = c(ORCID = "0000-0001-9508-9623")) ) Maintainer: Kyle Messier -Description: amadeus is designed to faciliate access to and utiltiy with large scale, publicly available environmental data in R. The package contains funtions for downloading (download_data()) raw data files from web URLs, processing (process_covariates()) the raw data files into interpretable spatial objects, and extracting values (calc_covariates()) from the spatial data objects at point and polygon locations. These functions call a series of source-specific functions which are tailored to each data sources/datasets particular URL structure, data format, and spatio-temporal resolution. amadeus functions are tested, versioned, and open source and open access. +Description: Functions are designed to facilitate access to and utility with large scale, publicly available environmental data in R. The package contains functions for downloading (download_data()) raw data files from web URLs, processing (process_covariates()) the raw data files into clean spatial objects, and extracting values (calc_covariates()) from the spatial data objects at point and polygon locations. These functions call a series of source-specific functions which are tailored to each data sources/datasets particular URL structure, data format, and spatial/temporal resolution. amadeus functions are tested, versioned, and open source and open access. Depends: R (>= 4.1.0) Imports: dplyr, sf, sftime, stats, terra, methods, data.table, httr, rvest, exactextractr, utils, stringi, testthat (>= 3.0.0), parallelly, stars, future, future.apply, tidyr, rlang, rstac, nhdplusTools, archive, collapse, Rdpack -Suggests: covr, withr, knitr, rmarkdown, lwgeom, FNN, doRNG +Suggests: covr, withr, knitr, rmarkdown, pandoc, lwgeom, FNN, doRNG RdMacros: Rdpack Encoding: UTF-8 VignetteBuilder: knitr, rmarkdown diff --git a/inst/REFERENCES.bib b/inst/REFERENCES.bib index 5ebde79b..a04cfaa1 100644 --- a/inst/REFERENCES.bib +++ b/inst/REFERENCES.bib @@ -1959,7 +1959,7 @@ @usgs.gov. urldate = {2024-06-24}, publisher = {U.S. Geological Survey}, author = {Dewitz, J.}, - month = feb, + month = {March}, year = {2024}, keywords = {land use change}, } @@ -1987,7 +1987,7 @@ @article{mesinger_north_2006 urldate = {2024-06-24}, journal = {Bulletin of the American Meteorological Society}, author = {Mesinger, Fedor and DiMego, Geoff and Kalnay, Eugenia and Mitchell, Kenneth and Shafran, Perry C. and Ebisuzaki, Wesley and Jović, Dušan and Woollen, Jack and Rogers, Eric and Berbery, Ernesto H. and Ek, Michael B. and Fan, Yun and Grumbine, Robert and Higgins, Wayne and Li, Hong and Lin, Ying and Manikin, Geoff and Parrish, David and Shi, Wei}, - month = mar, + month = {March}, year = {2006}, pages = {343--360} } @@ -2061,7 +2061,7 @@ @article{keller_description_2021 urldate = {2024-06-24}, journal = {Journal of Advances in Modeling Earth Systems}, author = {Keller, Christoph A. and Knowland, K. Emma and Duncan, Bryan N. and Liu, Junhua and Anderson, Daniel C. and Das, Sampa and Lucchesi, Robert A. and Lundgren, Elizabeth W. and Nicely, Julie M. and Nielsen, Eric and Ott, Lesley E. and Saunders, Emily and Strode, Sarah A. and Wales, Pamela A. and Jacob, Daniel J. and Pawson, Steven}, - month = apr, + month = {April}, year = {2021}, pages = {e2020MS002413}, } From e127c18695975b571316701280e8fa87a49a6df4 Mon Sep 17 00:00:00 2001 From: Insang Song Date: Fri, 26 Jul 2024 12:10:30 -0400 Subject: [PATCH 2/6] OLM functions removed - Modified: download/process functions, wrappers candidates, tests - olm functions are stored in R/olm_functions.R, which is added to .Rbuildignore - Roxygen rebuilt --- .Rbuildignore | 3 +- NAMESPACE | 8 - R/download.R | 187 +------------- R/download_auxiliary.R | 65 ----- R/olm_functions.R | 280 +++++++++++++++++++++ R/process.R | 39 +-- man/download_data.Rd | 5 +- man/download_olm.Rd | 143 ----------- man/list_stac_files.Rd | 42 ---- man/process_covariates.Rd | 3 +- man/process_olm.Rd | 35 --- tests/olmtests/test-download_olm.R | 49 ++++ tests/olmtests/test-list_stac_files.R | 31 +++ tests/olmtests/test-process_olm.R | 23 ++ tests/testthat/test-calculate_covariates.R | 2 +- tests/testthat/test-download_functions.R | 85 +------ tests/testthat/test-process.R | 22 +- 17 files changed, 401 insertions(+), 621 deletions(-) create mode 100644 R/olm_functions.R delete mode 100644 man/download_olm.Rd delete mode 100644 man/list_stac_files.Rd delete mode 100644 man/process_olm.Rd create mode 100644 tests/olmtests/test-download_olm.R create mode 100644 tests/olmtests/test-list_stac_files.R create mode 100644 tests/olmtests/test-process_olm.R diff --git a/.Rbuildignore b/.Rbuildignore index 6beba603..eb1a7d0a 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -18,4 +18,5 @@ output manuscript input tests -vignettes/epa_download.Rmd \ No newline at end of file +vignettes/epa_download.Rmd +R/olm_functions.R \ No newline at end of file diff --git a/NAMESPACE b/NAMESPACE index 72dc62fd..b6e5b6a9 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -50,7 +50,6 @@ export(download_modis) export(download_narr) export(download_nei) export(download_nlcd) -export(download_olm) export(download_permit) export(download_prism) export(download_remove_command) @@ -69,7 +68,6 @@ export(extract_urls) export(generate_date_sequence) export(generate_time_sequence) export(is_date_proper) -export(list_stac_files) export(narr_variable) export(process_aqs) export(process_blackmarble) @@ -99,7 +97,6 @@ export(process_modis_warp) export(process_narr) export(process_nei) export(process_nlcd) -export(process_olm) export(process_prism) export(process_sedac_codes) export(process_sedac_groads) @@ -161,11 +158,6 @@ importFrom(nhdplusTools,get_huc) importFrom(parallelly,availableWorkers) importFrom(rlang,inject) importFrom(rlang,sym) -importFrom(rstac,assets_url) -importFrom(rstac,links) -importFrom(rstac,read_collections) -importFrom(rstac,read_items) -importFrom(rstac,read_stac) importFrom(sf,st_as_sf) importFrom(sf,st_as_sfc) importFrom(sf,st_bbox) diff --git a/R/download.R b/R/download.R index ced1d302..205d9a75 100644 --- a/R/download.R +++ b/R/download.R @@ -34,6 +34,9 @@ #' * \code{\link{download_nei}}: `"nei"`, `"NEI"` #' * \code{\link{download_gridmet}}: `"gridMET"`, `"gridmet"` #' * \code{\link{download_terraclimate}}: `"TerraClimate"`, `"terraclimate"` +#' * \code{\link{download_huc}}: `"huc"` +#' * \code{\link{download_cropscape}}: `"cropscape"`, `"cdl"` +#' * \code{\link{download_prism}}: `"prism"` #' @return NULL #' @examples #' \dontrun{ @@ -57,7 +60,7 @@ download_data <- "sedac_population", "groads", "population", "hms", "smoke", "tri", "nei", "gridmet", "terraclimate", "huc", "cropscape", "cdl", - "prism", "olm", "openlandmap"), + "prism"), directory_to_save = NULL, acknowledgement = FALSE, ... @@ -94,9 +97,7 @@ download_data <- huc = download_huc, cropscape = download_cropscape, cdl = download_cropscape, - prism = download_prism, - olm = download_olm, - openlandmap = download_olm + prism = download_prism ) tryCatch( @@ -2819,184 +2820,6 @@ download_nei <- function( } -# nolint start -#' Download OpenLandMap data -#' @description -#' Accesses and downloads OpenLandMap data from the [OpenLandMap website](https://opengeohub.org/about-openlandmap/). -#' @param product character(1). Available collection name in OpenLandMap -#' STAC Catalog. [list_stac_files] with `id_only = TRUE` to see available collections. -#' * "no2_s5p.l3.trop.tmwm" -#' * "no2_s5p.l3.trop.tmwm.ltm" -#' * "log.oc_iso.10694" -#' * "evi_mod13q1.stl.trend.logit.ols.beta" -#' * "land.cover_esacci.lc.l4" -#' * "evi_mod13q1.tmwm.inpaint" -#' * "dtm.bareearth_ensemble" -#' * "fapar_essd.lstm" -#' * "fapar_essd.lstm.p95.beta" -#' * "pot.fapar_fapar.p95.eml.m" -#' * "pot.fapar_fapar.p95.eml" -#' * "snow.cover_esa.modis" -#' * "snow.cover_esa.modis.ltm" -#' * "wilderness_li2022.human.footprint" -#' * "wv_mcd19a2v061.seasconv" -#' * "wv_mcd19a2v061.seasconv.m_p50" -#' * "wv_mcd19a2v061.seasconv.m_p25" -#' * "wv_mcd19a2v061.seasconv.m_p75" -#' * "wv_mcd19a2v061.seasconv.m_std" -#' * "wv_mcd19a2v061.seasconv.m.yearly" -#' * "bulkdens.fineearth_usda.4a1h" -#' * "geom_merit.dem" -#' * "fapar_proba.v" -#' * "forest.cover_esacci.ifl" -#' * "grtgroup_usda.soiltax" -#' * "land.cover_copernicus" -#' * "organic.carbon.stock_msa.kgm2" -#' * "organic.carbon_usda.6a1c" -#' * "pft.landcover_esa.cci.lc" -#' * "precipitation_sm2rain.ltm" -#' * "ph.h2o_usda.4c1a2a" -#' * "pop.count_ghs.jrc" -#' * "sand.wfraction_usda.3a1a1a" -#' * "lc_mcd12q1v061.p1" -#' * "texture.class_usda.tt" -#' * "water.occurrence_jrc.surfacewater" -#' * "watercontent.33kPa_usda.4b1c" -#' * "dsm_glo30" -#' * "lc_glad.glcluc" -#' * "lc_glad.glcluc.change" -#' * "landuse.cropland_hyde" -#' * "landuse.pasture_hyde" -#' * "land.use.land.cover_hilda.plus" -#' * "lst_mod11a2.daytime.trend.logit.ols.beta" -#' * "lst_mod11a2.nighttime.trend.logit.ols.beta" -#' * "lst_mod11a2.daytime.annual" -#' * "lst_mod11a2.nighttime.annual" -#' * "lst_mod11a2.daytime" -#' * "lst_mod11a2.nighttime" -#' * "landform_usgs.ecotapestry" -#' * "lithology_usgs.ecotapestry" -#' * "grtgroup_usda.soiltax.hapludalfs" -#' * "biome.type_biome00k" -#' * "biomes_biome6k.tropical.evergreen.broadleaf.forest" -#' * "biomes_biome6k.tropical.evergreen.broadleaf.forest.rcp26" -#' * "biomes_biome6k.tropical.evergreen.broadleaf.forest.rcp45" -#' * "biomes_biome6k.tropical.evergreen.broadleaf.forest.rcp85" -#' * "biomes_biome6k.tropical.savanna" -#' * "biomes_biome6k.tropical.savanna.rcp26" -#' * "biomes_biome6k.tropical.savanna.rcp45" -#' * "biomes_biome6k.tropical.savanna.rcp85" -#' * "lc_glc.fcs30d" -#' * "nightlights.average_viirs.v21" -#' * "nightlights.difference_viirs.v21" -#' * "l2a.gedi" -#' * "fluxnet" -#' * "gbov" -#' * "geowiki.lc" -#' * "geowiki.forest.loss" -#' * "veg.plot" -#' * "obis" -#' * "fapar.eml" -#' @param format character(1). File format to query. Default is "tif". -#' It could be used as a pattern search for the file names. -#' @param directory_to_save character(1). Directory to download files. -#' @param acknowledgement logical(1). By setting \code{TRUE} the -#' user acknowledges that the data downloaded using this function may be very -#' large and use lots of machine storage and memory. -#' @param download logical(1). \code{FALSE} will generate a *.txt file -#' containing all download commands. By setting \code{TRUE} the function -#' will download all of the requested data files. -#' @param remove_command logical(1). -#' Remove (\code{TRUE}) or keep (\code{FALSE}) -#' the text file containing download commands. -#' @author Insang Song -#' @note `extdata/openlandmap_assets.rds` contains the available assets in OpenLandMap. -#' Users may want to check the available assets to download data directly. -#' For developers: JSON files should be found at STAC catalog of OpenLandMap when updated. -#' @return NULL; GeoTIFF (.tif) files will be stored in -#' \code{directory_to_save}. -#' @seealso [list_stac_files] -#' @importFrom Rdpack reprompt -#' @references -#' \insertRef{data_hengl2023openlandmap}{amadeus} -#' @examples -#' \dontrun{ -#' download_olm( -#' product = "no2_s5p.l3.trop.tmwm", -#' format = "tif", -#' directory_to_save = "./data", -#' acknowledgement = TRUE, -#' download = TRUE, -#' remove_command = TRUE -#' ) -#' } -#' @export -# nolint end -download_olm <- function( - product = NULL, - format = "tif", - directory_to_save = NULL, - acknowledgement = FALSE, - download = FALSE, - remove_command = FALSE -) { - #### 1. check for data download acknowledgement - download_permit(acknowledgement = acknowledgement) - #### 2. directory setup - download_setup_dir(directory_to_save) - directory_to_save <- download_sanitize_path(directory_to_save) - - #### 3. define measurement data paths - - download_urls <- - list_stac_files( - which = product, - format = format, - id_only = FALSE - ) - url_filenames <- strsplit(download_urls, "/", fixed = TRUE)[[1]] - url_filenames <- sapply(url_filenames, \(x) x[length(x)]) - - download_names <- paste0(directory_to_save, url_filenames) - #### 4. build download command - download_commands <- - paste0("wget -e robots=off -np ", - download_urls, - " -O ", - download_names, - "\n") - - #### 5. initiate "..._curl_commands.txt" - commands_txt <- paste0( - directory_to_save, - "OLM_queried_", - product, - "_", - Sys.Date(), - "_wget_commands.txt" - ) - download_sink(commands_txt) - #### 6. concatenate and print download commands to "..._curl_commands.txt" - writeLines(download_commands) - #### 7. finish "..._curl_commands.txt" file - sink() - #### 8. build system command - system_command <- paste0( - ". ", - commands_txt, - "\n" - ) - #### 9. download data - download_run(download = download, - system_command = system_command) - - message("Requests were processed.\n") - #### 10. remove download commands - download_remove_command(commands_txt = commands_txt, - remove = remove_command) - -} - # nolint start #' Download National Hydrography Dataset (NHD) data diff --git a/R/download_auxiliary.R b/R/download_auxiliary.R index 904754de..acbdfccf 100644 --- a/R/download_auxiliary.R +++ b/R/download_auxiliary.R @@ -451,71 +451,6 @@ test_download_functions <- function( } } -#' Retrieve file links from SpatioTemporal Assets Catalog (STAC) -#' @description -#' Read file links from SpatioTemporal Assets Catalog (STAC) JSON file. -#' @note -#' Retrieving URLs may take a while depending on the spatial tiling, -#' temporal resolution, and other assets. Users are encouraged to use -#' `which` parameter to select a specific collection. -#' @param stac_json character(1). Full path of STAC JSON file. -#' @param format character(1). Format of target files. Default is "tif". -#' @param which numeric/character. Index or name of collection to retrieve. -#' @param id_only logical(1). Return collection IDs only. -#' @return character vector of file links. -#' @examples -#' \dontrun{ -#' read_stac_json() -#' } -#' @export -#' @keywords auxiliary -#' @author Insang Song -#' @importFrom rstac read_collections -#' @importFrom rstac read_items -#' @importFrom rstac assets_url -#' @importFrom rstac read_stac -#' @importFrom rstac links -list_stac_files <- - function( - stac_json = - "https://s3.eu-central-1.wasabisys.com/stac/openlandmap/catalog.json", - format = "tif", - which = NULL, - id_only = FALSE - ) { - allcollections <- - rstac::read_stac( - stac_json - ) |> - rstac:::read_collections.catalog() - collection <- allcollections$collections - collection_ids <- sapply(collection, \(x) x$id) - if (id_only) { - return(collection_ids) - } - - message( - sprintf( - "Names of collections include: %s", - paste( - collection_ids, collapse = "\n" - ) - ) - ) - - which <- if (!is.numeric(which)) grep(which, collection_ids) else which - collection <- if (is.null(which)) collection else collection[which] - - collection_assets <- - lapply( - collection, - \(x) rstac::assets_url(rstac:::read_items.doc_collection(x)) - ) - collection_assets <- unlist(collection_assets) - list_assets <- grep(sprintf("%s$", format), collection_assets, value = TRUE) - - return(list_assets) - } #' Sort NOAA NARR variables #' @description diff --git a/R/olm_functions.R b/R/olm_functions.R new file mode 100644 index 00000000..cfcacd25 --- /dev/null +++ b/R/olm_functions.R @@ -0,0 +1,280 @@ + +# nolint start +#' Download OpenLandMap data +#' @description +#' Accesses and downloads OpenLandMap data from the [OpenLandMap website](https://opengeohub.org/about-openlandmap/). +#' @param product character(1). Available collection name in OpenLandMap +#' STAC Catalog. [list_stac_files] with `id_only = TRUE` to see available collections. +#' * "no2_s5p.l3.trop.tmwm" +#' * "no2_s5p.l3.trop.tmwm.ltm" +#' * "log.oc_iso.10694" +#' * "evi_mod13q1.stl.trend.logit.ols.beta" +#' * "land.cover_esacci.lc.l4" +#' * "evi_mod13q1.tmwm.inpaint" +#' * "dtm.bareearth_ensemble" +#' * "fapar_essd.lstm" +#' * "fapar_essd.lstm.p95.beta" +#' * "pot.fapar_fapar.p95.eml.m" +#' * "pot.fapar_fapar.p95.eml" +#' * "snow.cover_esa.modis" +#' * "snow.cover_esa.modis.ltm" +#' * "wilderness_li2022.human.footprint" +#' * "wv_mcd19a2v061.seasconv" +#' * "wv_mcd19a2v061.seasconv.m_p50" +#' * "wv_mcd19a2v061.seasconv.m_p25" +#' * "wv_mcd19a2v061.seasconv.m_p75" +#' * "wv_mcd19a2v061.seasconv.m_std" +#' * "wv_mcd19a2v061.seasconv.m.yearly" +#' * "bulkdens.fineearth_usda.4a1h" +#' * "geom_merit.dem" +#' * "fapar_proba.v" +#' * "forest.cover_esacci.ifl" +#' * "grtgroup_usda.soiltax" +#' * "land.cover_copernicus" +#' * "organic.carbon.stock_msa.kgm2" +#' * "organic.carbon_usda.6a1c" +#' * "pft.landcover_esa.cci.lc" +#' * "precipitation_sm2rain.ltm" +#' * "ph.h2o_usda.4c1a2a" +#' * "pop.count_ghs.jrc" +#' * "sand.wfraction_usda.3a1a1a" +#' * "lc_mcd12q1v061.p1" +#' * "texture.class_usda.tt" +#' * "water.occurrence_jrc.surfacewater" +#' * "watercontent.33kPa_usda.4b1c" +#' * "dsm_glo30" +#' * "lc_glad.glcluc" +#' * "lc_glad.glcluc.change" +#' * "landuse.cropland_hyde" +#' * "landuse.pasture_hyde" +#' * "land.use.land.cover_hilda.plus" +#' * "lst_mod11a2.daytime.trend.logit.ols.beta" +#' * "lst_mod11a2.nighttime.trend.logit.ols.beta" +#' * "lst_mod11a2.daytime.annual" +#' * "lst_mod11a2.nighttime.annual" +#' * "lst_mod11a2.daytime" +#' * "lst_mod11a2.nighttime" +#' * "landform_usgs.ecotapestry" +#' * "lithology_usgs.ecotapestry" +#' * "grtgroup_usda.soiltax.hapludalfs" +#' * "biome.type_biome00k" +#' * "biomes_biome6k.tropical.evergreen.broadleaf.forest" +#' * "biomes_biome6k.tropical.evergreen.broadleaf.forest.rcp26" +#' * "biomes_biome6k.tropical.evergreen.broadleaf.forest.rcp45" +#' * "biomes_biome6k.tropical.evergreen.broadleaf.forest.rcp85" +#' * "biomes_biome6k.tropical.savanna" +#' * "biomes_biome6k.tropical.savanna.rcp26" +#' * "biomes_biome6k.tropical.savanna.rcp45" +#' * "biomes_biome6k.tropical.savanna.rcp85" +#' * "lc_glc.fcs30d" +#' * "nightlights.average_viirs.v21" +#' * "nightlights.difference_viirs.v21" +#' * "l2a.gedi" +#' * "fluxnet" +#' * "gbov" +#' * "geowiki.lc" +#' * "geowiki.forest.loss" +#' * "veg.plot" +#' * "obis" +#' * "fapar.eml" +#' @param format character(1). File format to query. Default is "tif". +#' It could be used as a pattern search for the file names. +#' @param directory_to_save character(1). Directory to download files. +#' @param acknowledgement logical(1). By setting \code{TRUE} the +#' user acknowledges that the data downloaded using this function may be very +#' large and use lots of machine storage and memory. +#' @param download logical(1). \code{FALSE} will generate a *.txt file +#' containing all download commands. By setting \code{TRUE} the function +#' will download all of the requested data files. +#' @param remove_command logical(1). +#' Remove (\code{TRUE}) or keep (\code{FALSE}) +#' the text file containing download commands. +#' @author Insang Song +#' @note `extdata/openlandmap_assets.rds` contains the available assets in OpenLandMap. +#' Users may want to check the available assets to download data directly. +#' For developers: JSON files should be found at STAC catalog of OpenLandMap when updated. +#' @return NULL; GeoTIFF (.tif) files will be stored in +#' \code{directory_to_save}. +#' @seealso [list_stac_files] +#' @importFrom Rdpack reprompt +#' @references +#' \insertRef{data_hengl2023openlandmap}{amadeus} +#' @examples +#' \dontrun{ +#' download_olm( +#' product = "no2_s5p.l3.trop.tmwm", +#' format = "tif", +#' directory_to_save = "./data", +#' acknowledgement = TRUE, +#' download = TRUE, +#' remove_command = TRUE +#' ) +#' } +#' @export +# nolint end +download_olm <- function( + product = NULL, + format = "tif", + directory_to_save = NULL, + acknowledgement = FALSE, + download = FALSE, + remove_command = FALSE +) { + #### 1. check for data download acknowledgement + download_permit(acknowledgement = acknowledgement) + #### 2. directory setup + download_setup_dir(directory_to_save) + directory_to_save <- download_sanitize_path(directory_to_save) + + #### 3. define measurement data paths + + download_urls <- + list_stac_files( + which = product, + format = format, + id_only = FALSE + ) + url_filenames <- strsplit(download_urls, "/", fixed = TRUE)[[1]] + url_filenames <- sapply(url_filenames, \(x) x[length(x)]) + + download_names <- paste0(directory_to_save, url_filenames) + #### 4. build download command + download_commands <- + paste0("wget -e robots=off -np ", + download_urls, + " -O ", + download_names, + "\n") + + #### 5. initiate "..._curl_commands.txt" + commands_txt <- paste0( + directory_to_save, + "OLM_queried_", + product, + "_", + Sys.Date(), + "_wget_commands.txt" + ) + download_sink(commands_txt) + #### 6. concatenate and print download commands to "..._curl_commands.txt" + writeLines(download_commands) + #### 7. finish "..._curl_commands.txt" file + sink() + #### 8. build system command + system_command <- paste0( + ". ", + commands_txt, + "\n" + ) + #### 9. download data + download_run(download = download, + system_command = system_command) + + message("Requests were processed.\n") + #### 10. remove download commands + download_remove_command(commands_txt = commands_txt, + remove = remove_command) + +} + + +#' Retrieve file links from SpatioTemporal Assets Catalog (STAC) +#' @description +#' Read file links from SpatioTemporal Assets Catalog (STAC) JSON file. +#' @note +#' Retrieving URLs may take a while depending on the spatial tiling, +#' temporal resolution, and other assets. Users are encouraged to use +#' `which` parameter to select a specific collection. +#' @param stac_json character(1). Full path of STAC JSON file. +#' @param format character(1). Format of target files. Default is "tif". +#' @param which numeric/character. Index or name of collection to retrieve. +#' @param id_only logical(1). Return collection IDs only. +#' @return character vector of file links. +#' @examples +#' \dontrun{ +#' read_stac_json() +#' } +#' @export +#' @keywords auxiliary +#' @author Insang Song +#' @importFrom rstac read_collections +#' @importFrom rstac read_items +#' @importFrom rstac assets_url +#' @importFrom rstac read_stac +#' @importFrom rstac links +list_stac_files <- + function( + stac_json = + "https://s3.eu-central-1.wasabisys.com/stac/openlandmap/catalog.json", + format = "tif", + which = NULL, + id_only = FALSE + ) { + allcollections <- + rstac::read_stac( + stac_json + ) |> + rstac:::read_collections.catalog() + collection <- allcollections$collections + collection_ids <- sapply(collection, \(x) x$id) + if (id_only) { + return(collection_ids) + } + + message( + sprintf( + "Names of collections include: %s", + paste( + collection_ids, collapse = "\n" + ) + ) + ) + + which <- if (!is.numeric(which)) grep(which, collection_ids) else which + collection <- if (is.null(which)) collection else collection[which] + + collection_assets <- + lapply( + collection, + \(x) rstac::assets_url(rstac:::read_items.doc_collection(x)) + ) + collection_assets <- unlist(collection_assets) + list_assets <- grep(sprintf("%s$", format), collection_assets, value = TRUE) + + return(list_assets) + } + + +#' Process OpenLandMap data +#' @param path character giving OpenLandMap data path +#' @param extent numeric(4) or SpatExtent giving the extent of the raster +#' if `NULL` (default), the entire raster is loaded +#' @param ... Placeholders. +#' @return a `SpatRaster` object +#' @author Insang Song +#' @importFrom terra rast +#' @examples +#' \dontrun{ +#' olm <- process_olm( +#' path = paste0( +#' "./data/no2_s5p.l3.trop.tmwm.p50_p90_2km_a_", +#' "20180501_20221130_go_epsg.4326_v20221219.tif" +#' ) +#' ) +#' } +#' @export +process_olm <- + function( + path = NULL, + extent = NULL, + ... + ) { + # check inputs + if (!is.character(path) || is.null(path)) { + stop("path is not a character.") + } + olm <- terra::rast(path, win = extent) + return(olm) + } + + diff --git a/R/process.R b/R/process.R index 4217f8b0..3f1cb0f9 100644 --- a/R/process.R +++ b/R/process.R @@ -32,7 +32,6 @@ #' * \code{\link{process_huc}}: "huc", "HUC" #' * \code{\link{process_cropscape}}: "cropscape", "cdl" #' * \code{\link{process_prism}}: "prism", "PRISM" -#' * \code{\link{process_olm}}: "olm", "openlandmap" #' @return `SpatVector`, `SpatRaster`, `sf`, or `character` depending on #' covariate type and selections. #' @author Insang Song @@ -60,7 +59,7 @@ process_covariates <- "nlcd", "tri", "narr", "nei", "ecoregions", "ecoregion", "merra", "merra2", "gridmet", "terraclimate", - "huc", "cropscape", "cdl", "prism", "olm", "openlandmap"), + "huc", "cropscape", "cdl", "prism"), path = NULL, ... ) { @@ -98,9 +97,7 @@ process_covariates <- huc = process_huc, cropscape = process_cropscape, cdl = process_cropscape, - prism = process_prism, - olm = process_olm, - openlandmap = process_olm + prism = process_prism ) res_covariate <- @@ -2940,35 +2937,3 @@ process_prism <- } # nolint end - -#' Process OpenLandMap data -#' @param path character giving OpenLandMap data path -#' @param extent numeric(4) or SpatExtent giving the extent of the raster -#' if `NULL` (default), the entire raster is loaded -#' @param ... Placeholders. -#' @return a `SpatRaster` object -#' @author Insang Song -#' @importFrom terra rast -#' @examples -#' \dontrun{ -#' olm <- process_olm( -#' path = paste0( -#' "./data/no2_s5p.l3.trop.tmwm.p50_p90_2km_a_", -#' "20180501_20221130_go_epsg.4326_v20221219.tif" -#' ) -#' ) -#' } -#' @export -process_olm <- - function( - path = NULL, - extent = NULL, - ... - ) { - # check inputs - if (!is.character(path) || is.null(path)) { - stop("path is not a character.") - } - olm <- terra::rast(path, win = extent) - return(olm) - } diff --git a/man/download_data.Rd b/man/download_data.Rd index 59285b77..8717fc23 100644 --- a/man/download_data.Rd +++ b/man/download_data.Rd @@ -8,7 +8,7 @@ download_data( dataset_name = c("aqs", "ecoregion", "ecoregions", "geos", "gmted", "koppen", "koppengeiger", "merra2", "merra", "modis", "narr", "nlcd", "noaa", "sedac_groads", "sedac_population", "groads", "population", "hms", "smoke", "tri", "nei", "gridmet", - "terraclimate", "huc", "cropscape", "cdl", "prism", "olm", "openlandmap"), + "terraclimate", "huc", "cropscape", "cdl", "prism"), directory_to_save = NULL, acknowledgement = FALSE, ... @@ -68,6 +68,9 @@ Please refer to: \item \code{\link{download_nei}}: \code{"nei"}, \code{"NEI"} \item \code{\link{download_gridmet}}: \code{"gridMET"}, \code{"gridmet"} \item \code{\link{download_terraclimate}}: \code{"TerraClimate"}, \code{"terraclimate"} +\item \code{\link{download_huc}}: \code{"huc"} +\item \code{\link{download_cropscape}}: \code{"cropscape"}, \code{"cdl"} +\item \code{\link{download_prism}}: \code{"prism"} } } \author{ diff --git a/man/download_olm.Rd b/man/download_olm.Rd deleted file mode 100644 index a56ffd85..00000000 --- a/man/download_olm.Rd +++ /dev/null @@ -1,143 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/download.R -\name{download_olm} -\alias{download_olm} -\title{Download OpenLandMap data} -\usage{ -download_olm( - product = NULL, - format = "tif", - directory_to_save = NULL, - acknowledgement = FALSE, - download = FALSE, - remove_command = FALSE -) -} -\arguments{ -\item{product}{character(1). Available collection name in OpenLandMap -STAC Catalog. \link{list_stac_files} with \code{id_only = TRUE} to see available collections. -\itemize{ -\item "no2_s5p.l3.trop.tmwm" -\item "no2_s5p.l3.trop.tmwm.ltm" -\item "log.oc_iso.10694" -\item "evi_mod13q1.stl.trend.logit.ols.beta" -\item "land.cover_esacci.lc.l4" -\item "evi_mod13q1.tmwm.inpaint" -\item "dtm.bareearth_ensemble" -\item "fapar_essd.lstm" -\item "fapar_essd.lstm.p95.beta" -\item "pot.fapar_fapar.p95.eml.m" -\item "pot.fapar_fapar.p95.eml" -\item "snow.cover_esa.modis" -\item "snow.cover_esa.modis.ltm" -\item "wilderness_li2022.human.footprint" -\item "wv_mcd19a2v061.seasconv" -\item "wv_mcd19a2v061.seasconv.m_p50" -\item "wv_mcd19a2v061.seasconv.m_p25" -\item "wv_mcd19a2v061.seasconv.m_p75" -\item "wv_mcd19a2v061.seasconv.m_std" -\item "wv_mcd19a2v061.seasconv.m.yearly" -\item "bulkdens.fineearth_usda.4a1h" -\item "geom_merit.dem" -\item "fapar_proba.v" -\item "forest.cover_esacci.ifl" -\item "grtgroup_usda.soiltax" -\item "land.cover_copernicus" -\item "organic.carbon.stock_msa.kgm2" -\item "organic.carbon_usda.6a1c" -\item "pft.landcover_esa.cci.lc" -\item "precipitation_sm2rain.ltm" -\item "ph.h2o_usda.4c1a2a" -\item "pop.count_ghs.jrc" -\item "sand.wfraction_usda.3a1a1a" -\item "lc_mcd12q1v061.p1" -\item "texture.class_usda.tt" -\item "water.occurrence_jrc.surfacewater" -\item "watercontent.33kPa_usda.4b1c" -\item "dsm_glo30" -\item "lc_glad.glcluc" -\item "lc_glad.glcluc.change" -\item "landuse.cropland_hyde" -\item "landuse.pasture_hyde" -\item "land.use.land.cover_hilda.plus" -\item "lst_mod11a2.daytime.trend.logit.ols.beta" -\item "lst_mod11a2.nighttime.trend.logit.ols.beta" -\item "lst_mod11a2.daytime.annual" -\item "lst_mod11a2.nighttime.annual" -\item "lst_mod11a2.daytime" -\item "lst_mod11a2.nighttime" -\item "landform_usgs.ecotapestry" -\item "lithology_usgs.ecotapestry" -\item "grtgroup_usda.soiltax.hapludalfs" -\item "biome.type_biome00k" -\item "biomes_biome6k.tropical.evergreen.broadleaf.forest" -\item "biomes_biome6k.tropical.evergreen.broadleaf.forest.rcp26" -\item "biomes_biome6k.tropical.evergreen.broadleaf.forest.rcp45" -\item "biomes_biome6k.tropical.evergreen.broadleaf.forest.rcp85" -\item "biomes_biome6k.tropical.savanna" -\item "biomes_biome6k.tropical.savanna.rcp26" -\item "biomes_biome6k.tropical.savanna.rcp45" -\item "biomes_biome6k.tropical.savanna.rcp85" -\item "lc_glc.fcs30d" -\item "nightlights.average_viirs.v21" -\item "nightlights.difference_viirs.v21" -\item "l2a.gedi" -\item "fluxnet" -\item "gbov" -\item "geowiki.lc" -\item "geowiki.forest.loss" -\item "veg.plot" -\item "obis" -\item "fapar.eml" -}} - -\item{format}{character(1). File format to query. Default is "tif". -It could be used as a pattern search for the file names.} - -\item{directory_to_save}{character(1). Directory to download files.} - -\item{acknowledgement}{logical(1). By setting \code{TRUE} the -user acknowledges that the data downloaded using this function may be very -large and use lots of machine storage and memory.} - -\item{download}{logical(1). \code{FALSE} will generate a *.txt file -containing all download commands. By setting \code{TRUE} the function -will download all of the requested data files.} - -\item{remove_command}{logical(1). -Remove (\code{TRUE}) or keep (\code{FALSE}) -the text file containing download commands.} -} -\value{ -NULL; GeoTIFF (.tif) files will be stored in -\code{directory_to_save}. -} -\description{ -Accesses and downloads OpenLandMap data from the \href{https://opengeohub.org/about-openlandmap/}{OpenLandMap website}. -} -\note{ -\code{extdata/openlandmap_assets.rds} contains the available assets in OpenLandMap. -Users may want to check the available assets to download data directly. -For developers: JSON files should be found at STAC catalog of OpenLandMap when updated. -} -\examples{ -\dontrun{ -download_olm( - product = "no2_s5p.l3.trop.tmwm", - format = "tif", - directory_to_save = "./data", - acknowledgement = TRUE, - download = TRUE, - remove_command = TRUE -) -} -} -\references{ -\insertRef{data_hengl2023openlandmap}{amadeus} -} -\seealso{ -\link{list_stac_files} -} -\author{ -Insang Song -} diff --git a/man/list_stac_files.Rd b/man/list_stac_files.Rd deleted file mode 100644 index 3138ac41..00000000 --- a/man/list_stac_files.Rd +++ /dev/null @@ -1,42 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/download_auxiliary.R -\name{list_stac_files} -\alias{list_stac_files} -\title{Retrieve file links from SpatioTemporal Assets Catalog (STAC)} -\usage{ -list_stac_files( - stac_json = "https://s3.eu-central-1.wasabisys.com/stac/openlandmap/catalog.json", - format = "tif", - which = NULL, - id_only = FALSE -) -} -\arguments{ -\item{stac_json}{character(1). Full path of STAC JSON file.} - -\item{format}{character(1). Format of target files. Default is "tif".} - -\item{which}{numeric/character. Index or name of collection to retrieve.} - -\item{id_only}{logical(1). Return collection IDs only.} -} -\value{ -character vector of file links. -} -\description{ -Read file links from SpatioTemporal Assets Catalog (STAC) JSON file. -} -\note{ -Retrieving URLs may take a while depending on the spatial tiling, -temporal resolution, and other assets. Users are encouraged to use -\code{which} parameter to select a specific collection. -} -\examples{ -\dontrun{ -read_stac_json() -} -} -\author{ -Insang Song -} -\keyword{auxiliary} diff --git a/man/process_covariates.Rd b/man/process_covariates.Rd index ff9c045c..fd5f63b4 100644 --- a/man/process_covariates.Rd +++ b/man/process_covariates.Rd @@ -9,7 +9,7 @@ process_covariates( "koeppen-geiger", "koppen", "koeppen", "geos", "dummies", "gmted", "hms", "smoke", "sedac_population", "population", "sedac_groads", "groads", "roads", "nlcd", "tri", "narr", "nei", "ecoregions", "ecoregion", "merra", "merra2", "gridmet", - "terraclimate", "huc", "cropscape", "cdl", "prism", "olm", "openlandmap"), + "terraclimate", "huc", "cropscape", "cdl", "prism"), path = NULL, ... ) @@ -66,7 +66,6 @@ narr <- process_covariates( \item \code{\link{process_huc}}: "huc", "HUC" \item \code{\link{process_cropscape}}: "cropscape", "cdl" \item \code{\link{process_prism}}: "prism", "PRISM" -\item \code{\link{process_olm}}: "olm", "openlandmap" } } \author{ diff --git a/man/process_olm.Rd b/man/process_olm.Rd deleted file mode 100644 index 422ae211..00000000 --- a/man/process_olm.Rd +++ /dev/null @@ -1,35 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/process.R -\name{process_olm} -\alias{process_olm} -\title{Process OpenLandMap data} -\usage{ -process_olm(path = NULL, extent = NULL, ...) -} -\arguments{ -\item{path}{character giving OpenLandMap data path} - -\item{extent}{numeric(4) or SpatExtent giving the extent of the raster -if \code{NULL} (default), the entire raster is loaded} - -\item{...}{Placeholders.} -} -\value{ -a \code{SpatRaster} object -} -\description{ -Process OpenLandMap data -} -\examples{ -\dontrun{ -olm <- process_olm( - path = paste0( - "./data/no2_s5p.l3.trop.tmwm.p50_p90_2km_a_", - "20180501_20221130_go_epsg.4326_v20221219.tif" - ) -) -} -} -\author{ -Insang Song -} diff --git a/tests/olmtests/test-download_olm.R b/tests/olmtests/test-download_olm.R new file mode 100644 index 00000000..1e6e7d8f --- /dev/null +++ b/tests/olmtests/test-download_olm.R @@ -0,0 +1,49 @@ + +testthat::test_that( + "Download OpenLandMap using STAC", + { + withr::local_package("rstac") + links <- + readRDS( + system.file("extdata", "openlandmap_assets.rds", package = "amadeus") + ) + product <- "no2_s5p.l3.trop.tmwm" + format <- "p50_p90_2km*.*tif" + directory_to_save <- testthat::test_path("..", "testdata", "olm_temp/") + acknowledgement <- TRUE + download <- FALSE + + testthat::expect_no_error( + download_olm( + product = product, + format = format, + directory_to_save = directory_to_save, + acknowledgement = acknowledgement, + download = download, + remove_command = FALSE + ) + ) + + commands_path <- paste0( + directory_to_save, + "OLM_queried_", + product, + "_", + Sys.Date(), + "_wget_commands.txt" + ) + # import commands + commands <- read_commands(commands_path = commands_path) + # extract urls + urls <- extract_urls(commands = commands, position = 5) + # check HTTP URL status + url_status <- check_urls(urls = urls, size = 1L, method = "HEAD") + # implement unit tests + test_download_functions(directory_to_save = directory_to_save, + commands_path = commands_path, + url_status = url_status) + # remove file with commands after test + file.remove(commands_path) + unlink(directory_to_save, recursive = TRUE) + } +) diff --git a/tests/olmtests/test-list_stac_files.R b/tests/olmtests/test-list_stac_files.R new file mode 100644 index 00000000..c7518bb3 --- /dev/null +++ b/tests/olmtests/test-list_stac_files.R @@ -0,0 +1,31 @@ + +testthat::test_that("list_stac_files returns a character vector of file links", { + withr::local_package("rstac") + # Set up test data + stac_json <- "https://s3.eu-central-1.wasabisys.com/stac/openlandmap/catalog.json" + format <- "tif" + which <- 35 + + # Call the function + testthat::expect_message( + result <- list_stac_files(stac_json, format, which) + ) + # Check the return type + testthat::expect_true(is.character(result)) + # Check if all elements end with the specified format + testthat::expect_true(all(grepl(sprintf("%s$", format), result))) + + # string search keyword + keyword <- "bulkdens" + testthat::expect_message( + result1 <- list_stac_files(stac_json, format, keyword) + ) + testthat::expect_true(is.character(result1)) + + # retrieve ids only + testthat::expect_no_error( + result2 <- list_stac_files(stac_json, format, keyword, id_only = TRUE) + ) + testthat::expect_true(is.character(result2)) + +}) \ No newline at end of file diff --git a/tests/olmtests/test-process_olm.R b/tests/olmtests/test-process_olm.R new file mode 100644 index 00000000..68b4103f --- /dev/null +++ b/tests/olmtests/test-process_olm.R @@ -0,0 +1,23 @@ + +# test OpenLandMap #### +# nolint start +testthat::test_that("process_olm", { + withr::local_package("terra") + tmwm <- testthat::test_path("..", "testdata", "openlandmap", + "no2_s5p.l3.trop.tmwm.p50_p90_2km_a_20180501_20221130_go_epsg.4326_v20221219_test.tif") + testthat::expect_no_error( + olm <- process_olm(path = tmwm) + ) + testthat::expect_s4_class(olm, "SpatRaster") + testthat::expect_error( + process_olm(path = 1L) + ) + + # test with cropping extent + testthat::expect_no_error( + olm_ext <- process_olm(path = tmwm, extent = terra::ext(olm)) + ) +}) +# nolint end + + diff --git a/tests/testthat/test-calculate_covariates.R b/tests/testthat/test-calculate_covariates.R index d8601067..da955f12 100644 --- a/tests/testthat/test-calculate_covariates.R +++ b/tests/testthat/test-calculate_covariates.R @@ -1824,7 +1824,7 @@ testthat::test_that("calc_covariates wrapper works", { "gmted", "narr", "geos", "sedac_population", "population", "nlcd", "merra", "MERRA", "merra2", "MERRA2", - "tri", "nei") + "tri", "nei", "prism", "huc", "cdl") for (cand in candidates) { testthat::expect_error( calc_covariates(covariate = cand) diff --git a/tests/testthat/test-download_functions.R b/tests/testthat/test-download_functions.R index 372b85b8..cc12aaa8 100644 --- a/tests/testthat/test-download_functions.R +++ b/tests/testthat/test-download_functions.R @@ -6,8 +6,7 @@ testthat::test_that("Error when acknowledgement = FALSE", { "nlcd", "noaa", "sedac_groads", "sedac_population", "groads", "population", "hms", "smoke", "gridmet", - "terraclimate", "huc", "cropscape", "cdl", "prism", - "olm", "openlandmap") + "terraclimate", "huc", "cropscape", "cdl", "prism") for (d in seq_along(download_datasets)) { expect_error( download_data(dataset_name = download_datasets[d], @@ -24,8 +23,7 @@ testthat::test_that("Error when one parameter is NULL.", { "nlcd", "noaa", "sedac_groads", "sedac_population", "groads", "population", "hms", "smoke", "gridmet", - "terraclimate", "huc", "cropscape", "cdl", "prism", - "olm", "openlandmap") + "terraclimate", "huc", "cropscape", "cdl", "prism") for (d in seq_along(download_datasets)) { expect_error( download_data(dataset_name = download_datasets[d], @@ -1667,36 +1665,6 @@ testthat::test_that("download_prism downloads the correct data files", { }) -testthat::test_that("list_stac_files returns a character vector of file links", { - withr::local_package("rstac") - # Set up test data - stac_json <- "https://s3.eu-central-1.wasabisys.com/stac/openlandmap/catalog.json" - format <- "tif" - which <- 35 - - # Call the function - testthat::expect_message( - result <- list_stac_files(stac_json, format, which) - ) - # Check the return type - testthat::expect_true(is.character(result)) - # Check if all elements end with the specified format - testthat::expect_true(all(grepl(sprintf("%s$", format), result))) - - # string search keyword - keyword <- "bulkdens" - testthat::expect_message( - result1 <- list_stac_files(stac_json, format, keyword) - ) - testthat::expect_true(is.character(result1)) - - # retrieve ids only - testthat::expect_no_error( - result2 <- list_stac_files(stac_json, format, keyword, id_only = TRUE) - ) - testthat::expect_true(is.character(result2)) - -}) testthat::test_that("download_huc works", @@ -1757,55 +1725,6 @@ testthat::test_that("download_huc works", }) -testthat::test_that( - "Download OpenLandMap using STAC", - { - withr::local_package("rstac") - links <- - readRDS( - system.file("extdata", "openlandmap_assets.rds", package = "amadeus") - ) - product <- "no2_s5p.l3.trop.tmwm" - format <- "p50_p90_2km*.*tif" - directory_to_save <- testthat::test_path("..", "testdata", "olm_temp/") - acknowledgement <- TRUE - download <- FALSE - - testthat::expect_no_error( - download_olm( - product = product, - format = format, - directory_to_save = directory_to_save, - acknowledgement = acknowledgement, - download = download, - remove_command = FALSE - ) - ) - - commands_path <- paste0( - directory_to_save, - "OLM_queried_", - product, - "_", - Sys.Date(), - "_wget_commands.txt" - ) - # import commands - commands <- read_commands(commands_path = commands_path) - # extract urls - urls <- extract_urls(commands = commands, position = 5) - # check HTTP URL status - url_status <- check_urls(urls = urls, size = 1L, method = "HEAD") - # implement unit tests - test_download_functions(directory_to_save = directory_to_save, - commands_path = commands_path, - url_status = url_status) - # remove file with commands after test - file.remove(commands_path) - unlink(directory_to_save, recursive = TRUE) - } -) - testthat::test_that("download_sink test", { testfile <- testthat::test_path("../testdata", "sink_test.txt") file.create(testfile) diff --git a/tests/testthat/test-process.R b/tests/testthat/test-process.R index 0a77bf29..cbe1982a 100644 --- a/tests/testthat/test-process.R +++ b/tests/testthat/test-process.R @@ -74,7 +74,7 @@ testthat::test_that("test generic process_covariates", { "sedac_groads", "groads", "roads", "nlcd", "narr", "nei", "ecoregions", "ecoregion", "huc", "cropscape", "cdl", - "prism", "olm", "openlandmap", "terraclimate", "gridmet") + "prism", "terraclimate", "gridmet") for (cty in covar_types) { testthat::expect_error( process_covariates( @@ -1757,26 +1757,6 @@ testthat::test_that("process_huc", } ) -# test OpenLandMap #### -# nolint start -testthat::test_that("process_olm", { - withr::local_package("terra") - tmwm <- testthat::test_path("..", "testdata", "openlandmap", - "no2_s5p.l3.trop.tmwm.p50_p90_2km_a_20180501_20221130_go_epsg.4326_v20221219_test.tif") - testthat::expect_no_error( - olm <- process_olm(path = tmwm) - ) - testthat::expect_s4_class(olm, "SpatRaster") - testthat::expect_error( - process_olm(path = 1L) - ) - - # test with cropping extent - testthat::expect_no_error( - olm_ext <- process_olm(path = tmwm, extent = terra::ext(olm)) - ) -}) -# nolint end # AUX tests #### testthat::test_that("loc_radius tests", { From 4c8822a819ead785424f78f416bb2d4d1f6009a8 Mon Sep 17 00:00:00 2001 From: mitchellmanware Date: Fri, 26 Jul 2024 12:14:00 -0400 Subject: [PATCH 3/6] cran debug 2 --- .Rbuildignore | 8 +- .gitignore | 4 + DESCRIPTION | 5 +- README.md | 2 +- man/figures/logo.svg | 236 +++++++++++++++++++++++++++++++ vignettes/.gitignore | 2 + vignettes/download_functions.Rmd | 14 +- vignettes/protected_datasets.Rmd | 2 +- 8 files changed, 262 insertions(+), 11 deletions(-) create mode 100644 man/figures/logo.svg create mode 100644 vignettes/.gitignore diff --git a/.Rbuildignore b/.Rbuildignore index 6beba603..50af6ea4 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -17,5 +17,9 @@ pkgdown output manuscript input -tests -vignettes/epa_download.Rmd \ No newline at end of file +test +^doc$ +^Meta$ +docs/ +help/ +^vignettes/ \ No newline at end of file diff --git a/.gitignore b/.gitignore index 74a93ac0..e90e453f 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,7 @@ # pkgdown site docs/ +doc/ # translation temp files po/*~ @@ -50,3 +51,6 @@ rsconnect/ # custom linter dev \.github/linters +/doc/ +/Meta/ +inst/doc diff --git a/DESCRIPTION b/DESCRIPTION index 97f8063b..87695116 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -9,13 +9,12 @@ Authors@R: c( person("Kyle", "Messier", role = c("aut", "cre"), email = "kyle.messier@nih.gov", comment = c(ORCID = "0000-0001-9508-9623")) ) Maintainer: Kyle Messier -Description: Functions are designed to facilitate access to and utility with large scale, publicly available environmental data in R. The package contains functions for downloading (download_data()) raw data files from web URLs, processing (process_covariates()) the raw data files into clean spatial objects, and extracting values (calc_covariates()) from the spatial data objects at point and polygon locations. These functions call a series of source-specific functions which are tailored to each data sources/datasets particular URL structure, data format, and spatial/temporal resolution. amadeus functions are tested, versioned, and open source and open access. +Description: Functions are designed to facilitate access to and utility with large scale, publicly available environmental data in R. The package contains functions for downloading (download_data()) raw data files from web URLs, processing (process_covariates()) the raw data files into clean spatial objects, and extracting values (calc_covariates()) from the spatial data objects at point and polygon locations. These functions call a series of source-specific functions which are tailored to each data sources/datasets particular URL structure, data format, and spatial/temporal resolution. The functions are tested, versioned, and open source and open access. Depends: R (>= 4.1.0) Imports: dplyr, sf, sftime, stats, terra, methods, data.table, httr, rvest, exactextractr, utils, stringi, testthat (>= 3.0.0), parallelly, stars, future, future.apply, tidyr, rlang, rstac, nhdplusTools, archive, collapse, Rdpack -Suggests: covr, withr, knitr, rmarkdown, pandoc, lwgeom, FNN, doRNG +Suggests: covr, withr, knitr, rmarkdown, lwgeom, FNN, doRNG, devtools, stringr RdMacros: Rdpack Encoding: UTF-8 -VignetteBuilder: knitr, rmarkdown Roxygen: list(markdown = TRUE) RoxygenNote: 7.3.2 Config/Needs/website: tidyverse/tidytemplate diff --git a/README.md b/README.md index b49d3979..a825a149 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# amadeus +# amadeus amadeus website [![R-CMD-check](https://github.com/NIEHS/amadeus/actions/workflows/check-standard.yaml/badge.svg)](https://github.com/NIEHS/amadeus/actions/workflows/check-standard.yaml) [![cov](https://NIEHS.github.io/amadeus/badges/coverage.svg)](https://github.com/NIEHS/amadeus/actions) diff --git a/man/figures/logo.svg b/man/figures/logo.svg new file mode 100644 index 00000000..53d73f51 --- /dev/null +++ b/man/figures/logo.svg @@ -0,0 +1,236 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + www. + + + + + + + + + + + + + + \ No newline at end of file diff --git a/vignettes/.gitignore b/vignettes/.gitignore new file mode 100644 index 00000000..097b2416 --- /dev/null +++ b/vignettes/.gitignore @@ -0,0 +1,2 @@ +*.html +*.R diff --git a/vignettes/download_functions.Rmd b/vignettes/download_functions.Rmd index 5101f5fb..3b602612 100644 --- a/vignettes/download_functions.Rmd +++ b/vignettes/download_functions.Rmd @@ -9,13 +9,19 @@ date: "2024-07-09" author: "Mitchell Manware" --- -```{r, echo = FALSE, warning = FALSE, message = FALSE, results = "hide"} +```{r, include = FALSE} # packages +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>" +) library(knitr) library(testthat) library(devtools) -# source functions -devtools::load_all("../") +``` + +```{r setup} +library(amadeus) ``` ## Motivation @@ -86,7 +92,7 @@ link <- c( "https://gmao.gsfc.nasa.gov/reanalysis/MERRA-2/", "https://psl.noaa.gov/data/gridded/data.narr.html", "https://www.mrlc.gov/data", - "https://www.ospo.noaa.gov/Products/land/hms.html#0", + "https://www.ospo.noaa.gov/products/land/hms.html#0", paste0( "https://sedac.ciesin.columbia.edu/data/set/groads-", "global-roads-open-access-v1/data-download" diff --git a/vignettes/protected_datasets.Rmd b/vignettes/protected_datasets.Rmd index 41d3d226..384ae91e 100644 --- a/vignettes/protected_datasets.Rmd +++ b/vignettes/protected_datasets.Rmd @@ -20,7 +20,7 @@ This vignette will demonstrate how to create and log into a NASA EarthData Accou ## NASA EarthData Account -Visit [https://urs.earthdata.nasa.gov/](http://urs.earthdata.nasa.gov) to register for or log into a NASA EarthData account. +Visit [https://urs.earthdata.nasa.gov/](https://urs.earthdata.nasa.gov/) to register for or log into a NASA EarthData account. ![NASA EarthData Account Landing Page](images/NASA_EarthData_login.png) From fd969d12c246e427093942e6827a7b4668bad78d Mon Sep 17 00:00:00 2001 From: mitchellmanware Date: Fri, 26 Jul 2024 12:56:56 -0400 Subject: [PATCH 4/6] cran debug 3 --- DESCRIPTION | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 87695116..feaefb5b 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -11,10 +11,11 @@ Authors@R: c( Maintainer: Kyle Messier Description: Functions are designed to facilitate access to and utility with large scale, publicly available environmental data in R. The package contains functions for downloading (download_data()) raw data files from web URLs, processing (process_covariates()) the raw data files into clean spatial objects, and extracting values (calc_covariates()) from the spatial data objects at point and polygon locations. These functions call a series of source-specific functions which are tailored to each data sources/datasets particular URL structure, data format, and spatial/temporal resolution. The functions are tested, versioned, and open source and open access. Depends: R (>= 4.1.0) -Imports: dplyr, sf, sftime, stats, terra, methods, data.table, httr, rvest, exactextractr, utils, stringi, testthat (>= 3.0.0), parallelly, stars, future, future.apply, tidyr, rlang, rstac, nhdplusTools, archive, collapse, Rdpack +Imports: dplyr, sf, sftime, stats, terra, methods, data.table, httr, rvest, exactextractr, utils, stringi, testthat (>= 3.0.0), parallelly, stars, future, future.apply, tidyr, rlang, nhdplusTools, archive, collapse, Rdpack Suggests: covr, withr, knitr, rmarkdown, lwgeom, FNN, doRNG, devtools, stringr RdMacros: Rdpack Encoding: UTF-8 +VignetteBuilder: knitr, rmarkdown Roxygen: list(markdown = TRUE) RoxygenNote: 7.3.2 Config/Needs/website: tidyverse/tidytemplate From 896dac24e9ae7a127442bc6a1d7a345925521834 Mon Sep 17 00:00:00 2001 From: mitchellmanware Date: Fri, 26 Jul 2024 13:17:40 -0400 Subject: [PATCH 5/6] lint --- R/olm_functions.R | 2 -- R/process.R | 1 - tests/olmtests/test-list_stac_files.R | 7 ++++--- tests/olmtests/test-process_olm.R | 2 -- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/R/olm_functions.R b/R/olm_functions.R index cfcacd25..7bec7838 100644 --- a/R/olm_functions.R +++ b/R/olm_functions.R @@ -276,5 +276,3 @@ process_olm <- olm <- terra::rast(path, win = extent) return(olm) } - - diff --git a/R/process.R b/R/process.R index 3f1cb0f9..c52a8e8c 100644 --- a/R/process.R +++ b/R/process.R @@ -2936,4 +2936,3 @@ process_prism <- return(prism) } # nolint end - diff --git a/tests/olmtests/test-list_stac_files.R b/tests/olmtests/test-list_stac_files.R index c7518bb3..1b43a10b 100644 --- a/tests/olmtests/test-list_stac_files.R +++ b/tests/olmtests/test-list_stac_files.R @@ -1,8 +1,9 @@ -testthat::test_that("list_stac_files returns a character vector of file links", { +testthat::test_that("list_stac_files returns character vector of file links", { withr::local_package("rstac") # Set up test data - stac_json <- "https://s3.eu-central-1.wasabisys.com/stac/openlandmap/catalog.json" + stac_json <- + "https://s3.eu-central-1.wasabisys.com/stac/openlandmap/catalog.json" format <- "tif" which <- 35 @@ -28,4 +29,4 @@ testthat::test_that("list_stac_files returns a character vector of file links", ) testthat::expect_true(is.character(result2)) -}) \ No newline at end of file +}) diff --git a/tests/olmtests/test-process_olm.R b/tests/olmtests/test-process_olm.R index 68b4103f..8ea5d54d 100644 --- a/tests/olmtests/test-process_olm.R +++ b/tests/olmtests/test-process_olm.R @@ -19,5 +19,3 @@ testthat::test_that("process_olm", { ) }) # nolint end - - From 433b0993a0032b1ede42700de2b14ab9862465c5 Mon Sep 17 00:00:00 2001 From: mitchellmanware Date: Fri, 26 Jul 2024 13:20:28 -0400 Subject: [PATCH 6/6] reorganize tests --- tests/{olmtests => testthat}/test-download_olm.R | 0 tests/{olmtests => testthat}/test-list_stac_files.R | 0 tests/{olmtests => testthat}/test-process_olm.R | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename tests/{olmtests => testthat}/test-download_olm.R (100%) rename tests/{olmtests => testthat}/test-list_stac_files.R (100%) rename tests/{olmtests => testthat}/test-process_olm.R (100%) diff --git a/tests/olmtests/test-download_olm.R b/tests/testthat/test-download_olm.R similarity index 100% rename from tests/olmtests/test-download_olm.R rename to tests/testthat/test-download_olm.R diff --git a/tests/olmtests/test-list_stac_files.R b/tests/testthat/test-list_stac_files.R similarity index 100% rename from tests/olmtests/test-list_stac_files.R rename to tests/testthat/test-list_stac_files.R diff --git a/tests/olmtests/test-process_olm.R b/tests/testthat/test-process_olm.R similarity index 100% rename from tests/olmtests/test-process_olm.R rename to tests/testthat/test-process_olm.R