From 6bfe7dfd8ef9ff7f44c2fae0d96a6079aadc0740 Mon Sep 17 00:00:00 2001 From: Pedro Ribeiro de Almeida Date: Fri, 9 Aug 2024 10:49:59 +1000 Subject: [PATCH 01/20] Fix show_after error when sizeof fails to determine DataType size This error was preventing users from creating YAXArrays of Strings} and Unions that contain Strings. --- src/Cubes/Cubes.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Cubes/Cubes.jl b/src/Cubes/Cubes.jl index 6666cde6..457609c2 100644 --- a/src/Cubes/Cubes.jl +++ b/src/Cubes/Cubes.jl @@ -510,7 +510,11 @@ function DD.show_after(io::IO, mime, c::YAXArray) blockwidth = get(io, :blockwidth, 0) DD.print_block_separator(io, "file size", blockwidth, blockwidth) println(io, " ") - println(io, " file size: ", formatbytes(cubesize(c))) + try + println(io, " file size: ", formatbytes(cubesize(c))) + catch e + e isa ErrorException ? println(" could not determine DataType size.") : rethrow(e) + end DD.print_block_close(io, blockwidth) # And if you want the array data to print: # ndims(c) > 0 && println(io) From 51d5f829f7f3a5d1ac264c1eb996f69058b0f510 Mon Sep 17 00:00:00 2001 From: Pedro Ribeiro de Almeida Date: Thu, 5 Sep 2024 13:57:00 +1000 Subject: [PATCH 02/20] Remove try catch and only print cube size for a few selected types --- src/Cubes/Cubes.jl | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/Cubes/Cubes.jl b/src/Cubes/Cubes.jl index 457609c2..854ebb81 100644 --- a/src/Cubes/Cubes.jl +++ b/src/Cubes/Cubes.jl @@ -508,12 +508,10 @@ getCubeDes(::Type{T}) where {T} = string(T) function DD.show_after(io::IO, mime, c::YAXArray) blockwidth = get(io, :blockwidth, 0) - DD.print_block_separator(io, "file size", blockwidth, blockwidth) - println(io, " ") - try - println(io, " file size: ", formatbytes(cubesize(c))) - catch e - e isa ErrorException ? println(" could not determine DataType size.") : rethrow(e) + # This are some types for which sizeof is known to return a meaninful value + if isconcretetype(eltype(c)) && <:(eltype(c),Union{AbstractFloat,Signed,Unsigned,Bool}) + DD.print_block_separator(io, "file size", blockwidth, blockwidth) + println(io, "\n file size: ", formatbytes(cubesize(c))) end DD.print_block_close(io, blockwidth) # And if you want the array data to print: From 287fcc5a0bdb72068b1bba540f04c9476239086a Mon Sep 17 00:00:00 2001 From: Lazaro Alonso Date: Mon, 16 Sep 2024 13:51:44 +0200 Subject: [PATCH 03/20] ignore tif --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 87baef2c..073f2754 100644 --- a/.gitignore +++ b/.gitignore @@ -28,4 +28,5 @@ docs/node_modules docs/.vitepress/cache docs/.vitepress/dist docs/.DS_Store -docs/package-lock.json \ No newline at end of file +docs/package-lock.json +*.tif \ No newline at end of file From a739257a189947ef301d8a104c742d815a85ef05 Mon Sep 17 00:00:00 2001 From: Lazaro Alonso Date: Mon, 16 Sep 2024 13:57:13 +0200 Subject: [PATCH 04/20] dont use DiskArrays 0.4, readblocks outputs values in wrong positions in some cases --- Project.toml | 2 +- docs/Project.toml | 2 -- docs/src/UserGuide/debug.jl | 8 ++++++++ 3 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 docs/src/UserGuide/debug.jl diff --git a/Project.toml b/Project.toml index a3ebb7d6..ea2c178d 100644 --- a/Project.toml +++ b/Project.toml @@ -36,7 +36,7 @@ CFTime = "0.0, 0.1" DataStructures = "0.17, 0.18" DimensionalData = "0.27" DiskArrayTools = "0.1" -DiskArrays = "0.3,0.4" +DiskArrays = "0.3" DocStringExtensions = "0.8, 0.9" Glob = "1.3" Interpolations = "0.12, 0.13, 0.14, 0.15" diff --git a/docs/Project.toml b/docs/Project.toml index 63bbbb9c..db53bf1e 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -6,7 +6,6 @@ CFTime = "179af706-886a-5703-950a-314cd64e0468" CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" -DimensionalData = "0703355e-b756-11e9-17c0-8b28908087d0" DiskArrayTools = "fcd2136c-9f69-4db6-97e5-f31981721d63" DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" @@ -30,6 +29,5 @@ Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" TimeSeries = "9e3dc215-6440-5c97-bce1-76c03772f85e" WGLMakie = "276b4fcb-3e11-5398-bf8b-a0c2d153d008" WeightedOnlineStats = "bbac0a1f-7c9d-5672-960b-c6ca726e5d5d" -YAXArrayBase = "90b8fcef-0c2d-428d-9c56-5f86629e9d14" YAXArrays = "c21b50f5-aa40-41ea-b809-c0f5e47bfa5c" Zarr = "0a941bbe-ad1d-11e8-39d9-ab76183a1d99" diff --git a/docs/src/UserGuide/debug.jl b/docs/src/UserGuide/debug.jl new file mode 100644 index 00000000..6c824071 --- /dev/null +++ b/docs/src/UserGuide/debug.jl @@ -0,0 +1,8 @@ +using Zarr +using YAXArrays + +da = open_dataset("http://data.rsc4earth.de/EarthSystemDataCube/v3.0.2/esdc-8d-0.25deg-1x720x1440-3.0.2.zarr") +da.kndvi.properties["long_name"] + +da = Cube("http://data.rsc4earth.de/EarthSystemDataCube/v3.0.2/esdc-8d-0.25deg-1x720x1440-3.0.2.zarr") +da[Variable = At("kndvi")].properties["long_name"] \ No newline at end of file From e26d5078ea6ba8fa3cbb1ff53a6d69e98d82ce41 Mon Sep 17 00:00:00 2001 From: Lazaro Alonso Date: Mon, 16 Sep 2024 17:05:49 +0200 Subject: [PATCH 05/20] fixes outdims default driver --- Project.toml | 3 +-- docs/Project.toml | 6 ++---- docs/src/UserGuide/debug.jl | 8 -------- src/DAT/DAT.jl | 12 ++++++++---- test/Datasets/datasets.jl | 6 ++++++ 5 files changed, 17 insertions(+), 18 deletions(-) delete mode 100644 docs/src/UserGuide/debug.jl diff --git a/Project.toml b/Project.toml index ea2c178d..04516e26 100644 --- a/Project.toml +++ b/Project.toml @@ -34,7 +34,6 @@ YAXArrayBase = "90b8fcef-0c2d-428d-9c56-5f86629e9d14" [compat] CFTime = "0.0, 0.1" DataStructures = "0.17, 0.18" -DimensionalData = "0.27" DiskArrayTools = "0.1" DiskArrays = "0.3" DocStringExtensions = "0.8, 0.9" @@ -54,5 +53,5 @@ Statistics = "1" StatsBase = "0.32, 0.33, 0.34" Tables = "0.2, 1.0" WeightedOnlineStats = "0.3, 0.4, 0.5, 0.6" -YAXArrayBase = "0.6,0.7" +YAXArrayBase = "0.6, 0.7" julia = "1.9" diff --git a/docs/Project.toml b/docs/Project.toml index db53bf1e..9733f5f6 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,11 +1,11 @@ [deps] -ArchGDAL = "c9ce4bd3-c3d5-55b8-8973-c0e20141b8c3" BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" Bonito = "824d6782-a2ef-11e9-3a09-e5662e0c26f8" CFTime = "179af706-886a-5703-950a-314cd64e0468" CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +DimensionalData = "0703355e-b756-11e9-17c0-8b28908087d0" DiskArrayTools = "fcd2136c-9f69-4db6-97e5-f31981721d63" DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" @@ -20,14 +20,12 @@ LaTeXStrings = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306" MarketData = "945b72a4-3b13-509d-9b46-1525bb5c06de" MultivariateStats = "6f286f6a-111f-5878-ab1e-185364afe411" -NetCDF = "30363a11-5582-574a-97bb-aa9a979735b9" OnlineStats = "a15396b6-48d5-5d58-9928-6d29437db91e" PlotUtils = "995b91a9-d308-5afd-9ec6-746e21dbc043" -Rasters = "a3a2b9e3-a471-40c9-b274-f788e487c689" SkipNan = "aed68c70-c8b0-4309-8cd1-d392a74f991a" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" TimeSeries = "9e3dc215-6440-5c97-bce1-76c03772f85e" WGLMakie = "276b4fcb-3e11-5398-bf8b-a0c2d153d008" WeightedOnlineStats = "bbac0a1f-7c9d-5672-960b-c6ca726e5d5d" +YAXArrayBase = "90b8fcef-0c2d-428d-9c56-5f86629e9d14" YAXArrays = "c21b50f5-aa40-41ea-b809-c0f5e47bfa5c" -Zarr = "0a941bbe-ad1d-11e8-39d9-ab76183a1d99" diff --git a/docs/src/UserGuide/debug.jl b/docs/src/UserGuide/debug.jl deleted file mode 100644 index 6c824071..00000000 --- a/docs/src/UserGuide/debug.jl +++ /dev/null @@ -1,8 +0,0 @@ -using Zarr -using YAXArrays - -da = open_dataset("http://data.rsc4earth.de/EarthSystemDataCube/v3.0.2/esdc-8d-0.25deg-1x720x1440-3.0.2.zarr") -da.kndvi.properties["long_name"] - -da = Cube("http://data.rsc4earth.de/EarthSystemDataCube/v3.0.2/esdc-8d-0.25deg-1x720x1440-3.0.2.zarr") -da[Variable = At("kndvi")].properties["long_name"] \ No newline at end of file diff --git a/src/DAT/DAT.jl b/src/DAT/DAT.jl index e18a594a..a30c3e4b 100644 --- a/src/DAT/DAT.jl +++ b/src/DAT/DAT.jl @@ -796,14 +796,18 @@ function getbackend(oc, ispar, max_cache) outsize = sizeof(elementtype) * (length(oc.allAxes) > 0 ? prod(map(length, oc.allAxes)) : 1) rt = oc.desc.backend - if rt == :auto - if ispar[] || outsize > max_cache - rt = :zarr + ispath = get(oc.desc.backendargs, :path, nothing) + + b = if rt == :auto + if ispar[] || outsize > max_cache || !isnothing(ispath) + YAXArrayBase.backendfrompath(ispath) # if backend is not available, it defaults to last available, usually zarr. We need a warning for such case. else rt = :array + YAXArrayBase.backendlist[Symbol(rt)] end + else + YAXArrayBase.backendlist[Symbol(rt)] # Handle non-auto rt case end - b = YAXArrayBase.backendlist[Symbol(rt)] if !allow_parallel_write(b) ispar[] = false end diff --git a/test/Datasets/datasets.jl b/test/Datasets/datasets.jl index e871c0f5..f848c286 100644 --- a/test/Datasets/datasets.jl +++ b/test/Datasets/datasets.jl @@ -410,6 +410,12 @@ end end +@testset "Saving, OutDims" begin + using YAXArrays, Zarr, NetCDF, ArchGDAL + using Dates + +end + @testset "Caching" begin using YAXArrays.Cubes.DiskArrays.TestTypes using YAXArrays.Cubes: DiskArrays From 0ad550bd322ca94877f820d381b2b715f1dc68cc Mon Sep 17 00:00:00 2001 From: Lazaro Alonso Date: Mon, 16 Sep 2024 17:38:32 +0200 Subject: [PATCH 06/20] tests --- docs/Project.toml | 4 ++++ test/Datasets/datasets.jl | 42 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/docs/Project.toml b/docs/Project.toml index 9733f5f6..63bbbb9c 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,4 +1,5 @@ [deps] +ArchGDAL = "c9ce4bd3-c3d5-55b8-8973-c0e20141b8c3" BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" Bonito = "824d6782-a2ef-11e9-3a09-e5662e0c26f8" CFTime = "179af706-886a-5703-950a-314cd64e0468" @@ -20,8 +21,10 @@ LaTeXStrings = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306" MarketData = "945b72a4-3b13-509d-9b46-1525bb5c06de" MultivariateStats = "6f286f6a-111f-5878-ab1e-185364afe411" +NetCDF = "30363a11-5582-574a-97bb-aa9a979735b9" OnlineStats = "a15396b6-48d5-5d58-9928-6d29437db91e" PlotUtils = "995b91a9-d308-5afd-9ec6-746e21dbc043" +Rasters = "a3a2b9e3-a471-40c9-b274-f788e487c689" SkipNan = "aed68c70-c8b0-4309-8cd1-d392a74f991a" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" TimeSeries = "9e3dc215-6440-5c97-bce1-76c03772f85e" @@ -29,3 +32,4 @@ WGLMakie = "276b4fcb-3e11-5398-bf8b-a0c2d153d008" WeightedOnlineStats = "bbac0a1f-7c9d-5672-960b-c6ca726e5d5d" YAXArrayBase = "90b8fcef-0c2d-428d-9c56-5f86629e9d14" YAXArrays = "c21b50f5-aa40-41ea-b809-c0f5e47bfa5c" +Zarr = "0a941bbe-ad1d-11e8-39d9-ab76183a1d99" diff --git a/test/Datasets/datasets.jl b/test/Datasets/datasets.jl index f848c286..8425a16b 100644 --- a/test/Datasets/datasets.jl +++ b/test/Datasets/datasets.jl @@ -413,7 +413,47 @@ end @testset "Saving, OutDims" begin using YAXArrays, Zarr, NetCDF, ArchGDAL using Dates - + flolat(lo, la, t) = (lo + la + Dates.dayofyear(t)) + + function g(xout, lo, la, t) + xout .= flolat.(lo, la, t) + end + lon = YAXArray(Dim{:lon}(range(1, 15))) + lat = YAXArray(Dim{:lat}(range(1, 10))) + tspan = Date("2022-01-01"):Day(1):Date("2022-01-30") + time = YAXArray(Dim{:time}(tspan)) + + gen_cube = mapCube(g, (lon, lat, time); + indims = (InDims(), InDims(), InDims("time")), + outdims = OutDims("time", + outtype = Float32) + # max_cache=1e9 + ) + # test saves, zarr + gen_zarr = mapCube(g, (lon, lat, time); + indims = (InDims(), InDims(), InDims("time")), + outdims = OutDims("time", overwrite=true, path="my_gen_cube.zarr", + outtype = Float32) + # max_cache=1e9 + ) + ds_zarr = open_dataset("my_gen_cube.zarr") + # test saves, nc + gen_nc = mapCube(g, (lon, lat, time); + indims = (InDims(), InDims(), InDims("time")), + outdims = OutDims("time", overwrite=true, path="my_gen_cube.nc", + outtype = Float32) + # max_cache=1e9 + ) + ds_nc = open_dataset("my_gen_cube.nc") + # test saves, tif, once we update to YAXArrayBase 0.7.3 + # gen_zarr = mapCube(g, (lon, lat, time); + # indims = (InDims(), InDims(), InDims("time")), + # outdims = OutDims("time", overwrite=true, path="my_gen_cube.tif", + # outtype = Float32) + # # max_cache=1e9 + # ) + @test gen_cube.data[:,:] == ds_zarr["layer"].data[:,:] + @test gen_cube.data[:,:] == ds_nc["layer"].data[:,:] end @testset "Caching" begin From 4d21127ef7e117d1f41aa5845edac0bcb0298051 Mon Sep 17 00:00:00 2001 From: Lazaro Alonso Date: Mon, 16 Sep 2024 18:59:20 +0200 Subject: [PATCH 07/20] do ENV[JULIA_PKG_SERVER_REGISTRY_PREFERENCE] = eager --- docs/Project.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/Project.toml b/docs/Project.toml index 63bbbb9c..e6ae7333 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -24,7 +24,6 @@ MultivariateStats = "6f286f6a-111f-5878-ab1e-185364afe411" NetCDF = "30363a11-5582-574a-97bb-aa9a979735b9" OnlineStats = "a15396b6-48d5-5d58-9928-6d29437db91e" PlotUtils = "995b91a9-d308-5afd-9ec6-746e21dbc043" -Rasters = "a3a2b9e3-a471-40c9-b274-f788e487c689" SkipNan = "aed68c70-c8b0-4309-8cd1-d392a74f991a" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" TimeSeries = "9e3dc215-6440-5c97-bce1-76c03772f85e" From 84eecb0e2e986289237d6feffe73dd81df9c5b6d Mon Sep 17 00:00:00 2001 From: Lazaro Alonso Date: Mon, 16 Sep 2024 19:31:32 +0200 Subject: [PATCH 08/20] fix tests --- docs/Project.toml | 1 + test/Datasets/datasets.jl | 58 +++++++++++++++++++++++---------------- 2 files changed, 35 insertions(+), 24 deletions(-) diff --git a/docs/Project.toml b/docs/Project.toml index e6ae7333..63bbbb9c 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -24,6 +24,7 @@ MultivariateStats = "6f286f6a-111f-5878-ab1e-185364afe411" NetCDF = "30363a11-5582-574a-97bb-aa9a979735b9" OnlineStats = "a15396b6-48d5-5d58-9928-6d29437db91e" PlotUtils = "995b91a9-d308-5afd-9ec6-746e21dbc043" +Rasters = "a3a2b9e3-a471-40c9-b274-f788e487c689" SkipNan = "aed68c70-c8b0-4309-8cd1-d392a74f991a" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" TimeSeries = "9e3dc215-6440-5c97-bce1-76c03772f85e" diff --git a/test/Datasets/datasets.jl b/test/Datasets/datasets.jl index 8425a16b..f2d7be7a 100644 --- a/test/Datasets/datasets.jl +++ b/test/Datasets/datasets.jl @@ -413,47 +413,57 @@ end @testset "Saving, OutDims" begin using YAXArrays, Zarr, NetCDF, ArchGDAL using Dates + flolat(lo, la, t) = (lo + la + Dates.dayofyear(t)) + flola(lo, la) = lo + la + 1 function g(xout, lo, la, t) xout .= flolat.(lo, la, t) end + + function g2d(xout, lo, la) + xout .= flola.(lo, la) + end + lon = YAXArray(Dim{:lon}(range(1, 15))) lat = YAXArray(Dim{:lat}(range(1, 10))) tspan = Date("2022-01-01"):Day(1):Date("2022-01-30") time = YAXArray(Dim{:time}(tspan)) gen_cube = mapCube(g, (lon, lat, time); - indims = (InDims(), InDims(), InDims("time")), - outdims = OutDims("time", - outtype = Float32) - # max_cache=1e9 - ) + indims = (InDims(), InDims(), InDims("time")), + outdims = OutDims("time", + outtype = Float32) + # max_cache=1e9 + ) + + gen_cube2d = mapCube(g2d, (lon, lat); + indims = (InDims(), InDims()), + outdims = OutDims(outtype = Float32) + # max_cache=1e9 + ) + # test saves, zarr gen_zarr = mapCube(g, (lon, lat, time); - indims = (InDims(), InDims(), InDims("time")), - outdims = OutDims("time", overwrite=true, path="my_gen_cube.zarr", - outtype = Float32) - # max_cache=1e9 - ) + indims = (InDims(), InDims(), InDims("time")), + outdims = OutDims("time", overwrite=true, path="my_gen_cube.zarr", + outtype = Float32) + # max_cache=1e9 + ) ds_zarr = open_dataset("my_gen_cube.zarr") # test saves, nc gen_nc = mapCube(g, (lon, lat, time); - indims = (InDims(), InDims(), InDims("time")), - outdims = OutDims("time", overwrite=true, path="my_gen_cube.nc", - outtype = Float32) - # max_cache=1e9 - ) + indims = (InDims(), InDims(), InDims("time")), + outdims = OutDims("time", overwrite=true, path="my_gen_cube.nc", + outtype = Float32) + # max_cache=1e9 + ) ds_nc = open_dataset("my_gen_cube.nc") - # test saves, tif, once we update to YAXArrayBase 0.7.3 - # gen_zarr = mapCube(g, (lon, lat, time); - # indims = (InDims(), InDims(), InDims("time")), - # outdims = OutDims("time", overwrite=true, path="my_gen_cube.tif", - # outtype = Float32) - # # max_cache=1e9 - # ) - @test gen_cube.data[:,:] == ds_zarr["layer"].data[:,:] - @test gen_cube.data[:,:] == ds_nc["layer"].data[:,:] + + # TODO: fix tif for general inputs, so that writing also works. + + @test gen_cube.data[:,:,:] == ds_zarr["layer"].data[:,:,:] + @test gen_cube.data[:,:,:] == ds_nc["layer"].data[:,:,:] end @testset "Caching" begin From 1edd024df33be07d56f64259b22202e8da0eb8c0 Mon Sep 17 00:00:00 2001 From: Lazaro Alonso Date: Mon, 16 Sep 2024 20:24:39 +0200 Subject: [PATCH 09/20] update toml test --- docs/src/UserGuide/compute_debug.jl | 12 ++++++++++++ docs/src/UserGuide/read.md | 2 +- test/Project.toml | 1 + 3 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 docs/src/UserGuide/compute_debug.jl diff --git a/docs/src/UserGuide/compute_debug.jl b/docs/src/UserGuide/compute_debug.jl new file mode 100644 index 00000000..2cf8eefb --- /dev/null +++ b/docs/src/UserGuide/compute_debug.jl @@ -0,0 +1,12 @@ +using YAXArrays, Dates + +axlist = ( + Dim{:time}(Date("2022-01-01"):Day(1):Date("2022-01-10")), + Dim{:lon}(range(1, 10, length=10)), + Dim{:lat}(range(1, 5, length=15)), + Dim{:variables}(["a", "b"]) +) +data = rand(10, 10, 15, 2) +properties = Dict(:description => "multi dimensional test cube") +a = YAXArray(axlist, data, properties) + diff --git a/docs/src/UserGuide/read.md b/docs/src/UserGuide/read.md index 43bd7164..9b621219 100644 --- a/docs/src/UserGuide/read.md +++ b/docs/src/UserGuide/read.md @@ -53,6 +53,6 @@ using ArchGDAL using Downloads: download path = download("https://github.com/yeesian/ArchGDALDatasets/raw/307f8f0e584a39a050c042849004e6a2bd674f99/gdalworkshop/world.tif", "world.tif") -# ds = open_dataset(path) # this is broken +ds = open_dataset(path) nothing ```` \ No newline at end of file diff --git a/test/Project.toml b/test/Project.toml index a7486127..0810e4b4 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,4 +1,5 @@ [deps] +ArchGDAL = "c9ce4bd3-c3d5-55b8-8973-c0e20141b8c3" CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" From 3d693d8201d0f30469054a715d3ddba050c390a3 Mon Sep 17 00:00:00 2001 From: Lazaro Alonso Date: Mon, 16 Sep 2024 22:40:37 +0200 Subject: [PATCH 10/20] maybe add properties also to genCube from OutDims --- docs/src/UserGuide/compute_debug.jl | 20 ++++++++++++++++++-- docs/src/UserGuide/read.md | 1 - src/DAT/DAT.jl | 3 ++- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/docs/src/UserGuide/compute_debug.jl b/docs/src/UserGuide/compute_debug.jl index 2cf8eefb..fb07423c 100644 --- a/docs/src/UserGuide/compute_debug.jl +++ b/docs/src/UserGuide/compute_debug.jl @@ -6,7 +6,23 @@ axlist = ( Dim{:lat}(range(1, 5, length=15)), Dim{:variables}(["a", "b"]) ) -data = rand(10, 10, 15, 2) +data = 2*ones(10, 10, 15, 2) properties = Dict(:description => "multi dimensional test cube") -a = YAXArray(axlist, data, properties) +yax_test = YAXArray(axlist, data, properties) +function one_to_many(xout_sqrt, xout_quad, xout_flat, xin_one) + xout_sqrt .= xin_one .^2 + xout_quad .= xin_one .^4 + xout_flat .= sum(xin_one) # this will reduce the time dimension if we set outdims = OutDims() + return nothing +end + +indims_one = InDims("Time") +# outputs dimension +outdims_sqrt = OutDims("Time") +outdims_quad = OutDims("Time", name="quads", units ="double") +outdims_flat = OutDims(; name="flat") # space + +ds = mapCube(one_to_many, yax_test, + indims = indims_one, + outdims = (outdims_sqrt, outdims_quad, outdims_flat)); diff --git a/docs/src/UserGuide/read.md b/docs/src/UserGuide/read.md index 9b621219..cf4024bc 100644 --- a/docs/src/UserGuide/read.md +++ b/docs/src/UserGuide/read.md @@ -54,5 +54,4 @@ using Downloads: download path = download("https://github.com/yeesian/ArchGDALDatasets/raw/307f8f0e584a39a050c042849004e6a2bd674f99/gdalworkshop/world.tif", "world.tif") ds = open_dataset(path) -nothing ```` \ No newline at end of file diff --git a/src/DAT/DAT.jl b/src/DAT/DAT.jl index a30c3e4b..901e1279 100644 --- a/src/DAT/DAT.jl +++ b/src/DAT/DAT.jl @@ -853,7 +853,8 @@ function generateOutCube( newsize = map(length, oc.allAxes) outar = Array{elementtype}(undef, newsize...) fill!(outar,_zero(elementtype)) - oc.cube = YAXArray(tuple(oc.allAxes...), outar) + properties = Dict{String, Any}(string(k) => v for (k, v) in oc.desc.backendargs) + oc.cube = YAXArray(tuple(oc.allAxes...), outar, properties) # ? here, including properties! oc.cube_unpermuted = oc.cube end _zero(T) = zero(T) From f5d8a84370abb7cb522b1e5c7dbb954cce095e2b Mon Sep 17 00:00:00 2001 From: Lazaro Alonso Date: Tue, 17 Sep 2024 16:03:40 +0200 Subject: [PATCH 11/20] more on OutDims and InDims --- docs/src/UserGuide/compute_debug.jl | 145 +++++++++++++++++++++++++--- 1 file changed, 134 insertions(+), 11 deletions(-) diff --git a/docs/src/UserGuide/compute_debug.jl b/docs/src/UserGuide/compute_debug.jl index fb07423c..6ac6c12f 100644 --- a/docs/src/UserGuide/compute_debug.jl +++ b/docs/src/UserGuide/compute_debug.jl @@ -1,28 +1,151 @@ using YAXArrays, Dates +using Random axlist = ( - Dim{:time}(Date("2022-01-01"):Day(1):Date("2022-01-10")), - Dim{:lon}(range(1, 10, length=10)), - Dim{:lat}(range(1, 5, length=15)), + Dim{:time}(Date("2022-01-01"):Day(1):Date("2022-01-05")), + Dim{:lon}(range(1, 4, length=4)), + Dim{:lat}(range(1, 3, length=3)), Dim{:variables}(["a", "b"]) ) -data = 2*ones(10, 10, 15, 2) -properties = Dict(:description => "multi dimensional test cube") + +Random.seed!(123) +data = rand(1:5, 5, 4, 3, 2) + +properties = Dict("description" => "multi dimensional test cube") yax_test = YAXArray(axlist, data, properties) -function one_to_many(xout_sqrt, xout_quad, xout_flat, xin_one) - xout_sqrt .= xin_one .^2 - xout_quad .= xin_one .^4 +properties_2d = Dict("description" => "2d dimensional test cube") +yax_2d = YAXArray(axlist[2:end], rand(-1:1, 4, 3, 2), properties_2d) + +f1(xin) = xin + 1 +f2(xin) = xin + 2 +function one_to_many(xout_one, xout_two, xout_flat, xin_one) + xout_one .= f1.(xin_one) + xout_two .= f2.(xin_one) xout_flat .= sum(xin_one) # this will reduce the time dimension if we set outdims = OutDims() return nothing end indims_one = InDims("Time") # outputs dimension -outdims_sqrt = OutDims("Time") -outdims_quad = OutDims("Time", name="quads", units ="double") +outdims_one = OutDims("Time", name="plus_one") +outdims_two = OutDims("Time", name="plus_two", units ="double") outdims_flat = OutDims(; name="flat") # space ds = mapCube(one_to_many, yax_test, indims = indims_one, - outdims = (outdims_sqrt, outdims_quad, outdims_flat)); + outdims = (outdims_one, outdims_two, outdims_flat)); + +# many to many. mix input dimensions + +f2mix(xin_xyt, xin_xy) = xin_xyt - xin_xy + +function many_to_many(xout_one, xout_two, xout_flat, xin_one, xin_two, xin_drei) + xout_one .= f1.(xin_one) + xout_two .= f2mix.(xin_one, xin_two) + xout_flat .= sum(xin_drei) # this will reduce the time dimension if we set outdims = OutDims() + return nothing +end + +indims_one = InDims("Time") +indims_2d = InDims() # it matches only to the other 2 dimensions and uses the same values for each time step + +outdims_one = OutDims("Time", name="plus_one") +outdims_two = OutDims("Time", name="plus_two", units ="double") +outdims_flat = OutDims(; name="flat") # space + +ds = mapCube(many_to_many, (yax_test, yax_2d, yax_test), + indims = (indims_one, indims_2d, indims_one), + outdims = (outdims_one, outdims_two, outdims_flat)); + + +properties_2dz = Dict("description" => "2d-z dimensional test cube") + +axlist = ( + Dim{:lon}(range(1, 4, length=4)), + Dim{:lat}(range(1, 3, length=3)), + Dim{:depth}(range(1, 2, length=2)), + Dim{:variables}(["a", "b"]) +) + +yax_2dz = YAXArray(axlist, rand(-2:2, 4, 3, 2, 2), properties_2dz) + +function f2mix_depth(xin_xyt, xin_xyz) + @show xin_xyz + return xin_xyt + sum(abs.(xin_xyz)) +end + +function many_to_many_depth(xout_one, xout_two, xout_flat, xin_one, xin_two) + xout_one .= f1.(xin_one) + xout_two .= f2mix_depth.(xin_one, xin_two) + xout_flat .= sum(abs.(xin_two)) # this will reduce the time dimension if we set outdims = OutDims() + return nothing +end + +indims_one = InDims("Time") +indims_2d = InDims() # it matches only to the other 2 dimensions and uses the same values for each time step +outdims_one = OutDims("Time") +outdims_two = OutDims("Time") +outdims_flat = OutDims() # space + +ds = mapCube(many_to_many_depth, (yax_test, yax_2dz), + indims = (indims_one, indims_2d), + outdims = (outdims_one, outdims_two, outdims_flat)) + + +function many_depth(xout_two, xin_one, xin_two) + xout_two .= f2mix_depth.(xin_one, xin_two) + return nothing +end +function f2mix_depth(xin_xyt, xin_xyz) + s = sum(abs.(xin_xyz)) + # @show s # is not doing what I think is doing! + # @show xin_xyz + return xin_xyt + s +end + +ds = mapCube(many_depth, (yax_test[Variable= At("a")], yax_2dz[Variable= At("a")]), + indims = (InDims("Time"), InDims()), + outdims = OutDims("Time")) + +yax_s = sum(yax_2dz[Variable= At("a")], dims=:depth) +yax_s = dropdims(yax_s, dims=:depth) + +# use case, things should operate at the pixel level, (lon, lat) and extract the corresponding values there. + +axlist = ( + Dim{:lon}(1:4), + Dim{:lat}(1:3), + Dim{:depth}(1:7), +) + +yax_2d = YAXArray(axlist, rand(-3:0, 4, 3, 7)) + +axlist = ( + Dim{:time}(Date("2022-01-01"):Day(1):Date("2022-01-05")), + Dim{:lon}(1:4), + Dim{:lat}(1:3), +) + +Random.seed!(123) +data = rand(3:5, 5, 4, 3) + +properties = Dict("description" => "multi dimensional test cube") +yax_test = YAXArray(axlist, data, properties) + +function mix_time_depth(xin_xyt, xin_xyz) + s = sum(abs.(xin_xyz)) + @show s # ? is doing what I think is doing! + @show xin_xyz + return xin_xyt.^2 .+ s +end + +function time_depth(xout, xin_one, xin_two) + xout .= mix_time_depth(xin_one, xin_two) # ? note also here, no dot anymore! + return nothing +end + +ds = mapCube(time_depth, (yax_test, yax_2dz), + indims = (InDims("Time"), InDims("depth")), + # ? it shouldn't be better to say "lon", "lat". Logic? well, you specify an anchor dimension and then map over the others. + outdims = OutDims("Time")) \ No newline at end of file From 775ce9c7260a68c611eda1dd321beb778e659fe4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Pasquier?= <4486578+briochemc@users.noreply.github.com> Date: Thu, 19 Sep 2024 15:52:56 +1000 Subject: [PATCH 12/20] Fix rebuild --- src/Cubes/Cubes.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Cubes/Cubes.jl b/src/Cubes/Cubes.jl index ff1cb239..d609cf23 100644 --- a/src/Cubes/Cubes.jl +++ b/src/Cubes/Cubes.jl @@ -200,7 +200,7 @@ function DD.rebuild(A::YAXArray, data::AbstractArray, dims::Tuple, refdims::Tupl #end YAXArray(dims, data, metadata; cleaner=A.cleaner)#, chunks=GridChunks(chunks)) end -function DD.rebuild(A::YAXArray; data=parent(A), dims=dims(A), metadata=DD.metadata(A), kw...) +function DD.rebuild(A::YAXArray; data=parent(A), dims=DD.dims(A), metadata=DD.metadata(A), kw...) YAXArray(dims, data, metadata; cleaner=A.cleaner) end From e3e4d76a911d3cc2c38f3523321e40cfd3242ec5 Mon Sep 17 00:00:00 2001 From: Lazaro Alonso Date: Thu, 19 Sep 2024 20:07:45 +0200 Subject: [PATCH 13/20] up versions --- .gitignore | 1 + Project.toml | 4 ++-- docs/package.json | 1 + docs/src/.vitepress/theme/style.css | 14 ++++++++++++++ docs/src/UserGuide/read.md | 2 +- 5 files changed, 19 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 87baef2c..86a93f47 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ build .vscode *.zarr *.nc +*.tif var *.ipynb_checkpoints diff --git a/Project.toml b/Project.toml index a3ebb7d6..a549797f 100644 --- a/Project.toml +++ b/Project.toml @@ -34,7 +34,7 @@ YAXArrayBase = "90b8fcef-0c2d-428d-9c56-5f86629e9d14" [compat] CFTime = "0.0, 0.1" DataStructures = "0.17, 0.18" -DimensionalData = "0.27" +DimensionalData = "0.27, 0.28" DiskArrayTools = "0.1" DiskArrays = "0.3,0.4" DocStringExtensions = "0.8, 0.9" @@ -54,5 +54,5 @@ Statistics = "1" StatsBase = "0.32, 0.33, 0.34" Tables = "0.2, 1.0" WeightedOnlineStats = "0.3, 0.4, 0.5, 0.6" -YAXArrayBase = "0.6,0.7" +YAXArrayBase = "0.6, 0.7" julia = "1.9" diff --git a/docs/package.json b/docs/package.json index f908fde5..904caf08 100644 --- a/docs/package.json +++ b/docs/package.json @@ -8,6 +8,7 @@ "markdown-it": "^14.0.0", "markdown-it-mathjax3": "^4.3.2", "vitepress-plugin-tabs": "^0.5.0", + "vitepress": "^1.3.4", "vitest": "^1.3.0" } } diff --git a/docs/src/.vitepress/theme/style.css b/docs/src/.vitepress/theme/style.css index b0eec417..80723753 100644 --- a/docs/src/.vitepress/theme/style.css +++ b/docs/src/.vitepress/theme/style.css @@ -264,4 +264,18 @@ mjx-container > svg { /* Style for output code blocks */ .language- { background-color: var(--vp-c-bg-output) !important; +} + +/* Component: Docstring Custom Block */ + +.jldocstring.custom-block { + border: 1px solid var(--vp-c-gray-2); + color: var(--vp-c-text-1) +} + +.jldocstring.custom-block summary { + font-weight: 700; + cursor: pointer; + user-select: none; + margin: 0 0 8px; } \ No newline at end of file diff --git a/docs/src/UserGuide/read.md b/docs/src/UserGuide/read.md index 43bd7164..9b621219 100644 --- a/docs/src/UserGuide/read.md +++ b/docs/src/UserGuide/read.md @@ -53,6 +53,6 @@ using ArchGDAL using Downloads: download path = download("https://github.com/yeesian/ArchGDALDatasets/raw/307f8f0e584a39a050c042849004e6a2bd674f99/gdalworkshop/world.tif", "world.tif") -# ds = open_dataset(path) # this is broken +ds = open_dataset(path) nothing ```` \ No newline at end of file From 30cd05e938dd9be3f8c306b94f5a38e0aeed221b Mon Sep 17 00:00:00 2001 From: Lazaro Alonso Date: Thu, 19 Sep 2024 21:41:30 +0200 Subject: [PATCH 14/20] add version select --- docs/src/.vitepress/config.mts | 136 +++++++++++-------- docs/src/.vitepress/theme/VersionPicker.vue | 142 ++++++++++++++++++++ docs/src/.vitepress/theme/index.ts | 4 +- 3 files changed, 224 insertions(+), 58 deletions(-) create mode 100644 docs/src/.vitepress/theme/VersionPicker.vue diff --git a/docs/src/.vitepress/config.mts b/docs/src/.vitepress/config.mts index 0be65ceb..3cb2a7c7 100644 --- a/docs/src/.vitepress/config.mts +++ b/docs/src/.vitepress/config.mts @@ -2,6 +2,82 @@ import { defineConfig } from 'vitepress' import { tabsMarkdownPlugin } from 'vitepress-plugin-tabs' import mathjax3 from "markdown-it-mathjax3"; +function getBaseRepository(base: string): string { + if (!base) return '/'; + // I guess if deploy_url is available. From where do I check this ? + const parts = base.split('/').filter(Boolean); + return parts.length > 0 ? `/${parts[0]}/` : '/'; +} + +const baseTemp = { + base: 'REPLACE_ME_DOCUMENTER_VITEPRESS', +} + +const navTemp = { + nav: [ + { text: 'Home', link: '/' }, + { text: 'Get Started', link: '/get_started' }, + { + text: 'User Guide', + items: [ + { text: 'Types', link: '/UserGuide/types' }, + { text: 'Read', link: '/UserGuide/read' }, + { text: 'Write', link: '/UserGuide/write' }, + { text: 'Convert', link: '/UserGuide/convert' }, + { text: 'Create', link: '/UserGuide/create' }, + { text: 'Select', link: '/UserGuide/select' }, + { text: 'Compute', link: '/UserGuide/compute' }, + { text: 'Chunk', link: '/UserGuide/chunk' }, + { text: 'Cache', link: '/UserGuide/cache' }, + { text: 'Group', link: '/UserGuide/group' }, + { text: 'Combine', link: '/UserGuide/combine' }, + { text: 'FAQ', link: '/UserGuide/faq' } + ] + }, + { + text: 'Tutorials', + items: [ + { text: 'Overview', link: '/tutorials/tutorial' }, + { text: 'Plotting maps', link: '/tutorials/plottingmaps' }, + { text: 'Mean Seasonal Cycle', link: '/tutorials/mean_seasonal_cycle' }, + { + text: 'ESDL studies', + items: [ + { text: 'ESDL study 1', link: 'https://github.com/JuliaDataCubes/YAXArrays.jl/blob/master/docs/src/tutorials/esdl/examples_from_esdl_study_1.jl' }, + { text: 'ESDL study 2', link: 'https://github.com/JuliaDataCubes/YAXArrays.jl/blob/master/docs/src/tutorials/esdl/examples_from_esdl_study_2.jl' }, + { text: 'ESDL study 3', link: 'https://github.com/JuliaDataCubes/YAXArrays.jl/blob/master/docs/src/tutorials/esdl/examples_from_esdl_study_3.jl' }, + { text: 'ESDL study 4', link: 'https://github.com/JuliaDataCubes/YAXArrays.jl/blob/master/docs/src/tutorials/esdl/examples_from_esdl_study_4.jl' }, + ] + }, + { text: 'Other Tutorials', link: '/tutorials/other_tutorials' }, + ] + }, + { text: 'Ecosystem', + items: [ + { text: 'DimensionalData.jl', link: 'https://rafaqz.github.io/DimensionalData.jl/dev/' }, + { text: 'NetCDF.jl', link: 'https://juliageo.org/NetCDF.jl/stable/'}, + { text: 'Zarr.jl', link: 'https://juliaio.github.io/Zarr.jl/latest/'}, + { text: 'ArchGDAL.jl', link: 'https://yeesian.com/ArchGDAL.jl/stable/' }, + { text: 'GeoMakie.jl', link: 'https://geo.makie.org/dev/' }, + { text: 'Makie.jl', link: 'https://docs.makie.org/dev/' }, + ] + }, + { + text: 'Development', + items: [ + { text: 'Contribute', link: 'development/contribute' }, + { text: 'Contributors', link: 'development/contributors' } + ] + }, + ], +} + +const nav = [ + ...navTemp.nav, + { + component: 'VersionPicker' + } +] // https://vitepress.dev/reference/site-config export default defineConfig({ base: 'REPLACE_ME_DOCUMENTER_VITEPRESS', @@ -12,6 +88,8 @@ export default defineConfig({ outDir: 'REPLACE_ME_DOCUMENTER_VITEPRESS', // This is required for MarkdownVitepress to work correctly... head: [ ['link', { rel: 'icon', href: 'REPLACE_ME_DOCUMENTER_VITEPRESS_FAVICON' }], + ['script', {src: `${getBaseRepository(baseTemp.base)}versions.js`}], + ['script', {src: `${baseTemp.base}siteinfo.js`}] ], // ignoreDeadLinks: true, @@ -36,63 +114,7 @@ export default defineConfig({ detailedView: true } }, - nav: [ - { text: 'Home', link: '/' }, - { text: 'Get Started', link: '/get_started' }, - { - text: 'User Guide', - items: [ - { text: 'Types', link: '/UserGuide/types' }, - { text: 'Read', link: '/UserGuide/read' }, - { text: 'Write', link: '/UserGuide/write' }, - { text: 'Convert', link: '/UserGuide/convert' }, - { text: 'Create', link: '/UserGuide/create' }, - { text: 'Select', link: '/UserGuide/select' }, - { text: 'Compute', link: '/UserGuide/compute' }, - { text: 'Chunk', link: '/UserGuide/chunk' }, - { text: 'Cache', link: '/UserGuide/cache' }, - { text: 'Group', link: '/UserGuide/group' }, - { text: 'Combine', link: '/UserGuide/combine' }, - { text: 'FAQ', link: '/UserGuide/faq' } - ] - }, - { - text: 'Tutorials', - items: [ - { text: 'Overview', link: '/tutorials/tutorial' }, - { text: 'Plotting maps', link: '/tutorials/plottingmaps' }, - { text: 'Mean Seasonal Cycle', link: '/tutorials/mean_seasonal_cycle' }, - { - text: 'ESDL studies', - items: [ - { text: 'ESDL study 1', link: 'https://github.com/JuliaDataCubes/YAXArrays.jl/blob/master/docs/src/tutorials/esdl/examples_from_esdl_study_1.jl' }, - { text: 'ESDL study 2', link: 'https://github.com/JuliaDataCubes/YAXArrays.jl/blob/master/docs/src/tutorials/esdl/examples_from_esdl_study_2.jl' }, - { text: 'ESDL study 3', link: 'https://github.com/JuliaDataCubes/YAXArrays.jl/blob/master/docs/src/tutorials/esdl/examples_from_esdl_study_3.jl' }, - { text: 'ESDL study 4', link: 'https://github.com/JuliaDataCubes/YAXArrays.jl/blob/master/docs/src/tutorials/esdl/examples_from_esdl_study_4.jl' }, - ] - }, - { text: 'Other Tutorials', link: '/tutorials/other_tutorials' }, - ] - }, - { text: 'Ecosystem', - items: [ - { text: 'DimensionalData.jl', link: 'https://rafaqz.github.io/DimensionalData.jl/dev/' }, - { text: 'NetCDF.jl', link: 'https://juliageo.org/NetCDF.jl/stable/'}, - { text: 'Zarr.jl', link: 'https://juliaio.github.io/Zarr.jl/latest/'}, - { text: 'ArchGDAL.jl', link: 'https://yeesian.com/ArchGDAL.jl/stable/' }, - { text: 'GeoMakie.jl', link: 'https://geo.makie.org/dev/' }, - { text: 'Makie.jl', link: 'https://docs.makie.org/dev/' }, - ] - }, - { - text: 'Development', - items: [ - { text: 'Contribute', link: 'development/contribute' }, - { text: 'Contributors', link: 'development/contributors' } - ] - }, - ], - + nav, sidebar: [ { text: 'Get Started', link: '/get_started' }, { text: 'API Reference', link: '/api' }, diff --git a/docs/src/.vitepress/theme/VersionPicker.vue b/docs/src/.vitepress/theme/VersionPicker.vue new file mode 100644 index 00000000..b6505402 --- /dev/null +++ b/docs/src/.vitepress/theme/VersionPicker.vue @@ -0,0 +1,142 @@ + + + + + + + diff --git a/docs/src/.vitepress/theme/index.ts b/docs/src/.vitepress/theme/index.ts index d305f4af..a689558a 100644 --- a/docs/src/.vitepress/theme/index.ts +++ b/docs/src/.vitepress/theme/index.ts @@ -3,6 +3,7 @@ import { h } from 'vue' import type { Theme } from 'vitepress' import DefaultTheme from 'vitepress/theme' import AsideTrustees from '../../components/AsideTrustees.vue' +import VersionPicker from "./VersionPicker.vue" import { enhanceAppWithTabs } from 'vitepress-plugin-tabs/client' import './style.css' @@ -15,6 +16,7 @@ export default { }) }, enhanceApp({ app, router, siteData }) { - enhanceAppWithTabs(app) + enhanceAppWithTabs(app); + app.component('VersionPicker', VersionPicker); } } satisfies Theme \ No newline at end of file From c438001ef44b4a784093e42d91ab4c7a0b8fca73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Pasquier?= <4486578+briochemc@users.noreply.github.com> Date: Thu, 19 Sep 2024 15:52:56 +1000 Subject: [PATCH 15/20] Fix rebuild --- src/Cubes/Cubes.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Cubes/Cubes.jl b/src/Cubes/Cubes.jl index ff1cb239..d609cf23 100644 --- a/src/Cubes/Cubes.jl +++ b/src/Cubes/Cubes.jl @@ -200,7 +200,7 @@ function DD.rebuild(A::YAXArray, data::AbstractArray, dims::Tuple, refdims::Tupl #end YAXArray(dims, data, metadata; cleaner=A.cleaner)#, chunks=GridChunks(chunks)) end -function DD.rebuild(A::YAXArray; data=parent(A), dims=dims(A), metadata=DD.metadata(A), kw...) +function DD.rebuild(A::YAXArray; data=parent(A), dims=DD.dims(A), metadata=DD.metadata(A), kw...) YAXArray(dims, data, metadata; cleaner=A.cleaner) end From 75d77e5ff847e2e97d1e26503e4dd67bf1f7635a Mon Sep 17 00:00:00 2001 From: Lazaro Alonso Date: Thu, 19 Sep 2024 20:07:45 +0200 Subject: [PATCH 16/20] up versions --- .gitignore | 1 + Project.toml | 1 + docs/package.json | 1 + docs/src/.vitepress/theme/style.css | 14 ++++++++++++++ docs/src/UserGuide/read.md | 2 ++ 5 files changed, 19 insertions(+) diff --git a/.gitignore b/.gitignore index 073f2754..349cc91d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ build .vscode *.zarr *.nc +*.tif var *.ipynb_checkpoints diff --git a/Project.toml b/Project.toml index 04516e26..b7d71070 100644 --- a/Project.toml +++ b/Project.toml @@ -34,6 +34,7 @@ YAXArrayBase = "90b8fcef-0c2d-428d-9c56-5f86629e9d14" [compat] CFTime = "0.0, 0.1" DataStructures = "0.17, 0.18" +DimensionalData = "0.27, 0.28" DiskArrayTools = "0.1" DiskArrays = "0.3" DocStringExtensions = "0.8, 0.9" diff --git a/docs/package.json b/docs/package.json index f908fde5..904caf08 100644 --- a/docs/package.json +++ b/docs/package.json @@ -8,6 +8,7 @@ "markdown-it": "^14.0.0", "markdown-it-mathjax3": "^4.3.2", "vitepress-plugin-tabs": "^0.5.0", + "vitepress": "^1.3.4", "vitest": "^1.3.0" } } diff --git a/docs/src/.vitepress/theme/style.css b/docs/src/.vitepress/theme/style.css index b0eec417..80723753 100644 --- a/docs/src/.vitepress/theme/style.css +++ b/docs/src/.vitepress/theme/style.css @@ -264,4 +264,18 @@ mjx-container > svg { /* Style for output code blocks */ .language- { background-color: var(--vp-c-bg-output) !important; +} + +/* Component: Docstring Custom Block */ + +.jldocstring.custom-block { + border: 1px solid var(--vp-c-gray-2); + color: var(--vp-c-text-1) +} + +.jldocstring.custom-block summary { + font-weight: 700; + cursor: pointer; + user-select: none; + margin: 0 0 8px; } \ No newline at end of file diff --git a/docs/src/UserGuide/read.md b/docs/src/UserGuide/read.md index cf4024bc..f5c58b47 100644 --- a/docs/src/UserGuide/read.md +++ b/docs/src/UserGuide/read.md @@ -54,4 +54,6 @@ using Downloads: download path = download("https://github.com/yeesian/ArchGDALDatasets/raw/307f8f0e584a39a050c042849004e6a2bd674f99/gdalworkshop/world.tif", "world.tif") ds = open_dataset(path) +nothing +ds = open_dataset(path) ```` \ No newline at end of file From 73024fde146d9ab9aeda50e1dcc63b22ee43ec43 Mon Sep 17 00:00:00 2001 From: Lazaro Alonso Date: Thu, 19 Sep 2024 21:41:30 +0200 Subject: [PATCH 17/20] add version select --- docs/src/.vitepress/config.mts | 136 +++++++++++-------- docs/src/.vitepress/theme/VersionPicker.vue | 142 ++++++++++++++++++++ docs/src/.vitepress/theme/index.ts | 4 +- 3 files changed, 224 insertions(+), 58 deletions(-) create mode 100644 docs/src/.vitepress/theme/VersionPicker.vue diff --git a/docs/src/.vitepress/config.mts b/docs/src/.vitepress/config.mts index 0be65ceb..3cb2a7c7 100644 --- a/docs/src/.vitepress/config.mts +++ b/docs/src/.vitepress/config.mts @@ -2,6 +2,82 @@ import { defineConfig } from 'vitepress' import { tabsMarkdownPlugin } from 'vitepress-plugin-tabs' import mathjax3 from "markdown-it-mathjax3"; +function getBaseRepository(base: string): string { + if (!base) return '/'; + // I guess if deploy_url is available. From where do I check this ? + const parts = base.split('/').filter(Boolean); + return parts.length > 0 ? `/${parts[0]}/` : '/'; +} + +const baseTemp = { + base: 'REPLACE_ME_DOCUMENTER_VITEPRESS', +} + +const navTemp = { + nav: [ + { text: 'Home', link: '/' }, + { text: 'Get Started', link: '/get_started' }, + { + text: 'User Guide', + items: [ + { text: 'Types', link: '/UserGuide/types' }, + { text: 'Read', link: '/UserGuide/read' }, + { text: 'Write', link: '/UserGuide/write' }, + { text: 'Convert', link: '/UserGuide/convert' }, + { text: 'Create', link: '/UserGuide/create' }, + { text: 'Select', link: '/UserGuide/select' }, + { text: 'Compute', link: '/UserGuide/compute' }, + { text: 'Chunk', link: '/UserGuide/chunk' }, + { text: 'Cache', link: '/UserGuide/cache' }, + { text: 'Group', link: '/UserGuide/group' }, + { text: 'Combine', link: '/UserGuide/combine' }, + { text: 'FAQ', link: '/UserGuide/faq' } + ] + }, + { + text: 'Tutorials', + items: [ + { text: 'Overview', link: '/tutorials/tutorial' }, + { text: 'Plotting maps', link: '/tutorials/plottingmaps' }, + { text: 'Mean Seasonal Cycle', link: '/tutorials/mean_seasonal_cycle' }, + { + text: 'ESDL studies', + items: [ + { text: 'ESDL study 1', link: 'https://github.com/JuliaDataCubes/YAXArrays.jl/blob/master/docs/src/tutorials/esdl/examples_from_esdl_study_1.jl' }, + { text: 'ESDL study 2', link: 'https://github.com/JuliaDataCubes/YAXArrays.jl/blob/master/docs/src/tutorials/esdl/examples_from_esdl_study_2.jl' }, + { text: 'ESDL study 3', link: 'https://github.com/JuliaDataCubes/YAXArrays.jl/blob/master/docs/src/tutorials/esdl/examples_from_esdl_study_3.jl' }, + { text: 'ESDL study 4', link: 'https://github.com/JuliaDataCubes/YAXArrays.jl/blob/master/docs/src/tutorials/esdl/examples_from_esdl_study_4.jl' }, + ] + }, + { text: 'Other Tutorials', link: '/tutorials/other_tutorials' }, + ] + }, + { text: 'Ecosystem', + items: [ + { text: 'DimensionalData.jl', link: 'https://rafaqz.github.io/DimensionalData.jl/dev/' }, + { text: 'NetCDF.jl', link: 'https://juliageo.org/NetCDF.jl/stable/'}, + { text: 'Zarr.jl', link: 'https://juliaio.github.io/Zarr.jl/latest/'}, + { text: 'ArchGDAL.jl', link: 'https://yeesian.com/ArchGDAL.jl/stable/' }, + { text: 'GeoMakie.jl', link: 'https://geo.makie.org/dev/' }, + { text: 'Makie.jl', link: 'https://docs.makie.org/dev/' }, + ] + }, + { + text: 'Development', + items: [ + { text: 'Contribute', link: 'development/contribute' }, + { text: 'Contributors', link: 'development/contributors' } + ] + }, + ], +} + +const nav = [ + ...navTemp.nav, + { + component: 'VersionPicker' + } +] // https://vitepress.dev/reference/site-config export default defineConfig({ base: 'REPLACE_ME_DOCUMENTER_VITEPRESS', @@ -12,6 +88,8 @@ export default defineConfig({ outDir: 'REPLACE_ME_DOCUMENTER_VITEPRESS', // This is required for MarkdownVitepress to work correctly... head: [ ['link', { rel: 'icon', href: 'REPLACE_ME_DOCUMENTER_VITEPRESS_FAVICON' }], + ['script', {src: `${getBaseRepository(baseTemp.base)}versions.js`}], + ['script', {src: `${baseTemp.base}siteinfo.js`}] ], // ignoreDeadLinks: true, @@ -36,63 +114,7 @@ export default defineConfig({ detailedView: true } }, - nav: [ - { text: 'Home', link: '/' }, - { text: 'Get Started', link: '/get_started' }, - { - text: 'User Guide', - items: [ - { text: 'Types', link: '/UserGuide/types' }, - { text: 'Read', link: '/UserGuide/read' }, - { text: 'Write', link: '/UserGuide/write' }, - { text: 'Convert', link: '/UserGuide/convert' }, - { text: 'Create', link: '/UserGuide/create' }, - { text: 'Select', link: '/UserGuide/select' }, - { text: 'Compute', link: '/UserGuide/compute' }, - { text: 'Chunk', link: '/UserGuide/chunk' }, - { text: 'Cache', link: '/UserGuide/cache' }, - { text: 'Group', link: '/UserGuide/group' }, - { text: 'Combine', link: '/UserGuide/combine' }, - { text: 'FAQ', link: '/UserGuide/faq' } - ] - }, - { - text: 'Tutorials', - items: [ - { text: 'Overview', link: '/tutorials/tutorial' }, - { text: 'Plotting maps', link: '/tutorials/plottingmaps' }, - { text: 'Mean Seasonal Cycle', link: '/tutorials/mean_seasonal_cycle' }, - { - text: 'ESDL studies', - items: [ - { text: 'ESDL study 1', link: 'https://github.com/JuliaDataCubes/YAXArrays.jl/blob/master/docs/src/tutorials/esdl/examples_from_esdl_study_1.jl' }, - { text: 'ESDL study 2', link: 'https://github.com/JuliaDataCubes/YAXArrays.jl/blob/master/docs/src/tutorials/esdl/examples_from_esdl_study_2.jl' }, - { text: 'ESDL study 3', link: 'https://github.com/JuliaDataCubes/YAXArrays.jl/blob/master/docs/src/tutorials/esdl/examples_from_esdl_study_3.jl' }, - { text: 'ESDL study 4', link: 'https://github.com/JuliaDataCubes/YAXArrays.jl/blob/master/docs/src/tutorials/esdl/examples_from_esdl_study_4.jl' }, - ] - }, - { text: 'Other Tutorials', link: '/tutorials/other_tutorials' }, - ] - }, - { text: 'Ecosystem', - items: [ - { text: 'DimensionalData.jl', link: 'https://rafaqz.github.io/DimensionalData.jl/dev/' }, - { text: 'NetCDF.jl', link: 'https://juliageo.org/NetCDF.jl/stable/'}, - { text: 'Zarr.jl', link: 'https://juliaio.github.io/Zarr.jl/latest/'}, - { text: 'ArchGDAL.jl', link: 'https://yeesian.com/ArchGDAL.jl/stable/' }, - { text: 'GeoMakie.jl', link: 'https://geo.makie.org/dev/' }, - { text: 'Makie.jl', link: 'https://docs.makie.org/dev/' }, - ] - }, - { - text: 'Development', - items: [ - { text: 'Contribute', link: 'development/contribute' }, - { text: 'Contributors', link: 'development/contributors' } - ] - }, - ], - + nav, sidebar: [ { text: 'Get Started', link: '/get_started' }, { text: 'API Reference', link: '/api' }, diff --git a/docs/src/.vitepress/theme/VersionPicker.vue b/docs/src/.vitepress/theme/VersionPicker.vue new file mode 100644 index 00000000..b6505402 --- /dev/null +++ b/docs/src/.vitepress/theme/VersionPicker.vue @@ -0,0 +1,142 @@ + + + + + + + diff --git a/docs/src/.vitepress/theme/index.ts b/docs/src/.vitepress/theme/index.ts index d305f4af..a689558a 100644 --- a/docs/src/.vitepress/theme/index.ts +++ b/docs/src/.vitepress/theme/index.ts @@ -3,6 +3,7 @@ import { h } from 'vue' import type { Theme } from 'vitepress' import DefaultTheme from 'vitepress/theme' import AsideTrustees from '../../components/AsideTrustees.vue' +import VersionPicker from "./VersionPicker.vue" import { enhanceAppWithTabs } from 'vitepress-plugin-tabs/client' import './style.css' @@ -15,6 +16,7 @@ export default { }) }, enhanceApp({ app, router, siteData }) { - enhanceAppWithTabs(app) + enhanceAppWithTabs(app); + app.component('VersionPicker', VersionPicker); } } satisfies Theme \ No newline at end of file From e956cab6a9b723a154ba0d5fe91b0aec89c142e1 Mon Sep 17 00:00:00 2001 From: Lazaro Alonso Date: Fri, 20 Sep 2024 13:12:29 +0200 Subject: [PATCH 18/20] outdims and properties --- Project.toml | 2 +- docs/src/UserGuide/compute_debug.jl | 151 ---------------------------- docs/src/UserGuide/faq.md | 6 ++ docs/src/UserGuide/read.md | 2 - src/DAT/DAT.jl | 23 +++-- src/DatasetAPI/Datasets.jl | 4 +- test/Datasets/datasets.jl | 34 +++++-- 7 files changed, 46 insertions(+), 176 deletions(-) delete mode 100644 docs/src/UserGuide/compute_debug.jl diff --git a/Project.toml b/Project.toml index b7d71070..2b87892d 100644 --- a/Project.toml +++ b/Project.toml @@ -36,7 +36,7 @@ CFTime = "0.0, 0.1" DataStructures = "0.17, 0.18" DimensionalData = "0.27, 0.28" DiskArrayTools = "0.1" -DiskArrays = "0.3" +DiskArrays = "0.3, 0.4" DocStringExtensions = "0.8, 0.9" Glob = "1.3" Interpolations = "0.12, 0.13, 0.14, 0.15" diff --git a/docs/src/UserGuide/compute_debug.jl b/docs/src/UserGuide/compute_debug.jl deleted file mode 100644 index 6ac6c12f..00000000 --- a/docs/src/UserGuide/compute_debug.jl +++ /dev/null @@ -1,151 +0,0 @@ -using YAXArrays, Dates -using Random - -axlist = ( - Dim{:time}(Date("2022-01-01"):Day(1):Date("2022-01-05")), - Dim{:lon}(range(1, 4, length=4)), - Dim{:lat}(range(1, 3, length=3)), - Dim{:variables}(["a", "b"]) -) - -Random.seed!(123) -data = rand(1:5, 5, 4, 3, 2) - -properties = Dict("description" => "multi dimensional test cube") -yax_test = YAXArray(axlist, data, properties) - -properties_2d = Dict("description" => "2d dimensional test cube") -yax_2d = YAXArray(axlist[2:end], rand(-1:1, 4, 3, 2), properties_2d) - -f1(xin) = xin + 1 -f2(xin) = xin + 2 -function one_to_many(xout_one, xout_two, xout_flat, xin_one) - xout_one .= f1.(xin_one) - xout_two .= f2.(xin_one) - xout_flat .= sum(xin_one) # this will reduce the time dimension if we set outdims = OutDims() - return nothing -end - -indims_one = InDims("Time") -# outputs dimension -outdims_one = OutDims("Time", name="plus_one") -outdims_two = OutDims("Time", name="plus_two", units ="double") -outdims_flat = OutDims(; name="flat") # space - -ds = mapCube(one_to_many, yax_test, - indims = indims_one, - outdims = (outdims_one, outdims_two, outdims_flat)); - -# many to many. mix input dimensions - -f2mix(xin_xyt, xin_xy) = xin_xyt - xin_xy - -function many_to_many(xout_one, xout_two, xout_flat, xin_one, xin_two, xin_drei) - xout_one .= f1.(xin_one) - xout_two .= f2mix.(xin_one, xin_two) - xout_flat .= sum(xin_drei) # this will reduce the time dimension if we set outdims = OutDims() - return nothing -end - -indims_one = InDims("Time") -indims_2d = InDims() # it matches only to the other 2 dimensions and uses the same values for each time step - -outdims_one = OutDims("Time", name="plus_one") -outdims_two = OutDims("Time", name="plus_two", units ="double") -outdims_flat = OutDims(; name="flat") # space - -ds = mapCube(many_to_many, (yax_test, yax_2d, yax_test), - indims = (indims_one, indims_2d, indims_one), - outdims = (outdims_one, outdims_two, outdims_flat)); - - -properties_2dz = Dict("description" => "2d-z dimensional test cube") - -axlist = ( - Dim{:lon}(range(1, 4, length=4)), - Dim{:lat}(range(1, 3, length=3)), - Dim{:depth}(range(1, 2, length=2)), - Dim{:variables}(["a", "b"]) -) - -yax_2dz = YAXArray(axlist, rand(-2:2, 4, 3, 2, 2), properties_2dz) - -function f2mix_depth(xin_xyt, xin_xyz) - @show xin_xyz - return xin_xyt + sum(abs.(xin_xyz)) -end - -function many_to_many_depth(xout_one, xout_two, xout_flat, xin_one, xin_two) - xout_one .= f1.(xin_one) - xout_two .= f2mix_depth.(xin_one, xin_two) - xout_flat .= sum(abs.(xin_two)) # this will reduce the time dimension if we set outdims = OutDims() - return nothing -end - -indims_one = InDims("Time") -indims_2d = InDims() # it matches only to the other 2 dimensions and uses the same values for each time step -outdims_one = OutDims("Time") -outdims_two = OutDims("Time") -outdims_flat = OutDims() # space - -ds = mapCube(many_to_many_depth, (yax_test, yax_2dz), - indims = (indims_one, indims_2d), - outdims = (outdims_one, outdims_two, outdims_flat)) - - -function many_depth(xout_two, xin_one, xin_two) - xout_two .= f2mix_depth.(xin_one, xin_two) - return nothing -end -function f2mix_depth(xin_xyt, xin_xyz) - s = sum(abs.(xin_xyz)) - # @show s # is not doing what I think is doing! - # @show xin_xyz - return xin_xyt + s -end - -ds = mapCube(many_depth, (yax_test[Variable= At("a")], yax_2dz[Variable= At("a")]), - indims = (InDims("Time"), InDims()), - outdims = OutDims("Time")) - -yax_s = sum(yax_2dz[Variable= At("a")], dims=:depth) -yax_s = dropdims(yax_s, dims=:depth) - -# use case, things should operate at the pixel level, (lon, lat) and extract the corresponding values there. - -axlist = ( - Dim{:lon}(1:4), - Dim{:lat}(1:3), - Dim{:depth}(1:7), -) - -yax_2d = YAXArray(axlist, rand(-3:0, 4, 3, 7)) - -axlist = ( - Dim{:time}(Date("2022-01-01"):Day(1):Date("2022-01-05")), - Dim{:lon}(1:4), - Dim{:lat}(1:3), -) - -Random.seed!(123) -data = rand(3:5, 5, 4, 3) - -properties = Dict("description" => "multi dimensional test cube") -yax_test = YAXArray(axlist, data, properties) - -function mix_time_depth(xin_xyt, xin_xyz) - s = sum(abs.(xin_xyz)) - @show s # ? is doing what I think is doing! - @show xin_xyz - return xin_xyt.^2 .+ s -end - -function time_depth(xout, xin_one, xin_two) - xout .= mix_time_depth(xin_one, xin_two) # ? note also here, no dot anymore! - return nothing -end - -ds = mapCube(time_depth, (yax_test, yax_2dz), - indims = (InDims("Time"), InDims("depth")), - # ? it shouldn't be better to say "lon", "lat". Logic? well, you specify an anchor dimension and then map over the others. - outdims = OutDims("Time")) \ No newline at end of file diff --git a/docs/src/UserGuide/faq.md b/docs/src/UserGuide/faq.md index 96ea356c..4895303a 100644 --- a/docs/src/UserGuide/faq.md +++ b/docs/src/UserGuide/faq.md @@ -313,6 +313,12 @@ nothing # hide ds = YAXArrays.Dataset(; (keylist .=> varlist)...) ```` +::: warning + +You will not be able to save this dataset, first you will need to rename those `dimensions` with the `same name` but different values. + +::: + ## Ho do I construct a `Dataset` from a TimeArray In this section we will use `MarketData.jl` and `TimeSeries.jl` to simulate some stocks. diff --git a/docs/src/UserGuide/read.md b/docs/src/UserGuide/read.md index f5c58b47..cf4024bc 100644 --- a/docs/src/UserGuide/read.md +++ b/docs/src/UserGuide/read.md @@ -54,6 +54,4 @@ using Downloads: download path = download("https://github.com/yeesian/ArchGDALDatasets/raw/307f8f0e584a39a050c042849004e6a2bd674f99/gdalworkshop/world.tif", "world.tif") ds = open_dataset(path) -nothing -ds = open_dataset(path) ```` \ No newline at end of file diff --git a/src/DAT/DAT.jl b/src/DAT/DAT.jl index 901e1279..706324fa 100644 --- a/src/DAT/DAT.jl +++ b/src/DAT/DAT.jl @@ -798,15 +798,16 @@ function getbackend(oc, ispar, max_cache) rt = oc.desc.backend ispath = get(oc.desc.backendargs, :path, nothing) - b = if rt == :auto - if ispar[] || outsize > max_cache || !isnothing(ispath) - YAXArrayBase.backendfrompath(ispath) # if backend is not available, it defaults to last available, usually zarr. We need a warning for such case. + b = if rt == :auto && !isnothing(ispath) + YAXArrayBase.backendfrompath(ispath) + elseif rt == :auto + if ispar[] || outsize > max_cache + YAXArrayBase.backendlist[:zarr] + else + YAXArrayBase.backendlist[:array] + end else - rt = :array - YAXArrayBase.backendlist[Symbol(rt)] - end - else - YAXArrayBase.backendlist[Symbol(rt)] # Handle non-auto rt case + YAXArrayBase.backendlist[Symbol(rt)] # Handle non-auto rt case end if !allow_parallel_write(b) ispar[] = false @@ -853,8 +854,10 @@ function generateOutCube( newsize = map(length, oc.allAxes) outar = Array{elementtype}(undef, newsize...) fill!(outar,_zero(elementtype)) - properties = Dict{String, Any}(string(k) => v for (k, v) in oc.desc.backendargs) - oc.cube = YAXArray(tuple(oc.allAxes...), outar, properties) # ? here, including properties! + # @show oc.desc.backendargs[:properties] + properties = get(oc.desc.backendargs, :properties, Dict{String, Any}()) + # @show properties + oc.cube = YAXArray(tuple(oc.allAxes...), outar, properties) # properties ? include properties! oc.cube_unpermuted = oc.cube end _zero(T) = zero(T) diff --git a/src/DatasetAPI/Datasets.jl b/src/DatasetAPI/Datasets.jl index 4cbf06e8..766bc560 100644 --- a/src/DatasetAPI/Datasets.jl +++ b/src/DatasetAPI/Datasets.jl @@ -678,7 +678,7 @@ function savecube( if chunks !== nothing error("Setting chunks in savecube is not supported anymore. Rechunk using `setchunks` before saving. ") end - + ds = to_dataset(c; layername, datasetaxis) ds = savedataset(ds; path, max_cache, driver, overwrite, append,skeleton, writefac, kwargs...) Cube(ds, joinname = datasetaxis) @@ -718,7 +718,7 @@ function createdataset( properties = Dict{String,Any}(), globalproperties = Dict{String,Any}(), datasetaxis = "Variable", - layername = "layer", + layername = get(properties, "name", "layer"), kwargs..., ) if persist === nothing diff --git a/test/Datasets/datasets.jl b/test/Datasets/datasets.jl index f2d7be7a..e7789cb0 100644 --- a/test/Datasets/datasets.jl +++ b/test/Datasets/datasets.jl @@ -430,40 +430,54 @@ end tspan = Date("2022-01-01"):Day(1):Date("2022-01-30") time = YAXArray(Dim{:time}(tspan)) + properties = Dict{String, Any}("name" => "out_array") + gen_cube = mapCube(g, (lon, lat, time); indims = (InDims(), InDims(), InDims("time")), - outdims = OutDims("time", + outdims = OutDims("time"; properties, outtype = Float32) # max_cache=1e9 ) gen_cube2d = mapCube(g2d, (lon, lat); indims = (InDims(), InDims()), - outdims = OutDims(outtype = Float32) + outdims = OutDims(; outtype = Float32) # max_cache=1e9 ) - + properties = Dict{String, Any}("name" => "out_zarr") # test saves, zarr - gen_zarr = mapCube(g, (lon, lat, time); + mapCube(g, (lon, lat, time); indims = (InDims(), InDims(), InDims("time")), - outdims = OutDims("time", overwrite=true, path="my_gen_cube.zarr", + outdims = OutDims("time"; overwrite=true, path="my_gen_cube.zarr", + properties, outtype = Float32) # max_cache=1e9 ) ds_zarr = open_dataset("my_gen_cube.zarr") # test saves, nc - gen_nc = mapCube(g, (lon, lat, time); + properties = Dict{String, Any}("name" => "out_nc") + mapCube(g, (lon, lat, time); indims = (InDims(), InDims(), InDims("time")), - outdims = OutDims("time", overwrite=true, path="my_gen_cube.nc", + outdims = OutDims("time"; overwrite=true, path="my_gen_cube.nc", + properties, outtype = Float32) # max_cache=1e9 ) + mapCube(g, (lon, lat, time); + indims = (InDims(), InDims(), InDims("time")), + outdims = OutDims("time"; overwrite=true, path="my_gen_cube_no_p.nc", + outtype = Float32) + # max_cache=1e9 + ) ds_nc = open_dataset("my_gen_cube.nc") + ds_nc_no_p = open_dataset("my_gen_cube_no_p.nc") # TODO: fix tif for general inputs, so that writing also works. - - @test gen_cube.data[:,:,:] == ds_zarr["layer"].data[:,:,:] - @test gen_cube.data[:,:,:] == ds_nc["layer"].data[:,:,:] + + @test gen_cube.properties["name"] == "out_array" + @test gen_cube.data[:,:,:] == ds_zarr["out_zarr"].data[:,:,:] + @test gen_cube.data[:,:,:] == ds_nc["out_nc"].data[:,:,:] + @test gen_cube.data[:,:,:] == ds_nc_no_p["layer"].data[:,:,:] end @testset "Caching" begin From ffc57e477c606022a8f095ea5d8ade1028a5b455 Mon Sep 17 00:00:00 2001 From: Lazaro Alonso Date: Sun, 22 Sep 2024 12:34:24 +0200 Subject: [PATCH 19/20] eltypes and fallback --- src/Cubes/Cubes.jl | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Cubes/Cubes.jl b/src/Cubes/Cubes.jl index 711ecb1c..5068ec60 100644 --- a/src/Cubes/Cubes.jl +++ b/src/Cubes/Cubes.jl @@ -508,19 +508,25 @@ getCubeDes(::Type{T}) where {T} = string(T) function DD.show_after(io::IO, mime, c::YAXArray) blockwidth = get(io, :blockwidth, 0) - # This are some types for which sizeof is known to return a meaninful value - if isconcretetype(eltype(c)) && <:(eltype(c),Union{AbstractFloat,Signed,Unsigned,Bool}) + # ? sizeof : Check if the element type is a bitstype or a union of bitstypes + if (isconcretetype(eltype(c)) && isbitstype(eltype(c))) || + (eltype(c) isa Union && all(isbitstype, Base.uniontypes(eltype(c)))) + DD.print_block_separator(io, "file size", blockwidth, blockwidth) println(io, "\n file size: ", formatbytes(cubesize(c))) + else # fallback + DD.print_block_separator(io, "memory size", blockwidth, blockwidth) + println(io, "\n summarysize size: ", formatbytes(Base.summarysize(parent(c)))) end DD.print_block_close(io, blockwidth) - # And if you want the array data to print: + # Uncomment to print array data if needed # ndims(c) > 0 && println(io) # DD.print_array(io, mime, c) end + include("TransformedCubes.jl") include("Slices.jl") include("Rechunker.jl") From 773d6271552094e2b3716c6a6a1411bd55ecd64b Mon Sep 17 00:00:00 2001 From: Lazaro Alonso Date: Sun, 22 Sep 2024 13:32:54 +0200 Subject: [PATCH 20/20] examples --- docs/src/UserGuide/faq.md | 22 +++++++++++++++++++++- src/Cubes/Cubes.jl | 2 +- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/docs/src/UserGuide/faq.md b/docs/src/UserGuide/faq.md index 4895303a..cd8d394e 100644 --- a/docs/src/UserGuide/faq.md +++ b/docs/src/UserGuide/faq.md @@ -363,4 +363,24 @@ and voilà ds = Dataset(; (d_keys .=> yax_list)...) ```` -now they are printed together, showing that is exactly the same axis structure for all variables. \ No newline at end of file +now they are printed together, showing that is exactly the same axis structure for all variables. + +## Create a `YAXArray` with unions containing `Strings` + +````@example howdoi +test_x = stack(Vector{Union{Int,String}}[[1, "Test"], [2, "Test2"]]) +yax_string = YAXArray(test_x) +```` + +or simply with an `Any` type + +````@example howdoi +test_bool = ["Test1" 1 false; 2 "Test2" true; 1 2f0 1f2] +yax_bool = YAXArray(test_bool) +```` + +::: warning + +Note that although their creation is allowed, it is not possible to save these types into Zarr or NetCDF. + +::: \ No newline at end of file diff --git a/src/Cubes/Cubes.jl b/src/Cubes/Cubes.jl index 5068ec60..9a047bac 100644 --- a/src/Cubes/Cubes.jl +++ b/src/Cubes/Cubes.jl @@ -516,7 +516,7 @@ function DD.show_after(io::IO, mime, c::YAXArray) println(io, "\n file size: ", formatbytes(cubesize(c))) else # fallback DD.print_block_separator(io, "memory size", blockwidth, blockwidth) - println(io, "\n summarysize size: ", formatbytes(Base.summarysize(parent(c)))) + println(io, "\n summarysize: ", formatbytes(Base.summarysize(parent(c)))) end DD.print_block_close(io, blockwidth) # Uncomment to print array data if needed