Merge branch 'master' into sd/ci-1.10

.github/workflows/reference_tests.yml

@@ -198,9 +198,10 @@ jobs:
 
           # Loop through the directories and concatenate the files, and copy recorded folders
           for dir in WGLMakie CairoMakie GLMakie; do
-              # Concatenate scores.tsv and new_files.txt
+              # Concatenate scores.tsv, new_files.txt and missing_files.txt
               cat "${baseDir}/${dir}/scores.tsv" >> "./ReferenceImagesCombined/scores.tsv"
               cat "${baseDir}/${dir}/new_files.txt" >> "./ReferenceImagesCombined/new_files.txt"
+              cat "${baseDir}/${dir}/missing_files.txt" >> "./ReferenceImagesCombined/missing_files.txt"
 
               # Copy recorded folder
               mkdir -p "./ReferenceImagesCombined/recorded/${dir}/"

CairoMakie/test/runtests.jl

@@ -193,7 +193,7 @@ functions = [:volume, :volume!, :uv_mesh]
     CairoMakie.activate!(type = "png", px_per_unit = 1)
     ReferenceTests.mark_broken_tests(excludes, functions=functions)
     recorded_files, recording_dir = @include_reference_tests CairoMakie "refimages.jl"
-    missing_images, scores = ReferenceTests.record_comparison(recording_dir)
+    missing_images, scores = ReferenceTests.record_comparison(recording_dir, "CairoMakie")
     ReferenceTests.test_comparison(scores; threshold = 0.05)
 end
 

GLMakie/test/runtests.jl

@@ -30,7 +30,7 @@ include("unit_tests.jl")
     @testset "refimages" begin
         ReferenceTests.mark_broken_tests()
         recorded_files, recording_dir = @include_reference_tests GLMakie "refimages.jl" joinpath(@__DIR__, "glmakie_refimages.jl")
-        missing_images, scores = ReferenceTests.record_comparison(recording_dir)
+        missing_images, scores = ReferenceTests.record_comparison(recording_dir, "GLMakie")
         ReferenceTests.test_comparison(scores; threshold = 0.05)
     end
 

ReferenceTests/src/database.jl

@@ -17,6 +17,7 @@ const RECORDING_DIR = Base.RefValue{String}()
 const SKIP_TITLES = Set{String}()
 const SKIP_FUNCTIONS = Set{Symbol}()
 const COUNTER = Ref(0)
+const SKIPPED_NAMES = Set{String}() # names skipped due to title exclusion or function exclusion
 
 """
     @reference_test(name, code)
@@ -32,6 +33,7 @@ macro reference_test(name, code)
         @testset $(title) begin
             if $skip
                 @test_broken false
+                mark_skipped!($title)
             else
                 t1 = time()
                 if $title in $REGISTERED_TESTS
@@ -84,10 +86,13 @@ end
 function mark_broken_tests(title_excludes = []; functions=[])
     empty!(SKIP_TITLES)
     empty!(SKIP_FUNCTIONS)
+    empty!(SKIPPED_NAMES)
     union!(SKIP_TITLES, title_excludes)
     union!(SKIP_FUNCTIONS, functions)
 end
 
+mark_skipped!(name::String) = push!(SKIPPED_NAMES, name)
+
 macro include_reference_tests(backend::Symbol, path, paths...)
     toplevel_folder = dirname(string(__source__.file))
     return esc(quote

ReferenceTests/src/runtests.jl

@@ -81,7 +81,7 @@ function get_all_relative_filepaths_recursively(dir)
     end
 end
 
-function record_comparison(base_folder::String; record_folder_name="recorded", tag=last_major_version())
+function record_comparison(base_folder::String, backend::String; record_folder_name="recorded", tag=last_major_version())
     record_folder = joinpath(base_folder, record_folder_name)
     @info "Downloading reference images"
     reference_folder = download_refimages(tag)
@@ -99,6 +99,19 @@ function record_comparison(base_folder::String; record_folder_name="recorded", t
             println(file, path)
         end
     end
+
+    open(joinpath(base_folder, "missing_files.txt"), "w") do file
+        backend_ref_dir = joinpath(reference_folder, backend)
+        recorded_paths = mapreduce(vcat, walkdir(backend_ref_dir)) do (root, dirs, files)
+            relpath.(joinpath.(root, files), reference_folder)
+        end
+        skipped = Set([joinpath(backend, "$name.png") for name in SKIPPED_NAMES])
+        missing_recordings = setdiff(Set(recorded_paths), Set(testimage_paths), skipped)
+
+        for path in missing_recordings
+            println(file, path)
+        end
+    end
 
     open(joinpath(base_folder, "scores.tsv"), "w") do file
         paths_scores = sort(collect(pairs(scores)), by = last, rev = true)
@@ -120,7 +133,12 @@ function test_comparison(scores; threshold)
     end
 end
 
-function compare(relative_test_paths::Vector{String}, reference_dir::String, record_dir; o_refdir=reference_dir, missing_refimages=String[], scores=Dict{String,Float64}())
+function compare(
+        relative_test_paths::Vector{String}, reference_dir::String, record_dir; 
+        o_refdir = reference_dir, missing_refimages = String[], 
+        scores = Dict{String,Float64}()
+    )
+
     for relative_test_path in relative_test_paths
         ref_path = joinpath(reference_dir, relative_test_path)
         rec_path = joinpath(record_dir, relative_test_path)

ReferenceUpdater/Project.toml

@@ -4,6 +4,7 @@ authors = ["Julius Krumbiegel <julius.krumbiegel@gmail.com>"]
 version = "0.1.0"
 
 [deps]
+Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"

ReferenceUpdater/src/ReferenceUpdater.jl

@@ -8,6 +8,7 @@ import JSON3
 import ZipFile
 import REPL
 import TOML
+using Dates
 
 function github_token()
     get(ENV, "GITHUB_TOKEN") do

ReferenceUpdater/src/image_download.jl

@@ -18,3 +18,13 @@ function upload_reference_images(path=basedir("recorded"), tag=last_major_versio
         upload_release("MakieOrg", "Makie.jl", github_token(), tag, tarfile)
     end
 end
+
+function download_refimages(tag=last_major_version())
+    url = "https://github.com/MakieOrg/Makie.jl/releases/download/$(tag)/reference_images.tar"
+    images_tar = Downloads.download(url)
+    images = tempname()
+    isdir(images) && rm(images, recursive=true, force=true)
+    Tar.extract(images_tar, images)
+    rm(images_tar)
+    return images
+end

ReferenceUpdater/src/local_server.jl

@@ -12,31 +12,44 @@ function serve_update_page_from_dir(folder)
 
     folder = realpath(folder)
     @assert isdir(folder) "$folder is not a valid directory."
-    split_scores(folder)
+    group_scores(folder)
+    group_files(folder, "new_files.txt", "new_files_grouped.txt")
+    group_files(folder, "missing_files.txt", "missing_files_grouped.txt")
 
     router = HTTP.Router()
 
     function receive_update(req)
         data = JSON3.read(req.body)
-        images = data["images"]
+        images_to_update = data["images_to_update"]
+        images_to_delete = data["images_to_delete"]
         tag = data["tag"]
 
-        tempdir = tempname()
         recorded_folder = joinpath(folder, "recorded")
-        reference_folder = joinpath(folder, "reference")
 
-        @info "Copying reference folder to \"$tempdir\""
-        cp(reference_folder, tempdir)
+        @info "Downloading latest reference folder for $tag"
+        tempdir = download_refimages(tag)
+
+        @info "Updating files in $tempdir"
 
-        for image in images
-            @info "Overwriting \"$image\" in new reference folder"
+        for image in images_to_update
+            @info "Overwriting or adding $image"
             copy_filepath = joinpath(tempdir, image)
             copy_dir = splitdir(copy_filepath)[1]
             # make the path in case a new refimage is in a not yet existing folder
             mkpath(copy_dir)
             cp(joinpath(recorded_folder, image), copy_filepath, force = true)
         end
 
+        for image in images_to_delete
+            @info "Deleting $image"
+            copy_filepath = joinpath(tempdir, image)
+            if isfile(copy_filepath)
+                rm(copy_filepath, recursive = true)
+            else
+                @warn "Cannot delete $image - it has already been deleted."
+            end
+        end
+
         @info "Uploading updated reference images under tag \"$tag\""
         try
             upload_reference_images(tempdir, tag)
@@ -121,14 +134,20 @@ function serve_update_page(; commit = nothing, pr = nothing)
             return false
         end
     end
+
     if isempty(checkruns)
         error("\"Merge artifacts\" run is not available.")
     end
     if length(checkruns) > 1
-        error("Found multiple checkruns for \"Merge artifacts\", this is unexpected.")
-    end
-
+        datetimes = map(checkruns) do checkrun
+            DateTime(checkrun["completed_at"], dateformat"y-m-dTH:M:SZ")
+        end
+        datetime, idx = findmax(datetimes)
+        @warn("Found multiple checkruns for \"Merge artifacts\". Using latest with timestamp: $datetime")
+        check = checkruns[idx]
+    else
     check = only(checkruns)
+    end
 
     job = JSON3.read(authget("https://github.com/gitapi/repos/MakieOrg/Makie.jl/actions/jobs/$(check["id"])").body)
     run = JSON3.read(authget(job["run_url"]).body)
@@ -185,7 +204,7 @@ function unzip(file, exdir = "")
 end
 
 
-function split_scores(path)
+function group_scores(path)
     isfile(joinpath(path, "scores_table.tsv")) && return
 
     # Load all refimg scores into a Dict
@@ -231,3 +250,42 @@ function split_scores(path)
 
     return
 end
+
+function group_files(path, input_filename, output_filename)
+    isfile(joinpath(path, output_filename)) && return
+
+    # Group files in new_files/missing_files into a table like layout:
+    #  GLMakie  CairoMakie  WGLMakie
+
+    # collect refimg names and which backends they exist for
+    data = Dict{String, Vector{Bool}}()
+    open(joinpath(path, input_filename), "r") do file
+        for filepath in eachline(file)
+            pieces = split(filepath, '/')
+            backend = pieces[1]
+            if !(backend in ("GLMakie", "CairoMakie", "WGLMakie"))
+                error("Failed to parse backend in \"$line\", got \"$backend\"")
+            end
+
+            filename = join(pieces[2:end], '/')
+            exists = get!(data, filename, [false, false, false])
+
+            exists[1] |= backend == "GLMakie"
+            exists[2] |= backend == "CairoMakie"
+            exists[3] |= backend == "WGLMakie"
+        end
+    end
+
+    # generate new structed file
+    open(joinpath(path, output_filename), "w") do file
+        for (filename, valid) in data
+            println(file,
+                ifelse(valid[1], "GLMakie/$filename", "INVALID"), '\t',
+                ifelse(valid[2], "CairoMakie/$filename", "INVALID"), '\t',
+                ifelse(valid[3], "WGLMakie/$filename", "INVALID")
+            )
+        end
+    end
+
+    return
+end