added torchmeta sources to the repo and fixed dependencies

.gitignore

@@ -3,3 +3,5 @@
 bagofmaml.egg-info
 meteor.egg-info
 data.npz
+venv
+build

experiments/common/model.py

@@ -30,11 +30,11 @@ def reset_indices(targets):
     return torch.stack(rows)
 
 
-def get_model(modelname, snapshot_path, inplanes, select_bands=None):
+def get_model(modelname, snapshot_path, inplanes, select_bands=None, device="cuda"):
 
     if modelname == "meteor":
         basemodel = meteormodels.get_model("maml_resnet12", subset_bands=select_bands)
-        model = METEOR(basemodel, verbose=False, device="cuda")
+        model = METEOR(basemodel, verbose=False, device=device)
 
     elif modelname == "ssl4eo-dinorn50":
         ''' model from https://github.com/zhu-xlab/SSL4EO-S12 '''

experiments/config.yaml

@@ -93,3 +93,4 @@ data:
 results:
   table2resultsfolder: '/data/meteor-paper/results/table2'
   table3resultsfolder: '/data/meteor-paper/results/table3'
+  runtimeresultsfolder: '/data/meteor-paper/results/runtime'

experiments/runtime_dfc2020.py

@@ -0,0 +1,132 @@
+import numpy as np
+import torch
+import yaml
+import os
+from sklearn.metrics import classification_report
+
+from common.dfc2020.data import regions, bands
+from common.model import get_model
+from common.aggregate_results import save_results
+from common.data.dfc2020 import get_data
+import time
+import pandas as pd
+
+
+CONFIG_YAML = "/home/marc/projects/meteor/experiments/config.yaml"
+
+compare_models = ["meteor", "MOSAIKS-localfeatures", "SSLTransformerRS-resnet50", "swav", "dino", "seco",
+                  "imagenet", "proto", "scratch", "ssl4eo-mocorn50", "baseline-resnet18"]
+
+
+def aggregate_results():
+    from glob import glob
+
+    config = yaml.safe_load(open(CONFIG_YAML))
+    outputfolder = config['results']['runtimeresultsfolder']
+
+    times_files = glob(f"{outputfolder}/*/*/*/*/times.csv")
+
+    stats = []
+    for times_file in times_files:
+        df = pd.read_csv(times_file)
+
+        stats.append(dict(
+                model = times_file.split("/")[-2],
+                seed = times_file.split("/")[-3],
+                region = times_file.split("/")[-4],
+                shots = times_file.split("/")[-5],
+                fit_timedelta = df.fit_timedelta.values[0],
+                predict_timedelta = df.predict_timedelta.values[0],
+                total_timedelta = df.total_timedelta.values[0]
+            )
+        )
+
+    aggregated_df = pd.DataFrame(stats)
+    csvfile = os.path.join(outputfolder, "runtimes.csv")
+    print(f"writing {csvfile}")
+    aggregated_df.to_csv(csvfile)
+
+def fit():
+    config = yaml.safe_load(open(CONFIG_YAML))
+    dfc_path = config['data']['dfc2020']['datapath']
+    outputfolder = config['results']['runtimeresultsfolder']
+    os.makedirs(outputfolder, exist_ok=True)
+
+    random_states = [0]
+    num_shots = [1, 2, 5, 10, 15]
+
+    for shots in num_shots:
+        for region in regions:
+            for random_state in random_states:
+
+                outfolder = os.path.join(outputfolder, str(shots), "-".join(region), str(random_state))
+                os.makedirs(outfolder, exist_ok=True)
+                print(f'writing results in {outfolder}')
+
+                support_input, support_target, query_input, query_target, present_classes, s2bands, dataset_stats \
+                    = get_data(dfc_path, shots, region, random_state=random_state, return_info=True)
+                test_models(support_input, support_target, query_input, query_target, outfolder, present_classes)
+
+
+def test_models(support_input, support_target, query_input, query_target, outfolder, classes):
+    config = yaml.safe_load(open(CONFIG_YAML))
+
+    device = "cuda"
+
+    model = get_model("meteor", snapshot_path=None, inplanes=13, select_bands=bands, device=device)
+
+    start_time = time.time()
+    model.fit(support_input.to(device), support_target)
+    fit_time = time.time()
+    y_pred, y_score = model.predict(query_input.to(device))
+    end_time = time.time()
+
+    times = dict(
+        fit_timedelta=fit_time - start_time,
+        predict_timedelta=end_time - fit_time,
+        total_timedelta=end_time - start_time
+    )
+
+    os.makedirs(os.path.join(outfolder, "meteor"), exist_ok=True)
+    save_results(os.path.join(outfolder, "meteor"), y_pred, query_target, y_score, classes)
+    pd.DataFrame([times]).to_csv(os.path.join(outfolder, "meteor", "times.csv"))
+    print(classification_report(y_pred=y_pred, y_true=query_target, target_names=classes))
+
+    print(f"METEOR-CPU")
+    device = "cpu"
+
+    model = get_model("meteor", snapshot_path=None, inplanes=13, select_bands=bands, device=device)
+    support_input = support_input.to(device)
+    query_input = query_input.to(device)
+
+    start_time = time.time()
+    model.fit(support_input, support_target)
+    fit_time = time.time()
+    y_pred, y_score = model.predict(query_input)
+    end_time = time.time()
+
+    times = dict(
+        fit_timedelta = fit_time-start_time,
+        predict_timedelta = end_time-fit_time,
+        total_timedelta = end_time-start_time
+    )
+
+    os.makedirs(os.path.join(outfolder, "meteor-cpu"), exist_ok=True)
+    save_results(os.path.join(outfolder, "meteor-cpu"), y_pred, query_target, y_score, classes)
+    pd.DataFrame([times]).to_csv(os.path.join(outfolder, "meteor-cpu", "times.csv"))
+
+    print(classification_report(y_pred=y_pred, y_true=query_target, target_names=classes))
+
+
+def reset_indices(targets, class_ids):
+    """
+    resets absolute class indices (1,7,5,3) with relative ones (0,1,2,3)
+    """
+    row = torch.clone(targets)
+    for idx, id in enumerate(class_ids):
+        row[row == id] = idx
+    return row
+
+if __name__ == '__main__':
+    #fit()
+    aggregate_results()

meteor/examples/beirut.py

@@ -68,12 +68,19 @@ def example():
     y_support = torch.hstack([torch.zeros(shot), torch.ones(shot)]).long()
 
     # get model
-    model = models.get_model("maml_resnet12_s2")
-    taskmodel = METEOR(model, verbose=True, inner_step_size=0.32)
+    # get model
+    s2bands = ["S2B1", "S2B2", "S2B3", "S2B4", "S2B5", "S2B6", "S2B7", "S2B8", "S2B8A", "S2B9", "S2B10", "S2B11",
+               "S2B12"]
+    model = models.get_model("maml_resnet12", subset_bands=s2bands)
+    taskmodel = METEOR(model, verbose=True, inner_step_size=0.4, gradient_steps=20)
 
     # fit and predict
     taskmodel.fit(X_support, y_support)
     y_pred, y_score = taskmodel.predict(timeseries)
 
     # plot score
     plot(y_score, dates_dt)
+
+if __name__ == '__main__':
+    example()
+    plt.show()

requirements.txt

@@ -1,4 +1,4 @@
-torch==1.9.1
-torchmeta==1.8.0
-torchvision==0.10.1
+torch==2.0.0
+torchvision>=0.10.1
 tqdm
+ordered-set

setup.py

@@ -7,8 +7,8 @@
     name='meteor',
     version='0.1.0',
     author="Marc Rußwurm",
-    author_email="marc.russwurm@epfl.ch",
-    packages=find_packages(include=['meteor', 'meteor.*']),
+    author_email="marc.russwurm@wur.nl",
+    packages=find_packages(include=['meteor', 'meteor.*', 'torchmeta']),
     install_requires=requirements,
     extras_require={"examples": ["numpy", "pandas", "matplotlib"]}
 )

torchmeta/__init__.py

@@ -0,0 +1,7 @@
+from torchmeta import datasets
+from torchmeta import modules
+from torchmeta import toy
+from torchmeta import transforms
+from torchmeta import utils
+
+from torchmeta.version import VERSION as __version__

torchmeta/datasets/__init__.py

@@ -0,0 +1,39 @@
+from torchmeta.datasets.triplemnist import TripleMNIST
+from torchmeta.datasets.doublemnist import DoubleMNIST
+from torchmeta.datasets.cub import CUB
+from torchmeta.datasets.cifar100 import CIFARFS, FC100
+from torchmeta.datasets.miniimagenet import MiniImagenet
+from torchmeta.datasets.omniglot import Omniglot
+from torchmeta.datasets.tieredimagenet import TieredImagenet
+from torchmeta.datasets.tcga import TCGA
+from torchmeta.datasets.pascal5i import Pascal5i
+from torchmeta.datasets.letter import Letter
+from torchmeta.datasets.one_hundred_plants_texture import PlantsTexture
+from torchmeta.datasets.one_hundred_plants_shape import PlantsShape
+from torchmeta.datasets.one_hundred_plants_margin import PlantsMargin
+from torchmeta.datasets.bach import Bach
+
+from torchmeta.datasets import helpers
+from torchmeta.datasets import helpers_tabular
+
+__all__ = [
+    # image data
+    'TCGA',
+    'Omniglot',
+    'MiniImagenet',
+    'TieredImagenet',
+    'CIFARFS',
+    'FC100',
+    'CUB',
+    'DoubleMNIST',
+    'TripleMNIST',
+    'Pascal5i',
+    'helpers',
+    # tabular data
+    'Letter',
+    'PlantsTexture',
+    'PlantsShape',
+    'PlantsMargin',
+    'Bach',
+    'helpers_tabular'
+]

torchmeta/datasets/assets/bach/test.json

@@ -0,0 +1 @@
+[" Dbd", " E_M", " G_M", " C_M6", " C#M7", " A#d7", " D_M7", " B_m7", " A_M", " BbM7", " F_m", " G#m", " F#M4", " F_d7", " F_M4", " B_m6", " Dbd7", " EbM7", " B_M", " G#d7", " D_M6"]

torchmeta/datasets/assets/bach/train.json

@@ -0,0 +1 @@
+[" C#d", " E_m", " Dbm7", " D#d7", " C_m6", " C_d7", " G#M", " D#d", " A_m7", " A#d", " B_d7", " C#d6", " G_M7", " F_d", " F#d", " A_M6", " F#m6", " C#m", " Abd", " Bbm6", " G_m6", " D_d7", " BbM", " E_m6", " D#M", " G_d", " B_M4", " F#M", " C_d6", " A_m6", " F_M7", " AbM", " G_m7", " D_m6", " C#M4", " E_M4", " A_M7", " F#m", " E_M7", " C_M", " B_M7", " G#d", " C#m7", " A_m", " C_m7", " Ebd", " C#M", " B_m", " F_M6", " Bbm", " F#d7", " D_M", " Abm", " Bbd", " A_M4", " B_d", " C_M7", " A_m4", " F#m7", " A_d", " E_d"]

torchmeta/datasets/assets/bach/val.json

@@ -0,0 +1 @@
+[" C_M4", " F_m7", " D_m7", " G_M4", " D_m", " D_M4", " E_m7", " D#d6", " EbM", " Dbm", " G_m", " F_M", " C_m", " DbM7", " D#m", " F_m6", " DbM", " F#M7", " C#d7", " G_M6"]

torchmeta/datasets/assets/cifar100/cifar-fs/test.json

@@ -0,0 +1 @@
+[["aquatic_mammals","whale"],["flowers","poppy"],["flowers","rose"],["fruit_and_vegetables","sweet_pepper"],["household_electrical_devices","telephone"],["household_furniture","bed"],["household_furniture","table"],["household_furniture","wardrobe"],["large_carnivores","leopard"],["large_natural_outdoor_scenes","plain"],["large_omnivores_and_herbivores","chimpanzee"],["medium_mammals","fox"],["non-insect_invertebrates","snail"],["non-insect_invertebrates","worm"],["people","baby"],["people","man"],["people","woman"],["vehicles_1","bicycle"],["vehicles_1","pickup_truck"],["vehicles_2","rocket"]]

torchmeta/datasets/assets/cifar100/cifar-fs/train.json

@@ -0,0 +1 @@
+[["aquatic_mammals","dolphin"],["aquatic_mammals","seal"],["fish","aquarium_fish"],["fish","ray"],["fish","trout"],["flowers","orchid"],["flowers","sunflower"],["flowers","tulip"],["food_containers","bottle"],["food_containers","bowl"],["food_containers","can"],["food_containers","cup"],["food_containers","plate"],["fruit_and_vegetables","apple"],["fruit_and_vegetables","mushroom"],["fruit_and_vegetables","orange"],["fruit_and_vegetables","pear"],["household_electrical_devices","clock"],["household_electrical_devices","keyboard"],["household_furniture","chair"],["household_furniture","couch"],["insects","bee"],["insects","caterpillar"],["insects","cockroach"],["large_carnivores","bear"],["large_carnivores","lion"],["large_carnivores","tiger"],["large_carnivores","wolf"],["large_man-made_outdoor_things","bridge"],["large_man-made_outdoor_things","castle"],["large_man-made_outdoor_things","house"],["large_man-made_outdoor_things","road"],["large_man-made_outdoor_things","skyscraper"],["large_natural_outdoor_scenes","cloud"],["large_natural_outdoor_scenes","forest"],["large_natural_outdoor_scenes","mountain"],["large_omnivores_and_herbivores","elephant"],["large_omnivores_and_herbivores","kangaroo"],["medium_mammals","porcupine"],["medium_mammals","possum"],["medium_mammals","raccoon"],["medium_mammals","skunk"],["non-insect_invertebrates","lobster"],["non-insect_invertebrates","spider"],["people","boy"],["people","girl"],["reptiles","dinosaur"],["reptiles","lizard"],["reptiles","snake"],["reptiles","turtle"],["small_mammals","hamster"],["small_mammals","mouse"],["small_mammals","rabbit"],["small_mammals","shrew"],["small_mammals","squirrel"],["trees","oak_tree"],["trees","palm_tree"],["trees","pine_tree"],["trees","willow_tree"],["vehicles_1","bus"],["vehicles_1","train"],["vehicles_2","lawn_mower"],["vehicles_2","streetcar"],["vehicles_2","tank"]]

torchmeta/datasets/assets/cifar100/cifar-fs/val.json

@@ -0,0 +1 @@
+[["aquatic_mammals","beaver"],["aquatic_mammals","otter"],["fish","flatfish"],["fish","shark"],["household_electrical_devices","lamp"],["household_electrical_devices","television"],["insects","beetle"],["insects","butterfly"],["large_natural_outdoor_scenes","sea"],["large_omnivores_and_herbivores","camel"],["large_omnivores_and_herbivores","cattle"],["non-insect_invertebrates","crab"],["reptiles","crocodile"],["trees","maple_tree"],["vehicles_1","motorcycle"],["vehicles_2","tractor"]]

torchmeta/datasets/assets/cifar100/fc100/test.json

@@ -0,0 +1 @@
+["aquatic_mammals","insects","medium_mammals","people"]

torchmeta/datasets/assets/cifar100/fc100/train.json

@@ -0,0 +1 @@
+["fish","flowers","food_containers","fruit_and_vegetables","household_electrical_devices","household_furniture","large_man-made_outdoor_things","large_natural_outdoor_scenes","reptiles","trees","vehicles_1","vehicles_2"]

torchmeta/datasets/assets/cifar100/fc100/val.json

@@ -0,0 +1 @@
+["large_carnivores","large_omnivores_and_herbivores","non-insect_invertebrates","small_mammals"]

torchmeta/datasets/assets/cub/test.json

@@ -0,0 +1 @@
+["004.Groove_billed_Ani","008.Rhinoceros_Auklet","012.Yellow_headed_Blackbird","016.Painted_Bunting","020.Yellow_breasted_Chat","024.Red_faced_Cormorant","028.Brown_Creeper","032.Mangrove_Cuckoo","036.Northern_Flicker","040.Olive_sided_Flycatcher","044.Frigatebird","048.European_Goldfinch","052.Pied_billed_Grebe","056.Pine_Grosbeak","060.Glaucous_winged_Gull","064.Ring_billed_Gull","068.Ruby_throated_Hummingbird","072.Pomarine_Jaeger","076.Dark_eyed_Junco","080.Green_Kingfisher","084.Red_legged_Kittiwake","088.Western_Meadowlark","092.Nighthawk","096.Hooded_Oriole","100.Brown_Pelican","104.American_Pipit","108.White_necked_Raven","112.Great_Grey_Shrike","116.Chipping_Sparrow","120.Fox_Sparrow","124.Le_Conte_Sparrow","128.Seaside_Sparrow","132.White_crowned_Sparrow","136.Barn_Swallow","140.Summer_Tanager","144.Common_Tern","148.Green_tailed_Towhee","152.Blue_headed_Vireo","156.White_eyed_Vireo","160.Black_throated_Blue_Warbler","164.Cerulean_Warbler","168.Kentucky_Warbler","172.Nashville_Warbler","176.Prairie_Warbler","180.Wilson_Warbler","184.Louisiana_Waterthrush","188.Pileated_Woodpecker","192.Downy_Woodpecker","196.House_Wren","200.Common_Yellowthroat"]

torchmeta/datasets/assets/cub/train.json

@@ -0,0 +1 @@
+["001.Black_footed_Albatross","003.Sooty_Albatross","005.Crested_Auklet","007.Parakeet_Auklet","009.Brewer_Blackbird","011.Rusty_Blackbird","013.Bobolink","015.Lazuli_Bunting","017.Cardinal","019.Gray_Catbird","021.Eastern_Towhee","023.Brandt_Cormorant","025.Pelagic_Cormorant","027.Shiny_Cowbird","029.American_Crow","031.Black_billed_Cuckoo","033.Yellow_billed_Cuckoo","035.Purple_Finch","037.Acadian_Flycatcher","039.Least_Flycatcher","041.Scissor_tailed_Flycatcher","043.Yellow_bellied_Flycatcher","045.Northern_Fulmar","047.American_Goldfinch","049.Boat_tailed_Grackle","051.Horned_Grebe","053.Western_Grebe","055.Evening_Grosbeak","057.Rose_breasted_Grosbeak","059.California_Gull","061.Heermann_Gull","063.Ivory_Gull","065.Slaty_backed_Gull","067.Anna_Hummingbird","069.Rufous_Hummingbird","071.Long_tailed_Jaeger","073.Blue_Jay","075.Green_Jay","077.Tropical_Kingbird","079.Belted_Kingfisher","081.Pied_Kingfisher","083.White_breasted_Kingfisher","085.Horned_Lark","087.Mallard","089.Hooded_Merganser","091.Mockingbird","093.Clark_Nutcracker","095.Baltimore_Oriole","097.Orchard_Oriole","099.Ovenbird","101.White_Pelican","103.Sayornis","105.Whip_poor_Will","107.Common_Raven","109.American_Redstart","111.Loggerhead_Shrike","113.Baird_Sparrow","115.Brewer_Sparrow","117.Clay_colored_Sparrow","119.Field_Sparrow","121.Grasshopper_Sparrow","123.Henslow_Sparrow","125.Lincoln_Sparrow","127.Savannah_Sparrow","129.Song_Sparrow","131.Vesper_Sparrow","133.White_throated_Sparrow","135.Bank_Swallow","137.Cliff_Swallow","139.Scarlet_Tanager","141.Artic_Tern","143.Caspian_Tern","145.Elegant_Tern","147.Least_Tern","149.Brown_Thrasher","151.Black_capped_Vireo","153.Philadelphia_Vireo","155.Warbling_Vireo","157.Yellow_throated_Vireo","159.Black_and_white_Warbler","161.Blue_winged_Warbler","163.Cape_May_Warbler","165.Chestnut_sided_Warbler","167.Hooded_Warbler","169.Magnolia_Warbler","171.Myrtle_Warbler","173.Orange_crowned_Warbler","175.Pine_Warbler","177.Prothonotary_Warbler","179.Tennessee_Warbler","181.Worm_eating_Warbler","183.Northern_Waterthrush","185.Bohemian_Waxwing","187.American_Three_toed_Woodpecker","189.Red_bellied_Woodpecker","191.Red_headed_Woodpecker","193.Bewick_Wren","195.Carolina_Wren","197.Marsh_Wren","199.Winter_Wren"]

torchmeta/datasets/assets/cub/val.json

@@ -0,0 +1 @@
+["002.Laysan_Albatross","006.Least_Auklet","010.Red_winged_Blackbird","014.Indigo_Bunting","018.Spotted_Catbird","022.Chuck_will_Widow","026.Bronzed_Cowbird","030.Fish_Crow","034.Gray_crowned_Rosy_Finch","038.Great_Crested_Flycatcher","042.Vermilion_Flycatcher","046.Gadwall","050.Eared_Grebe","054.Blue_Grosbeak","058.Pigeon_Guillemot","062.Herring_Gull","066.Western_Gull","070.Green_Violetear","074.Florida_Jay","078.Gray_Kingbird","082.Ringed_Kingfisher","086.Pacific_Loon","090.Red_breasted_Merganser","094.White_breasted_Nuthatch","098.Scott_Oriole","102.Western_Wood_Pewee","106.Horned_Puffin","110.Geococcyx","114.Black_throated_Sparrow","118.House_Sparrow","122.Harris_Sparrow","126.Nelson_Sharp_tailed_Sparrow","130.Tree_Sparrow","134.Cape_Glossy_Starling","138.Tree_Swallow","142.Black_Tern","146.Forsters_Tern","150.Sage_Thrasher","154.Red_eyed_Vireo","158.Bay_breasted_Warbler","162.Canada_Warbler","166.Golden_winged_Warbler","170.Mourning_Warbler","174.Palm_Warbler","178.Swainson_Warbler","182.Yellow_Warbler","186.Cedar_Waxwing","190.Red_cockaded_Woodpecker","194.Cactus_Wren","198.Rock_Wren"]

torchmeta/datasets/assets/doublemnist/test.json

@@ -0,0 +1 @@
+["02", "17", "25", "32", "36", "46", "47", "49", "55", "57", "66", "67", "68", "73", "78", "80", "83", "86", "92", "96"]

torchmeta/datasets/assets/doublemnist/train.json

@@ -0,0 +1 @@
+["00", "01", "04", "05", "06", "08", "09", "11", "12", "13", "14", "15", "16", "18", "19", "20", "21", "23", "24", "26", "28", "29", "30", "31", "33", "35", "37", "38", "41", "42", "43", "44", "45", "50", "51", "53", "54", "56", "59", "60", "62", "63", "65", "69", "70", "72", "74", "75", "76", "77", "79", "81", "82", "84", "85", "87", "88", "89", "90", "91", "94", "95", "97", "98"]

torchmeta/datasets/assets/doublemnist/val.json

@@ -0,0 +1 @@
+["03", "07", "10", "22", "27", "34", "39", "40", "48", "52", "58", "61", "64", "71", "93", "99"]

torchmeta/datasets/assets/letter/test.json

@@ -0,0 +1 @@
+["U", "H", "K", "O", "T", "G"]

torchmeta/datasets/assets/letter/train.json

@@ -0,0 +1 @@
+["I", "Q", "A", "Y", "L", "J", "N", "B", "X", "F", "C", "M", "P", "D", "E"]

torchmeta/datasets/assets/letter/val.json

@@ -0,0 +1 @@
+["V", "R", "W", "S", "Z"]