Step 4 fit the model problem

[Philip642]

Hey, here is another problem

[jgraving]

Hi, Again can you provide more details? What system are you running this on? What dataset are you attempting to train the model with? Did the rest of the notebook run ok?

[Philip642]

Hi, Ubuntu 18.04 and your code with your data, changed just the directory
No, there are more errors codes, but I want to solve these first, maybe the others(step 5a and 5b) will then disappear

[Philip642]

Hi, this still doesn't work (On Ubuntu 18.04, with your data and on jupyter notebook):
train_generator = TrainingGenerator(datapath=annotations[2],
downsample_factor=2,
augmenter=augmenter,
sigma=5,
validation_split=0.1,
use_graph=True,
random_seed=1,
graph_scale=0.1)
train_generator.get_config()

{'shuffle': True,
'downsample_factor': 2,
'sigma': 5,
'use_graph': True,
'graph_scale': 0.1,
'validation_split': 0.1,
'datapath': '/home/einat/deepposekit-master/deepposekit-data/datasets/fly/example_annotation_set.h5',
'dataset': 'images',
'output_shape': (48, 48),
'n_validation': 0,
'random_seed': 1,
'n_output_channels': 66,
'augmenter': True,
'n_keypoints': 32}

Then I changed log_densenet.h5 to example_annotation_set.h5 for the logger:
logger = Logger(HOME + '/deepposekit-data/datasets/fly/example_annotation_set.h5',
validation_batch_size=10)
while the rest remained like you had it.
With the settings:
model.fit(batch_size=16,
validation_batch_size=10,
callbacks=callbacks,
epochs=100,
n_workers=8,
steps_per_epoch=None)
I get the following error messages:
Epoch 1/100

---

AttributeError Traceback (most recent call last)
in
4 epochs=100,
5 n_workers=8,
----> 6 steps_per_epoch=None)

~/anaconda3/envs/whipspider/lib/python3.7/site-packages/deepposekit/models/engine.py in fit(self, batch_size, validation_batch_size, callbacks, epochs, use_multiprocessing, n_workers, steps_per_epoch, **kwargs)
143 validation_data=validation_generator,
144 validation_steps=len(validation_generator),
--> 145 **kwargs)
146
147 def evaluate(self, batch_size):

~/anaconda3/envs/whipspider/lib/python3.7/site-packages/keras/legacy/interfaces.py in wrapper(*args, **kwargs)
89 warnings.warn('Update your ' + object_name + ' call to the ' +
90 'Keras 2 API: ' + signature, stacklevel=2)
---> 91 return func(*args, **kwargs)
92 wrapper._original_function = func
93 return wrapper

~/anaconda3/envs/whipspider/lib/python3.7/site-packages/keras/engine/training.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
1416 use_multiprocessing=use_multiprocessing,
1417 shuffle=shuffle,
-> 1418 initial_epoch=initial_epoch)
1419
1420 @interfaces.legacy_generator_methods_support

~/anaconda3/envs/whipspider/lib/python3.7/site-packages/keras/engine/training_generator.py in fit_generator(model, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
249 break
250
--> 251 callbacks.on_epoch_end(epoch, epoch_logs)
252 epoch += 1
253 if callback_model.stop_training:

~/anaconda3/envs/whipspider/lib/python3.7/site-packages/keras/callbacks.py in on_epoch_end(self, epoch, logs)
77 logs = logs or {}
78 for callback in self.callbacks:
---> 79 callback.on_epoch_end(epoch, logs)
80
81 def on_batch_begin(self, batch, logs=None):

~/anaconda3/envs/whipspider/lib/python3.7/site-packages/keras/callbacks.py in on_epoch_end(self, epoch, logs)
336 self.log_values.append((k, logs[k]))
337 if self.verbose:
--> 338 self.progbar.update(self.seen, self.log_values)
339
340

AttributeError: 'ProgbarLogger' object has no attribute 'log_values'

[jgraving]

logger = Logger(HOME + '/deepposekit-data/datasets/fly/example_annotation_set.h5', validation_batch_size=10)
Should stay as log_densenet.h5 not switch to example_annotation_set.h5. Otherwise it will erase your annotation set example_annotation_set.h5. filepath tells the Logger where to save the training log data, so you need to set this as log_densenet.h5 or some file name that is not your training set.

[Philip642]

Teh logger worked now but I still have this Error:
AttributeError: 'ProgbarLogger' object has no attribute 'log_values'

[jgraving]

Does it still give the error if you remove the callbacks from the model training? Set callbacks=None when calling model.fit

[Philip642]

Then i get this error:
TypeError Traceback (most recent call last)
in
4 epochs=100,
5 n_workers=8,
----> 6 steps_per_epoch=None)

~/anaconda3/envs/whipspider/lib/python3.7/site-packages/deepposekit/models/engine.py in fit(self, batch_size, validation_batch_size, callbacks, epochs, use_multiprocessing, n_workers, steps_per_epoch, **kwargs)
126 confidence=True)
127 activated_callbacks = []
--> 128 if len(callbacks) > 0:
129 for callback in callbacks:
130 if hasattr(callback, 'pass_model'):

TypeError: object of type 'NoneType' has no len()

[jgraving]

Sorry, there's a bug when callbacks=None. Pass an empty list instead callbacks=[].

[Philip642]

AttributeError Traceback (most recent call last)
in
4 epochs=100,
5 n_workers=8,
----> 6 steps_per_epoch=None)

~/anaconda3/envs/whipspider/lib/python3.7/site-packages/deepposekit/models/engine.py in fit(self, batch_size, validation_batch_size, callbacks, epochs, use_multiprocessing, n_workers, steps_per_epoch, **kwargs)
143 validation_data=validation_generator,
144 validation_steps=len(validation_generator),
--> 145 **kwargs)
146
147 def evaluate(self, batch_size):

~/anaconda3/envs/whipspider/lib/python3.7/site-packages/keras/legacy/interfaces.py in wrapper(*args, **kwargs)
89 warnings.warn('Update your ' + object_name + ' call to the ' +
90 'Keras 2 API: ' + signature, stacklevel=2)
---> 91 return func(*args, **kwargs)
92 wrapper._original_function = func
93 return wrapper

~/anaconda3/envs/whipspider/lib/python3.7/site-packages/keras/engine/training.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
1416 use_multiprocessing=use_multiprocessing,
1417 shuffle=shuffle,
-> 1418 initial_epoch=initial_epoch)
1419
1420 @interfaces.legacy_generator_methods_support

~/anaconda3/envs/whipspider/lib/python3.7/site-packages/keras/engine/training_generator.py in fit_generator(model, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
249 break
250
--> 251 callbacks.on_epoch_end(epoch, epoch_logs)
252 epoch += 1
253 if callback_model.stop_training:

~/anaconda3/envs/whipspider/lib/python3.7/site-packages/keras/callbacks.py in on_epoch_end(self, epoch, logs)
77 logs = logs or {}
78 for callback in self.callbacks:
---> 79 callback.on_epoch_end(epoch, logs)
80
81 def on_batch_begin(self, batch, logs=None):

~/anaconda3/envs/whipspider/lib/python3.7/site-packages/keras/callbacks.py in on_epoch_end(self, epoch, logs)
336 self.log_values.append((k, logs[k]))
337 if self.verbose:
--> 338 self.progbar.update(self.seen, self.log_values)
339
340

AttributeError: 'ProgbarLogger' object has no attribute 'log_values'

[jgraving]

It appears you don't have a validation set. You should increase validation_split so you have at least 1 validation example. This will be fixed in a forthcoming update.

[Philip642]

When i set validation_split here 1 or above:
train_generator = TrainingGenerator(datapath=annotations[2],
downsample_factor=2,
augmenter=augmenter,
sigma=5,
validation_split=1,
use_graph=True,
random_seed=1,
graph_scale=0.1)
train_generator.get_config()

I get the following error:

---

ValueError Traceback (most recent call last)
in
6 use_graph=True,
7 random_seed=1,
----> 8 graph_scale=0.1)
9 train_generator.get_config()

~/anaconda3/envs/whipspider/lib/python3.7/site-packages/deepposekit/io/TrainingGenerator.py in init(self, datapath, dataset, downsample_factor, use_graph, augmenter, shuffle, sigma, validation_split, graph_scale, random_seed)
113 self.confidence = True
114 self._init_augmenter(augmenter)
--> 115 self._init_data(datapath, dataset)
116 self.on_epoch_end()
117

~/anaconda3/envs/whipspider/lib/python3.7/site-packages/deepposekit/io/TrainingGenerator.py in _init_data(self, datapath, dataset)
173 self.n_branches = np.unique(graph_to_edges(self.graph)).shape[0]
174 self.on_epoch_end()
--> 175 X, y = self.getitem(0)
176 self.n_edges = y[..., self.n_keypoints + self.n_branches:-2].shape[-1]
177 self.n_output_channels = y.shape[-1]

~/anaconda3/envs/whipspider/lib/python3.7/site-packages/deepposekit/io/TrainingGenerator.py in getitem(self, index)
228
229 # Generate data
--> 230 X, y = self.generate_batch(indexes)
231
232 return X, y

~/anaconda3/envs/whipspider/lib/python3.7/site-packages/deepposekit/io/TrainingGenerator.py in generate_batch(self, indexes)
265 def generate_batch(self, indexes):
266 """Generates data containing batch_size samples"""
--> 267 X, y = self.load_batch(indexes)
268 if self.augmenter and not self.validation:
269 X, y = self.augment(X, y)

~/anaconda3/envs/whipspider/lib/python3.7/site-packages/deepposekit/io/TrainingGenerator.py in load_batch(self, indexes)
245 else:
246 batch_index = self.train_index[indexes]
--> 247 return self.generator[batch_index]
248
249 def augment(self, images, keypoints):

~/anaconda3/envs/whipspider/lib/python3.7/site-packages/deepposekit/io/DataGenerator.py in getitem(self, key)
184
185 def getitem(self, key):
--> 186 idx = self._check_index(key)
187 return self.get_data(idx)
188

~/anaconda3/envs/whipspider/lib/python3.7/site-packages/deepposekit/io/DataGenerator.py in _check_index(self, key)
170 raise IndexError
171 elif isinstance(key, np.ndarray):
--> 172 if np.max(key) < len(self):
173 idx = key.tolist()
174 else:

~/anaconda3/envs/whipspider/lib/python3.7/site-packages/numpy/core/fromnumeric.py in amax(a, axis, out, keepdims, initial)
2503 """
2504 return _wrapreduction(a, np.maximum, 'max', axis, None, out, keepdims=keepdims,
-> 2505 initial=initial)
2506
2507

~/anaconda3/envs/whipspider/lib/python3.7/site-packages/numpy/core/fromnumeric.py in _wrapreduction(obj, ufunc, method, axis, dtype, out, **kwargs)
84 return reduction(axis=axis, out=out, **passkwargs)
85
---> 86 return ufunc.reduce(obj, axis, dtype, out, **passkwargs)
87
88

ValueError: zero-size array to reduction operation maximum which has no identity

[jgraving]

This is the opposite problem. validation_split is the proportion of the data to use for validation during training. If you set it to validation_split=1 then there will be no training data for training to model. Try validation_split=0.5. If you still get 'n_validation': 0 for the generator config then there is something wrong with your dataset, which might have been erased when you set the Logger to overwrite the example_annotation_set.h5. I'd recommend re-downloading the example or trying one of the other datasets.
I'll add some sanity checks for the 0.2 update to avoid this in the future.

[jgraving]

This should now be fixed the latest version 0.1.1, but please reopen this issue if you're still having problems.

[Philip642]

Hi, unfortunately it's still not working. I downloaded everything again, compared it with your code here and tried it with the locust and fly data. The error occurs at the model.fit part

Epoch 1/1000

---

AttributeError Traceback (most recent call last)
in
4 epochs=1000,
5 n_workers=8,
----> 6 steps_per_epoch=None)

~/anaconda3/envs/whipspider/lib/python3.7/site-packages/deepposekit/models/engine.py in fit(self, batch_size, validation_batch_size, callbacks, epochs, use_multiprocessing, n_workers, steps_per_epoch, **kwargs)
156 callbacks=activated_callbacks,
157 validation_data=validation_generator,
--> 158 **kwargs
159 )
160

~/anaconda3/envs/whipspider/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
1431 shuffle=shuffle,
1432 initial_epoch=initial_epoch,
-> 1433 steps_name='steps_per_epoch')
1434
1435 def evaluate_generator(self,

~/anaconda3/envs/whipspider/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_generator.py in model_iteration(model, data, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch, mode, batch_size, steps_name, **kwargs)
258 # Callbacks batch begin.
259 batch_logs = {'batch': step, 'size': batch_size}
--> 260 callbacks._call_batch_hook(mode, 'begin', step, batch_logs)
261 progbar.on_batch_begin(step, batch_logs)
262

~/anaconda3/envs/whipspider/lib/python3.7/site-packages/tensorflow/python/keras/callbacks.py in _call_batch_hook(self, mode, hook, batch, logs)
245 t_before_callbacks = time.time()
246 for callback in self.callbacks:
--> 247 batch_hook = getattr(callback, hook_name)
248 batch_hook(batch, logs)
249 self._delta_ts[hook_name].append(time.time() - t_before_callbacks)

AttributeError: 'EarlyStopping' object has no attribute 'on_train_batch_begin'

For both fly and locust data I get this error

[jgraving]

We recently updated to tensorflow.keras because stand-alone keras will be deprecated. Make sure you're using the updated notebooks where EarlyStopping is imported from tensorflow.keras.callbacks. If that doesn't fix it then try removing the EarlyStopping from the list of callbacks. ModelCheckpoint will automatically save the best model, and if you run model.fit for long enough you'll end up with a model that performs well. The v0.3.0 update will be pushed soon with further updates and better documentation.