WIP: Add support for ConfigMap Gone status

References:
* ManageIQ/kubeclient#452 (comment)
* fabric8io/kubernetes-client#1800 (comment)
* kubernetes/kubernetes#25151 (comment)
* kubernetes/kubernetes#35068 (comment)
* https://github.com/kubernetes/kubernetes/blob/dde6e8e7465468c32642659cb708a5cc922add64/test/e2e/apimachinery/protocol.go#L68-L75
* https://kubernetes.io/docs/reference/using-api/api-concepts/#410-gone-responses
* https://kubernetes.io/docs/reference/using-api/api-concepts/#efficient-detection-of-changes
* https://www.baeldung.com/java-kubernetes-watch#1-resource-versions

kubernetes-discovery-client/src/main/java/io/micronaut/kubernetes/client/v1/KubernetesObject.java

@@ -26,8 +26,23 @@
 @Introspected
 public abstract class KubernetesObject {
 
+    private String kind;
     private Metadata metadata;
 
+    /**
+     * @return The Kind
+     */
+    public String getKind() {
+        return kind;
+    }
+
+    /**
+     * @param kind The Kind
+     */
+    public void setKind(String kind) {
+        this.kind = kind;
+    }
+
     /**
      * @return The Metadata
      */

kubernetes-discovery-client/src/main/java/io/micronaut/kubernetes/client/v1/configmaps/ConfigMap.java

@@ -31,6 +31,10 @@
 public class ConfigMap extends KubernetesObject {
 
     private Map<String, String> data = new HashMap<>();
+    private String status;
+    private String message;
+    private String reason;
+    private int code;
 
     /**
      * @return A Map where the key is the file name, and the value is a string with all the properties
@@ -46,11 +50,73 @@ public void setData(Map<String, String> data) {
         this.data = data;
     }
 
+    /**
+     * @return The Status
+     */
+    public String getStatus() {
+        return status;
+    }
+
+    /**
+     * @param status The Status
+     */
+    public void setStatus(String status) {
+        this.status = status;
+    }
+
+    /**
+     * @return The Message
+     */
+    public String getMessage() {
+        return message;
+    }
+
+    /**
+     * @param message The Message
+     */
+    public void setMessage(String message) {
+      this.message = message;
+    }
+
+    /**
+     * @return The Reason
+     */
+    public String getReason() {
+      return reason;
+    }
+
+    /**
+     * @param reason The Reason
+     */
+
+    public void setReason(String reason) {
+      this.reason = reason;
+    }
+
+    /**
+     * @return The Code
+     */
+    public int getCode() {
+      return code;
+    }
+
+    /**
+     * @param code The Code
+     */
+    public void setCode(int code) {
+      this.code = code;
+    }
+
     @Override
     public String toString() {
         return "ConfigMap{" +
-                "metadata=" + getMetadata() +
+                "kind=" + getKind() +
+                ", metadata=" + getMetadata() +
                 ", data=" + data +
+                ", status='" + status + '\'' +
+                ", message='" + message + '\'' +
+                ", reason='" + reason + '\'' +
+                ", code=" + code +
                 '}';
     }
 }

kubernetes-discovery-client/src/main/java/io/micronaut/kubernetes/configuration/KubernetesConfigMapWatcher.java

@@ -36,9 +36,12 @@
 import java.util.Collection;
 import java.util.Map;
 import java.util.concurrent.ExecutorService;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import static io.micronaut.kubernetes.configuration.KubernetesConfigurationClient.KUBERNETES_CONFIG_MAP_NAME_SUFFIX;
 import static io.micronaut.kubernetes.util.KubernetesUtils.computePodLabelSelector;
+import static java.net.HttpURLConnection.HTTP_GONE;
 
 /**
  * Watches for ConfigMap changes and makes the appropriate changes to the {@link Environment} by adding or removing
@@ -54,6 +57,7 @@
 public class KubernetesConfigMapWatcher implements ApplicationEventListener<ServiceReadyEvent> {
 
     private static final Logger LOG = LoggerFactory.getLogger(KubernetesConfigMapWatcher.class);
+    private static final Pattern LAST_RESOURCE_VERSION_PATTERN = Pattern.compile("too old resource version: \\d+ \\((\\d+)\\)");
 
     private Environment environment;
     private final KubernetesClient client;
@@ -77,10 +81,13 @@ public KubernetesConfigMapWatcher(Environment environment, KubernetesClient clie
         this.executorService = executorService;
     }
 
-    @SuppressWarnings("ResultOfMethodCallIgnored")
     @Override
     public void onApplicationEvent(ServiceReadyEvent event) {
-        long lastResourceVersion = computeLastResourceVersion();
+        watch(computeLastResourceVersion());
+    }
+
+    @SuppressWarnings("ResultOfMethodCallIgnored")
+    private void watch(long lastResourceVersion) {
         Map<String, String> labels = configuration.getConfigMaps().getLabels();
         Flowable<String> singleLabelSelector = computePodLabelSelector(client, configuration.getConfigMaps().getPodLabels(), configuration.getNamespace(), labels);
 
@@ -94,9 +101,11 @@ public void onApplicationEvent(ServiceReadyEvent event) {
                         LOG.trace("Received ConfigMap watch event: {}", e);
                     }
                 })
-                .doOnError(throwable -> LOG.error("Error while watching ConfigMap events", throwable))
+                .onErrorReturn(throwable -> {
+                    LOG.error("Error while watching ConfigMap events", throwable);
+                    return new ConfigMapWatchEvent(ConfigMapWatchEvent.EventType.ERROR);
+                })
                 .subscribeOn(Schedulers.from(this.executorService))
-                .onErrorReturnItem(new ConfigMapWatchEvent(ConfigMapWatchEvent.EventType.ERROR))
                 .subscribe(this::processEvent);
     }
 
@@ -178,6 +187,29 @@ private void processConfigMapDeleted(ConfigMap configMap) {
 
     private void processConfigMapErrored(ConfigMapWatchEvent event) {
         LOG.error("Kubernetes API returned an error for a ConfigMap watch event: {}", event.toString());
+        final ConfigMap object = event.getObject();
+        if ("Status".equals(object.getKind()) && object.getCode() == HTTP_GONE) {
+            /*"kind":"Status",
+            "apiVersion":"v1",
+            "metadata":{},
+            "status":"Failure",
+            "message":"too old resource version: 1770726893 (1771132522)",
+            "reason":"Expired",
+            "code":410}}*/
+            this.environment.getPropertySources().forEach(src -> LOG.warn("WTF: src={}", src));
+            this.environment = environment.refresh();
+            final String message = object.getMessage();
+            if (message != null) {
+                final Matcher matcher = LAST_RESOURCE_VERSION_PATTERN.matcher(message);
+                if (matcher.find()) {
+                    long lastResourceVersion = Long.parseLong(matcher.group(1));
+                    if (LOG.isDebugEnabled()) {
+                        LOG.debug("Latest resourceVersion from {} is: {}", message, lastResourceVersion);
+                    }
+                    watch(lastResourceVersion);
+                }
+            }
+        }
     }
 
     private boolean passesIncludesExcludesLabelsFilters(ConfigMap configMap) {