Update to kryo 5.6.0

This eliminates a few more very old dependencies that aren't in Maven Central.

Our direct usage of the unsupported sun.reflect.ReflectionFactory JDK API (which newer compilers complain about) is no longer needed as Kryo now has a SerializingInstantiatorStrategy that does roughly the same thing.

commons/pom.xml

@@ -145,20 +145,7 @@
 		<dependency>
 			<groupId>com.esotericsoftware</groupId>
 			<artifactId>kryo</artifactId>
-			<version>1.01</version>
-			<scope>compile</scope>
-		</dependency>
-		<dependency>
-			<groupId>com.esotericsoftware</groupId>
-			<artifactId>reflectasm</artifactId>
-			<version>0.8</version>
-			<scope>runtime</scope>
-		</dependency>
-		<dependency>
-			<groupId>com.esotericsoftware</groupId>
-			<artifactId>minlog</artifactId>
-			<version>1.2</version>
-			<scope>runtime</scope>
+			<version>5.6.0</version>
 		</dependency>
 
 		<dependency>

commons/src/main/java/org/archive/bdb/AutoKryo.java

@@ -1,94 +1,52 @@
 package org.archive.bdb;
 
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
 import java.util.ArrayList;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.HashSet;
+import java.util.Set;
 
 import com.esotericsoftware.kryo.Kryo;
-import com.esotericsoftware.kryo.SerializationException;
-
-import sun.reflect.ReflectionFactory;
+import com.esotericsoftware.kryo.util.DefaultInstantiatorStrategy;
+import com.esotericsoftware.kryo.util.MapReferenceResolver;
+import org.objenesis.strategy.SerializingInstantiatorStrategy;
 
 /**
  * Extensions to Kryo to let classes control their own registration, suggest
- * other classes to register together, and use the same (Sun-JVM-only) trick
+ * other classes to register together, and use SerializingInstantiatorStrategy
  * for deserializing classes without no-arg constructors.
- * 
- * newInstance technique and constructor caching inspired by the 
- * KryoReflectionFactorySupport class of Martin Grotzke's kryo-serializers 
- * project. <a href="https://github.com/magro/kryo-serializers">https://github.com/magro/kryo-serializers</a>
- * 
- * TODO: more comments!
- * 
+ *
  * @author gojomo
  */
 @SuppressWarnings("unchecked")
 public class AutoKryo extends Kryo {
-    protected ArrayList<Class<?>> registeredClasses = new ArrayList<Class<?>>(); 
-
-    @Override
-    protected void handleUnregisteredClass(@SuppressWarnings("rawtypes") Class type) {
-        System.err.println("UNREGISTERED FOR KRYO "+type+" in "+registeredClasses.get(0));
-        super.handleUnregisteredClass(type);
+    protected ArrayList<Class<?>> registeredClasses = new ArrayList<>();
+    protected Set<Class<?>> referenceClasses = new HashSet<>();
+
+    public AutoKryo() {
+        super();
+        setInstantiatorStrategy(new DefaultInstantiatorStrategy(new SerializingInstantiatorStrategy()));
     }
 
     public void autoregister(Class<?> type) {
         if (registeredClasses.contains(type)) {
             return;
         }
-        registeredClasses.add(type); 
+        registeredClasses.add(type);
         try {
-            invokeStatic(
-                "autoregisterTo", 
-                type,
-                new Class[]{ ((Class<?>)AutoKryo.class), }, 
-                new Object[] { this, });
+            type.getMethod("autoregisterTo", new Class[]{AutoKryo.class}).invoke(null, this);
         } catch (Exception e) {
             register(type); 
         }
     }
 
-    protected static final ReflectionFactory REFLECTION_FACTORY = ReflectionFactory.getReflectionFactory();
-    protected static final Object[] INITARGS = new Object[0];
-    protected static final Map<Class<?>, Constructor<?>> CONSTRUCTOR_CACHE = new ConcurrentHashMap<Class<?>, Constructor<?>>();
-
-    @Override
-    public <T> T newInstance(Class<T> type) {
-        SerializationException ex = null; 
-        try {
-            return super.newInstance(type);
-        } catch (SerializationException se) {
-            ex = se;
+    public void useReferencesFor(Class<?> clazz) {
+        if (!getReferences()) {
+            setReferenceResolver(new MapReferenceResolver() {
+                @Override
+                public boolean useReferences(Class type) {
+                    return referenceClasses.contains(type);
+                }
+            });
         }
-        try {
-            Constructor<?> constructor = CONSTRUCTOR_CACHE.get(type);
-            if(constructor == null) {
-                constructor = REFLECTION_FACTORY.newConstructorForSerialization( 
-                        type, Object.class.getDeclaredConstructor( new Class[0] ) );
-                constructor.setAccessible( true );
-                CONSTRUCTOR_CACHE.put(type, constructor);
-            }
-            Object inst = constructor.newInstance( INITARGS );
-            return (T) inst;
-        } catch (SecurityException e) {
-            e.printStackTrace();
-        } catch (NoSuchMethodException e) {
-            e.printStackTrace();
-        } catch (IllegalArgumentException e) {
-             e.printStackTrace();
-        } catch (InstantiationException e) {
-            e.printStackTrace();
-        } catch (IllegalAccessException e) {
-            e.printStackTrace();
-        } catch (InvocationTargetException e) {
-            e.printStackTrace();
-        }
-        throw ex;
-    }
-
-    protected Object invokeStatic(String method, Class<?> clazz, Class<?>[] types, Object[] args) throws Exception {
-        return clazz.getMethod(method, types).invoke(null, args);
+        referenceClasses.add(clazz);
     }
 }

commons/src/main/java/org/archive/bdb/KryoBinding.java

@@ -1,8 +1,8 @@
 /*
  *  This file is part of the Heritrix web crawler (crawler.archive.org).
  *
- *  Licensed to the Internet Archive (IA) by one or more individual 
- *  contributors. 
+ *  Licensed to the Internet Archive (IA) by one or more individual
+ *  contributors.
  *
  *  The IA licenses this file to You under the Apache License, Version 2.0
  *  (the "License"); you may not use this file except in compliance with
@@ -18,69 +18,71 @@
  */
 package org.archive.bdb;
 
-import java.lang.ref.WeakReference;
-
 import com.esotericsoftware.kryo.Kryo;
-import com.esotericsoftware.kryo.ObjectBuffer;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+import com.esotericsoftware.kryo.util.Pool;
 import com.sleepycat.bind.EntryBinding;
 import com.sleepycat.je.DatabaseEntry;
 
 /**
  * Binding for use with BerkeleyDB-JE that uses Kryo serialization rather
  * than BDB's (custom version of) Java serialization.
- * 
+ *
  * @author gojomo
  */
 public class KryoBinding<K> implements EntryBinding<K> {
+    private static final int POOL_SIZE = 8;
 
-    protected Class<K> baseClass;
-    protected AutoKryo kryo = new AutoKryo(); 
-    protected ThreadLocal<WeakReference<ObjectBuffer>> threadBuffer = new ThreadLocal<WeakReference<ObjectBuffer>>() {
-        @Override
-        protected WeakReference<ObjectBuffer> initialValue() {
-            return new WeakReference<ObjectBuffer>(new ObjectBuffer(kryo,16*1024,Integer.MAX_VALUE));
+    protected final Class<K> baseClass;
+
+    Pool<AutoKryo> kryoPool = new Pool<AutoKryo>(true, false, POOL_SIZE) {
+        protected AutoKryo create () {
+            AutoKryo kryo = new AutoKryo();
+            kryo.autoregister(baseClass);
+            kryo.setRegistrationRequired(false);
+            kryo.setWarnUnregisteredClasses(true);
+            return kryo;
         }
     };
-
-    /**
-     * Constructor. Save parameters locally, as superclass 
-     * fields are private. 
-     * 
-     * @param baseClass is the base class for serialized objects stored using
-     * this binding
-     */
+
+    Pool<Output> outputPool = new Pool<Output>(true, false, POOL_SIZE) {
+        protected Output create () {
+            return new Output(16 * 1024, -1);
+        }
+    };
+
     public KryoBinding(Class<K> baseClass) {
         this.baseClass = baseClass;
-        kryo.autoregister(baseClass);
-        // TODO: reevaluate if explicit registration should be required
-        kryo.setRegistrationOptional(true);
     }
 
-    public Kryo getKryo() {
-        return kryo;
-    }
-
-    private ObjectBuffer getBuffer() {
-        WeakReference<ObjectBuffer> ref = threadBuffer.get();
-        ObjectBuffer ob = ref.get();
-        if (ob == null) {
-            ob = new ObjectBuffer(kryo,16*1024,Integer.MAX_VALUE);
-            threadBuffer.set(new WeakReference<ObjectBuffer>(ob));
-        }
-        return ob;        
-    }
-
     /**
      * Copies superclass simply to allow different source for FastOoutputStream.
-     * 
+     *
      * @see com.sleepycat.bind.serial.SerialBinding#entryToObject
      */
     public void objectToEntry(K object, DatabaseEntry entry) {
-        entry.setData(getBuffer().writeObjectData(object));
+        AutoKryo kryo = kryoPool.obtain();
+        try {
+            Output output = outputPool.obtain();
+            try {
+                kryo.writeObject(output, object);
+                entry.setData(output.toBytes());
+            } finally {
+                outputPool.free(output);
+            }
+        } finally {
+            kryoPool.free(kryo);
+        }
     }
 
     @Override
     public K entryToObject(DatabaseEntry entry) {
-        return getBuffer().readObjectData(entry.getData(), baseClass);
+        AutoKryo kryo = kryoPool.obtain();
+        try {
+            return kryo.readObject(new Input(entry.getData()), baseClass);
+        } finally {
+            kryoPool.free(kryo);
+        }
     }
 }

commons/src/main/java/org/archive/net/UURI.java

@@ -21,21 +21,41 @@
 import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
-import java.nio.ByteBuffer;
 
+import com.esotericsoftware.kryo.DefaultSerializer;
+import com.esotericsoftware.kryo.Serializer;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
 import org.apache.commons.httpclient.URIException;
 import org.archive.url.UsableURI;
 
-import com.esotericsoftware.kryo.CustomSerialization;
 import com.esotericsoftware.kryo.Kryo;
-import com.esotericsoftware.kryo.serialize.StringSerializer;
 
 /**
  * Usable URI. The bulk of the functionality of this class has moved to
  * {@link UsableURI} in the archive-commons project. This class adds Kryo
  * serialization.
  */
-public class UURI extends UsableURI implements CustomSerialization {
+@DefaultSerializer(UURI.KryoSerializer.class)
+public class UURI extends UsableURI {
+
+    public static class KryoSerializer extends Serializer<UURI> {
+        @Override
+        public void write(Kryo kryo, Output output, UURI uuri) {
+            output.writeString(uuri.toCustomString());
+        }
+
+        @Override
+        public UURI read(Kryo kryo, Input input, Class<? extends UURI> aClass) {
+            UURI uuri = new UURI();
+            try {
+                uuri.parseUriReference(input.readString(), true);
+            } catch (URIException e) {
+                throw new RuntimeException("Error deserializing UURI", e);
+            }
+            return uuri;
+        }
+    }
 
     private static final long serialVersionUID = -8946640480772772310L;
 
@@ -52,21 +72,6 @@ protected UURI() {
         super();
     }
 
-    @Override
-    public void writeObjectData(Kryo kryo, ByteBuffer buffer) {
-        StringSerializer.put(buffer, toCustomString());
-    }
-
-    @Override
-    public void readObjectData(Kryo kryo, ByteBuffer buffer) {
-        try {
-            parseUriReference(StringSerializer.get(buffer), true);
-        } catch (URIException e) {
-            // TODO Auto-generated catch block
-            e.printStackTrace();
-        }
-    }
-
     private void writeObject(ObjectOutputStream stream) throws IOException {
         stream.writeUTF(toCustomString());
     }

commons/src/main/java/org/archive/util/Histotable.java

@@ -41,7 +41,14 @@
  */
 public class Histotable<K> extends TreeMap<K,Long> {    
     private static final long serialVersionUID = 310306238032568623L;
-
+
+    public Histotable() {
+    }
+
+    protected Histotable(Comparator<K> comparator) {
+        super(comparator);
+    }
+
     /**
      * Record one more occurrence of the given object key.
      * 

commons/src/test/java/org/archive/util/IdentityCacheableWrapper.java

@@ -69,6 +69,5 @@ public void setIdentityCache(ObjectIdentityCache<?> cache) {
     //
     public static void autoregisterTo(AutoKryo kryo) {
         kryo.register(IdentityCacheableWrapper.class);
-        kryo.setRegistrationOptional(true); 
     }
 }

engine/src/main/java/org/archive/crawler/frontier/BdbWorkQueue.java

@@ -178,6 +178,5 @@ public static void autoregisterTo(AutoKryo kryo) {
         kryo.autoregister(HashSet.class);
         kryo.autoregister(SimplePrecedenceProvider.class);
         kryo.autoregister(byte[].class);
-        kryo.setRegistrationOptional(true); 
     }
 }

modules/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java

@@ -19,6 +19,7 @@
 
 package org.archive.crawler.util;
 
+import java.util.Comparator;
 import java.util.Map;
 
 import org.archive.io.warc.WARCWriter;
@@ -51,6 +52,10 @@ public CrawledBytesHistotable() {
         super();
     }
 
+    protected CrawledBytesHistotable(Comparator<String> comparator) {
+        super(comparator);
+    }
+
     @SuppressWarnings("unchecked")
     public void accumulate(CrawlURI curi) {
         if (curi.getRevisitProfile() instanceof ServerNotModifiedRevisit) {

modules/src/main/java/org/archive/modules/CrawlURI.java

@@ -1705,7 +1705,6 @@ public static void autoregisterTo(AutoKryo kryo) {
         kryo.autoregister(org.apache.commons.httpclient.NameValuePair.class);
         kryo.autoregister(org.apache.commons.httpclient.NameValuePair[].class);
         kryo.autoregister(FetchType.class);
-        kryo.setRegistrationOptional(true);
     }
 
     /**