Consume Input stream instead of file in split and to-csv

src/main/java/HdrToCsv.java

@@ -1,11 +1,8 @@
-import org.HdrHistogram.Histogram;
 import org.kohsuke.args4j.Option;
-import psy.lob.saw.OrderedHistogramLogReader;
+import psy.lob.saw.CsvConverter;
 
-import java.io.File;
-import java.io.FileNotFoundException;
+import java.io.*;
 import java.nio.file.Paths;
-import java.util.Locale;
 
 public class HdrToCsv implements Runnable
 {
@@ -34,34 +31,13 @@ public void setInputFile(String fileName)
     @Override
     public void run()
     {
-        OrderedHistogramLogReader reader = null;
-        try
+        try(InputStream inputStream = new FileInputStream(inputFile))
         {
-            reader = new OrderedHistogramLogReader(inputFile);
+            new CsvConverter(inputStream, System.out).convert();
         }
-        catch (FileNotFoundException e)
+        catch (IOException e)
         {
             throw new RuntimeException(e);
         }
-
-        System.out.println(
-            "#Absolute timestamp,Relative timestamp,Throughput,Min,Avg,p50,p90,p95,p99,p999,p9999,Max");
-        while (reader.hasNext())
-        {
-            Histogram interval = (Histogram) reader.nextIntervalHistogram();
-            System.out.printf(Locale.US,
-                "%.3f,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d%n",
-                interval.getStartTimeStamp() / 1000.0,
-                interval.getStartTimeStamp() / 1000 - (long) reader.getStartTimeSec(),
-                interval.getTotalCount(), interval.getMinValue(),
-                (long) interval.getMean(),
-                interval.getValueAtPercentile(50),
-                interval.getValueAtPercentile(90),
-                interval.getValueAtPercentile(95),
-                interval.getValueAtPercentile(99),
-                interval.getValueAtPercentile(99.9),
-                interval.getValueAtPercentile(99.99),
-                interval.getMaxValue());
-        }
     }
 }

src/main/java/SplitHistogramLogs.java

@@ -1,14 +1,11 @@
-import org.HdrHistogram.Histogram;
-import org.HdrHistogram.HistogramLogWriter;
 import org.kohsuke.args4j.Option;
-import psy.lob.saw.HdrHistogramUtil;
-import psy.lob.saw.OrderedHistogramLogReader;
+import psy.lob.saw.HistogramsSplitter;
 
 import java.io.File;
-import java.io.FileNotFoundException;
-import java.util.HashMap;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.nio.file.Path;
 import java.util.HashSet;
-import java.util.Map;
 import java.util.Set;
 
 import static psy.lob.saw.HdrHistogramUtil.logHistogramForVerbose;
@@ -27,6 +24,7 @@ public class SplitHistogramLogs implements Runnable
     private File inputFile;
     private Set<String> excludeTags = new HashSet<>();
     private Set<String> includeTags = new HashSet<>();
+    private String fileName;
 
     public static void main(String[] args) throws Exception
     {
@@ -91,61 +89,21 @@ public void run()
                 System.out.printf("start:%.2f end: MAX path:%s file:%s \n", start, absolutePath, name);
             }
         }
-        try
+        try(InputStream inputStream = new FileInputStream(inputFile))
         {
-            split();
+            Path outputDir = inputFile.toPath().getParent();
+            fileName = inputFile.getName();
+            new HistogramsSplitter(fileName, inputStream, start, end, verbose, this::shouldSkipTag, outputDir).split();
         }
         catch (Exception e)
         {
             throw new RuntimeException(e);
         }
     }
 
-    private void split() throws FileNotFoundException
-    {
-        OrderedHistogramLogReader reader = new OrderedHistogramLogReader(
-            inputFile,
-            start,
-            end,
-            tag -> shouldSkipTag(tag));
-        Map<String, HistogramLogWriter> writerByTag = new HashMap<>();
-        Histogram interval;
-        int i = 0;
-        while (reader.hasNext())
-        {
-            interval = (Histogram) reader.nextIntervalHistogram();
-            if (interval == null)
-            {
-                continue;
-            }
-            String ntag = interval.getTag();
-            if (shouldSkipTag(ntag))
-            {
-                throw new IllegalStateException("Should be filtered upfront by the reader");
-            }
-            if (verbose)
-            {
-                logHistogramForVerbose(System.out, interval, i++);
-            }
-            interval.setTag(null);
-            HistogramLogWriter writer = writerByTag.computeIfAbsent(ntag, k -> createWriterForTag(reader, k));
-            writer.outputIntervalHistogram(interval);
-
-        }
-    }
-
     private boolean shouldSkipTag(String ntag)
     {
         ntag = (ntag == null) ? "default" : ntag;
         return excludeTags.contains(ntag) || (!includeTags.isEmpty() && !includeTags.contains(ntag));
     }
-
-    private HistogramLogWriter createWriterForTag(OrderedHistogramLogReader reader, String tag)
-    {
-        tag = (tag == null) ? "default" : tag;
-        File outputFile = new File(tag + "." + inputFile.getName());
-        String comment = "Splitting of:" + inputFile.getName() + " start:" + start + " end:" + end;
-        HistogramLogWriter writer = HdrHistogramUtil.createLogWriter(outputFile, comment, reader.getStartTimeSec());
-        return writer;
-    }
-}
+}

src/main/java/SummarizeHistogramLogs.java

@@ -3,10 +3,7 @@
 import org.kohsuke.args4j.Option;
 import psy.lob.saw.OrderedHistogramLogReader;
 
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.PrintStream;
+import java.io.*;
 import java.util.*;
 import java.util.function.Predicate;
 import java.util.regex.Pattern;
@@ -134,69 +131,77 @@ private void summarizeAndPrint() throws FileNotFoundException
         long intervalLengthSum = 0;
         for (File inputFile : inputFiles)
         {
-            if (verbose)
+            try(InputStream inputStream = new FileInputStream(inputFile))
             {
-                System.out.println("Summarizing file: " + inputFile.getName());
-            }
-            OrderedHistogramLogReader reader = new OrderedHistogramLogReader(
-                inputFile,
-                start,
-                end,
-                tag -> shouldSkipTag(tag));
-            Histogram interval;
-            int i = 0;
-            boolean first = true;
-            long startTime = 0;
-
-            while (reader.hasNext())
-            {
-                interval = (Histogram) reader.nextIntervalHistogram();
-                if (interval == null)
+                if (verbose)
                 {
-                    continue;
+                    System.out.println("Summarizing file: " + inputFile.getName());
                 }
-                if (first)
+                OrderedHistogramLogReader reader = new OrderedHistogramLogReader(
+                        inputStream,
+                        start,
+                        end,
+                        tag -> shouldSkipTag(tag));
+                Histogram interval;
+                int i = 0;
+                boolean first = true;
+                long startTime = 0;
+
+                while (reader.hasNext())
                 {
-                    first = false;
-                    startTime = interval.getStartTimeStamp();
-                    if (verbose)
+                    interval = (Histogram) reader.nextIntervalHistogram();
+                    if (interval == null)
                     {
-                        System.out.println("StartTime: " + new Date(startTime));
+                        continue;
                     }
+                    if (first)
+                    {
+                        first = false;
+                        startTime = interval.getStartTimeStamp();
+                        if (verbose)
+                        {
+                            System.out.println("StartTime: " + new Date(startTime));
+                        }
 
+                    }
+                    String ntag = ignoreTag ? null : interval.getTag();
+                    final int numberOfSignificantValueDigits = interval.getNumberOfSignificantValueDigits();
+                    Histogram sum = sumByTag.computeIfAbsent(ntag, k ->
+                    {
+                        Histogram h = new Histogram(numberOfSignificantValueDigits);
+                        h.setTag(k);
+                        return h;
+                    });
+                    final long intervalLength = interval.getEndTimeStamp() - interval.getStartTimeStamp();
+                    intervalLengthSum += intervalLength;
+                    sum.add(interval);
+                    if (verbose)
+                    {
+                        logHistogramForVerbose(System.out, interval, i++, outputValueUnitRatio);
+                    }
                 }
-                String ntag = ignoreTag ? null : interval.getTag();
-                final int numberOfSignificantValueDigits = interval.getNumberOfSignificantValueDigits();
-                Histogram sum = sumByTag.computeIfAbsent(ntag, k ->
-                {
-                    Histogram h = new Histogram(numberOfSignificantValueDigits);
-                    h.setTag(k);
-                    return h;
-                });
-                final long intervalLength = interval.getEndTimeStamp() - interval.getStartTimeStamp();
-                intervalLengthSum += intervalLength;
-                sum.add(interval);
-                if (verbose)
+                // calculate period
+                long maxPeriod = 0;
+                for (Histogram sum : sumByTag.values())
                 {
-                    logHistogramForVerbose(System.out, interval, i++, outputValueUnitRatio);
+                    long sumPeriod = (sum.getEndTimeStamp() - sum.getStartTimeStamp());
+                    if (verbose)
+                    {
+                        System.out.print(inputFile.getName());
+                        System.out.print(", ");
+                        logHistogramForVerbose(System.out, sum, i++, outputValueUnitRatio);
+                    }
+                    sum.setEndTimeStamp(0);
+                    sum.setStartTimeStamp(Long.MAX_VALUE);
+                    maxPeriod = Math.max(maxPeriod, sumPeriod);
                 }
+                period += maxPeriod;
             }
-            // calculate period
-            long maxPeriod = 0;
-            for (Histogram sum : sumByTag.values())
+            catch (IOException e)
             {
-                long sumPeriod = (sum.getEndTimeStamp() - sum.getStartTimeStamp());
-                if (verbose)
-                {
-                    System.out.print(inputFile.getName());
-                    System.out.print(", ");
-                    logHistogramForVerbose(System.out, sum, i++, outputValueUnitRatio);
-                }
-                sum.setEndTimeStamp(0);
-                sum.setStartTimeStamp(Long.MAX_VALUE);
-                maxPeriod = Math.max(maxPeriod, sumPeriod);
+                throw new RuntimeException(e);
             }
-            period += maxPeriod;
+
         }
         if (ignoreTimeStamps)
         {

src/main/java/UnionHistogramLogs.java

@@ -6,9 +6,7 @@
 import psy.lob.saw.OrderedHistogramLogReader;
 import psy.lob.saw.UnionHistograms;
 
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.PrintStream;
+import java.io.*;
 import java.util.*;
 import java.util.function.Predicate;
 import java.util.regex.Pattern;
@@ -152,10 +150,13 @@ public void run()
             List<HistogramIterator> ins = new ArrayList<>();
             for (File inputFile : inputFiles)
             {
-                ins.add(new HistogramIterator(
-                    new OrderedHistogramLogReader(inputFile, start, end),
-                    inputFilesTags.get(inputFile),
-                    relative));
+                try(InputStream inputStream = new FileInputStream(inputFile))
+                {
+                    ins.add(new HistogramIterator(
+                            new OrderedHistogramLogReader(inputStream, start, end),
+                            inputFilesTags.get(inputFile),
+                            relative));
+                }
             }
             UnionHistograms unionHistograms = new UnionHistograms(verbose, System.out, ins, new HistogramSink()
             {

src/main/java/psy/lob/saw/CsvConverter.java

@@ -0,0 +1,44 @@
+package psy.lob.saw;
+
+import org.HdrHistogram.Histogram;
+
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+import java.io.PrintStream;
+import java.util.Locale;
+
+public class CsvConverter
+{
+    private final InputStream inputStream;
+    private final PrintStream outputStream;
+
+    public CsvConverter(InputStream inputStream, PrintStream outputStream)
+    {
+        this.inputStream = inputStream;
+        this.outputStream = outputStream;
+    }
+
+    public void convert() throws FileNotFoundException
+    {
+        OrderedHistogramLogReader reader = new OrderedHistogramLogReader(inputStream);
+        outputStream.println(
+                "#Absolute timestamp,Relative timestamp,Throughput,Min,Avg,p50,p90,p95,p99,p999,p9999,Max");
+        while (reader.hasNext())
+        {
+            Histogram interval = (Histogram) reader.nextIntervalHistogram();
+            outputStream.printf(Locale.US,
+                    "%.3f,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d%n",
+                    interval.getStartTimeStamp() / 1000.0,
+                    interval.getStartTimeStamp() / 1000 - (long) reader.getStartTimeSec(),
+                    interval.getTotalCount(), interval.getMinValue(),
+                    (long) interval.getMean(),
+                    interval.getValueAtPercentile(50),
+                    interval.getValueAtPercentile(90),
+                    interval.getValueAtPercentile(95),
+                    interval.getValueAtPercentile(99),
+                    interval.getValueAtPercentile(99.9),
+                    interval.getValueAtPercentile(99.99),
+                    interval.getMaxValue());
+        }
+    }
+}