I thought it would be helpful to show an implementation based on the recommendations of Aleksey and Rafael.
The key changes:
The same set of random data is supplied to all benchmarks.
This is achieved by serializing the dataset to temp file, supplying the path to the setup() method via the @Param mechanism and then deserializing the data into instance fields.
Each benchmark run the methods against the entire dataset. We use the operationsPerInvocation feature to get accurate times.
The results of all operations are consumed via the black hole mechanism.
I created two examples, one based on the original question using a Serializable data set class that can be used directly and another that tests everyone's favorite non-serializable class, Optional.
If Aleksey or Rafael (or anyone) has any suggestions they would be much appreciated.
With Serializable data set.
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Comparator;
import java.util.Random;
import java.util.concurrent.TimeUnit;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.infra.Blackhole;
import org.openjdk.jmh.runner.Runner;
import org.openjdk.jmh.runner.options.Options;
import org.openjdk.jmh.runner.options.OptionsBuilder;
/**
* In this example each benchmark loops over the entire randomly generated data set.
* The same data set is used for all benchmarks.
* And we black hole the results.
*/
@SuppressWarnings("javadoc")
@State(Scope.Benchmark)
public class JmhBenchmark {
static final int DATA_SET_SAMPLE_SIZE = 1024 * 1024;
static final Random RANDOM = new Random();
static final Grisu g = Grisu.fmt;
double[] random_values;
double[] lowp_values;
byte[] bresults;
@Param("dataSetFilename")
String dataSetFilename;
@Setup
public void setup() throws FileNotFoundException, IOException, ClassNotFoundException {
try (FileInputStream fis = new FileInputStream(new File(this.dataSetFilename));
ObjectInputStream ois = new ObjectInputStream(fis)) {
final DataSet dataSet = (DataSet) ois.readObject();
this.random_values = dataSet.random_values;
this.lowp_values = dataSet.lowp_values;
}
this.bresults = new byte[30];
}
@Benchmark
public void test_rand_doubleto(final Blackhole bh) {
for (double random_value : this.random_values) {
bh.consume(Double.toString(random_value));
}
}
@Benchmark
public void test_lowp_doubleto(final Blackhole bh) {
for (double lowp_value : this.lowp_values) {
bh.consume(Double.toString(lowp_value));
}
}
@Benchmark
public void test_rand_grisustr(final Blackhole bh) {
for (double random_value : this.random_values) {
bh.consume(g.doubleToString(random_value));
}
}
@Benchmark
public void test_lowp_grisustr(final Blackhole bh) {
for (double lowp_value : this.lowp_values) {
bh.consume(g.doubleToString(lowp_value));
}
}
@Benchmark
public void test_rand_grisubuf(final Blackhole bh) {
for (double random_value : this.random_values) {
bh.consume(g.doubleToBytes(this.bresults, 0, random_value));
}
}
@Benchmark
public void test_lowp_grisubuf(final Blackhole bh) {
for (double lowp_value : this.lowp_values) {
bh.consume(g.doubleToBytes(this.bresults, 0, lowp_value));
}
}
/**
* Serializes an object containing random data. This data will be the same for all benchmarks.
* We pass the file name via the "dataSetFilename" parameter.
*
* @param args the arguments
*/
public static void main(final String[] args) {
try {
// clean up any old runs as data set files can be large
deleteTmpDirs(JmhBenchmark.class.getSimpleName());
// create a tempDir for the benchmark
final Path tempDirPath = createTempDir(JmhBenchmark.class.getSimpleName());
// create a data set file
final Path dateSetFilePath = Files.createTempFile(tempDirPath,
JmhBenchmark.class.getSimpleName() + "DataSet", ".ser");
final File dateSetFile = dateSetFilePath.toFile();
dateSetFile.deleteOnExit();
// create the data
final DataSet dataset = new DataSet();
try (FileOutputStream fos = new FileOutputStream(dateSetFile);
ObjectOutputStream oos = new ObjectOutputStream(fos)) {
oos.writeObject(dataset);
oos.flush();
oos.close();
}
final Options opt = new OptionsBuilder().include(JmhBenchmark.class.getSimpleName())
.param("dataSetFilename", dateSetFile.getAbsolutePath())
.operationsPerInvocation(DATA_SET_SAMPLE_SIZE)
.mode(org.openjdk.jmh.annotations.Mode.All)
.timeUnit(TimeUnit.MICROSECONDS)
.forks(1)
.build();
new Runner(opt).run();
} catch (final Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
throw new RuntimeException(e);
}
}
static Path createTempDir(String prefix) throws IOException {
final Path tempDirPath = Files.createTempDirectory(prefix);
tempDirPath.toFile()
.deleteOnExit();
return tempDirPath;
}
static void deleteTmpDirs(final String prefix) throws IOException {
for (Path dir : Files.newDirectoryStream(new File(System.getProperty("java.io.tmpdir")).toPath(),
prefix + "*")) {
for (Path toDelete : Files.walk(dir)
.sorted(Comparator.reverseOrder())
.toArray(Path[]::new)) {
Files.delete(toDelete);
}
}
}
static final class DataSet implements Serializable {
private static final long serialVersionUID = 2194487667134930491L;
private static final int[] pows = new int[] { 1, 10, 100, 1000, 10000, 100000, 1000000 };
final double[] random_values = new double[DATA_SET_SAMPLE_SIZE];
final double[] lowp_values = new double[DATA_SET_SAMPLE_SIZE];
DataSet() {
for (int i = 0; i < DATA_SET_SAMPLE_SIZE; i++) {
this.random_values[i] = RANDOM.nextDouble();
}
for (int i = 0; i < DATA_SET_SAMPLE_SIZE; i++) {
this.lowp_values[i] = (1 + RANDOM.nextInt(10000)) / pows[RANDOM.nextInt(pows.length)];
}
}
}
}
With a non-serializable test subject (Optional)
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Comparator;
import java.util.List;
import java.util.Optional;
import java.util.Random;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.infra.Blackhole;
import org.openjdk.jmh.runner.Runner;
import org.openjdk.jmh.runner.options.Options;
import org.openjdk.jmh.runner.options.OptionsBuilder;
@SuppressWarnings("javadoc")
@State(Scope.Benchmark)
public class NonSerializable {
static final int DATA_SET_SAMPLE_SIZE = 20000;
static final Random RANDOM = new Random();
Optional<Integer>[] optionals;
@Param("dataSetFilename")
String dataSetFilename;
@Setup
public void setup() throws FileNotFoundException, IOException, ClassNotFoundException {
try (FileInputStream fis = new FileInputStream(new File(this.dataSetFilename));
ObjectInputStream ois = new ObjectInputStream(fis)) {
@SuppressWarnings("unchecked")
List<Integer> strings = (List<Integer>) ois.readObject();
this.optionals = strings.stream()
.map(Optional::ofNullable)
.toArray(Optional[]::new);
}
}
@Benchmark
public void mapAndIfPresent(final Blackhole bh) {
for (int i = 0; i < this.optionals.length; i++) {
this.optionals[i].map(integer -> integer.toString())
.ifPresent(bh::consume);
}
}
@Benchmark
public void explicitGet(final Blackhole bh) {
for (int i = 0; i < this.optionals.length; i++) {
final Optional<Integer> optional = this.optionals[i];
if (optional.isPresent()) {
bh.consume(optional.get()
.toString());
}
}
}
/**
* Serializes a list of integers containing random data or null. This data will be the same for all benchmarks.
* We pass the file name via the "dataSetFilename" parameter.
*
* @param args the arguments
*/
public static void main(final String[] args) {
try {
// clean up any old runs as data set files can be large
deleteTmpDirs(NonSerializable.class.getSimpleName());
// create a tempDir for the benchmark
final Path tempDirPath = createTempDir(NonSerializable.class.getSimpleName());
// create a data set file
final Path dateSetFilePath = Files.createTempFile(tempDirPath,
NonSerializable.class.getSimpleName() + "DataSet", ".ser");
final File dateSetFile = dateSetFilePath.toFile();
dateSetFile.deleteOnExit();
final List<Integer> dataSet = IntStream.range(0, DATA_SET_SAMPLE_SIZE)
.mapToObj(i -> RANDOM.nextBoolean() ? RANDOM.nextInt() : null)
.collect(Collectors.toList());
try (FileOutputStream fos = new FileOutputStream(dateSetFile);
ObjectOutputStream oos = new ObjectOutputStream(fos)) {
oos.writeObject(dataSet);
oos.flush();
oos.close();
}
final Options opt = new OptionsBuilder().include(NonSerializable.class.getSimpleName())
.param("dataSetFilename", dateSetFile.getAbsolutePath())
.operationsPerInvocation(DATA_SET_SAMPLE_SIZE)
.mode(org.openjdk.jmh.annotations.Mode.All)
.timeUnit(TimeUnit.MICROSECONDS)
.forks(1)
.build();
new Runner(opt).run();
} catch (final Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
throw new RuntimeException(e);
}
}
static Path createTempDir(String prefix) throws IOException {
final Path tempDirPath = Files.createTempDirectory(prefix);
tempDirPath.toFile()
.deleteOnExit();
return tempDirPath;
}
static void deleteTmpDirs(final String prefix) throws IOException {
for (Path dir : Files.newDirectoryStream(new File(System.getProperty("java.io.tmpdir")).toPath(),
prefix + "*")) {
for (Path toDelete : Files.walk(dir)
.sorted(Comparator.reverseOrder())
.toArray(Path[]::new)) {
Files.delete(toDelete);
}
}
}
}
