daily pastebin goal
22%
SHARE
TWEET

Untitled

a guest Oct 18th, 2018 89 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. /*
  2.  * Licensed to the Apache Software Foundation (ASF) under one
  3.  * or more contributor license agreements.  See the NOTICE file
  4.  * distributed with this work for additional information
  5.  * regarding copyright ownership.  The ASF licenses this file
  6.  * to you under the Apache License, Version 2.0 (the
  7.  * "License"); you may not use this file except in compliance
  8.  * with the License.  You may obtain a copy of the License at
  9.  *
  10.  *     http://www.apache.org/licenses/LICENSE-2.0
  11.  *
  12.  * Unless required by applicable law or agreed to in writing, software
  13.  * distributed under the License is distributed on an "AS IS" BASIS,
  14.  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15.  * See the License for the specific language governing permissions and
  16.  * limitations under the License.
  17.  */
  18.  
  19. package org.apache.cassandra.test.microbench;
  20.  
  21. import java.nio.ByteBuffer;
  22. import java.util.Random;
  23. import java.util.concurrent.TimeUnit;
  24.  
  25. import net.jpountz.lz4.LZ4Compressor;
  26. import net.jpountz.lz4.LZ4Factory;
  27.  
  28. import net.jpountz.lz4.LZ4FastDecompressor;
  29. import org.openjdk.jmh.annotations.Benchmark;
  30. import org.openjdk.jmh.annotations.BenchmarkMode;
  31. import org.openjdk.jmh.annotations.Fork;
  32. import org.openjdk.jmh.annotations.Level;
  33. import org.openjdk.jmh.annotations.Measurement;
  34. import org.openjdk.jmh.annotations.Mode;
  35. import org.openjdk.jmh.annotations.OperationsPerInvocation;
  36. import org.openjdk.jmh.annotations.OutputTimeUnit;
  37. import org.openjdk.jmh.annotations.Scope;
  38. import org.openjdk.jmh.annotations.Setup;
  39. import org.openjdk.jmh.annotations.State;
  40. import org.openjdk.jmh.annotations.Threads;
  41. import org.openjdk.jmh.annotations.Warmup;
  42. import org.openjdk.jmh.infra.Blackhole;
  43. import org.xerial.snappy.Snappy;
  44.  
  45. @BenchmarkMode(Mode.Throughput)
  46. @OutputTimeUnit(TimeUnit.SECONDS)
  47. @Warmup(iterations = 0, time = 1, timeUnit = TimeUnit.SECONDS)
  48. @Measurement(iterations = 2, time = 10, timeUnit = TimeUnit.SECONDS)
  49. @Fork(value = 1,jvmArgsAppend = { "-Xmx8g",  "-ea"})
  50. @Threads(1) // make sure this matches the number of _physical_cores_
  51. @State(Scope.Benchmark)
  52. public class CompactIntegerSequenceBench
  53. {
  54.     private static final int NUM_ELEMENTS = 1024 * 1024 * 1024 / 2;
  55.  
  56.     private ByteBuffer uncompressedCorpus;
  57.     private ByteBuffer compressedCorpusLZ4High;
  58.     private ByteBuffer compressedCorpusLZ4Fast8k;
  59.     private ByteBuffer compressedCorpusLZ4Fast16k;
  60.     private ByteBuffer compressedCorpusLZ4Fast32k;
  61.     private ByteBuffer compressedCorpusLZ4Fast64k;
  62.     private ByteBuffer compressedCorpusSnappy;
  63.     private ByteBuffer output;
  64.     private LZ4Compressor highCompressor = LZ4Factory.fastestInstance().highCompressor(9);
  65.     private LZ4Compressor fastCompressor = LZ4Factory.fastestInstance().fastCompressor();
  66.     private LZ4FastDecompressor fastDecompressor = LZ4Factory.fastestInstance().fastDecompressor();
  67.  
  68.     @Setup(Level.Trial)
  69.     public void setup()
  70.     {
  71.         uncompressedCorpus = ByteBuffer.allocateDirect(1024 * 1024 * 1024);
  72.         Random r = new Random();
  73.         while (uncompressedCorpus.hasRemaining())
  74.         {
  75.             uncompressedCorpus.put((byte)(r.nextInt() % 3));
  76.         }
  77.         uncompressedCorpus.flip();
  78.         output = ByteBuffer.allocateDirect(Math.max(highCompressor.maxCompressedLength(1024 * 1024 * 32), Snappy.maxCompressedLength(1024 * 1024 * 32)));
  79. //        compressedCorpusLZ4High = ByteBuffer.allocateDirect(Math.max(highCompressor.maxCompressedLength(uncompressedCorpus.capacity()), Snappy.maxCompressedLength(uncompressedCorpus.capacity())));
  80.         compressedCorpusLZ4Fast8k = ByteBuffer.allocateDirect(Math.max(fastCompressor.maxCompressedLength(uncompressedCorpus.capacity()), Snappy.maxCompressedLength(uncompressedCorpus.capacity())));
  81.         compressedCorpusLZ4Fast16k = ByteBuffer.allocateDirect(Math.max(fastCompressor.maxCompressedLength(uncompressedCorpus.capacity()), Snappy.maxCompressedLength(uncompressedCorpus.capacity())));
  82.         compressedCorpusLZ4Fast32k = ByteBuffer.allocateDirect(Math.max(fastCompressor.maxCompressedLength(uncompressedCorpus.capacity()), Snappy.maxCompressedLength(uncompressedCorpus.capacity())));
  83.         compressedCorpusLZ4Fast64k = ByteBuffer.allocateDirect(Math.max(fastCompressor.maxCompressedLength(uncompressedCorpus.capacity()), Snappy.maxCompressedLength(uncompressedCorpus.capacity())));
  84. //        compressedCorpusSnappy = ByteBuffer.allocateDirect(Math.max(fastCompressor.maxCompressedLength(uncompressedCorpus.capacity()), Snappy.maxCompressedLength(uncompressedCorpus.capacity())));
  85. //
  86.         while (uncompressedCorpus.hasRemaining())
  87.         {
  88.             compressedCorpusLZ4Fast8k.position(compressedCorpusLZ4Fast8k.position() + fastCompressor.compress(uncompressedCorpus, uncompressedCorpus.position(), 1024 * 8, compressedCorpusLZ4Fast8k, compressedCorpusLZ4Fast8k.position(), compressedCorpusLZ4Fast8k.remaining()));
  89.             uncompressedCorpus.position(uncompressedCorpus.position() + 1024 * 8);
  90.         }
  91.         uncompressedCorpus.flip();
  92.  
  93.         while (uncompressedCorpus.hasRemaining())
  94.         {
  95.             compressedCorpusLZ4Fast16k.position(compressedCorpusLZ4Fast16k.position() + fastCompressor.compress(uncompressedCorpus, uncompressedCorpus.position(), 1024 * 16, compressedCorpusLZ4Fast16k, compressedCorpusLZ4Fast16k.position(), compressedCorpusLZ4Fast16k.remaining()));
  96.             uncompressedCorpus.position(uncompressedCorpus.position() + 1024 * 16);
  97.         }
  98.         uncompressedCorpus.flip();
  99.  
  100.         while (uncompressedCorpus.hasRemaining())
  101.         {
  102.             compressedCorpusLZ4Fast32k.position(compressedCorpusLZ4Fast32k.position() + fastCompressor.compress(uncompressedCorpus, uncompressedCorpus.position(), 1024 * 32, compressedCorpusLZ4Fast32k, compressedCorpusLZ4Fast32k.position(), compressedCorpusLZ4Fast32k.remaining()));
  103.             uncompressedCorpus.position(uncompressedCorpus.position() + 1024 * 32);
  104.         }
  105.         uncompressedCorpus.flip();
  106.  
  107.         while (uncompressedCorpus.hasRemaining())
  108.         {
  109.             compressedCorpusLZ4Fast64k.position(compressedCorpusLZ4Fast64k.position() + fastCompressor.compress(uncompressedCorpus, uncompressedCorpus.position(), 1024 * 64, compressedCorpusLZ4Fast64k, compressedCorpusLZ4Fast64k.position(), compressedCorpusLZ4Fast64k.remaining()));
  110.             uncompressedCorpus.position(uncompressedCorpus.position() + 1024 * 64);
  111.         }
  112.         uncompressedCorpus.flip();
  113.  
  114.         compressedCorpusLZ4Fast8k.flip();
  115.         compressedCorpusLZ4Fast16k.flip();
  116.         compressedCorpusLZ4Fast32k.flip();
  117.         compressedCorpusLZ4Fast64k.flip();
  118.     }
  119.  
  120.     public static void main(String args[]) throws Exception
  121.     {
  122.         ByteBuffer corpus = ByteBuffer.allocateDirect(1024 * 1024 * 1024);
  123.         Random r = new Random();
  124.  
  125.         LZ4Factory factory = LZ4Factory.fastestInstance();
  126.         LZ4Compressor compressor = factory.fastCompressor();
  127.         ByteBuffer output = ByteBuffer.allocateDirect(compressor.maxCompressedLength(1024 * 1024 * 64));
  128.  
  129.         while (corpus.hasRemaining())
  130.         {
  131.             corpus.put((byte)(r.nextInt() % 3));
  132.         }
  133.  
  134.         int sizes[] = new int[] { 1024 * 8, 1024 * 16, 1024 * 32, 1024 * 64 };
  135.         for (int size : sizes)
  136.         {
  137.             long compressedSize = 0;
  138.             corpus.clear();
  139.             while (corpus.hasRemaining())
  140.             {
  141.                 compressedSize += compressor.compress(corpus, 0, size, output, 0, output.capacity());
  142.                 corpus.position(corpus.position() + size);
  143.             }
  144.             System.out.printf("Chunk size %d, ratio %f%n", size, compressedSize / (double)(1024 * 1024 * 1024));
  145.         }
  146.     }
  147.  
  148.     @Benchmark
  149.     @OperationsPerInvocation(1024 * 8)
  150.     public void benchCompressLZ4Fast8k(Blackhole bh)
  151.     {
  152.         if (!uncompressedCorpus.hasRemaining())
  153.             uncompressedCorpus.position(0);
  154.         bh.consume(fastCompressor.compress(uncompressedCorpus, uncompressedCorpus.position(), 1024 * 8, output, 0, output.remaining()));
  155.         uncompressedCorpus.position(uncompressedCorpus.position() + 1024 * 8);
  156.     }
  157.  
  158.     @Benchmark
  159.     @OperationsPerInvocation(1024 * 16)
  160.     public void benchCompressLZ4Fast16k(Blackhole bh)
  161.     {
  162.         if (!uncompressedCorpus.hasRemaining())
  163.             uncompressedCorpus.position(0);
  164.         bh.consume(fastCompressor.compress(uncompressedCorpus, uncompressedCorpus.position(), 1024 * 16, output, 0, output.remaining()));
  165.         uncompressedCorpus.position(uncompressedCorpus.position() + 1024 * 16);
  166.     }
  167.  
  168.     @Benchmark
  169.     @OperationsPerInvocation(1024 * 32)
  170.     public void benchCompressLZ4Fast32k(Blackhole bh)
  171.     {
  172.         if (!uncompressedCorpus.hasRemaining())
  173.             uncompressedCorpus.position(0);
  174.         bh.consume(fastCompressor.compress(uncompressedCorpus, uncompressedCorpus.position(), 1024 * 32, output, 0,  output.remaining()));
  175.         uncompressedCorpus.position(uncompressedCorpus.position() + 1024 * 32);
  176.     }
  177.  
  178.     @Benchmark
  179.     @OperationsPerInvocation(1024 * 64)
  180.     public void benchCompressLZ4Fast64k(Blackhole bh)
  181.     {
  182.         if (!uncompressedCorpus.hasRemaining())
  183.             uncompressedCorpus.position(0);
  184.         bh.consume(fastCompressor.compress(uncompressedCorpus, uncompressedCorpus.position(), 1024 * 64, output, 0, output.remaining()));
  185.         uncompressedCorpus.position(uncompressedCorpus.position() + 1024 * 64);
  186.     }
  187.  
  188.     @Benchmark
  189.     @OperationsPerInvocation(1024 * 8)
  190.     public void benchDecompressLZ4Fast8k()
  191.     {
  192.         if (!compressedCorpusLZ4Fast8k.hasRemaining())
  193.             compressedCorpusLZ4Fast8k.position(0);
  194.         compressedCorpusLZ4Fast8k.position(compressedCorpusLZ4Fast8k.position() + fastDecompressor.decompress(compressedCorpusLZ4Fast8k, compressedCorpusLZ4Fast8k.position(), output, 0, 1024 * 8));
  195.     }
  196.  
  197.     @Benchmark
  198.     @OperationsPerInvocation(1024 * 16)
  199.     public void benchDecompressLZ4Fast16k()
  200.     {
  201.         if (!compressedCorpusLZ4Fast16k.hasRemaining())
  202.             compressedCorpusLZ4Fast16k.position(0);
  203.         compressedCorpusLZ4Fast16k.position(compressedCorpusLZ4Fast16k.position() + fastDecompressor.decompress(compressedCorpusLZ4Fast16k, compressedCorpusLZ4Fast16k.position(), output, 0, 1024 * 16));
  204.     }
  205.  
  206.     @Benchmark
  207.     @OperationsPerInvocation(1024 * 32)
  208.     public void benchDecompressLZ4Fast32k()
  209.     {
  210.         if (!compressedCorpusLZ4Fast32k.hasRemaining())
  211.             compressedCorpusLZ4Fast32k.position(0);
  212.         compressedCorpusLZ4Fast32k.position(compressedCorpusLZ4Fast32k.position() + fastDecompressor.decompress(compressedCorpusLZ4Fast32k, compressedCorpusLZ4Fast32k.position(), output, 0, 1024 * 32));
  213.     }
  214.  
  215.     @Benchmark
  216.     @OperationsPerInvocation(1024 * 64)
  217.     public void benchDecompressLZ4Fast64k()
  218.     {
  219.         if (!compressedCorpusLZ4Fast64k.hasRemaining())
  220.             compressedCorpusLZ4Fast64k.position(0);
  221.         compressedCorpusLZ4Fast64k.position(compressedCorpusLZ4Fast64k.position() + fastDecompressor.decompress(compressedCorpusLZ4Fast64k, compressedCorpusLZ4Fast64k.position(), output, 0, 1024 * 64));
  222.     }
  223.  
  224.  
  225. //    @Benchmark
  226. //    @OperationsPerInvocation(1024 * 8)
  227. //    public void benchCompresssLZ4High8k(Blackhole bh)
  228. //    {
  229. //        if (!uncompressedCorpus.hasRemaining())
  230. //            uncompressedCorpus.position(0);
  231. //        bh.consume(highCompressor.compress(uncompressedCorpus, uncompressedCorpus.position(), 1024 * 8, output, 0, output.remaining()));
  232. //        uncompressedCorpus.position(uncompressedCorpus.position() + 1024 * 8);
  233. //    }
  234. //
  235. //    @Benchmark
  236. //    @OperationsPerInvocation(1024 * 16)
  237. //    public void benchCompressLZ4High16k(Blackhole bh)
  238. //    {
  239. //        if (!uncompressedCorpus.hasRemaining())
  240. //            uncompressedCorpus.position(0);
  241. //        bh.consume(highCompressor.compress(uncompressedCorpus, uncompressedCorpus.position(), 1024 * 16, output, 0, output.remaining()));
  242. //        uncompressedCorpus.position(uncompressedCorpus.position() + 1024 * 16);
  243. //    }
  244. //
  245. //    @Benchmark
  246. //    @OperationsPerInvocation(1024 * 32)
  247. //    public void benchCompressLZ4High32k(Blackhole bh)
  248. //    {
  249. //        if (!uncompressedCorpus.hasRemaining())
  250. //            uncompressedCorpus.position(0);
  251. //        bh.consume(highCompressor.compress(uncompressedCorpus, uncompressedCorpus.position(), 1024 * 32, output, 0, output.remaining()));
  252. //        uncompressedCorpus.position(uncompressedCorpus.position() + 1024 * 32);
  253. //    }
  254. //
  255. //    @Benchmark
  256. //    @OperationsPerInvocation(1024 * 64)
  257. //    public void benchCompressLZ4High64k(Blackhole bh)
  258. //    {
  259. //        if (!uncompressedCorpus.hasRemaining())
  260. //            uncompressedCorpus.position(0);
  261. //        bh.consume(highCompressor.compress(uncompressedCorpus, uncompressedCorpus.position(), 1024 * 64, output, 0, output.remaining()));
  262. //        uncompressedCorpus.position(uncompressedCorpus.position() + 1024 * 64);
  263. //    }
  264. //
  265. //    @Benchmark
  266. //    @OperationsPerInvocation(1024 * 8)
  267. //    public void benchCompressSnappy8k(Blackhole bh) throws Exception
  268. //    {
  269. //        if (uncompressedCorpus.position() == uncompressedCorpus.capacity())
  270. //            uncompressedCorpus.position(0);
  271. //        output.clear();
  272. //        uncompressedCorpus.limit(uncompressedCorpus.position() + 1024 * 8);
  273. //        bh.consume(Snappy.compress(uncompressedCorpus, output));
  274. //        uncompressedCorpus.position(uncompressedCorpus.position() + 1024 * 8);
  275. //    }
  276. //
  277. //    @Benchmark
  278. //    @OperationsPerInvocation(1024 * 16)
  279. //    public void benchCompressSnappy16k(Blackhole bh) throws Exception
  280. //    {
  281. //        if (uncompressedCorpus.position() == uncompressedCorpus.capacity())
  282. //            uncompressedCorpus.position(0);
  283. //        output.clear();
  284. //        uncompressedCorpus.limit(uncompressedCorpus.position() + 1024 * 16);
  285. //        bh.consume(Snappy.compress(uncompressedCorpus, output));
  286. //        uncompressedCorpus.position(uncompressedCorpus.position() + 1024 * 16);
  287. //    }
  288. //
  289. //    @Benchmark
  290. //    @OperationsPerInvocation(1024 * 32)
  291. //    public void benchCompressSnappy32k(Blackhole bh) throws Exception
  292. //    {
  293. //        if (uncompressedCorpus.position() == uncompressedCorpus.capacity())
  294. //            uncompressedCorpus.position(0);
  295. //        output.clear();
  296. //        uncompressedCorpus.limit(uncompressedCorpus.position() + 1024 * 32);
  297. //        bh.consume(Snappy.compress(uncompressedCorpus, output));
  298. //        uncompressedCorpus.position(uncompressedCorpus.position() + 1024 * 32);
  299. //    }
  300. //
  301. //    @Benchmark
  302. //    @OperationsPerInvocation(1024 * 64)
  303. //    public void benchCompressSnappy64k(Blackhole bh) throws Exception
  304. //    {
  305. //        if (uncompressedCorpus.position() == uncompressedCorpus.capacity())
  306. //            uncompressedCorpus.position(0);
  307. //        output.clear();
  308. //        uncompressedCorpus.limit(uncompressedCorpus.position() + 1024 * 64);
  309. //        bh.consume(Snappy.compress(uncompressedCorpus, output));
  310. //        uncompressedCorpus.position(uncompressedCorpus.position() + 1024 * 64);
  311. //    }
  312.  
  313. }
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
 
Top