API tools faq
paste
Advertisement
Guest User

Untitled

a guest
Jan 18th, 2017
203
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 238.61 KB | None | 0 0
raw download clone embed print report
  1. From 41021b064ef01867d5a977382b679e07dbdcf130 Mon Sep 17 00:00:00 2001
  2. From: Valery Tolstov <testautomail@mail.ru>
  3. Date: Fri, 6 Jan 2017 23:15:20 +0300
  4. Subject: [PATCH 1/5] Import lzfse code first time
  5.  
  6. ---
  7. include/linux/lzfse.h | 102 +++++
  8. lib/Makefile | 1 +
  9. lib/lzfse/Makefile | 3 +
  10. lib/lzfse/lzfse_decode.c | 50 +++
  11. lib/lzfse/lzfse_decode_base.c | 630 ++++++++++++++++++++++++++++++
  12. lib/lzfse/lzfse_encode.c | 141 +++++++
  13. lib/lzfse/lzfse_encode_base.c | 830 ++++++++++++++++++++++++++++++++++++++++
  14. lib/lzfse/lzfse_encode_tables.h | 218 +++++++++++
  15. lib/lzfse/lzfse_fse.c | 216 +++++++++++
  16. lib/lzfse/lzfse_fse.h | 631 ++++++++++++++++++++++++++++++
  17. lib/lzfse/lzfse_internal.h | 616 +++++++++++++++++++++++++++++
  18. lib/lzfse/lzfse_tunables.h | 60 +++
  19. lib/lzfse/lzvn_decode_base.c | 711 ++++++++++++++++++++++++++++++++++
  20. lib/lzfse/lzvn_decode_base.h | 68 ++++
  21. lib/lzfse/lzvn_encode_base.c | 593 ++++++++++++++++++++++++++++
  22. lib/lzfse/lzvn_encode_base.h | 116 ++++++
  23. 16 files changed, 4986 insertions(+)
  24. create mode 100644 include/linux/lzfse.h
  25. create mode 100644 lib/lzfse/Makefile
  26. create mode 100644 lib/lzfse/lzfse_decode.c
  27. create mode 100644 lib/lzfse/lzfse_decode_base.c
  28. create mode 100644 lib/lzfse/lzfse_encode.c
  29. create mode 100644 lib/lzfse/lzfse_encode_base.c
  30. create mode 100644 lib/lzfse/lzfse_encode_tables.h
  31. create mode 100644 lib/lzfse/lzfse_fse.c
  32. create mode 100644 lib/lzfse/lzfse_fse.h
  33. create mode 100644 lib/lzfse/lzfse_internal.h
  34. create mode 100644 lib/lzfse/lzfse_tunables.h
  35. create mode 100644 lib/lzfse/lzvn_decode_base.c
  36. create mode 100644 lib/lzfse/lzvn_decode_base.h
  37. create mode 100644 lib/lzfse/lzvn_encode_base.c
  38. create mode 100644 lib/lzfse/lzvn_encode_base.h
  39.  
  40. diff --git a/include/linux/lzfse.h b/include/linux/lzfse.h
  41. new file mode 100644
  42. index 0000000..7cfb138
  43. --- /dev/null
  44. +++ b/include/linux/lzfse.h
  45. @@ -0,0 +1,102 @@
  46. +/*
  47. +Copyright (c) 2015-2016, Apple Inc. All rights reserved.
  48. +
  49. +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
  50. +
  51. +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
  52. +
  53. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
  54. + in the documentation and/or other materials provided with the distribution.
  55. +
  56. +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
  57. + from this software without specific prior written permission.
  58. +
  59. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  60. +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
  61. +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  62. +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  63. +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  64. +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  65. +*/
  66. +
  67. +#ifndef LZFSE_H
  68. +#define LZFSE_H
  69. +
  70. +/*! @abstract Get the required scratch buffer size to compress using LZFSE. */
  71. +size_t lzfse_encode_scratch_size(void);
  72. +
  73. +/*! @abstract Compress a buffer using LZFSE.
  74. + *
  75. + * @param dst_buffer
  76. + * Pointer to the first byte of the destination buffer.
  77. + *
  78. + * @param dst_size
  79. + * Size of the destination buffer in bytes.
  80. + *
  81. + * @param src_buffer
  82. + * Pointer to the first byte of the source buffer.
  83. + *
  84. + * @param src_size
  85. + * Size of the source buffer in bytes.
  86. + *
  87. + * @param scratch_buffer
  88. + * If non-NULL, a pointer to scratch space for the routine to use as workspace;
  89. + * the routine may use up to lzfse_encode_scratch_size( ) bytes of workspace
  90. + * during its operation, and will not perform any internal allocations. If
  91. + * NULL, the routine may allocate its own memory to use during operation via
  92. + * a single call to malloc( ), and will release it by calling free( ) prior
  93. + * to returning. For most use, passing NULL is perfectly satisfactory, but if
  94. + * you require strict control over allocation, you will want to pass an
  95. + * explicit scratch buffer.
  96. + *
  97. + * @return
  98. + * The number of bytes written to the destination buffer if the input is
  99. + * successfully compressed. If the input cannot be compressed to fit into
  100. + * the provided buffer, or an error occurs, zero is returned, and the
  101. + * contents of dst_buffer are unspecified. */
  102. +size_t lzfse_encode_buffer(uint8_t *__restrict dst_buffer,
  103. + size_t dst_size,
  104. + const uint8_t *__restrict src_buffer,
  105. + size_t src_size,
  106. + void *__restrict scratch_buffer);
  107. +
  108. +/*! @abstract Get the required scratch buffer size to decompress using LZFSE. */
  109. +size_t lzfse_decode_scratch_size(void);
  110. +
  111. +/*! @abstract Decompress a buffer using LZFSE.
  112. + *
  113. + * @param dst_buffer
  114. + * Pointer to the first byte of the destination buffer.
  115. + *
  116. + * @param dst_size
  117. + * Size of the destination buffer in bytes.
  118. + *
  119. + * @param src_buffer
  120. + * Pointer to the first byte of the source buffer.
  121. + *
  122. + * @param src_size
  123. + * Size of the source buffer in bytes.
  124. + *
  125. + * @param scratch_buffer
  126. + * If non-NULL, a pointer to scratch space for the routine to use as workspace;
  127. + * the routine may use up to lzfse_decode_scratch_size( ) bytes of workspace
  128. + * during its operation, and will not perform any internal allocations. If
  129. + * NULL, the routine may allocate its own memory to use during operation via
  130. + * a single call to malloc( ), and will release it by calling free( ) prior
  131. + * to returning. For most use, passing NULL is perfectly satisfactory, but if
  132. + * you require strict control over allocation, you will want to pass an
  133. + * explicit scratch buffer.
  134. + *
  135. + * @return
  136. + * The number of bytes written to the destination buffer if the input is
  137. + * successfully decompressed. If there is not enough space in the destination
  138. + * buffer to hold the entire expanded output, only the first dst_size bytes
  139. + * will be written to the buffer and dst_size is returned. Note that this
  140. + * behavior differs from that of lzfse_encode_buffer. */
  141. +size_t lzfse_decode_buffer(uint8_t *__restrict dst_buffer,
  142. + size_t dst_size,
  143. + const uint8_t *__restrict src_buffer,
  144. + size_t src_size,
  145. + void *__restrict scratch_buffer);
  146. +
  147. +#endif /* LZFSE_H */
  148. diff --git a/lib/Makefile b/lib/Makefile
  149. index 7f1de26..9c49eeb 100644
  150. --- a/lib/Makefile
  151. +++ b/lib/Makefile
  152. @@ -96,6 +96,7 @@ obj-$(CONFIG_LZO_DECOMPRESS) += lzo/
  153. obj-$(CONFIG_LZ4_COMPRESS) += lz4/
  154. obj-$(CONFIG_LZ4HC_COMPRESS) += lz4/
  155. obj-$(CONFIG_LZ4_DECOMPRESS) += lz4/
  156. +obj-y += lzfse/
  157. obj-$(CONFIG_XZ_DEC) += xz/
  158. obj-$(CONFIG_RAID6_PQ) += raid6/
  159.  
  160. diff --git a/lib/lzfse/Makefile b/lib/lzfse/Makefile
  161. new file mode 100644
  162. index 0000000..c20857a
  163. --- /dev/null
  164. +++ b/lib/lzfse/Makefile
  165. @@ -0,0 +1,3 @@
  166. +obj-y += lzfse_encode.o lzfse_encode_base.o lzvn_encode_base.o \
  167. + lzfse_decode.o lzfse_decode_base.o lzvn_decode_base.o \
  168. + lzfse_fse.o
  169. \ No newline at end of file
  170. diff --git a/lib/lzfse/lzfse_decode.c b/lib/lzfse/lzfse_decode.c
  171. new file mode 100644
  172. index 0000000..88593d3
  173. --- /dev/null
  174. +++ b/lib/lzfse/lzfse_decode.c
  175. @@ -0,0 +1,50 @@
  176. +/*
  177. +Copyright (c) 2015-2016, Apple Inc. All rights reserved.
  178. +
  179. +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
  180. +
  181. +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
  182. +
  183. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
  184. + in the documentation and/or other materials provided with the distribution.
  185. +
  186. +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
  187. + from this software without specific prior written permission.
  188. +
  189. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  190. +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
  191. +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  192. +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  193. +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  194. +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  195. +*/
  196. +
  197. +// LZFSE decode API
  198. +
  199. +#include <linux/lzfse.h>
  200. +#include "lzfse_internal.h"
  201. +
  202. +size_t lzfse_decode_scratch_size() { return sizeof(lzfse_decoder_state); }
  203. +
  204. +size_t lzfse_decode_buffer(uint8_t *__restrict dst_buffer, size_t dst_size,
  205. + const uint8_t *__restrict src_buffer,
  206. + size_t src_size, void *__restrict scratch_buffer) {
  207. + lzfse_decoder_state *s = (lzfse_decoder_state *)scratch_buffer;
  208. + memset(s, 0x00, sizeof(*s));
  209. +
  210. + // Initialize state
  211. + s->src = src_buffer;
  212. + s->src_begin = src_buffer;
  213. + s->src_end = s->src + src_size;
  214. + s->dst = dst_buffer;
  215. + s->dst_begin = dst_buffer;
  216. + s->dst_end = dst_buffer + dst_size;
  217. +
  218. + // Decode
  219. + int status = lzfse_decode(s);
  220. + if (status == LZFSE_STATUS_DST_FULL)
  221. + return dst_size;
  222. + if (status != LZFSE_STATUS_OK)
  223. + return 0; // failed
  224. + return (size_t)(s->dst - dst_buffer); // bytes written
  225. +}
  226. diff --git a/lib/lzfse/lzfse_decode_base.c b/lib/lzfse/lzfse_decode_base.c
  227. new file mode 100644
  228. index 0000000..31fe859
  229. --- /dev/null
  230. +++ b/lib/lzfse/lzfse_decode_base.c
  231. @@ -0,0 +1,630 @@
  232. +/*
  233. +Copyright (c) 2015-2016, Apple Inc. All rights reserved.
  234. +
  235. +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
  236. +
  237. +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
  238. +
  239. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
  240. + in the documentation and/or other materials provided with the distribution.
  241. +
  242. +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
  243. + from this software without specific prior written permission.
  244. +
  245. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  246. +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
  247. +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  248. +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  249. +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  250. +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  251. +*/
  252. +
  253. +#include "lzfse_internal.h"
  254. +#include "lzvn_decode_base.h"
  255. +
  256. +/*! @abstract Decode an entry value from next bits of stream.
  257. + * Return \p value, and set \p *nbits to the number of bits to consume
  258. + * (starting with LSB). */
  259. +static inline int lzfse_decode_v1_freq_value(uint32_t bits, int *nbits) {
  260. + static const int8_t lzfse_freq_nbits_table[32] = {
  261. + 2, 3, 2, 5, 2, 3, 2, 8, 2, 3, 2, 5, 2, 3, 2, 14,
  262. + 2, 3, 2, 5, 2, 3, 2, 8, 2, 3, 2, 5, 2, 3, 2, 14};
  263. + static const int8_t lzfse_freq_value_table[32] = {
  264. + 0, 2, 1, 4, 0, 3, 1, -1, 0, 2, 1, 5, 0, 3, 1, -1,
  265. + 0, 2, 1, 6, 0, 3, 1, -1, 0, 2, 1, 7, 0, 3, 1, -1};
  266. +
  267. + uint32_t b = bits & 31; // lower 5 bits
  268. + int n = lzfse_freq_nbits_table[b];
  269. + *nbits = n;
  270. +
  271. + // Special cases for > 5 bits encoding
  272. + if (n == 8)
  273. + return 8 + ((bits >> 4) & 0xf);
  274. + if (n == 14)
  275. + return 24 + ((bits >> 4) & 0x3ff);
  276. +
  277. + // <= 5 bits encoding from table
  278. + return lzfse_freq_value_table[b];
  279. +}
  280. +
  281. +/*! @abstract Extracts up to 32 bits from a 64-bit field beginning at
  282. + * \p offset, and zero-extends them to a \p uint32_t.
  283. + *
  284. + * If we number the bits of \p v from 0 (least significant) to 63 (most
  285. + * significant), the result is bits \p offset to \p offset+nbits-1. */
  286. +static inline uint32_t get_field(uint64_t v, int offset, int nbits) {
  287. + assert(offset + nbits < 64 && offset >= 0 && nbits <= 32);
  288. + if (nbits == 32)
  289. + return (uint32_t)(v >> offset);
  290. + return (uint32_t)((v >> offset) & ((1 << nbits) - 1));
  291. +}
  292. +
  293. +/*! @abstract Return \c header_size field from a \c lzfse_compressed_block_header_v2. */
  294. +static inline uint32_t
  295. +lzfse_decode_v2_header_size(const lzfse_compressed_block_header_v2 *in) {
  296. + return get_field(in->packed_fields[2], 0, 32);
  297. +}
  298. +
  299. +/*! @abstract Decode all fields from a \c lzfse_compressed_block_header_v2 to a
  300. + * \c lzfse_compressed_block_header_v1.
  301. + * @return 0 on success.
  302. + * @return -1 on failure. */
  303. +static inline int lzfse_decode_v1(lzfse_compressed_block_header_v1 *out,
  304. + const lzfse_compressed_block_header_v2 *in) {
  305. + // Clear all fields
  306. + memset(out, 0x00, sizeof(lzfse_compressed_block_header_v1));
  307. +
  308. + uint64_t v0 = in->packed_fields[0];
  309. + uint64_t v1 = in->packed_fields[1];
  310. + uint64_t v2 = in->packed_fields[2];
  311. +
  312. + out->magic = LZFSE_COMPRESSEDV1_BLOCK_MAGIC;
  313. + out->n_raw_bytes = in->n_raw_bytes;
  314. +
  315. + // Literal state
  316. + out->n_literals = get_field(v0, 0, 20);
  317. + out->n_literal_payload_bytes = get_field(v0, 20, 20);
  318. + out->literal_bits = (int)get_field(v0, 60, 3) - 7;
  319. + out->literal_state[0] = get_field(v1, 0, 10);
  320. + out->literal_state[1] = get_field(v1, 10, 10);
  321. + out->literal_state[2] = get_field(v1, 20, 10);
  322. + out->literal_state[3] = get_field(v1, 30, 10);
  323. +
  324. + // L,M,D state
  325. + out->n_matches = get_field(v0, 40, 20);
  326. + out->n_lmd_payload_bytes = get_field(v1, 40, 20);
  327. + out->lmd_bits = (int)get_field(v1, 60, 3) - 7;
  328. + out->l_state = get_field(v2, 32, 10);
  329. + out->m_state = get_field(v2, 42, 10);
  330. + out->d_state = get_field(v2, 52, 10);
  331. +
  332. + // Total payload size
  333. + out->n_payload_bytes =
  334. + out->n_literal_payload_bytes + out->n_lmd_payload_bytes;
  335. +
  336. + // Freq tables
  337. + uint16_t *dst = &(out->l_freq[0]);
  338. + const uint8_t *src = &(in->freq[0]);
  339. + const uint8_t *src_end =
  340. + (const uint8_t *)in + get_field(v2, 0, 32); // first byte after header
  341. + uint32_t accum = 0;
  342. + int accum_nbits = 0;
  343. +
  344. + // No freq tables?
  345. + if (src_end == src)
  346. + return 0; // OK, freq tables were omitted
  347. +
  348. + for (int i = 0; i < LZFSE_ENCODE_L_SYMBOLS + LZFSE_ENCODE_M_SYMBOLS +
  349. + LZFSE_ENCODE_D_SYMBOLS + LZFSE_ENCODE_LITERAL_SYMBOLS;
  350. + i++) {
  351. + // Refill accum, one byte at a time, until we reach end of header, or accum
  352. + // is full
  353. + while (src < src_end && accum_nbits + 8 <= 32) {
  354. + accum |= (uint32_t)(*src) << accum_nbits;
  355. + accum_nbits += 8;
  356. + src++;
  357. + }
  358. +
  359. + // Decode and store value
  360. + int nbits = 0;
  361. + dst[i] = lzfse_decode_v1_freq_value(accum, &nbits);
  362. +
  363. + if (nbits > accum_nbits)
  364. + return -1; // failed
  365. +
  366. + // Consume nbits bits
  367. + accum >>= nbits;
  368. + accum_nbits -= nbits;
  369. + }
  370. +
  371. + if (accum_nbits >= 8 || src != src_end)
  372. + return -1; // we need to end up exactly at the end of header, with less than
  373. + // 8 bits in accumulator
  374. +
  375. + return 0;
  376. +}
  377. +
  378. +static inline void copy(uint8_t *dst, const uint8_t *src, size_t length) {
  379. + const uint8_t *dst_end = dst + length;
  380. + do {
  381. + copy8(dst, src);
  382. + dst += 8;
  383. + src += 8;
  384. + } while (dst < dst_end);
  385. +}
  386. +
  387. +static int lzfse_decode_lmd(lzfse_decoder_state *s) {
  388. + lzfse_compressed_block_decoder_state *bs = &(s->compressed_lzfse_block_state);
  389. + fse_state l_state = bs->l_state;
  390. + fse_state m_state = bs->m_state;
  391. + fse_state d_state = bs->d_state;
  392. + fse_in_stream in = bs->lmd_in_stream;
  393. + const uint8_t *src_start = s->src_begin;
  394. + const uint8_t *src = s->src + bs->lmd_in_buf;
  395. + const uint8_t *lit = bs->current_literal;
  396. + uint8_t *dst = s->dst;
  397. + uint32_t symbols = bs->n_matches;
  398. + int32_t L = bs->l_value;
  399. + int32_t M = bs->m_value;
  400. + int32_t D = bs->d_value;
  401. +
  402. + assert(l_state < LZFSE_ENCODE_L_STATES);
  403. + assert(m_state < LZFSE_ENCODE_M_STATES);
  404. + assert(d_state < LZFSE_ENCODE_D_STATES);
  405. +
  406. + // Number of bytes remaining in the destination buffer, minus 32 to
  407. + // provide a margin of safety for using overlarge copies on the fast path.
  408. + // This is a signed quantity, and may go negative when we are close to the
  409. + // end of the buffer. That's OK; we're careful about how we handle it
  410. + // in the slow-and-careful match execution path.
  411. + ptrdiff_t remaining_bytes = s->dst_end - dst - 32;
  412. +
  413. + // If L or M is non-zero, that means that we have already started decoding
  414. + // this block, and that we needed to interrupt decoding to get more space
  415. + // from the caller. There's a pending L, M, D triplet that we weren't
  416. + // able to completely process. Jump ahead to finish executing that symbol
  417. + // before decoding new values.
  418. + if (L || M)
  419. + goto ExecuteMatch;
  420. +
  421. + while (symbols > 0) {
  422. + int res;
  423. + // Decode the next L, M, D symbol from the input stream.
  424. + res = fse_in_flush(&in, &src, src_start);
  425. + if (res) {
  426. + return LZFSE_STATUS_ERROR;
  427. + }
  428. + L = fse_value_decode(&l_state, bs->l_decoder, &in);
  429. + assert(l_state < LZFSE_ENCODE_L_STATES);
  430. + if ((lit + L) >= (bs->literals + LZFSE_LITERALS_PER_BLOCK + 64)) {
  431. + return LZFSE_STATUS_ERROR;
  432. + }
  433. + res = fse_in_flush2(&in, &src, src_start);
  434. + if (res) {
  435. + return LZFSE_STATUS_ERROR;
  436. + }
  437. + M = fse_value_decode(&m_state, bs->m_decoder, &in);
  438. + assert(m_state < LZFSE_ENCODE_M_STATES);
  439. + res = fse_in_flush2(&in, &src, src_start);
  440. + if (res) {
  441. + return LZFSE_STATUS_ERROR;
  442. + }
  443. + int32_t new_d = fse_value_decode(&d_state, bs->d_decoder, &in);
  444. + assert(d_state < LZFSE_ENCODE_D_STATES);
  445. + D = new_d ? new_d : D;
  446. + symbols--;
  447. +
  448. + ExecuteMatch:
  449. + // Error if D is out of range, so that we avoid passing through
  450. + // uninitialized data or accesssing memory out of the destination
  451. + // buffer.
  452. + if ((uint32_t)D > dst + L - s->dst_begin)
  453. + return LZFSE_STATUS_ERROR;
  454. +
  455. + if (L + M <= remaining_bytes) {
  456. + // If we have plenty of space remaining, we can copy the literal
  457. + // and match with 16- and 32-byte operations, without worrying
  458. + // about writing off the end of the buffer.
  459. + remaining_bytes -= L + M;
  460. + copy(dst, lit, L);
  461. + dst += L;
  462. + lit += L;
  463. + // For the match, we have two paths; a fast copy by 16-bytes if
  464. + // the match distance is large enough to allow it, and a more
  465. + // careful path that applies a permutation to account for the
  466. + // possible overlap between source and destination if the distance
  467. + // is small.
  468. + if (D >= 8 || D >= M)
  469. + copy(dst, dst - D, M);
  470. + else
  471. + for (size_t i = 0; i < M; i++)
  472. + dst[i] = dst[i - D];
  473. + dst += M;
  474. + }
  475. +
  476. + else {
  477. + // Otherwise, we are very close to the end of the destination
  478. + // buffer, so we cannot use wide copies that slop off the end
  479. + // of the region that we are copying to. First, we restore
  480. + // the true length remaining, rather than the sham value we've
  481. + // been using so far.
  482. + remaining_bytes += 32;
  483. + // Now, we process the literal. Either there's space for it
  484. + // or there isn't; if there is, we copy the whole thing and
  485. + // update all the pointers and lengths to reflect the copy.
  486. + if (L <= remaining_bytes) {
  487. + for (size_t i = 0; i < L; i++)
  488. + dst[i] = lit[i];
  489. + dst += L;
  490. + lit += L;
  491. + remaining_bytes -= L;
  492. + L = 0;
  493. + }
  494. + // There isn't enough space to fit the whole literal. Copy as
  495. + // much of it as we can, update the pointers and the value of
  496. + // L, and report that the destination buffer is full. Note that
  497. + // we always write right up to the end of the destination buffer.
  498. + else {
  499. + for (size_t i = 0; i < remaining_bytes; i++)
  500. + dst[i] = lit[i];
  501. + dst += remaining_bytes;
  502. + lit += remaining_bytes;
  503. + L -= remaining_bytes;
  504. + goto DestinationBufferIsFull;
  505. + }
  506. + // The match goes just like the literal does. We copy as much as
  507. + // we can byte-by-byte, and if we reach the end of the buffer
  508. + // before finishing, we return to the caller indicating that
  509. + // the buffer is full.
  510. + if (M <= remaining_bytes) {
  511. + for (size_t i = 0; i < M; i++)
  512. + dst[i] = dst[i - D];
  513. + dst += M;
  514. + remaining_bytes -= M;
  515. + M = 0;
  516. + (void)M; // no dead store warning
  517. + // We don't need to update M = 0, because there's no partial
  518. + // symbol to continue executing. Either we're at the end of
  519. + // the block, in which case we will never need to resume with
  520. + // this state, or we're going to decode another L, M, D set,
  521. + // which will overwrite M anyway.
  522. + //
  523. + // But we still set M = 0, to maintain the post-condition.
  524. + } else {
  525. + for (size_t i = 0; i < remaining_bytes; i++)
  526. + dst[i] = dst[i - D];
  527. + dst += remaining_bytes;
  528. + M -= remaining_bytes;
  529. + DestinationBufferIsFull:
  530. + // Because we want to be able to resume decoding where we've left
  531. + // off (even in the middle of a literal or match), we need to
  532. + // update all of the block state fields with the current values
  533. + // so that we can resume execution from this point once the
  534. + // caller has given us more space to write into.
  535. + bs->l_value = L;
  536. + bs->m_value = M;
  537. + bs->d_value = D;
  538. + bs->l_state = l_state;
  539. + bs->m_state = m_state;
  540. + bs->d_state = d_state;
  541. + bs->lmd_in_stream = in;
  542. + bs->n_matches = symbols;
  543. + bs->lmd_in_buf = (uint32_t)(src - s->src);
  544. + bs->current_literal = lit;
  545. + s->dst = dst;
  546. + return LZFSE_STATUS_DST_FULL;
  547. + }
  548. + // Restore the "sham" decremented value of remaining_bytes and
  549. + // continue to the next L, M, D triple. We'll just be back in
  550. + // the careful path again, but this only happens at the very end
  551. + // of the buffer, so a little minor inefficiency here is a good
  552. + // tradeoff for simpler code.
  553. + remaining_bytes -= 32;
  554. + }
  555. + }
  556. + // Because we've finished with the whole block, we don't need to update
  557. + // any of the blockstate fields; they will not be used again. We just
  558. + // update the destination pointer in the state object and return.
  559. + s->dst = dst;
  560. + return LZFSE_STATUS_OK;
  561. +}
  562. +
  563. +int lzfse_decode(lzfse_decoder_state *s) {
  564. + while (1) {
  565. + // Are we inside a block?
  566. + switch (s->block_magic) {
  567. + case LZFSE_NO_BLOCK_MAGIC: {
  568. + // We need at least 4 bytes of magic number to identify next block
  569. + if (s->src + 4 > s->src_end)
  570. + return LZFSE_STATUS_SRC_EMPTY; // SRC truncated
  571. + uint32_t magic = load4(s->src);
  572. +
  573. + if (magic == LZFSE_ENDOFSTREAM_BLOCK_MAGIC) {
  574. + s->src += 4;
  575. + s->end_of_stream = 1;
  576. + return LZFSE_STATUS_OK; // done
  577. + }
  578. +
  579. + if (magic == LZFSE_UNCOMPRESSED_BLOCK_MAGIC) {
  580. + if (s->src + sizeof(uncompressed_block_header) > s->src_end)
  581. + return LZFSE_STATUS_SRC_EMPTY; // SRC truncated
  582. + // Setup state for uncompressed block
  583. + uncompressed_block_decoder_state *bs = &(s->uncompressed_block_state);
  584. + bs->n_raw_bytes =
  585. + load4(s->src + offsetof(uncompressed_block_header, n_raw_bytes));
  586. + s->src += sizeof(uncompressed_block_header);
  587. + s->block_magic = magic;
  588. + break;
  589. + }
  590. +
  591. + if (magic == LZFSE_COMPRESSEDLZVN_BLOCK_MAGIC) {
  592. + if (s->src + sizeof(lzvn_compressed_block_header) > s->src_end)
  593. + return LZFSE_STATUS_SRC_EMPTY; // SRC truncated
  594. + // Setup state for compressed LZVN block
  595. + lzvn_compressed_block_decoder_state *bs =
  596. + &(s->compressed_lzvn_block_state);
  597. + bs->n_raw_bytes =
  598. + load4(s->src + offsetof(lzvn_compressed_block_header, n_raw_bytes));
  599. + bs->n_payload_bytes = load4(
  600. + s->src + offsetof(lzvn_compressed_block_header, n_payload_bytes));
  601. + bs->d_prev = 0;
  602. + s->src += sizeof(lzvn_compressed_block_header);
  603. + s->block_magic = magic;
  604. + break;
  605. + }
  606. +
  607. + if (magic == LZFSE_COMPRESSEDV1_BLOCK_MAGIC ||
  608. + magic == LZFSE_COMPRESSEDV2_BLOCK_MAGIC) {
  609. + lzfse_compressed_block_header_v1 header1;
  610. + size_t header_size = 0;
  611. +
  612. + // Decode compressed headers
  613. + if (magic == LZFSE_COMPRESSEDV2_BLOCK_MAGIC) {
  614. + // Check we have the fixed part of the structure
  615. + if (s->src + offsetof(lzfse_compressed_block_header_v2, freq) > s->src_end)
  616. + return LZFSE_STATUS_SRC_EMPTY; // SRC truncated
  617. +
  618. + // Get size, and check we have the entire structure
  619. + const lzfse_compressed_block_header_v2 *header2 =
  620. + (const lzfse_compressed_block_header_v2 *)s->src; // not aligned, OK
  621. + header_size = lzfse_decode_v2_header_size(header2);
  622. + if (s->src + header_size > s->src_end)
  623. + return LZFSE_STATUS_SRC_EMPTY; // SRC truncated
  624. + int decodeStatus = lzfse_decode_v1(&header1, header2);
  625. + if (decodeStatus != 0)
  626. + return LZFSE_STATUS_ERROR; // failed
  627. + } else {
  628. + if (s->src + sizeof(lzfse_compressed_block_header_v1) > s->src_end)
  629. + return LZFSE_STATUS_SRC_EMPTY; // SRC truncated
  630. + memcpy(&header1, s->src, sizeof(lzfse_compressed_block_header_v1));
  631. + header_size = sizeof(lzfse_compressed_block_header_v1);
  632. + }
  633. +
  634. + // We require the header + entire encoded block to be present in SRC
  635. + // during the entire block decoding.
  636. + // This can be relaxed somehow, if it becomes a limiting factor, at the
  637. + // price of a more complex state maintenance.
  638. + // For DST, we can't easily require space for the entire decoded block,
  639. + // because it may expand to something very very large.
  640. + if (s->src + header_size + header1.n_literal_payload_bytes +
  641. + header1.n_lmd_payload_bytes >
  642. + s->src_end)
  643. + return LZFSE_STATUS_SRC_EMPTY; // need all encoded block
  644. +
  645. + // Sanity checks
  646. + if (lzfse_check_block_header_v1(&header1) != 0) {
  647. + return LZFSE_STATUS_ERROR;
  648. + }
  649. +
  650. + // Skip header
  651. + s->src += header_size;
  652. +
  653. + // Setup state for compressed V1 block from header
  654. + lzfse_compressed_block_decoder_state *bs =
  655. + &(s->compressed_lzfse_block_state);
  656. + bs->n_lmd_payload_bytes = header1.n_lmd_payload_bytes;
  657. + bs->n_matches = header1.n_matches;
  658. + fse_init_decoder_table(LZFSE_ENCODE_LITERAL_STATES,
  659. + LZFSE_ENCODE_LITERAL_SYMBOLS,
  660. + header1.literal_freq, bs->literal_decoder);
  661. + fse_init_value_decoder_table(
  662. + LZFSE_ENCODE_L_STATES, LZFSE_ENCODE_L_SYMBOLS, header1.l_freq,
  663. + l_extra_bits, l_base_value, bs->l_decoder);
  664. + fse_init_value_decoder_table(
  665. + LZFSE_ENCODE_M_STATES, LZFSE_ENCODE_M_SYMBOLS, header1.m_freq,
  666. + m_extra_bits, m_base_value, bs->m_decoder);
  667. + fse_init_value_decoder_table(
  668. + LZFSE_ENCODE_D_STATES, LZFSE_ENCODE_D_SYMBOLS, header1.d_freq,
  669. + d_extra_bits, d_base_value, bs->d_decoder);
  670. +
  671. + // Decode literals
  672. + {
  673. + fse_in_stream in;
  674. + const uint8_t *buf_start = s->src_begin;
  675. + s->src += header1.n_literal_payload_bytes; // skip literal payload
  676. + const uint8_t *buf = s->src; // read bits backwards from the end
  677. + if (fse_in_init(&in, header1.literal_bits, &buf, buf_start) != 0)
  678. + return LZFSE_STATUS_ERROR;
  679. +
  680. + fse_state state0 = header1.literal_state[0];
  681. + fse_state state1 = header1.literal_state[1];
  682. + fse_state state2 = header1.literal_state[2];
  683. + fse_state state3 = header1.literal_state[3];
  684. +
  685. + for (uint32_t i = 0; i < header1.n_literals; i += 4) // n_literals is multiple of 4
  686. + {
  687. +#if FSE_IOSTREAM_64
  688. + if (fse_in_flush(&in, &buf, buf_start) != 0)
  689. + return LZFSE_STATUS_ERROR; // [57, 64] bits
  690. + bs->literals[i + 0] =
  691. + fse_decode(&state0, bs->literal_decoder, &in); // 10b max
  692. + bs->literals[i + 1] =
  693. + fse_decode(&state1, bs->literal_decoder, &in); // 10b max
  694. + bs->literals[i + 2] =
  695. + fse_decode(&state2, bs->literal_decoder, &in); // 10b max
  696. + bs->literals[i + 3] =
  697. + fse_decode(&state3, bs->literal_decoder, &in); // 10b max
  698. +#else
  699. + if (fse_in_flush(&in, &buf, buf_start) != 0)
  700. + return LZFSE_STATUS_ERROR; // [25, 23] bits
  701. + bs->literals[i + 0] =
  702. + fse_decode(&state0, bs->literal_decoder, &in); // 10b max
  703. + bs->literals[i + 1] =
  704. + fse_decode(&state1, bs->literal_decoder, &in); // 10b max
  705. + if (fse_in_flush(&in, &buf, buf_start) != 0)
  706. + return LZFSE_STATUS_ERROR; // [25, 23] bits
  707. + bs->literals[i + 2] =
  708. + fse_decode(&state2, bs->literal_decoder, &in); // 10b max
  709. + bs->literals[i + 3] =
  710. + fse_decode(&state3, bs->literal_decoder, &in); // 10b max
  711. +#endif
  712. + }
  713. +
  714. + bs->current_literal = bs->literals;
  715. + } // literals
  716. +
  717. + // SRC is not incremented to skip the LMD payload, since we need it
  718. + // during block decode.
  719. + // We will increment SRC at the end of the block only after this point.
  720. +
  721. + // Initialize the L,M,D decode stream, do not start decoding matches
  722. + // yet, and store decoder state
  723. + {
  724. + fse_in_stream in;
  725. + // read bits backwards from the end
  726. + const uint8_t *buf = s->src + header1.n_lmd_payload_bytes;
  727. + if (fse_in_init(&in, header1.lmd_bits, &buf, s->src) != 0)
  728. + return LZFSE_STATUS_ERROR;
  729. +
  730. + bs->l_state = header1.l_state;
  731. + bs->m_state = header1.m_state;
  732. + bs->d_state = header1.d_state;
  733. + bs->lmd_in_buf = (uint32_t)(buf - s->src);
  734. + bs->l_value = bs->m_value = 0;
  735. + // Initialize D to an illegal value so we can't erroneously use
  736. + // an uninitialized "previous" value.
  737. + bs->d_value = -1;
  738. + bs->lmd_in_stream = in;
  739. + }
  740. +
  741. + s->block_magic = magic;
  742. + break;
  743. + }
  744. +
  745. + // Here we have an invalid magic number
  746. + return LZFSE_STATUS_ERROR;
  747. + } // LZFSE_NO_BLOCK_MAGIC
  748. +
  749. + case LZFSE_UNCOMPRESSED_BLOCK_MAGIC: {
  750. + uncompressed_block_decoder_state *bs = &(s->uncompressed_block_state);
  751. +
  752. + // Compute the size (in bytes) of the data that we will actually copy.
  753. + // This size is minimum(bs->n_raw_bytes, space in src, space in dst).
  754. +
  755. + uint32_t copy_size = bs->n_raw_bytes; // bytes left to copy
  756. + if (copy_size == 0) {
  757. + s->block_magic = 0;
  758. + break;
  759. + } // end of block
  760. +
  761. + if (s->src_end <= s->src)
  762. + return LZFSE_STATUS_SRC_EMPTY; // need more SRC data
  763. + const size_t src_space = s->src_end - s->src;
  764. + if (copy_size > src_space)
  765. + copy_size = (uint32_t)src_space; // limit to SRC data (> 0)
  766. +
  767. + if (s->dst_end <= s->dst)
  768. + return LZFSE_STATUS_DST_FULL; // need more DST capacity
  769. + const size_t dst_space = s->dst_end - s->dst;
  770. + if (copy_size > dst_space)
  771. + copy_size = (uint32_t)dst_space; // limit to DST capacity (> 0)
  772. +
  773. + // Now that we know that the copy size is bounded to the source and
  774. + // dest buffers, go ahead and copy the data.
  775. + // We always have copy_size > 0 here
  776. + memcpy(s->dst, s->src, copy_size);
  777. + s->src += copy_size;
  778. + s->dst += copy_size;
  779. + bs->n_raw_bytes -= copy_size;
  780. +
  781. + break;
  782. + } // LZFSE_UNCOMPRESSED_BLOCK_MAGIC
  783. +
  784. + case LZFSE_COMPRESSEDV1_BLOCK_MAGIC:
  785. + case LZFSE_COMPRESSEDV2_BLOCK_MAGIC: {
  786. + lzfse_compressed_block_decoder_state *bs =
  787. + &(s->compressed_lzfse_block_state);
  788. + // Require the entire LMD payload to be in SRC
  789. + if (s->src_end <= s->src ||
  790. + bs->n_lmd_payload_bytes > (size_t)(s->src_end - s->src))
  791. + return LZFSE_STATUS_SRC_EMPTY;
  792. +
  793. + int status = lzfse_decode_lmd(s);
  794. + if (status != LZFSE_STATUS_OK)
  795. + return status;
  796. +
  797. + s->block_magic = LZFSE_NO_BLOCK_MAGIC;
  798. + s->src += bs->n_lmd_payload_bytes; // to next block
  799. + break;
  800. + } // LZFSE_COMPRESSEDV1_BLOCK_MAGIC || LZFSE_COMPRESSEDV2_BLOCK_MAGIC
  801. +
  802. + case LZFSE_COMPRESSEDLZVN_BLOCK_MAGIC: {
  803. + lzvn_compressed_block_decoder_state *bs =
  804. + &(s->compressed_lzvn_block_state);
  805. + if (bs->n_payload_bytes > 0 && s->src_end <= s->src)
  806. + return LZFSE_STATUS_SRC_EMPTY; // need more SRC data
  807. +
  808. + // Init LZVN decoder state
  809. + lzvn_decoder_state dstate;
  810. + memset(&dstate, 0x00, sizeof(dstate));
  811. + dstate.src = s->src;
  812. + dstate.src_end = s->src_end;
  813. + if (dstate.src_end - s->src > bs->n_payload_bytes)
  814. + dstate.src_end = s->src + bs->n_payload_bytes; // limit to payload bytes
  815. + dstate.dst_begin = s->dst_begin;
  816. + dstate.dst = s->dst;
  817. + dstate.dst_end = s->dst_end;
  818. + if (dstate.dst_end - s->dst > bs->n_raw_bytes)
  819. + dstate.dst_end = s->dst + bs->n_raw_bytes; // limit to raw bytes
  820. + dstate.d_prev = bs->d_prev;
  821. + dstate.end_of_stream = 0;
  822. +
  823. + // Run LZVN decoder
  824. + lzvn_decode(&dstate);
  825. +
  826. + // Update our state
  827. + size_t src_used = dstate.src - s->src;
  828. + size_t dst_used = dstate.dst - s->dst;
  829. + if (src_used > bs->n_payload_bytes || dst_used > bs->n_raw_bytes)
  830. + return LZFSE_STATUS_ERROR; // sanity check
  831. + s->src = dstate.src;
  832. + s->dst = dstate.dst;
  833. + bs->n_payload_bytes -= (uint32_t)src_used;
  834. + bs->n_raw_bytes -= (uint32_t)dst_used;
  835. + bs->d_prev = (uint32_t)dstate.d_prev;
  836. +
  837. + // Test end of block
  838. + if (bs->n_payload_bytes == 0 && bs->n_raw_bytes == 0 &&
  839. + dstate.end_of_stream) {
  840. + s->block_magic = 0;
  841. + break;
  842. + } // block done
  843. +
  844. + // Check for invalid state
  845. + if (bs->n_payload_bytes == 0 || bs->n_raw_bytes == 0 ||
  846. + dstate.end_of_stream)
  847. + return LZFSE_STATUS_ERROR;
  848. +
  849. + // Here, block is not done and state is valid, so we need more space in dst.
  850. + return LZFSE_STATUS_DST_FULL;
  851. + }
  852. +
  853. + default:
  854. + return LZFSE_STATUS_ERROR; // invalid magic
  855. +
  856. + } // switch magic
  857. +
  858. + } // block loop
  859. +
  860. + return LZFSE_STATUS_OK;
  861. +}
  862. diff --git a/lib/lzfse/lzfse_encode.c b/lib/lzfse/lzfse_encode.c
  863. new file mode 100644
  864. index 0000000..838dcad
  865. --- /dev/null
  866. +++ b/lib/lzfse/lzfse_encode.c
  867. @@ -0,0 +1,141 @@
  868. +/*
  869. +Copyright (c) 2015-2016, Apple Inc. All rights reserved.
  870. +
  871. +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
  872. +
  873. +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
  874. +
  875. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
  876. + in the documentation and/or other materials provided with the distribution.
  877. +
  878. +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
  879. + from this software without specific prior written permission.
  880. +
  881. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  882. +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
  883. +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  884. +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  885. +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  886. +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  887. +*/
  888. +
  889. +// LZFSE encode API
  890. +
  891. +#include <linux/lzfse.h>
  892. +#include "lzfse_internal.h"
  893. +
  894. +size_t lzfse_encode_scratch_size() {
  895. + size_t s1 = sizeof(lzfse_encoder_state);
  896. + size_t s2 = lzvn_encode_scratch_size();
  897. + return (s1 > s2) ? s1 : s2; // max(lzfse,lzvn)
  898. +}
  899. +
  900. +size_t lzfse_encode_buffer(uint8_t *__restrict dst_buffer, size_t dst_size,
  901. + const uint8_t *__restrict src_buffer,
  902. + size_t src_size, void *__restrict scratch_buffer) {
  903. + const size_t original_size = src_size;
  904. +
  905. + // If input is really really small, go directly to uncompressed buffer
  906. + // (because LZVN will refuse to encode it, and we will report a failure)
  907. + if (src_size < LZVN_ENCODE_MIN_SRC_SIZE)
  908. + goto try_uncompressed;
  909. +
  910. + // If input is too small, try encoding with LZVN
  911. + if (src_size < LZFSE_ENCODE_LZVN_THRESHOLD) {
  912. + // need header + end-of-stream marker
  913. + size_t extra_size = 4 + sizeof(lzvn_compressed_block_header);
  914. + if (dst_size <= extra_size)
  915. + goto try_uncompressed; // DST is really too small, give up
  916. +
  917. + size_t sz = lzvn_encode_buffer(
  918. + dst_buffer + sizeof(lzvn_compressed_block_header),
  919. + dst_size - extra_size, src_buffer, src_size, scratch_buffer);
  920. + if (sz == 0 || sz >= src_size)
  921. + goto try_uncompressed; // failed, or no compression, fall back to
  922. + // uncompressed block
  923. +
  924. + // If we could encode, setup header and end-of-stream marker (we left room
  925. + // for them, no need to test)
  926. + lzvn_compressed_block_header header;
  927. + header.magic = LZFSE_COMPRESSEDLZVN_BLOCK_MAGIC;
  928. + header.n_raw_bytes = (uint32_t)src_size;
  929. + header.n_payload_bytes = (uint32_t)sz;
  930. + memcpy(dst_buffer, &header, sizeof(header));
  931. + store4(dst_buffer + sizeof(lzvn_compressed_block_header) + sz,
  932. + LZFSE_ENDOFSTREAM_BLOCK_MAGIC);
  933. +
  934. + return sz + extra_size;
  935. + }
  936. +
  937. + // Try encoding with LZFSE
  938. + {
  939. + lzfse_encoder_state *state = scratch_buffer;
  940. + memset(state, 0x00, sizeof *state);
  941. + if (lzfse_encode_init(state) != LZFSE_STATUS_OK)
  942. + goto try_uncompressed;
  943. + state->dst = dst_buffer;
  944. + state->dst_begin = dst_buffer;
  945. + state->dst_end = &dst_buffer[dst_size];
  946. + state->src = src_buffer;
  947. + state->src_encode_i = 0;
  948. +
  949. + if (src_size >= 0xffffffffU) {
  950. + // lzfse only uses 32 bits for offsets internally, so if the input
  951. + // buffer is really huge, we need to process it in smaller chunks.
  952. + // Note that we switch over to this path for sizes much smaller
  953. + // 2GB because it's actually faster to change algorithms well before
  954. + // it's necessary for correctness.
  955. + // The first chunk, we just process normally.
  956. + const lzfse_offset encoder_block_size = 262144;
  957. + state->src_end = encoder_block_size;
  958. + if (lzfse_encode_base(state) != LZFSE_STATUS_OK)
  959. + goto try_uncompressed;
  960. + src_size -= encoder_block_size;
  961. + while (src_size >= encoder_block_size) {
  962. + // All subsequent chunks require a translation to keep the offsets
  963. + // from getting too big. Note that we are always going from
  964. + // encoder_block_size up to 2*encoder_block_size so that the
  965. + // offsets remain positive (as opposed to resetting to zero and
  966. + // having negative offsets).
  967. + state->src_end = 2 * encoder_block_size;
  968. + if (lzfse_encode_base(state) != LZFSE_STATUS_OK)
  969. + goto try_uncompressed;
  970. + lzfse_encode_translate(state, encoder_block_size);
  971. + src_size -= encoder_block_size;
  972. + }
  973. + // Set the end for the final chunk.
  974. + state->src_end = encoder_block_size + (lzfse_offset)src_size;
  975. + }
  976. + // If the source buffer is small enough to use 32-bit offsets, we simply
  977. + // encode the whole thing in a single chunk.
  978. + else
  979. + state->src_end = (lzfse_offset)src_size;
  980. + // This is either the trailing chunk (if the source file is huge), or
  981. + // the whole source file.
  982. + if (lzfse_encode_base(state) != LZFSE_STATUS_OK)
  983. + goto try_uncompressed;
  984. + if (lzfse_encode_finish(state) != LZFSE_STATUS_OK)
  985. + goto try_uncompressed;
  986. + // No error occured, return compressed size.
  987. + return state->dst - dst_buffer;
  988. + }
  989. +
  990. +try_uncompressed:
  991. + // Compression failed for some reason. If we can fit the data into the
  992. + // output buffer uncompressed, go ahead and do that instead.
  993. + if (original_size + 12 <= dst_size && original_size < INT32_MAX) {
  994. + uncompressed_block_header header = {.magic = LZFSE_UNCOMPRESSED_BLOCK_MAGIC,
  995. + .n_raw_bytes = (uint32_t)src_size};
  996. + uint8_t *dst_end = dst_buffer;
  997. + memcpy(dst_end, &header, sizeof header);
  998. + dst_end += sizeof header;
  999. + memcpy(dst_end, src_buffer, original_size);
  1000. + dst_end += original_size;
  1001. + store4(dst_end, LZFSE_ENDOFSTREAM_BLOCK_MAGIC);
  1002. + dst_end += 4;
  1003. + return dst_end - dst_buffer;
  1004. + }
  1005. +
  1006. + // Otherwise, there's nothing we can do, so return zero.
  1007. + return 0;
  1008. +}
  1009. diff --git a/lib/lzfse/lzfse_encode_base.c b/lib/lzfse/lzfse_encode_base.c
  1010. new file mode 100644
  1011. index 0000000..367d7a2
  1012. --- /dev/null
  1013. +++ b/lib/lzfse/lzfse_encode_base.c
  1014. @@ -0,0 +1,830 @@
  1015. +/*
  1016. +Copyright (c) 2015-2016, Apple Inc. All rights reserved.
  1017. +
  1018. +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
  1019. +
  1020. +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
  1021. +
  1022. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
  1023. + in the documentation and/or other materials provided with the distribution.
  1024. +
  1025. +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
  1026. + from this software without specific prior written permission.
  1027. +
  1028. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  1029. +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
  1030. +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  1031. +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  1032. +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  1033. +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  1034. +*/
  1035. +
  1036. +// LZFSE encoder
  1037. +
  1038. +#include "lzfse_internal.h"
  1039. +#include "lzfse_encode_tables.h"
  1040. +
  1041. +/*! @abstract Get hash in range [0, LZFSE_ENCODE_HASH_VALUES-1] from 4 bytes in X. */
  1042. +static inline uint32_t hashX(uint32_t x) {
  1043. + return (x * 2654435761U) >>
  1044. + (32 - LZFSE_ENCODE_HASH_BITS); // Knuth multiplicative hash
  1045. +}
  1046. +
  1047. +/*! @abstract Return value with all 0 except nbits<=32 unsigned bits from V
  1048. + * at bit offset OFFSET.
  1049. + * V is assumed to fit on nbits bits. */
  1050. +static inline uint64_t setField(uint32_t v, int offset, int nbits) {
  1051. + assert(offset + nbits < 64 && offset >= 0 && nbits <= 32);
  1052. + assert(nbits == 32 || (v < (1U << nbits)));
  1053. + return ((uint64_t)v << (uint64_t)offset);
  1054. +}
  1055. +
  1056. +/*! @abstract Encode all fields, except freq, from a
  1057. + * lzfse_compressed_block_header_v1 to a lzfse_compressed_block_header_v2.
  1058. + * All but the header_size and freq fields of the output are modified. */
  1059. +static inline void
  1060. +lzfse_encode_v1_state(lzfse_compressed_block_header_v2 *out,
  1061. + const lzfse_compressed_block_header_v1 *in) {
  1062. + out->magic = LZFSE_COMPRESSEDV2_BLOCK_MAGIC;
  1063. + out->n_raw_bytes = in->n_raw_bytes;
  1064. +
  1065. + // Literal state
  1066. + out->packed_fields[0] = setField(in->n_literals, 0, 20) |
  1067. + setField(in->n_literal_payload_bytes, 20, 20) |
  1068. + setField(in->n_matches, 40, 20) |
  1069. + setField(7 + in->literal_bits, 60, 3);
  1070. + out->packed_fields[1] = setField(in->literal_state[0], 0, 10) |
  1071. + setField(in->literal_state[1], 10, 10) |
  1072. + setField(in->literal_state[2], 20, 10) |
  1073. + setField(in->literal_state[3], 30, 10) |
  1074. + setField(in->n_lmd_payload_bytes, 40, 20) |
  1075. + setField(7 + in->lmd_bits, 60, 3);
  1076. + out->packed_fields[2] = out->packed_fields[2] // header_size already stored in v[2]
  1077. + | setField(in->l_state, 32, 10) | setField(in->m_state, 42, 10) |
  1078. + setField(in->d_state, 52, 10);
  1079. +}
  1080. +
  1081. +/*! @abstract Encode an entry value in a freq table. Return bits, and sets
  1082. + * *nbits to the number of bits to serialize. */
  1083. +static inline uint32_t lzfse_encode_v1_freq_value(int value, int *nbits) {
  1084. + // Fixed Huffman code, bits are read from LSB.
  1085. + // Note that we rely on the position of the first '0' bit providing the number
  1086. + // of bits.
  1087. + switch (value) {
  1088. + case 0:
  1089. + *nbits = 2;
  1090. + return 0; // 0.0
  1091. + case 1:
  1092. + *nbits = 2;
  1093. + return 2; // 1.0
  1094. + case 2:
  1095. + *nbits = 3;
  1096. + return 1; // 0.01
  1097. + case 3:
  1098. + *nbits = 3;
  1099. + return 5; // 1.01
  1100. + case 4:
  1101. + *nbits = 5;
  1102. + return 3; // 00.011
  1103. + case 5:
  1104. + *nbits = 5;
  1105. + return 11; // 01.011
  1106. + case 6:
  1107. + *nbits = 5;
  1108. + return 19; // 10.011
  1109. + case 7:
  1110. + *nbits = 5;
  1111. + return 27; // 11.011
  1112. + default:
  1113. + break;
  1114. + }
  1115. + if (value < 24) {
  1116. + *nbits = 8; // 4+4
  1117. + return 7 + ((value - 8) << 4); // xxxx.0111
  1118. + }
  1119. + // 24..1047
  1120. + *nbits = 14; // 4+10
  1121. + return ((value - 24) << 4) + 15; // xxxxxxxxxx.1111
  1122. +}
  1123. +
  1124. +/*! @abstract Encode all tables from a lzfse_compressed_block_header_v1
  1125. + * to a lzfse_compressed_block_header_v2.
  1126. + * Only the header_size and freq fields of the output are modified.
  1127. + * @return Size of the lzfse_compressed_block_header_v2 */
  1128. +static inline size_t
  1129. +lzfse_encode_v1_freq_table(lzfse_compressed_block_header_v2 *out,
  1130. + const lzfse_compressed_block_header_v1 *in) {
  1131. + uint32_t accum = 0;
  1132. + int accum_nbits = 0;
  1133. + const uint16_t *src = &(in->l_freq[0]); // first value of first table (struct
  1134. + // will not be modified, so this code
  1135. + // will remain valid)
  1136. + uint8_t *dst = &(out->freq[0]);
  1137. + for (int i = 0; i < LZFSE_ENCODE_L_SYMBOLS + LZFSE_ENCODE_M_SYMBOLS +
  1138. + LZFSE_ENCODE_D_SYMBOLS + LZFSE_ENCODE_LITERAL_SYMBOLS;
  1139. + i++) {
  1140. + // Encode one value to accum
  1141. + int nbits = 0;
  1142. + uint32_t bits = lzfse_encode_v1_freq_value(src[i], &nbits);
  1143. + assert(bits < (1 << nbits));
  1144. + accum |= bits << accum_nbits;
  1145. + accum_nbits += nbits;
  1146. +
  1147. + // Store bytes from accum to output buffer
  1148. + while (accum_nbits >= 8) {
  1149. + *dst = (uint8_t)(accum & 0xff);
  1150. + accum >>= 8;
  1151. + accum_nbits -= 8;
  1152. + dst++;
  1153. + }
  1154. + }
  1155. + // Store final byte if needed
  1156. + if (accum_nbits > 0) {
  1157. + *dst = (uint8_t)(accum & 0xff);
  1158. + dst++;
  1159. + }
  1160. +
  1161. + // Return final size of out
  1162. + uint32_t header_size = (uint32_t)(dst - (uint8_t *)out);
  1163. + out->packed_fields[0] = 0;
  1164. + out->packed_fields[1] = 0;
  1165. + out->packed_fields[2] = setField(header_size, 0, 32);
  1166. +
  1167. + return header_size;
  1168. +}
  1169. +
  1170. +// We need to limit forward match length to make sure it won't split into a too
  1171. +// large number of LMD.
  1172. +// The limit itself is quite large, so it doesn't really impact compression
  1173. +// ratio.
  1174. +// The matches may still be expanded backwards by a few bytes, so the final
  1175. +// length may be greater than this limit, which is OK.
  1176. +#define LZFSE_ENCODE_MAX_MATCH_LENGTH (100 * LZFSE_ENCODE_MAX_M_VALUE)
  1177. +
  1178. +// ===============================================================
  1179. +// Encoder back end
  1180. +
  1181. +/*! @abstract Encode matches stored in STATE into a compressed/uncompressed block.
  1182. + * @return LZFSE_STATUS_OK on success.
  1183. + * @return LZFSE_STATUS_DST_FULL and restore initial state if output buffer is
  1184. + * full. */
  1185. +static int lzfse_encode_matches(lzfse_encoder_state *s) {
  1186. + if (s->n_literals == 0 && s->n_matches == 0)
  1187. + return LZFSE_STATUS_OK; // nothing to store, OK
  1188. +
  1189. + uint32_t l_occ[LZFSE_ENCODE_L_SYMBOLS];
  1190. + uint32_t m_occ[LZFSE_ENCODE_M_SYMBOLS];
  1191. + uint32_t d_occ[LZFSE_ENCODE_D_SYMBOLS];
  1192. + uint32_t literal_occ[LZFSE_ENCODE_LITERAL_SYMBOLS];
  1193. + fse_encoder_entry l_encoder[LZFSE_ENCODE_L_SYMBOLS];
  1194. + fse_encoder_entry m_encoder[LZFSE_ENCODE_M_SYMBOLS];
  1195. + fse_encoder_entry d_encoder[LZFSE_ENCODE_D_SYMBOLS];
  1196. + fse_encoder_entry literal_encoder[LZFSE_ENCODE_LITERAL_SYMBOLS];
  1197. + int ok = 1;
  1198. + lzfse_compressed_block_header_v1 header1 = {0};
  1199. + lzfse_compressed_block_header_v2 *header2 = 0;
  1200. +
  1201. + // Keep initial state to be able to restore it if DST full
  1202. + uint8_t *dst0 = s->dst;
  1203. + uint32_t n_literals0 = s->n_literals;
  1204. +
  1205. + // Add 0x00 literals until n_literals multiple of 4, since we encode 4
  1206. + // interleaved literal streams.
  1207. + while (s->n_literals & 3) {
  1208. + uint32_t n = s->n_literals++;
  1209. + s->literals[n] = 0;
  1210. + }
  1211. +
  1212. + // Encode previous distance
  1213. + uint32_t d_prev = 0;
  1214. + for (uint32_t i = 0; i < s->n_matches; i++) {
  1215. + uint32_t d = s->d_values[i];
  1216. + if (d == d_prev)
  1217. + s->d_values[i] = 0;
  1218. + else
  1219. + d_prev = d;
  1220. + }
  1221. +
  1222. + // Clear occurrence tables
  1223. + memset(l_occ, 0, sizeof(l_occ));
  1224. + memset(m_occ, 0, sizeof(m_occ));
  1225. + memset(d_occ, 0, sizeof(d_occ));
  1226. + memset(literal_occ, 0, sizeof(literal_occ));
  1227. +
  1228. + // Update occurrence tables in all 4 streams (L,M,D,literals)
  1229. + uint32_t l_sum = 0;
  1230. + uint32_t m_sum = 0;
  1231. + for (uint32_t i = 0; i < s->n_matches; i++) {
  1232. + uint32_t l = s->l_values[i];
  1233. + l_sum += l;
  1234. + l_occ[l_base_from_value(l)]++;
  1235. + }
  1236. + for (uint32_t i = 0; i < s->n_matches; i++) {
  1237. + uint32_t m = s->m_values[i];
  1238. + m_sum += m;
  1239. + m_occ[m_base_from_value(m)]++;
  1240. + }
  1241. + for (uint32_t i = 0; i < s->n_matches; i++)
  1242. + d_occ[d_base_from_value(s->d_values[i])]++;
  1243. + for (uint32_t i = 0; i < s->n_literals; i++)
  1244. + literal_occ[s->literals[i]]++;
  1245. +
  1246. + // Make sure we have enough room for a _full_ V2 header
  1247. + if (s->dst + sizeof(lzfse_compressed_block_header_v2) > s->dst_end) {
  1248. + ok = 0;
  1249. + goto END;
  1250. + }
  1251. + header2 = (lzfse_compressed_block_header_v2 *)(s->dst);
  1252. +
  1253. + // Setup header V1
  1254. + header1.magic = LZFSE_COMPRESSEDV1_BLOCK_MAGIC;
  1255. + header1.n_raw_bytes = m_sum + l_sum;
  1256. + header1.n_matches = s->n_matches;
  1257. + header1.n_literals = s->n_literals;
  1258. +
  1259. + // Normalize occurrence tables to freq tables
  1260. + fse_normalize_freq(LZFSE_ENCODE_L_STATES, LZFSE_ENCODE_L_SYMBOLS, l_occ,
  1261. + header1.l_freq);
  1262. + fse_normalize_freq(LZFSE_ENCODE_M_STATES, LZFSE_ENCODE_M_SYMBOLS, m_occ,
  1263. + header1.m_freq);
  1264. + fse_normalize_freq(LZFSE_ENCODE_D_STATES, LZFSE_ENCODE_D_SYMBOLS, d_occ,
  1265. + header1.d_freq);
  1266. + fse_normalize_freq(LZFSE_ENCODE_LITERAL_STATES, LZFSE_ENCODE_LITERAL_SYMBOLS,
  1267. + literal_occ, header1.literal_freq);
  1268. +
  1269. + // Compress freq tables to V2 header, and get actual size of V2 header
  1270. + s->dst += lzfse_encode_v1_freq_table(header2, &header1);
  1271. +
  1272. + // Initialize encoder tables from freq tables
  1273. + fse_init_encoder_table(LZFSE_ENCODE_L_STATES, LZFSE_ENCODE_L_SYMBOLS,
  1274. + header1.l_freq, l_encoder);
  1275. + fse_init_encoder_table(LZFSE_ENCODE_M_STATES, LZFSE_ENCODE_M_SYMBOLS,
  1276. + header1.m_freq, m_encoder);
  1277. + fse_init_encoder_table(LZFSE_ENCODE_D_STATES, LZFSE_ENCODE_D_SYMBOLS,
  1278. + header1.d_freq, d_encoder);
  1279. + fse_init_encoder_table(LZFSE_ENCODE_LITERAL_STATES,
  1280. + LZFSE_ENCODE_LITERAL_SYMBOLS, header1.literal_freq,
  1281. + literal_encoder);
  1282. +
  1283. + // Encode literals
  1284. + {
  1285. + fse_out_stream out;
  1286. + fse_out_init(&out);
  1287. + fse_state state0, state1, state2, state3;
  1288. + state0 = state1 = state2 = state3 = 0;
  1289. +
  1290. + uint8_t *buf = s->dst;
  1291. + uint32_t i = s->n_literals; // I multiple of 4
  1292. + // We encode starting from the last literal so we can decode starting from
  1293. + // the first
  1294. + while (i > 0) {
  1295. + if (buf + 16 > s->dst_end) {
  1296. + ok = 0;
  1297. + goto END;
  1298. + } // out full
  1299. + i -= 4;
  1300. + fse_encode(&state3, literal_encoder, &out, s->literals[i + 3]); // 10b
  1301. + fse_encode(&state2, literal_encoder, &out, s->literals[i + 2]); // 10b
  1302. +#if !FSE_IOSTREAM_64
  1303. + fse_out_flush(&out, &buf);
  1304. +#endif
  1305. + fse_encode(&state1, literal_encoder, &out, s->literals[i + 1]); // 10b
  1306. + fse_encode(&state0, literal_encoder, &out, s->literals[i + 0]); // 10b
  1307. + fse_out_flush(&out, &buf);
  1308. + }
  1309. + fse_out_finish(&out, &buf);
  1310. +
  1311. + // Update header with final encoder state
  1312. + header1.literal_bits = out.accum_nbits; // [-7, 0]
  1313. + header1.n_literal_payload_bytes = (uint32_t)(buf - s->dst);
  1314. + header1.literal_state[0] = state0;
  1315. + header1.literal_state[1] = state1;
  1316. + header1.literal_state[2] = state2;
  1317. + header1.literal_state[3] = state3;
  1318. +
  1319. + // Update state
  1320. + s->dst = buf;
  1321. +
  1322. + } // literals
  1323. +
  1324. + // Encode L,M,D
  1325. + {
  1326. + fse_out_stream out;
  1327. + fse_out_init(&out);
  1328. + fse_state l_state, m_state, d_state;
  1329. + l_state = m_state = d_state = 0;
  1330. +
  1331. + uint8_t *buf = s->dst;
  1332. + uint32_t i = s->n_matches;
  1333. +
  1334. + // Add 8 padding bytes to the L,M,D payload
  1335. + if (buf + 8 > s->dst_end) {
  1336. + ok = 0;
  1337. + goto END;
  1338. + } // out full
  1339. + store8(buf, 0);
  1340. + buf += 8;
  1341. +
  1342. + // We encode starting from the last match so we can decode starting from the
  1343. + // first
  1344. + while (i > 0) {
  1345. + if (buf + 16 > s->dst_end) {
  1346. + ok = 0;
  1347. + goto END;
  1348. + } // out full
  1349. + i -= 1;
  1350. +
  1351. + // D requires 23b max
  1352. + int32_t d_value = s->d_values[i];
  1353. + uint8_t d_symbol = d_base_from_value(d_value);
  1354. + int32_t d_nbits = d_extra_bits[d_symbol];
  1355. + int32_t d_bits = d_value - d_base_value[d_symbol];
  1356. + fse_out_push(&out, d_nbits, d_bits);
  1357. + fse_encode(&d_state, d_encoder, &out, d_symbol);
  1358. +#if !FSE_IOSTREAM_64
  1359. + fse_out_flush(&out, &buf);
  1360. +#endif
  1361. +
  1362. + // M requires 17b max
  1363. + int32_t m_value = s->m_values[i];
  1364. + uint8_t m_symbol = m_base_from_value(m_value);
  1365. + int32_t m_nbits = m_extra_bits[m_symbol];
  1366. + int32_t m_bits = m_value - m_base_value[m_symbol];
  1367. + fse_out_push(&out, m_nbits, m_bits);
  1368. + fse_encode(&m_state, m_encoder, &out, m_symbol);
  1369. +#if !FSE_IOSTREAM_64
  1370. + fse_out_flush(&out, &buf);
  1371. +#endif
  1372. +
  1373. + // L requires 14b max
  1374. + int32_t l_value = s->l_values[i];
  1375. + uint8_t l_symbol = l_base_from_value(l_value);
  1376. + int32_t l_nbits = l_extra_bits[l_symbol];
  1377. + int32_t l_bits = l_value - l_base_value[l_symbol];
  1378. + fse_out_push(&out, l_nbits, l_bits);
  1379. + fse_encode(&l_state, l_encoder, &out, l_symbol);
  1380. + fse_out_flush(&out, &buf);
  1381. + }
  1382. + fse_out_finish(&out, &buf);
  1383. +
  1384. + // Update header with final encoder state
  1385. + header1.n_lmd_payload_bytes = (uint32_t)(buf - s->dst);
  1386. + header1.lmd_bits = out.accum_nbits; // [-7, 0]
  1387. + header1.l_state = l_state;
  1388. + header1.m_state = m_state;
  1389. + header1.d_state = d_state;
  1390. +
  1391. + // Update state
  1392. + s->dst = buf;
  1393. +
  1394. + } // L,M,D
  1395. +
  1396. + // Final state update, here we had enough space in DST, and are not going to
  1397. + // revert state
  1398. + s->n_literals = 0;
  1399. + s->n_matches = 0;
  1400. +
  1401. + // Final payload size
  1402. + header1.n_payload_bytes =
  1403. + header1.n_literal_payload_bytes + header1.n_lmd_payload_bytes;
  1404. +
  1405. + // Encode state info in V2 header (we previously encoded the tables, now we
  1406. + // set the other fields)
  1407. + lzfse_encode_v1_state(header2, &header1);
  1408. +
  1409. +END:
  1410. + if (!ok) {
  1411. + // Revert state, DST was full
  1412. +
  1413. + // Revert the d_prev encoding
  1414. + uint32_t d_prev = 0;
  1415. + for (uint32_t i = 0; i < s->n_matches; i++) {
  1416. + uint32_t d = s->d_values[i];
  1417. + if (d == 0)
  1418. + s->d_values[i] = d_prev;
  1419. + else
  1420. + d_prev = d;
  1421. + }
  1422. +
  1423. + // Revert literal count
  1424. + s->n_literals = n_literals0;
  1425. +
  1426. + // Revert DST
  1427. + s->dst = dst0;
  1428. +
  1429. + return LZFSE_STATUS_DST_FULL; // DST full
  1430. + }
  1431. +
  1432. + return LZFSE_STATUS_OK;
  1433. +}
  1434. +
  1435. +/*! @abstract Push a L,M,D match into the STATE.
  1436. + * @return LZFSE_STATUS_OK if OK.
  1437. + * @return LZFSE_STATUS_DST_FULL if the match can't be pushed, meaning one of
  1438. + * the buffers is full. In that case the state is not modified. */
  1439. +static inline int lzfse_push_lmd(lzfse_encoder_state *s, uint32_t L,
  1440. + uint32_t M, uint32_t D) {
  1441. + // Check if we have enough space to push the match (we add some margin to copy
  1442. + // literals faster here, and round final count later)
  1443. + if (s->n_matches + 1 + 8 > LZFSE_MATCHES_PER_BLOCK)
  1444. + return LZFSE_STATUS_DST_FULL; // state full
  1445. + if (s->n_literals + L + 16 > LZFSE_LITERALS_PER_BLOCK)
  1446. + return LZFSE_STATUS_DST_FULL; // state full
  1447. +
  1448. + // Store match
  1449. + uint32_t n = s->n_matches++;
  1450. + s->l_values[n] = L;
  1451. + s->m_values[n] = M;
  1452. + s->d_values[n] = D;
  1453. +
  1454. + // Store literals
  1455. + uint8_t *dst = s->literals + s->n_literals;
  1456. + const uint8_t *src = s->src + s->src_literal;
  1457. + uint8_t *dst_end = dst + L;
  1458. + if (s->src_literal + L + 16 > s->src_end) {
  1459. + // Careful at the end of SRC, we can't read 16 bytes
  1460. + if (L > 0)
  1461. + memcpy(dst, src, L);
  1462. + } else {
  1463. + copy16(dst, src);
  1464. + dst += 16;
  1465. + src += 16;
  1466. + while (dst < dst_end) {
  1467. + copy16(dst, src);
  1468. + dst += 16;
  1469. + src += 16;
  1470. + }
  1471. + }
  1472. + s->n_literals += L;
  1473. +
  1474. + // Update state
  1475. + s->src_literal += L + M;
  1476. +
  1477. + return LZFSE_STATUS_OK;
  1478. +}
  1479. +
  1480. +/*! @abstract Split MATCH into one or more L,M,D parts, and push to STATE.
  1481. + * @return LZFSE_STATUS_OK if OK.
  1482. + * @return LZFSE_STATUS_DST_FULL if the match can't be pushed, meaning one of the
  1483. + * buffers is full. In that case the state is not modified. */
  1484. +static int lzfse_push_match(lzfse_encoder_state *s, const lzfse_match *match) {
  1485. + // Save the initial n_matches, n_literals, src_literal
  1486. + uint32_t n_matches0 = s->n_matches;
  1487. + uint32_t n_literals0 = s->n_literals;
  1488. + lzfse_offset src_literals0 = s->src_literal;
  1489. +
  1490. + // L,M,D
  1491. + uint32_t L = (uint32_t)(match->pos - s->src_literal); // literal count
  1492. + uint32_t M = match->length; // match length
  1493. + uint32_t D = (uint32_t)(match->pos - match->ref); // match distance
  1494. + int ok = 1;
  1495. +
  1496. + // Split L if too large
  1497. + while (L > LZFSE_ENCODE_MAX_L_VALUE) {
  1498. + if (lzfse_push_lmd(s, LZFSE_ENCODE_MAX_L_VALUE, 0, 1) != 0) {
  1499. + ok = 0;
  1500. + goto END;
  1501. + } // take D=1 because most frequent, but not actually used
  1502. + L -= LZFSE_ENCODE_MAX_L_VALUE;
  1503. + }
  1504. +
  1505. + // Split if M too large
  1506. + while (M > LZFSE_ENCODE_MAX_M_VALUE) {
  1507. + if (lzfse_push_lmd(s, L, LZFSE_ENCODE_MAX_M_VALUE, D) != 0) {
  1508. + ok = 0;
  1509. + goto END;
  1510. + }
  1511. + L = 0;
  1512. + M -= LZFSE_ENCODE_MAX_M_VALUE;
  1513. + }
  1514. +
  1515. + // L,M in range
  1516. + if (L > 0 || M > 0) {
  1517. + if (lzfse_push_lmd(s, L, M, D) != 0) {
  1518. + ok = 0;
  1519. + goto END;
  1520. + }
  1521. + L = M = 0;
  1522. + (void)L;
  1523. + (void)M; // dead stores
  1524. + }
  1525. +
  1526. +END:
  1527. + if (!ok) {
  1528. + // Revert state
  1529. + s->n_matches = n_matches0;
  1530. + s->n_literals = n_literals0;
  1531. + s->src_literal = src_literals0;
  1532. +
  1533. + return LZFSE_STATUS_DST_FULL; // state tables full
  1534. + }
  1535. +
  1536. + return LZFSE_STATUS_OK; // OK
  1537. +}
  1538. +
  1539. +/*! @abstract Backend: add MATCH to state S. Encode block if necessary, when
  1540. + * state is full.
  1541. + * @return LZFSE_STATUS_OK if OK.
  1542. + * @return LZFSE_STATUS_DST_FULL if the match can't be added, meaning one of the
  1543. + * buffers is full. In that case the state is not modified. */
  1544. +static int lzfse_backend_match(lzfse_encoder_state *s,
  1545. + const lzfse_match *match) {
  1546. + // Try to push the match in state
  1547. + if (lzfse_push_match(s, match) == LZFSE_STATUS_OK)
  1548. + return LZFSE_STATUS_OK; // OK, match added to state
  1549. +
  1550. + // Here state tables are full, try to emit block
  1551. + if (lzfse_encode_matches(s) != LZFSE_STATUS_OK)
  1552. + return LZFSE_STATUS_DST_FULL; // DST full, match not added
  1553. +
  1554. + // Here block has been emitted, re-try to push the match in state
  1555. + return lzfse_push_match(s, match);
  1556. +}
  1557. +
  1558. +/*! @abstract Backend: add L literals to state S. Encode block if necessary,
  1559. + * when state is full.
  1560. + * @return LZFSE_STATUS_OK if OK.
  1561. + * @return LZFSE_STATUS_DST_FULL if the literals can't be added, meaning one of
  1562. + * the buffers is full. In that case the state is not modified. */
  1563. +static int lzfse_backend_literals(lzfse_encoder_state *s, lzfse_offset L) {
  1564. + // Create a fake match with M=0, D=1
  1565. + lzfse_match match;
  1566. + lzfse_offset pos = s->src_literal + L;
  1567. + match.pos = pos;
  1568. + match.ref = match.pos - 1;
  1569. + match.length = 0;
  1570. + return lzfse_backend_match(s, &match);
  1571. +}
  1572. +
  1573. +/*! @abstract Backend: flush final block, and emit end of stream
  1574. + * @return LZFSE_STATUS_OK if OK.
  1575. + * @return LZFSE_STATUS_DST_FULL if either the final block, or the end-of-stream
  1576. + * can't be added, meaning one of the buffers is full. If the block was emitted,
  1577. + * the state is updated to reflect this. Otherwise, it is left unchanged. */
  1578. +static int lzfse_backend_end_of_stream(lzfse_encoder_state *s) {
  1579. + // Final match triggers write, otherwise emit blocks when we have enough
  1580. + // matches stored
  1581. + if (lzfse_encode_matches(s) != LZFSE_STATUS_OK)
  1582. + return LZFSE_STATUS_DST_FULL; // DST full
  1583. +
  1584. + // Emit end-of-stream block
  1585. + if (s->dst + 4 > s->dst_end)
  1586. + return LZFSE_STATUS_DST_FULL; // DST full
  1587. + store4(s->dst, LZFSE_ENDOFSTREAM_BLOCK_MAGIC);
  1588. + s->dst += 4;
  1589. +
  1590. + return LZFSE_STATUS_OK; // OK
  1591. +}
  1592. +
  1593. +// ===============================================================
  1594. +// Encoder state management
  1595. +
  1596. +/*! @abstract Initialize state:
  1597. + * @code
  1598. + * - hash table with all invalid pos, and value 0.
  1599. + * - pending match to NO_MATCH.
  1600. + * - src_literal to 0.
  1601. + * - d_prev to 0.
  1602. + @endcode
  1603. + * @return LZFSE_STATUS_OK */
  1604. +int lzfse_encode_init(lzfse_encoder_state *s) {
  1605. + const lzfse_match NO_MATCH = {0};
  1606. + lzfse_history_set line;
  1607. + for (int i = 0; i < LZFSE_ENCODE_HASH_WIDTH; i++) {
  1608. + line.pos[i] = -4 * LZFSE_ENCODE_MAX_D_VALUE; // invalid pos
  1609. + line.value[i] = 0;
  1610. + }
  1611. + // Fill table
  1612. + for (int i = 0; i < LZFSE_ENCODE_HASH_VALUES; i++)
  1613. + s->history_table[i] = line;
  1614. + s->pending = NO_MATCH;
  1615. + s->src_literal = 0;
  1616. +
  1617. + return LZFSE_STATUS_OK; // OK
  1618. +}
  1619. +
  1620. +/*! @abstract Translate state \p src forward by \p delta > 0.
  1621. + * Offsets in \p src are updated backwards to point to the same positions.
  1622. + * @return LZFSE_STATUS_OK */
  1623. +int lzfse_encode_translate(lzfse_encoder_state *s, lzfse_offset delta) {
  1624. + assert(delta >= 0);
  1625. + if (delta == 0)
  1626. + return LZFSE_STATUS_OK; // OK
  1627. +
  1628. + // SRC
  1629. + s->src += delta;
  1630. +
  1631. + // Offsets in SRC
  1632. + s->src_end -= delta;
  1633. + s->src_encode_i -= delta;
  1634. + s->src_encode_end -= delta;
  1635. + s->src_literal -= delta;
  1636. +
  1637. + // Pending match
  1638. + s->pending.pos -= delta;
  1639. + s->pending.ref -= delta;
  1640. +
  1641. + // history_table positions, translated, and clamped to invalid pos
  1642. + int32_t invalidPos = -4 * LZFSE_ENCODE_MAX_D_VALUE;
  1643. + for (int i = 0; i < LZFSE_ENCODE_HASH_VALUES; i++) {
  1644. + int32_t *p = &(s->history_table[i].pos[0]);
  1645. + for (int j = 0; j < LZFSE_ENCODE_HASH_WIDTH; j++) {
  1646. + lzfse_offset newPos = p[j] - delta; // translate
  1647. + p[j] = (int32_t)((newPos < invalidPos) ? invalidPos : newPos); // clamp
  1648. + }
  1649. + }
  1650. +
  1651. + return LZFSE_STATUS_OK; // OK
  1652. +}
  1653. +
  1654. +// ===============================================================
  1655. +// Encoder front end
  1656. +
  1657. +int lzfse_encode_base(lzfse_encoder_state *s) {
  1658. + lzfse_history_set *history_table = s->history_table;
  1659. + lzfse_history_set *hashLine = 0;
  1660. + lzfse_history_set newH;
  1661. + const lzfse_match NO_MATCH = {0};
  1662. + int ok = 1;
  1663. +
  1664. + memset(&newH, 0x00, sizeof(newH));
  1665. +
  1666. + // 8 byte padding at end of buffer
  1667. + s->src_encode_end = s->src_end - 8;
  1668. + for (; s->src_encode_i < s->src_encode_end; s->src_encode_i++) {
  1669. + lzfse_offset pos = s->src_encode_i; // pos >= 0
  1670. +
  1671. + // Load 4 byte value and get hash line
  1672. + uint32_t x = load4(s->src + pos);
  1673. + hashLine = history_table + hashX(x);
  1674. + lzfse_history_set h = *hashLine;
  1675. +
  1676. + // Prepare next hash line (component 0 is the most recent) to prepare new
  1677. + // entries (stored later)
  1678. + {
  1679. + newH.pos[0] = (int32_t)pos;
  1680. + for (int k = 0; k < LZFSE_ENCODE_HASH_WIDTH - 1; k++)
  1681. + newH.pos[k + 1] = h.pos[k];
  1682. + newH.value[0] = x;
  1683. + for (int k = 0; k < LZFSE_ENCODE_HASH_WIDTH - 1; k++)
  1684. + newH.value[k + 1] = h.value[k];
  1685. + }
  1686. +
  1687. + // Do not look for a match if we are still covered by a previous match
  1688. + if (pos < s->src_literal)
  1689. + goto END_POS;
  1690. +
  1691. + // Search best incoming match
  1692. + lzfse_match incoming = {.pos = pos, .ref = 0, .length = 0};
  1693. +
  1694. + // Check for matches. We consider matches of length >= 4 only.
  1695. + for (int k = 0; k < LZFSE_ENCODE_HASH_WIDTH; k++) {
  1696. + uint32_t d = h.value[k] ^ x;
  1697. + if (d)
  1698. + continue; // no 4 byte match
  1699. + int32_t ref = h.pos[k];
  1700. + if (ref + LZFSE_ENCODE_MAX_D_VALUE < pos)
  1701. + continue; // too far
  1702. +
  1703. + const uint8_t *src_ref = s->src + ref;
  1704. + const uint8_t *src_pos = s->src + pos;
  1705. + uint32_t length = 4;
  1706. + uint32_t maxLength =
  1707. + (uint32_t)(s->src_end - pos - 8); // ensure we don't hit the end of SRC
  1708. + while (length < maxLength) {
  1709. + uint64_t d = load8(src_ref + length) ^ load8(src_pos + length);
  1710. + if (d == 0) {
  1711. + length += 8;
  1712. + continue;
  1713. + }
  1714. +
  1715. + length +=
  1716. + (__builtin_ctzll(d) >> 3); // ctzll must be called only with D != 0
  1717. + break;
  1718. + }
  1719. + if (length > incoming.length) {
  1720. + incoming.length = length;
  1721. + incoming.ref = ref;
  1722. + } // keep if longer
  1723. + }
  1724. +
  1725. + // No incoming match?
  1726. + if (incoming.length == 0) {
  1727. + // We may still want to emit some literals here, to not lag too far behind
  1728. + // the current search point, and avoid
  1729. + // ending up with a literal block not fitting in the state.
  1730. + lzfse_offset n_literals = pos - s->src_literal;
  1731. + // The threshold here should be larger than a couple of MAX_L_VALUE, and
  1732. + // much smaller than LITERALS_PER_BLOCK
  1733. + if (n_literals > 8 * LZFSE_ENCODE_MAX_L_VALUE) {
  1734. + // Here, we need to consume some literals. Emit pending match if there
  1735. + // is one
  1736. + if (s->pending.length > 0) {
  1737. + if (lzfse_backend_match(s, &s->pending) != LZFSE_STATUS_OK) {
  1738. + ok = 0;
  1739. + goto END;
  1740. + }
  1741. + s->pending = NO_MATCH;
  1742. + } else {
  1743. + // No pending match, emit a full LZFSE_ENCODE_MAX_L_VALUE block of
  1744. + // literals
  1745. + if (lzfse_backend_literals(s, LZFSE_ENCODE_MAX_L_VALUE) !=
  1746. + LZFSE_STATUS_OK) {
  1747. + ok = 0;
  1748. + goto END;
  1749. + }
  1750. + }
  1751. + }
  1752. + goto END_POS; // no incoming match
  1753. + }
  1754. +
  1755. + // Limit match length (it may still be expanded backwards, but this is
  1756. + // bounded by the limit on literals we tested before)
  1757. + if (incoming.length > LZFSE_ENCODE_MAX_MATCH_LENGTH) {
  1758. + incoming.length = LZFSE_ENCODE_MAX_MATCH_LENGTH;
  1759. + }
  1760. +
  1761. + // Expand backwards (since this is expensive, we do this for the best match
  1762. + // only)
  1763. + while (incoming.pos > s->src_literal && incoming.ref > 0 &&
  1764. + s->src[incoming.ref - 1] == s->src[incoming.pos - 1]) {
  1765. + incoming.pos--;
  1766. + incoming.ref--;
  1767. + }
  1768. + incoming.length += pos - incoming.pos; // update length after expansion
  1769. +
  1770. + // Match filtering heuristic (from LZVN). INCOMING is always defined here.
  1771. +
  1772. + // Incoming is 'good', emit incoming
  1773. + if (incoming.length >= LZFSE_ENCODE_GOOD_MATCH) {
  1774. + if (lzfse_backend_match(s, &incoming) != LZFSE_STATUS_OK) {
  1775. + ok = 0;
  1776. + goto END;
  1777. + }
  1778. + s->pending = NO_MATCH;
  1779. + goto END_POS;
  1780. + }
  1781. +
  1782. + // No pending, keep incoming
  1783. + if (s->pending.length == 0) {
  1784. + s->pending = incoming;
  1785. + goto END_POS;
  1786. + }
  1787. +
  1788. + // No overlap, emit pending, keep incoming
  1789. + if (s->pending.pos + s->pending.length <= incoming.pos) {
  1790. + if (lzfse_backend_match(s, &s->pending) != LZFSE_STATUS_OK) {
  1791. + ok = 0;
  1792. + goto END;
  1793. + }
  1794. + s->pending = incoming;
  1795. + goto END_POS;
  1796. + }
  1797. +
  1798. + // Overlap: emit longest
  1799. + if (incoming.length > s->pending.length) {
  1800. + if (lzfse_backend_match(s, &incoming) != LZFSE_STATUS_OK) {
  1801. + ok = 0;
  1802. + goto END;
  1803. + }
  1804. + } else {
  1805. + if (lzfse_backend_match(s, &s->pending) != LZFSE_STATUS_OK) {
  1806. + ok = 0;
  1807. + goto END;
  1808. + }
  1809. + }
  1810. + s->pending = NO_MATCH;
  1811. +
  1812. + END_POS:
  1813. + // We are done with this src_encode_i.
  1814. + // Update state now (s->pending has already been updated).
  1815. + *hashLine = newH;
  1816. + }
  1817. +
  1818. +END:
  1819. + return ok ? LZFSE_STATUS_OK : LZFSE_STATUS_DST_FULL;
  1820. +}
  1821. +
  1822. +int lzfse_encode_finish(lzfse_encoder_state *s) {
  1823. + const lzfse_match NO_MATCH = {0};
  1824. +
  1825. + // Emit pending match
  1826. + if (s->pending.length > 0) {
  1827. + if (lzfse_backend_match(s, &s->pending) != LZFSE_STATUS_OK)
  1828. + return LZFSE_STATUS_DST_FULL;
  1829. + s->pending = NO_MATCH;
  1830. + }
  1831. +
  1832. + // Emit final literals if any
  1833. + lzfse_offset L = s->src_end - s->src_literal;
  1834. + if (L > 0) {
  1835. + if (lzfse_backend_literals(s, L) != LZFSE_STATUS_OK)
  1836. + return LZFSE_STATUS_DST_FULL;
  1837. + }
  1838. +
  1839. + // Emit all matches, and end-of-stream block
  1840. + if (lzfse_backend_end_of_stream(s) != LZFSE_STATUS_OK)
  1841. + return LZFSE_STATUS_DST_FULL;
  1842. +
  1843. + return LZFSE_STATUS_OK;
  1844. +}
  1845. diff --git a/lib/lzfse/lzfse_encode_tables.h b/lib/lzfse/lzfse_encode_tables.h
  1846. new file mode 100644
  1847. index 0000000..1b78c52
  1848. --- /dev/null
  1849. +++ b/lib/lzfse/lzfse_encode_tables.h
  1850. @@ -0,0 +1,218 @@
  1851. +/*
  1852. +Copyright (c) 2015-2016, Apple Inc. All rights reserved.
  1853. +
  1854. +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
  1855. +
  1856. +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
  1857. +
  1858. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
  1859. + in the documentation and/or other materials provided with the distribution.
  1860. +
  1861. +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
  1862. + from this software without specific prior written permission.
  1863. +
  1864. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  1865. +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
  1866. +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  1867. +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  1868. +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  1869. +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  1870. +*/
  1871. +
  1872. +#ifndef LZFSE_ENCODE_TABLES_H
  1873. +#define LZFSE_ENCODE_TABLES_H
  1874. +
  1875. +#if defined(_MSC_VER) && !defined(__clang__)
  1876. +# define inline __inline
  1877. +#endif
  1878. +
  1879. +static inline uint8_t l_base_from_value(int32_t value) {
  1880. + static const uint8_t sym[LZFSE_ENCODE_MAX_L_VALUE + 1] = {
  1881. + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16,
  1882. + 16, 16, 17, 17, 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, 18, 18, 18, 18,
  1883. + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
  1884. + 18, 18, 18, 18, 18, 18, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1885. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1886. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1887. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1888. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1889. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1890. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1891. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1892. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1893. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1894. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1895. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1896. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1897. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1898. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19};
  1899. + return sym[value];
  1900. +}
  1901. +static inline uint8_t m_base_from_value(int32_t value) {
  1902. + static const uint8_t sym[LZFSE_ENCODE_MAX_M_VALUE + 1] = {
  1903. + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16,
  1904. + 16, 16, 16, 16, 16, 16, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17,
  1905. + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17,
  1906. + 17, 17, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
  1907. + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
  1908. + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
  1909. + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
  1910. + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
  1911. + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
  1912. + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
  1913. + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
  1914. + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
  1915. + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
  1916. + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
  1917. + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
  1918. + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
  1919. + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
  1920. + 18, 18, 18, 18, 18, 18, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1921. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1922. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1923. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1924. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1925. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1926. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1927. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1928. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1929. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1930. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1931. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1932. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1933. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1934. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1935. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1936. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1937. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1938. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1939. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1940. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1941. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1942. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1943. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1944. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1945. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1946. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1947. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1948. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1949. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1950. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1951. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1952. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1953. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1954. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1955. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1956. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1957. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1958. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1959. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1960. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1961. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1962. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1963. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1964. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1965. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1966. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1967. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1968. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1969. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1970. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1971. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1972. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1973. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1974. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1975. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1976. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1977. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1978. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1979. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1980. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1981. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1982. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1983. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1984. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1985. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1986. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1987. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1988. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1989. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1990. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1991. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1992. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1993. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1994. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1995. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1996. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1997. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1998. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  1999. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2000. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2001. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2002. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2003. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2004. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2005. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2006. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2007. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2008. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2009. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2010. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2011. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2012. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2013. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2014. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2015. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2016. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2017. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2018. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2019. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2020. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2021. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2022. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2023. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2024. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2025. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2026. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2027. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2028. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2029. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2030. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2031. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2032. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2033. + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
  2034. + 19, 19};
  2035. + return sym[value];
  2036. +}
  2037. +
  2038. +static inline uint8_t d_base_from_value(int32_t value) {
  2039. + static const uint8_t sym[64 * 4] = {
  2040. + 0, 1, 2, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 8, 8, 9, 9,
  2041. + 9, 9, 10, 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 12, 12, 12, 12, 12,
  2042. + 13, 13, 13, 13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 15, 15,
  2043. + 15, 15, 15, 15, 15, 15, 16, 16, 16, 16, 16, 17, 18, 19, 20, 20, 21, 21,
  2044. + 22, 22, 23, 23, 24, 24, 24, 24, 25, 25, 25, 25, 26, 26, 26, 26, 27, 27,
  2045. + 27, 27, 28, 28, 28, 28, 28, 28, 28, 28, 29, 29, 29, 29, 29, 29, 29, 29,
  2046. + 30, 30, 30, 30, 30, 30, 30, 30, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32,
  2047. + 32, 32, 32, 33, 34, 35, 36, 36, 37, 37, 38, 38, 39, 39, 40, 40, 40, 40,
  2048. + 41, 41, 41, 41, 42, 42, 42, 42, 43, 43, 43, 43, 44, 44, 44, 44, 44, 44,
  2049. + 44, 44, 45, 45, 45, 45, 45, 45, 45, 45, 46, 46, 46, 46, 46, 46, 46, 46,
  2050. + 47, 47, 47, 47, 47, 47, 47, 47, 48, 48, 48, 48, 48, 49, 50, 51, 52, 52,
  2051. + 53, 53, 54, 54, 55, 55, 56, 56, 56, 56, 57, 57, 57, 57, 58, 58, 58, 58,
  2052. + 59, 59, 59, 59, 60, 60, 60, 60, 60, 60, 60, 60, 61, 61, 61, 61, 61, 61,
  2053. + 61, 61, 62, 62, 62, 62, 62, 62, 62, 62, 63, 63, 63, 63, 63, 63, 63, 63,
  2054. + 0, 0, 0, 0};
  2055. + int index = 0;
  2056. + int in_range_k;
  2057. + in_range_k = (value >= 0 && value < 60);
  2058. + index |= (((value - 0) >> 0) + 0) & -in_range_k;
  2059. + in_range_k = (value >= 60 && value < 1020);
  2060. + index |= (((value - 60) >> 4) + 64) & -in_range_k;
  2061. + in_range_k = (value >= 1020 && value < 16380);
  2062. + index |= (((value - 1020) >> 8) + 128) & -in_range_k;
  2063. + in_range_k = (value >= 16380 && value < 262140);
  2064. + index |= (((value - 16380) >> 12) + 192) & -in_range_k;
  2065. + return sym[index & 255];
  2066. +}
  2067. +
  2068. +#endif // LZFSE_ENCODE_TABLES_H
  2069. diff --git a/lib/lzfse/lzfse_fse.c b/lib/lzfse/lzfse_fse.c
  2070. new file mode 100644
  2071. index 0000000..631ada0
  2072. --- /dev/null
  2073. +++ b/lib/lzfse/lzfse_fse.c
  2074. @@ -0,0 +1,216 @@
  2075. +/*
  2076. +Copyright (c) 2015-2016, Apple Inc. All rights reserved.
  2077. +
  2078. +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
  2079. +
  2080. +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
  2081. +
  2082. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
  2083. + in the documentation and/or other materials provided with the distribution.
  2084. +
  2085. +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
  2086. + from this software without specific prior written permission.
  2087. +
  2088. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  2089. +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
  2090. +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  2091. +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  2092. +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  2093. +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  2094. +*/
  2095. +
  2096. +#include "lzfse_internal.h"
  2097. +
  2098. +// Initialize encoder table T[NSYMBOLS].
  2099. +// NSTATES = sum FREQ[i] is the number of states (a power of 2)
  2100. +// NSYMBOLS is the number of symbols.
  2101. +// FREQ[NSYMBOLS] is a normalized histogram of symbol frequencies, with FREQ[i]
  2102. +// >= 0.
  2103. +// Some symbols may have a 0 frequency. In that case, they should not be
  2104. +// present in the data.
  2105. +void fse_init_encoder_table(int nstates, int nsymbols,
  2106. + const uint16_t *__restrict freq,
  2107. + fse_encoder_entry *__restrict t) {
  2108. + int offset = 0; // current offset
  2109. + int n_clz = __builtin_clz(nstates);
  2110. + for (int i = 0; i < nsymbols; i++) {
  2111. + int f = (int)freq[i];
  2112. + if (f == 0)
  2113. + continue; // skip this symbol, no occurrences
  2114. + int k =
  2115. + __builtin_clz(f) - n_clz; // shift needed to ensure N <= (F<<K) < 2*N
  2116. + t[i].s0 = (int16_t)((f << k) - nstates);
  2117. + t[i].k = (int16_t)k;
  2118. + t[i].delta0 = (int16_t)(offset - f + (nstates >> k));
  2119. + t[i].delta1 = (int16_t)(offset - f + (nstates >> (k - 1)));
  2120. + offset += f;
  2121. + }
  2122. +}
  2123. +
  2124. +// Initialize decoder table T[NSTATES].
  2125. +// NSTATES = sum FREQ[i] is the number of states (a power of 2)
  2126. +// NSYMBOLS is the number of symbols.
  2127. +// FREQ[NSYMBOLS] is a normalized histogram of symbol frequencies, with FREQ[i]
  2128. +// >= 0.
  2129. +// Some symbols may have a 0 frequency. In that case, they should not be
  2130. +// present in the data.
  2131. +int fse_init_decoder_table(int nstates, int nsymbols,
  2132. + const uint16_t *__restrict freq,
  2133. + int32_t *__restrict t) {
  2134. + assert(nsymbols <= 256);
  2135. + assert(fse_check_freq(freq, nsymbols, nstates) == 0);
  2136. + int n_clz = __builtin_clz(nstates);
  2137. + int sum_of_freq = 0;
  2138. + for (int i = 0; i < nsymbols; i++) {
  2139. + int f = (int)freq[i];
  2140. + if (f == 0)
  2141. + continue; // skip this symbol, no occurrences
  2142. +
  2143. + sum_of_freq += f;
  2144. +
  2145. + if (sum_of_freq > nstates) {
  2146. + return -1;
  2147. + }
  2148. +
  2149. + int k =
  2150. + __builtin_clz(f) - n_clz; // shift needed to ensure N <= (F<<K) < 2*N
  2151. + int j0 = ((2 * nstates) >> k) - f;
  2152. +
  2153. + // Initialize all states S reached by this symbol: OFFSET <= S < OFFSET + F
  2154. + for (int j = 0; j < f; j++) {
  2155. + fse_decoder_entry e;
  2156. +
  2157. + e.symbol = (uint8_t)i;
  2158. + if (j < j0) {
  2159. + e.k = (int8_t)k;
  2160. + e.delta = (int16_t)(((f + j) << k) - nstates);
  2161. + } else {
  2162. + e.k = (int8_t)(k - 1);
  2163. + e.delta = (int16_t)((j - j0) << (k - 1));
  2164. + }
  2165. +
  2166. + memcpy(t, &e, sizeof(e));
  2167. + t++;
  2168. + }
  2169. + }
  2170. +
  2171. + return 0; // OK
  2172. +}
  2173. +
  2174. +// Initialize value decoder table T[NSTATES].
  2175. +// NSTATES = sum FREQ[i] is the number of states (a power of 2)
  2176. +// NSYMBOLS is the number of symbols.
  2177. +// FREQ[NSYMBOLS] is a normalized histogram of symbol frequencies, with FREQ[i]
  2178. +// >= 0.
  2179. +// SYMBOL_VBITS[NSYMBOLS] and SYMBOLS_VBASE[NSYMBOLS] are the number of value
  2180. +// bits to read and the base value for each symbol.
  2181. +// Some symbols may have a 0 frequency. In that case, they should not be
  2182. +// present in the data.
  2183. +void fse_init_value_decoder_table(int nstates, int nsymbols,
  2184. + const uint16_t *__restrict freq,
  2185. + const uint8_t *__restrict symbol_vbits,
  2186. + const int32_t *__restrict symbol_vbase,
  2187. + fse_value_decoder_entry *__restrict t) {
  2188. + assert(nsymbols <= 256);
  2189. + assert(fse_check_freq(freq, nsymbols, nstates) == 0);
  2190. +
  2191. + int n_clz = __builtin_clz(nstates);
  2192. + for (int i = 0; i < nsymbols; i++) {
  2193. + int f = (int)freq[i];
  2194. + if (f == 0)
  2195. + continue; // skip this symbol, no occurrences
  2196. +
  2197. + int k =
  2198. + __builtin_clz(f) - n_clz; // shift needed to ensure N <= (F<<K) < 2*N
  2199. + int j0 = ((2 * nstates) >> k) - f;
  2200. +
  2201. + fse_value_decoder_entry ei = {0};
  2202. + ei.value_bits = symbol_vbits[i];
  2203. + ei.vbase = symbol_vbase[i];
  2204. +
  2205. + // Initialize all states S reached by this symbol: OFFSET <= S < OFFSET + F
  2206. + for (int j = 0; j < f; j++) {
  2207. + fse_value_decoder_entry e = ei;
  2208. +
  2209. + if (j < j0) {
  2210. + e.total_bits = (uint8_t)k + e.value_bits;
  2211. + e.delta = (int16_t)(((f + j) << k) - nstates);
  2212. + } else {
  2213. + e.total_bits = (uint8_t)(k - 1) + e.value_bits;
  2214. + e.delta = (int16_t)((j - j0) << (k - 1));
  2215. + }
  2216. +
  2217. + memcpy(t, &e, 8);
  2218. + t++;
  2219. + }
  2220. + }
  2221. +}
  2222. +
  2223. +// Remove states from symbols until the correct number of states is used.
  2224. +static void fse_adjust_freqs(uint16_t *freq, int overrun, int nsymbols) {
  2225. + for (int shift = 3; overrun != 0; shift--) {
  2226. + for (int sym = 0; sym < nsymbols; sym++) {
  2227. + if (freq[sym] > 1) {
  2228. + int n = (freq[sym] - 1) >> shift;
  2229. + if (n > overrun)
  2230. + n = overrun;
  2231. + freq[sym] -= n;
  2232. + overrun -= n;
  2233. + if (overrun == 0)
  2234. + break;
  2235. + }
  2236. + }
  2237. + }
  2238. +}
  2239. +
  2240. +// Normalize a table T[NSYMBOLS] of occurrences to FREQ[NSYMBOLS].
  2241. +void fse_normalize_freq(int nstates, int nsymbols, const uint32_t *__restrict t,
  2242. + uint16_t *__restrict freq) {
  2243. + uint32_t s_count = 0;
  2244. + int remaining = nstates; // must be signed; this may become < 0
  2245. + int max_freq = 0;
  2246. + int max_freq_sym = 0;
  2247. + int shift = __builtin_clz(nstates) - 1;
  2248. + uint32_t highprec_step;
  2249. +
  2250. + // Compute the total number of symbol occurrences
  2251. + for (int i = 0; i < nsymbols; i++)
  2252. + s_count += t[i];
  2253. +
  2254. + if (s_count == 0)
  2255. + highprec_step = 0; // no symbols used
  2256. + else
  2257. + highprec_step = ((uint32_t)1 << 31) / s_count;
  2258. +
  2259. + for (int i = 0; i < nsymbols; i++) {
  2260. +
  2261. + // Rescale the occurrence count to get the normalized frequency.
  2262. + // Round up if the fractional part is >= 0.5; otherwise round down.
  2263. + // For efficiency, we do this calculation using integer arithmetic.
  2264. + int f = (((t[i] * highprec_step) >> shift) + 1) >> 1;
  2265. +
  2266. + // If a symbol was used, it must be given a nonzero normalized frequency.
  2267. + if (f == 0 && t[i] != 0)
  2268. + f = 1;
  2269. +
  2270. + freq[i] = f;
  2271. + remaining -= f;
  2272. +
  2273. + // Remember the maximum frequency and which symbol had it.
  2274. + if (f > max_freq) {
  2275. + max_freq = f;
  2276. + max_freq_sym = i;
  2277. + }
  2278. + }
  2279. +
  2280. + // If there remain states to be assigned, then just assign them to the most
  2281. + // frequent symbol. Alternatively, if we assigned more states than were
  2282. + // actually available, then either remove states from the most frequent symbol
  2283. + // (for minor overruns) or use the slower adjustment algorithm (for major
  2284. + // overruns).
  2285. + if (-remaining < (max_freq >> 2)) {
  2286. + freq[max_freq_sym] += remaining;
  2287. + } else {
  2288. + fse_adjust_freqs(freq, -remaining, nsymbols);
  2289. + }
  2290. +}
  2291. diff --git a/lib/lzfse/lzfse_fse.h b/lib/lzfse/lzfse_fse.h
  2292. new file mode 100644
  2293. index 0000000..8846012
  2294. --- /dev/null
  2295. +++ b/lib/lzfse/lzfse_fse.h
  2296. @@ -0,0 +1,631 @@
  2297. +/*
  2298. +Copyright (c) 2015-2016, Apple Inc. All rights reserved.
  2299. +
  2300. +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
  2301. +
  2302. +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
  2303. +
  2304. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
  2305. + in the documentation and/or other materials provided with the distribution.
  2306. +
  2307. +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
  2308. + from this software without specific prior written permission.
  2309. +
  2310. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  2311. +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
  2312. +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  2313. +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  2314. +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  2315. +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  2316. +*/
  2317. +
  2318. +// Finite state entropy coding (FSE)
  2319. +// This is an implementation of the tANS algorithm described by Jarek Duda,
  2320. +// we use the more descriptive name "Finite State Entropy".
  2321. +
  2322. +#pragma once
  2323. +
  2324. +#include <assert.h>
  2325. +#include <stddef.h>
  2326. +#include <stdint.h>
  2327. +#include <stdlib.h>
  2328. +#include <string.h>
  2329. +
  2330. +// Select between 32/64-bit I/O streams for FSE. Note that the FSE stream
  2331. +// size need not match the word size of the machine, but in practice you
  2332. +// want to use 64b streams on 64b systems for better performance.
  2333. +#if defined(_M_AMD64) || defined(__x86_64__) || defined(__arm64__)
  2334. +#define FSE_IOSTREAM_64 1
  2335. +#else
  2336. +#define FSE_IOSTREAM_64 0
  2337. +#endif
  2338. +
  2339. +#if defined(_MSC_VER) && !defined(__clang__)
  2340. +# define FSE_INLINE __forceinline
  2341. +# define inline __inline
  2342. +# pragma warning(disable : 4068) // warning C4068: unknown pragma
  2343. +#else
  2344. +# define FSE_INLINE static inline __attribute__((__always_inline__))
  2345. +#endif
  2346. +
  2347. +// MARK: - Bit utils
  2348. +
  2349. +/*! @abstract Signed type used to represent bit count. */
  2350. +typedef int32_t fse_bit_count;
  2351. +
  2352. +/*! @abstract Unsigned type used to represent FSE state. */
  2353. +typedef uint16_t fse_state;
  2354. +
  2355. +// Mask the NBITS lsb of X. 0 <= NBITS < 64
  2356. +static inline uint64_t fse_mask_lsb64(uint64_t x, fse_bit_count nbits) {
  2357. + static const uint64_t mtable[65] = {
  2358. + 0x0000000000000000LLU, 0x0000000000000001LLU, 0x0000000000000003LLU,
  2359. + 0x0000000000000007LLU, 0x000000000000000fLLU, 0x000000000000001fLLU,
  2360. + 0x000000000000003fLLU, 0x000000000000007fLLU, 0x00000000000000ffLLU,
  2361. + 0x00000000000001ffLLU, 0x00000000000003ffLLU, 0x00000000000007ffLLU,
  2362. + 0x0000000000000fffLLU, 0x0000000000001fffLLU, 0x0000000000003fffLLU,
  2363. + 0x0000000000007fffLLU, 0x000000000000ffffLLU, 0x000000000001ffffLLU,
  2364. + 0x000000000003ffffLLU, 0x000000000007ffffLLU, 0x00000000000fffffLLU,
  2365. + 0x00000000001fffffLLU, 0x00000000003fffffLLU, 0x00000000007fffffLLU,
  2366. + 0x0000000000ffffffLLU, 0x0000000001ffffffLLU, 0x0000000003ffffffLLU,
  2367. + 0x0000000007ffffffLLU, 0x000000000fffffffLLU, 0x000000001fffffffLLU,
  2368. + 0x000000003fffffffLLU, 0x000000007fffffffLLU, 0x00000000ffffffffLLU,
  2369. + 0x00000001ffffffffLLU, 0x00000003ffffffffLLU, 0x00000007ffffffffLLU,
  2370. + 0x0000000fffffffffLLU, 0x0000001fffffffffLLU, 0x0000003fffffffffLLU,
  2371. + 0x0000007fffffffffLLU, 0x000000ffffffffffLLU, 0x000001ffffffffffLLU,
  2372. + 0x000003ffffffffffLLU, 0x000007ffffffffffLLU, 0x00000fffffffffffLLU,
  2373. + 0x00001fffffffffffLLU, 0x00003fffffffffffLLU, 0x00007fffffffffffLLU,
  2374. + 0x0000ffffffffffffLLU, 0x0001ffffffffffffLLU, 0x0003ffffffffffffLLU,
  2375. + 0x0007ffffffffffffLLU, 0x000fffffffffffffLLU, 0x001fffffffffffffLLU,
  2376. + 0x003fffffffffffffLLU, 0x007fffffffffffffLLU, 0x00ffffffffffffffLLU,
  2377. + 0x01ffffffffffffffLLU, 0x03ffffffffffffffLLU, 0x07ffffffffffffffLLU,
  2378. + 0x0fffffffffffffffLLU, 0x1fffffffffffffffLLU, 0x3fffffffffffffffLLU,
  2379. + 0x7fffffffffffffffLLU, 0xffffffffffffffffLLU,
  2380. + };
  2381. + return x & mtable[nbits];
  2382. +}
  2383. +
  2384. +// Mask the NBITS lsb of X. 0 <= NBITS < 32
  2385. +static inline uint32_t fse_mask_lsb32(uint32_t x, fse_bit_count nbits) {
  2386. + static const uint32_t mtable[33] = {
  2387. + 0x0000000000000000U, 0x0000000000000001U, 0x0000000000000003U,
  2388. + 0x0000000000000007U, 0x000000000000000fU, 0x000000000000001fU,
  2389. + 0x000000000000003fU, 0x000000000000007fU, 0x00000000000000ffU,
  2390. + 0x00000000000001ffU, 0x00000000000003ffU, 0x00000000000007ffU,
  2391. + 0x0000000000000fffU, 0x0000000000001fffU, 0x0000000000003fffU,
  2392. + 0x0000000000007fffU, 0x000000000000ffffU, 0x000000000001ffffU,
  2393. + 0x000000000003ffffU, 0x000000000007ffffU, 0x00000000000fffffU,
  2394. + 0x00000000001fffffU, 0x00000000003fffffU, 0x00000000007fffffU,
  2395. + 0x0000000000ffffffU, 0x0000000001ffffffU, 0x0000000003ffffffU,
  2396. + 0x0000000007ffffffU, 0x000000000fffffffU, 0x000000001fffffffU,
  2397. + 0x000000003fffffffU, 0x000000007fffffffU, 0x00000000ffffffffU,
  2398. + };
  2399. + return x & mtable[nbits];
  2400. +}
  2401. +
  2402. +/*! @abstract Select \c nbits at index \c start from \c x.
  2403. + * 0 <= start <= start+nbits <= 64 */
  2404. +FSE_INLINE uint64_t fse_extract_bits64(uint64_t x, fse_bit_count start,
  2405. + fse_bit_count nbits) {
  2406. +#if defined(__GNUC__)
  2407. + // If START and NBITS are constants, map to bit-field extraction instructions
  2408. + if (__builtin_constant_p(start) && __builtin_constant_p(nbits))
  2409. + return (x >> start) & ((1LLU << nbits) - 1LLU);
  2410. +#endif
  2411. +
  2412. + // Otherwise, shift and mask
  2413. + return fse_mask_lsb64(x >> start, nbits);
  2414. +}
  2415. +
  2416. +/*! @abstract Select \c nbits at index \c start from \c x.
  2417. + * 0 <= start <= start+nbits <= 32 */
  2418. +FSE_INLINE uint32_t fse_extract_bits32(uint32_t x, fse_bit_count start,
  2419. + fse_bit_count nbits) {
  2420. +#if defined(__GNUC__)
  2421. + // If START and NBITS are constants, map to bit-field extraction instructions
  2422. + if (__builtin_constant_p(start) && __builtin_constant_p(nbits))
  2423. + return (x >> start) & ((1U << nbits) - 1U);
  2424. +#endif
  2425. +
  2426. + // Otherwise, shift and mask
  2427. + return fse_mask_lsb32(x >> start, nbits);
  2428. +}
  2429. +
  2430. +// MARK: - Bit stream
  2431. +
  2432. +// I/O streams
  2433. +// The streams can be shared between several FSE encoders/decoders, which is why
  2434. +// they are not in the state struct
  2435. +
  2436. +/*! @abstract Output stream, 64-bit accum. */
  2437. +typedef struct {
  2438. + uint64_t accum; // Output bits
  2439. + fse_bit_count accum_nbits; // Number of valid bits in ACCUM, other bits are 0
  2440. +} fse_out_stream64;
  2441. +
  2442. +/*! @abstract Output stream, 32-bit accum. */
  2443. +typedef struct {
  2444. + uint32_t accum; // Output bits
  2445. + fse_bit_count accum_nbits; // Number of valid bits in ACCUM, other bits are 0
  2446. +} fse_out_stream32;
  2447. +
  2448. +/*! @abstract Object representing an input stream. */
  2449. +typedef struct {
  2450. + uint64_t accum; // Input bits
  2451. + fse_bit_count accum_nbits; // Number of valid bits in ACCUM, other bits are 0
  2452. +} fse_in_stream64;
  2453. +
  2454. +/*! @abstract Object representing an input stream. */
  2455. +typedef struct {
  2456. + uint32_t accum; // Input bits
  2457. + fse_bit_count accum_nbits; // Number of valid bits in ACCUM, other bits are 0
  2458. +} fse_in_stream32;
  2459. +
  2460. +/*! @abstract Initialize an output stream object. */
  2461. +FSE_INLINE void fse_out_init64(fse_out_stream64 *s) {
  2462. + s->accum = 0;
  2463. + s->accum_nbits = 0;
  2464. +}
  2465. +
  2466. +/*! @abstract Initialize an output stream object. */
  2467. +FSE_INLINE void fse_out_init32(fse_out_stream32 *s) {
  2468. + s->accum = 0;
  2469. + s->accum_nbits = 0;
  2470. +}
  2471. +
  2472. +/*! @abstract Write full bytes from the accumulator to output buffer, ensuring
  2473. + * accum_nbits is in [0, 7].
  2474. + * We assume we can write 8 bytes to the output buffer \c (*pbuf[0..7]) in all
  2475. + * cases.
  2476. + * @note *pbuf is incremented by the number of written bytes. */
  2477. +FSE_INLINE void fse_out_flush64(fse_out_stream64 *s, uint8_t **pbuf) {
  2478. + fse_bit_count nbits =
  2479. + s->accum_nbits & -8; // number of bits written, multiple of 8
  2480. +
  2481. + // Write 8 bytes of current accumulator
  2482. + memcpy(*pbuf, &(s->accum), 8);
  2483. + *pbuf += (nbits >> 3); // bytes
  2484. +
  2485. + // Update state
  2486. + s->accum >>= nbits; // remove nbits
  2487. + s->accum_nbits -= nbits;
  2488. +
  2489. + assert(s->accum_nbits >= 0 && s->accum_nbits <= 7);
  2490. + assert(s->accum_nbits == 64 || (s->accum >> s->accum_nbits) == 0);
  2491. +}
  2492. +
  2493. +/*! @abstract Write full bytes from the accumulator to output buffer, ensuring
  2494. + * accum_nbits is in [0, 7].
  2495. + * We assume we can write 4 bytes to the output buffer \c (*pbuf[0..3]) in all
  2496. + * cases.
  2497. + * @note *pbuf is incremented by the number of written bytes. */
  2498. +FSE_INLINE void fse_out_flush32(fse_out_stream32 *s, uint8_t **pbuf) {
  2499. + fse_bit_count nbits =
  2500. + s->accum_nbits & -8; // number of bits written, multiple of 8
  2501. +
  2502. + // Write 4 bytes of current accumulator
  2503. + memcpy(*pbuf, &(s->accum), 4);
  2504. + *pbuf += (nbits >> 3); // bytes
  2505. +
  2506. + // Update state
  2507. + s->accum >>= nbits; // remove nbits
  2508. + s->accum_nbits -= nbits;
  2509. +
  2510. + assert(s->accum_nbits >= 0 && s->accum_nbits <= 7);
  2511. + assert(s->accum_nbits == 32 || (s->accum >> s->accum_nbits) == 0);
  2512. +}
  2513. +
  2514. +/*! @abstract Write the last bytes from the accumulator to output buffer,
  2515. + * ensuring accum_nbits is in [-7, 0]. Bits are padded with 0 if needed.
  2516. + * We assume we can write 8 bytes to the output buffer \c (*pbuf[0..7]) in all
  2517. + * cases.
  2518. + * @note *pbuf is incremented by the number of written bytes. */
  2519. +FSE_INLINE void fse_out_finish64(fse_out_stream64 *s, uint8_t **pbuf) {
  2520. + fse_bit_count nbits =
  2521. + (s->accum_nbits + 7) & -8; // number of bits written, multiple of 8
  2522. +
  2523. + // Write 8 bytes of current accumulator
  2524. + memcpy(*pbuf, &(s->accum), 8);
  2525. + *pbuf += (nbits >> 3); // bytes
  2526. +
  2527. + // Update state
  2528. + s->accum = 0; // remove nbits
  2529. + s->accum_nbits -= nbits;
  2530. +
  2531. + assert(s->accum_nbits >= -7 && s->accum_nbits <= 0);
  2532. +}
  2533. +
  2534. +/*! @abstract Write the last bytes from the accumulator to output buffer,
  2535. + * ensuring accum_nbits is in [-7, 0]. Bits are padded with 0 if needed.
  2536. + * We assume we can write 4 bytes to the output buffer \c (*pbuf[0..3]) in all
  2537. + * cases.
  2538. + * @note *pbuf is incremented by the number of written bytes. */
  2539. +FSE_INLINE void fse_out_finish32(fse_out_stream32 *s, uint8_t **pbuf) {
  2540. + fse_bit_count nbits =
  2541. + (s->accum_nbits + 7) & -8; // number of bits written, multiple of 8
  2542. +
  2543. + // Write 8 bytes of current accumulator
  2544. + memcpy(*pbuf, &(s->accum), 4);
  2545. + *pbuf += (nbits >> 3); // bytes
  2546. +
  2547. + // Update state
  2548. + s->accum = 0; // remove nbits
  2549. + s->accum_nbits -= nbits;
  2550. +
  2551. + assert(s->accum_nbits >= -7 && s->accum_nbits <= 0);
  2552. +}
  2553. +
  2554. +/*! @abstract Accumulate \c n bits \c b to output stream \c s. We \b must have:
  2555. + * 0 <= b < 2^n, and N + s->accum_nbits <= 64.
  2556. + * @note The caller must ensure out_flush is called \b before the accumulator
  2557. + * overflows to more than 64 bits. */
  2558. +FSE_INLINE void fse_out_push64(fse_out_stream64 *s, fse_bit_count n,
  2559. + uint64_t b) {
  2560. + s->accum |= b << s->accum_nbits;
  2561. + s->accum_nbits += n;
  2562. +
  2563. + assert(s->accum_nbits >= 0 && s->accum_nbits <= 64);
  2564. + assert(s->accum_nbits == 64 || (s->accum >> s->accum_nbits) == 0);
  2565. +}
  2566. +
  2567. +/*! @abstract Accumulate \c n bits \c b to output stream \c s. We \b must have:
  2568. + * 0 <= n < 2^n, and n + s->accum_nbits <= 32.
  2569. + * @note The caller must ensure out_flush is called \b before the accumulator
  2570. + * overflows to more than 32 bits. */
  2571. +FSE_INLINE void fse_out_push32(fse_out_stream32 *s, fse_bit_count n,
  2572. + uint32_t b) {
  2573. + s->accum |= b << s->accum_nbits;
  2574. + s->accum_nbits += n;
  2575. +
  2576. + assert(s->accum_nbits >= 0 && s->accum_nbits <= 32);
  2577. + assert(s->accum_nbits == 32 || (s->accum >> s->accum_nbits) == 0);
  2578. +}
  2579. +
  2580. +#if FSE_IOSTREAM_64
  2581. +#define DEBUG_CHECK_INPUT_STREAM_PARAMETERS \
  2582. + assert(s->accum_nbits >= 56 && s->accum_nbits < 64); \
  2583. + assert((s->accum >> s->accum_nbits) == 0);
  2584. +#else
  2585. +#define DEBUG_CHECK_INPUT_STREAM_PARAMETERS \
  2586. + assert(s->accum_nbits >= 24 && s->accum_nbits < 32); \
  2587. + assert((s->accum >> s->accum_nbits) == 0);
  2588. +#endif
  2589. +
  2590. +/*! @abstract Initialize the fse input stream so that accum holds between 56
  2591. + * and 63 bits. We never want to have 64 bits in the stream, because that allows
  2592. + * us to avoid a special case in the fse_in_pull function (eliminating an
  2593. + * unpredictable branch), while not requiring any additional fse_flush
  2594. + * operations. This is why we have the special case for n == 0 (in which case
  2595. + * we want to load only 7 bytes instead of 8). */
  2596. +FSE_INLINE int fse_in_checked_init64(fse_in_stream64 *s, fse_bit_count n,
  2597. + const uint8_t **pbuf,
  2598. + const uint8_t *buf_start) {
  2599. + if (n) {
  2600. + if (*pbuf < buf_start + 8)
  2601. + return -1; // out of range
  2602. + *pbuf -= 8;
  2603. + memcpy(&(s->accum), *pbuf, 8);
  2604. + s->accum_nbits = n + 64;
  2605. + } else {
  2606. + if (*pbuf < buf_start + 7)
  2607. + return -1; // out of range
  2608. + *pbuf -= 7;
  2609. + memcpy(&(s->accum), *pbuf, 7);
  2610. + s->accum &= 0xffffffffffffff;
  2611. + s->accum_nbits = n + 56;
  2612. + }
  2613. +
  2614. + if ((s->accum_nbits < 56 || s->accum_nbits >= 64) ||
  2615. + ((s->accum >> s->accum_nbits) != 0)) {
  2616. + return -1; // the incoming input is wrong (encoder should have zeroed the
  2617. + // upper bits)
  2618. + }
  2619. +
  2620. + return 0; // OK
  2621. +}
  2622. +
  2623. +/*! @abstract Identical to previous function, but for 32-bit operation
  2624. + * (resulting bit count is between 24 and 31 bits). */
  2625. +FSE_INLINE int fse_in_checked_init32(fse_in_stream32 *s, fse_bit_count n,
  2626. + const uint8_t **pbuf,
  2627. + const uint8_t *buf_start) {
  2628. + if (n) {
  2629. + if (*pbuf < buf_start + 4)
  2630. + return -1; // out of range
  2631. + *pbuf -= 4;
  2632. + memcpy(&(s->accum), *pbuf, 4);
  2633. + s->accum_nbits = n + 32;
  2634. + } else {
  2635. + if (*pbuf < buf_start + 3)
  2636. + return -1; // out of range
  2637. + *pbuf -= 3;
  2638. + memcpy(&(s->accum), *pbuf, 3);
  2639. + s->accum &= 0xffffff;
  2640. + s->accum_nbits = n + 24;
  2641. + }
  2642. +
  2643. + if ((s->accum_nbits < 24 || s->accum_nbits >= 32) ||
  2644. + ((s->accum >> s->accum_nbits) != 0)) {
  2645. + return -1; // the incoming input is wrong (encoder should have zeroed the
  2646. + // upper bits)
  2647. + }
  2648. +
  2649. + return 0; // OK
  2650. +}
  2651. +
  2652. +/*! @abstract Read in new bytes from buffer to ensure that we have a full
  2653. + * complement of bits in the stream object (again, between 56 and 63 bits).
  2654. + * checking the new value of \c *pbuf remains >= \c buf_start.
  2655. + * @return 0 if OK.
  2656. + * @return -1 on failure. */
  2657. +FSE_INLINE int fse_in_checked_flush64(fse_in_stream64 *s, const uint8_t **pbuf,
  2658. + const uint8_t *buf_start) {
  2659. + // Get number of bits to add to bring us into the desired range.
  2660. + fse_bit_count nbits = (63 - s->accum_nbits) & -8;
  2661. + // Convert bits to bytes and decrement buffer address, then load new data.
  2662. + const uint8_t *buf = (*pbuf) - (nbits >> 3);
  2663. + if (buf < buf_start) {
  2664. + return -1; // out of range
  2665. + }
  2666. + *pbuf = buf;
  2667. + uint64_t incoming;
  2668. + memcpy(&incoming, buf, 8);
  2669. + // Update the state object and verify its validity (in DEBUG).
  2670. + s->accum = (s->accum << nbits) | fse_mask_lsb64(incoming, nbits);
  2671. + s->accum_nbits += nbits;
  2672. + DEBUG_CHECK_INPUT_STREAM_PARAMETERS
  2673. + return 0; // OK
  2674. +}
  2675. +
  2676. +/*! @abstract Identical to previous function (but again, we're only filling
  2677. + * a 32-bit field with between 24 and 31 bits). */
  2678. +FSE_INLINE int fse_in_checked_flush32(fse_in_stream32 *s, const uint8_t **pbuf,
  2679. + const uint8_t *buf_start) {
  2680. + // Get number of bits to add to bring us into the desired range.
  2681. + fse_bit_count nbits = (31 - s->accum_nbits) & -8;
  2682. +
  2683. + if (nbits > 0) {
  2684. + // Convert bits to bytes and decrement buffer address, then load new data.
  2685. + const uint8_t *buf = (*pbuf) - (nbits >> 3);
  2686. + if (buf < buf_start) {
  2687. + return -1; // out of range
  2688. + }
  2689. +
  2690. + *pbuf = buf;
  2691. +
  2692. + uint32_t incoming = *((uint32_t *)buf);
  2693. +
  2694. + // Update the state object and verify its validity (in DEBUG).
  2695. + s->accum = (s->accum << nbits) | fse_mask_lsb32(incoming, nbits);
  2696. + s->accum_nbits += nbits;
  2697. + }
  2698. + DEBUG_CHECK_INPUT_STREAM_PARAMETERS
  2699. + return 0; // OK
  2700. +}
  2701. +
  2702. +/*! @abstract Pull n bits out of the fse stream object. */
  2703. +FSE_INLINE uint64_t fse_in_pull64(fse_in_stream64 *s, fse_bit_count n) {
  2704. + assert(n >= 0 && n <= s->accum_nbits);
  2705. + s->accum_nbits -= n;
  2706. + uint64_t result = s->accum >> s->accum_nbits;
  2707. + s->accum = fse_mask_lsb64(s->accum, s->accum_nbits);
  2708. + return result;
  2709. +}
  2710. +
  2711. +/*! @abstract Pull n bits out of the fse stream object. */
  2712. +FSE_INLINE uint32_t fse_in_pull32(fse_in_stream32 *s, fse_bit_count n) {
  2713. + assert(n >= 0 && n <= s->accum_nbits);
  2714. + s->accum_nbits -= n;
  2715. + uint32_t result = s->accum >> s->accum_nbits;
  2716. + s->accum = fse_mask_lsb32(s->accum, s->accum_nbits);
  2717. + return result;
  2718. +}
  2719. +
  2720. +// MARK: - Encode/Decode
  2721. +
  2722. +// Map to 32/64-bit implementations and types for I/O
  2723. +#if FSE_IOSTREAM_64
  2724. +
  2725. +typedef uint64_t fse_bits;
  2726. +typedef fse_out_stream64 fse_out_stream;
  2727. +typedef fse_in_stream64 fse_in_stream;
  2728. +#define fse_mask_lsb fse_mask_lsb64
  2729. +#define fse_extract_bits fse_extract_bits64
  2730. +#define fse_out_init fse_out_init64
  2731. +#define fse_out_flush fse_out_flush64
  2732. +#define fse_out_finish fse_out_finish64
  2733. +#define fse_out_push fse_out_push64
  2734. +#define fse_in_init fse_in_checked_init64
  2735. +#define fse_in_checked_init fse_in_checked_init64
  2736. +#define fse_in_flush fse_in_checked_flush64
  2737. +#define fse_in_checked_flush fse_in_checked_flush64
  2738. +#define fse_in_flush2(_unused, _parameters, _unused2) 0 /* nothing */
  2739. +#define fse_in_checked_flush2(_unused, _parameters) /* nothing */
  2740. +#define fse_in_pull fse_in_pull64
  2741. +
  2742. +#else
  2743. +
  2744. +typedef uint32_t fse_bits;
  2745. +typedef fse_out_stream32 fse_out_stream;
  2746. +typedef fse_in_stream32 fse_in_stream;
  2747. +#define fse_mask_lsb fse_mask_lsb32
  2748. +#define fse_extract_bits fse_extract_bits32
  2749. +#define fse_out_init fse_out_init32
  2750. +#define fse_out_flush fse_out_flush32
  2751. +#define fse_out_finish fse_out_finish32
  2752. +#define fse_out_push fse_out_push32
  2753. +#define fse_in_init fse_in_checked_init32
  2754. +#define fse_in_checked_init fse_in_checked_init32
  2755. +#define fse_in_flush fse_in_checked_flush32
  2756. +#define fse_in_checked_flush fse_in_checked_flush32
  2757. +#define fse_in_flush2 fse_in_checked_flush32
  2758. +#define fse_in_checked_flush2 fse_in_checked_flush32
  2759. +#define fse_in_pull fse_in_pull32
  2760. +
  2761. +#endif
  2762. +
  2763. +/*! @abstract Entry for one symbol in the encoder table (64b). */
  2764. +typedef struct {
  2765. + int16_t s0; // First state requiring a K-bit shift
  2766. + int16_t k; // States S >= S0 are shifted K bits. States S < S0 are
  2767. + // shifted K-1 bits
  2768. + int16_t delta0; // Relative increment used to compute next state if S >= S0
  2769. + int16_t delta1; // Relative increment used to compute next state if S < S0
  2770. +} fse_encoder_entry;
  2771. +
  2772. +/*! @abstract Entry for one state in the decoder table (32b). */
  2773. +typedef struct { // DO NOT REORDER THE FIELDS
  2774. + int8_t k; // Number of bits to read
  2775. + uint8_t symbol; // Emitted symbol
  2776. + int16_t delta; // Signed increment used to compute next state (+bias)
  2777. +} fse_decoder_entry;
  2778. +
  2779. +/*! @abstract Entry for one state in the value decoder table (64b). */
  2780. +typedef struct { // DO NOT REORDER THE FIELDS
  2781. + uint8_t total_bits; // state bits + extra value bits = shift for next decode
  2782. + uint8_t value_bits; // extra value bits
  2783. + int16_t delta; // state base (delta)
  2784. + int32_t vbase; // value base
  2785. +} fse_value_decoder_entry;
  2786. +
  2787. +/*! @abstract Encode SYMBOL using the encoder table, and update \c *pstate,
  2788. + * \c out.
  2789. + * @note The caller must ensure we have enough bits available in the output
  2790. + * stream accumulator. */
  2791. +FSE_INLINE void fse_encode(fse_state *__restrict pstate,
  2792. + const fse_encoder_entry *__restrict encoder_table,
  2793. + fse_out_stream *__restrict out, uint8_t symbol) {
  2794. + int s = *pstate;
  2795. + fse_encoder_entry e = encoder_table[symbol];
  2796. + int s0 = e.s0;
  2797. + int k = e.k;
  2798. + int delta0 = e.delta0;
  2799. + int delta1 = e.delta1;
  2800. +
  2801. + // Number of bits to write
  2802. + int hi = s >= s0;
  2803. + fse_bit_count nbits = hi ? k : (k - 1);
  2804. + fse_state delta = hi ? delta0 : delta1;
  2805. +
  2806. + // Write lower NBITS of state
  2807. + fse_bits b = fse_mask_lsb(s, nbits);
  2808. + fse_out_push(out, nbits, b);
  2809. +
  2810. + // Update state with remaining bits and delta
  2811. + *pstate = delta + (s >> nbits);
  2812. +}
  2813. +
  2814. +/*! @abstract Decode and return symbol using the decoder table, and update
  2815. + * \c *pstate, \c in.
  2816. + * @note The caller must ensure we have enough bits available in the input
  2817. + * stream accumulator. */
  2818. +FSE_INLINE uint8_t fse_decode(fse_state *__restrict pstate,
  2819. + const int32_t *__restrict decoder_table,
  2820. + fse_in_stream *__restrict in) {
  2821. + int32_t e = decoder_table[*pstate];
  2822. +
  2823. + // Update state from K bits of input + DELTA
  2824. + *pstate = (fse_state)(e >> 16) + (fse_state)fse_in_pull(in, e & 0xff);
  2825. +
  2826. + // Return the symbol for this state
  2827. + return fse_extract_bits(e, 8, 8); // symbol
  2828. +}
  2829. +
  2830. +/*! @abstract Decode and return value using the decoder table, and update \c
  2831. + * *pstate, \c in.
  2832. + * \c value_decoder_table[nstates]
  2833. + * @note The caller must ensure we have enough bits available in the input
  2834. + * stream accumulator. */
  2835. +FSE_INLINE int32_t
  2836. +fse_value_decode(fse_state *__restrict pstate,
  2837. + const fse_value_decoder_entry *value_decoder_table,
  2838. + fse_in_stream *__restrict in) {
  2839. + fse_value_decoder_entry entry = value_decoder_table[*pstate];
  2840. + uint32_t state_and_value_bits = (uint32_t)fse_in_pull(in, entry.total_bits);
  2841. + *pstate =
  2842. + (fse_state)(entry.delta + (state_and_value_bits >> entry.value_bits));
  2843. + return (int32_t)(entry.vbase +
  2844. + fse_mask_lsb(state_and_value_bits, entry.value_bits));
  2845. +}
  2846. +
  2847. +// MARK: - Tables
  2848. +
  2849. +// IMPORTANT: To properly decode an FSE encoded stream, both encoder/decoder
  2850. +// tables shall be initialized with the same parameters, including the
  2851. +// FREQ[NSYMBOL] array.
  2852. +//
  2853. +
  2854. +/*! @abstract Sanity check on frequency table, verify sum of \c freq
  2855. + * is <= \c number_of_states. */
  2856. +FSE_INLINE int fse_check_freq(const uint16_t *freq_table,
  2857. + const size_t table_size,
  2858. + const size_t number_of_states) {
  2859. + size_t sum_of_freq = 0;
  2860. + for (int i = 0; i < table_size; i++) {
  2861. + sum_of_freq += freq_table[i];
  2862. + }
  2863. + return (sum_of_freq > number_of_states) ? -1 : 0;
  2864. +}
  2865. +
  2866. +/*! @abstract Initialize encoder table \c t[nsymbols].
  2867. + *
  2868. + * @param nstates
  2869. + * sum \c freq[i]; the number of states (a power of 2).
  2870. + *
  2871. + * @param nsymbols
  2872. + * the number of symbols.
  2873. + *
  2874. + * @param freq[nsymbols]
  2875. + * is a normalized histogram of symbol frequencies, with \c freq[i] >= 0.
  2876. + * Some symbols may have a 0 frequency. In that case they should not be
  2877. + * present in the data.
  2878. + */
  2879. +void fse_init_encoder_table(int nstates, int nsymbols,
  2880. + const uint16_t *__restrict freq,
  2881. + fse_encoder_entry *__restrict t);
  2882. +
  2883. +/*! @abstract Initialize decoder table \c t[nstates].
  2884. + *
  2885. + * @param nstates
  2886. + * sum \c freq[i]; the number of states (a power of 2).
  2887. + *
  2888. + * @param nsymbols
  2889. + * the number of symbols.
  2890. + *
  2891. + * @param feq[nsymbols]
  2892. + * a normalized histogram of symbol frequencies, with \c freq[i] >= 0.
  2893. + * Some symbols may have a 0 frequency. In that case they should not be
  2894. + * present in the data.
  2895. + *
  2896. + * @return 0 if OK.
  2897. + * @return -1 on failure.
  2898. + */
  2899. +int fse_init_decoder_table(int nstates, int nsymbols,
  2900. + const uint16_t *__restrict freq,
  2901. + int32_t *__restrict t);
  2902. +
  2903. +/*! @abstract Initialize value decoder table \c t[nstates].
  2904. + *
  2905. + * @param nstates
  2906. + * sum \cfreq[i]; the number of states (a power of 2).
  2907. + *
  2908. + * @param nsymbols
  2909. + * the number of symbols.
  2910. + *
  2911. + * @param freq[nsymbols]
  2912. + * a normalized histogram of symbol frequencies, with \c freq[i] >= 0.
  2913. + * \c symbol_vbits[nsymbols] and \c symbol_vbase[nsymbols] are the number of
  2914. + * value bits to read and the base value for each symbol.
  2915. + * Some symbols may have a 0 frequency. In that case they should not be
  2916. + * present in the data.
  2917. + */
  2918. +void fse_init_value_decoder_table(int nstates, int nsymbols,
  2919. + const uint16_t *__restrict freq,
  2920. + const uint8_t *__restrict symbol_vbits,
  2921. + const int32_t *__restrict symbol_vbase,
  2922. + fse_value_decoder_entry *__restrict t);
  2923. +
  2924. +/*! @abstract Normalize a table \c t[nsymbols] of occurrences to
  2925. + * \c freq[nsymbols]. */
  2926. +void fse_normalize_freq(int nstates, int nsymbols, const uint32_t *__restrict t,
  2927. + uint16_t *__restrict freq);
  2928. diff --git a/lib/lzfse/lzfse_internal.h b/lib/lzfse/lzfse_internal.h
  2929. new file mode 100644
  2930. index 0000000..7ea5a30
  2931. --- /dev/null
  2932. +++ b/lib/lzfse/lzfse_internal.h
  2933. @@ -0,0 +1,616 @@
  2934. +/*
  2935. +Copyright (c) 2015-2016, Apple Inc. All rights reserved.
  2936. +
  2937. +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
  2938. +
  2939. +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
  2940. +
  2941. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
  2942. + in the documentation and/or other materials provided with the distribution.
  2943. +
  2944. +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
  2945. + from this software without specific prior written permission.
  2946. +
  2947. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  2948. +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
  2949. +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  2950. +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  2951. +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  2952. +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  2953. +*/
  2954. +
  2955. +#ifndef LZFSE_INTERNAL_H
  2956. +#define LZFSE_INTERNAL_H
  2957. +
  2958. +// Unlike the tunable parameters defined in lzfse_tunables.h, you probably
  2959. +// should not modify the values defined in this header. Doing so will either
  2960. +// break the compressor, or result in a compressed data format that is
  2961. +// incompatible.
  2962. +
  2963. +#include "lzfse_fse.h"
  2964. +#include "lzfse_tunables.h"
  2965. +#include <limits.h>
  2966. +#include <stddef.h>
  2967. +#include <stdint.h>
  2968. +
  2969. +#if defined(_MSC_VER) && !defined(__clang__)
  2970. +# define LZFSE_INLINE __forceinline
  2971. +# define __builtin_expect(X, Y) (X)
  2972. +# define __attribute__(X)
  2973. +# pragma warning(disable : 4068) // warning C4068: unknown pragma
  2974. +#else
  2975. +# define LZFSE_INLINE static inline __attribute__((__always_inline__))
  2976. +#endif
  2977. +
  2978. +// Implement GCC bit scan builtins for MSVC
  2979. +#if defined(_MSC_VER) && !defined(__clang__)
  2980. +#include <intrin.h>
  2981. +
  2982. +LZFSE_INLINE int __builtin_clz(unsigned int val) {
  2983. + unsigned long r = 0;
  2984. + if (_BitScanReverse(&r, val)) {
  2985. + return 31 - r;
  2986. + }
  2987. + return 32;
  2988. +}
  2989. +
  2990. +LZFSE_INLINE int __builtin_ctzl(unsigned long val) {
  2991. + unsigned long r = 0;
  2992. + if (_BitScanForward(&r, val)) {
  2993. + return r;
  2994. + }
  2995. + return 32;
  2996. +}
  2997. +
  2998. +LZFSE_INLINE int __builtin_ctzll(uint64_t val) {
  2999. + unsigned long r = 0;
  3000. +#if defined(_M_AMD64) || defined(_M_ARM)
  3001. + if (_BitScanForward64(&r, val)) {
  3002. + return r;
  3003. + }
  3004. +#else
  3005. + if (_BitScanForward(&r, (uint32_t)val)) {
  3006. + return r;
  3007. + }
  3008. + if (_BitScanForward(&r, (uint32_t)(val >> 32))) {
  3009. + return 32 + r;
  3010. + }
  3011. +#endif
  3012. + return 64;
  3013. +}
  3014. +#endif
  3015. +
  3016. +// Throughout LZFSE we refer to "L", "M" and "D"; these will always appear as
  3017. +// a triplet, and represent a "usual" LZ-style literal and match pair. "L"
  3018. +// is the number of literal bytes, "M" is the number of match bytes, and "D"
  3019. +// is the match "distance"; the distance in bytes between the current pointer
  3020. +// and the start of the match.
  3021. +#define LZFSE_ENCODE_HASH_VALUES (1 << LZFSE_ENCODE_HASH_BITS)
  3022. +#define LZFSE_ENCODE_L_SYMBOLS 20
  3023. +#define LZFSE_ENCODE_M_SYMBOLS 20
  3024. +#define LZFSE_ENCODE_D_SYMBOLS 64
  3025. +#define LZFSE_ENCODE_LITERAL_SYMBOLS 256
  3026. +#define LZFSE_ENCODE_L_STATES 64
  3027. +#define LZFSE_ENCODE_M_STATES 64
  3028. +#define LZFSE_ENCODE_D_STATES 256
  3029. +#define LZFSE_ENCODE_LITERAL_STATES 1024
  3030. +#define LZFSE_MATCHES_PER_BLOCK 10000
  3031. +#define LZFSE_LITERALS_PER_BLOCK (4 * LZFSE_MATCHES_PER_BLOCK)
  3032. +#define LZFSE_DECODE_LITERALS_PER_BLOCK (4 * LZFSE_DECODE_MATCHES_PER_BLOCK)
  3033. +
  3034. +// LZFSE internal status. These values are used by internal LZFSE routines
  3035. +// as return codes. There should not be any good reason to change their
  3036. +// values; it is plausible that additional codes might be added in the
  3037. +// future.
  3038. +#define LZFSE_STATUS_OK 0
  3039. +#define LZFSE_STATUS_SRC_EMPTY -1
  3040. +#define LZFSE_STATUS_DST_FULL -2
  3041. +#define LZFSE_STATUS_ERROR -3
  3042. +
  3043. +// Type representing an offset between elements in a buffer. On 64-bit
  3044. +// systems, this is stored in a 64-bit container to avoid extra sign-
  3045. +// extension operations in addressing arithmetic, but the value is always
  3046. +// representable as a 32-bit signed value in LZFSE's usage.
  3047. +#if defined(_M_AMD64) || defined(__x86_64__) || defined(__arm64__)
  3048. +typedef int64_t lzfse_offset;
  3049. +#else
  3050. +typedef int32_t lzfse_offset;
  3051. +#endif
  3052. +
  3053. +/*! @abstract History table set. Each line of the history table represents a set
  3054. + * of candidate match locations, each of which begins with four bytes with the
  3055. + * same hash. The table contains not only the positions, but also the first
  3056. + * four bytes at each position. This doubles the memory footprint of the
  3057. + * table, but allows us to quickly eliminate false-positive matches without
  3058. + * doing any pointer chasing and without pulling in any additional cachelines.
  3059. + * This provides a large performance win in practice. */
  3060. +typedef struct {
  3061. + int32_t pos[LZFSE_ENCODE_HASH_WIDTH];
  3062. + uint32_t value[LZFSE_ENCODE_HASH_WIDTH];
  3063. +} lzfse_history_set;
  3064. +
  3065. +/*! @abstract An lzfse match is a sequence of bytes in the source buffer that
  3066. + * exactly matches an earlier (but possibly overlapping) sequence of bytes in
  3067. + * the same buffer.
  3068. + * @code
  3069. + * exeMPLARYexaMPLE
  3070. + * | | | ||-|--- lzfse_match2.length=3
  3071. + * | | | ||----- lzfse_match2.pos
  3072. + * | | |-|------ lzfse_match1.length=3
  3073. + * | | |-------- lzfse_match1.pos
  3074. + * | |-------------- lzfse_match2.ref
  3075. + * |----------------- lzfse_match1.ref
  3076. + * @endcode
  3077. + */
  3078. +typedef struct {
  3079. + // Offset of the first byte in the match.
  3080. + lzfse_offset pos;
  3081. + // First byte of the source -- the earlier location in the buffer with the
  3082. + // same contents.
  3083. + lzfse_offset ref;
  3084. + // Length of the match.
  3085. + uint32_t length;
  3086. +} lzfse_match;
  3087. +
  3088. +// MARK: - Encoder and Decoder state objects
  3089. +
  3090. +/*! @abstract Encoder state object. */
  3091. +typedef struct {
  3092. + // Pointer to first byte of the source buffer.
  3093. + const uint8_t *src;
  3094. + // Length of the source buffer in bytes. Note that this is not a size_t,
  3095. + // but rather lzfse_offset, which is a signed type. The largest
  3096. + // representable buffer is 2GB, but arbitrarily large buffers may be
  3097. + // handled by repeatedly calling the encoder function and "translating"
  3098. + // the state between calls. When doing this, it is beneficial to use
  3099. + // blocks smaller than 2GB in order to maintain residency in the last-level
  3100. + // cache. Consult the implementation of lzfse_encode_buffer for details.
  3101. + lzfse_offset src_end;
  3102. + // Offset of the first byte of the next literal to encode in the source
  3103. + // buffer.
  3104. + lzfse_offset src_literal;
  3105. + // Offset of the byte currently being checked for a match.
  3106. + lzfse_offset src_encode_i;
  3107. + // The last byte offset to consider for a match. In some uses it makes
  3108. + // sense to use a smaller offset than src_end.
  3109. + lzfse_offset src_encode_end;
  3110. + // Pointer to the next byte to be written in the destination buffer.
  3111. + uint8_t *dst;
  3112. + // Pointer to the first byte of the destination buffer.
  3113. + uint8_t *dst_begin;
  3114. + // Pointer to one byte past the end of the destination buffer.
  3115. + uint8_t *dst_end;
  3116. + // Pending match; will be emitted unless a better match is found.
  3117. + lzfse_match pending;
  3118. + // The number of matches written so far. Note that there is no problem in
  3119. + // using a 32-bit field for this quantity, because the state already limits
  3120. + // us to at most 2GB of data; there cannot possibly be more matches than
  3121. + // there are bytes in the input.
  3122. + uint32_t n_matches;
  3123. + // The number of literals written so far.
  3124. + uint32_t n_literals;
  3125. + // Lengths of found literals.
  3126. + uint32_t l_values[LZFSE_MATCHES_PER_BLOCK];
  3127. + // Lengths of found matches.
  3128. + uint32_t m_values[LZFSE_MATCHES_PER_BLOCK];
  3129. + // Distances of found matches.
  3130. + uint32_t d_values[LZFSE_MATCHES_PER_BLOCK];
  3131. + // Concatenated literal bytes.
  3132. + uint8_t literals[LZFSE_LITERALS_PER_BLOCK];
  3133. + // History table used to search for matches. Each entry of the table
  3134. + // corresponds to a group of four byte sequences in the input stream
  3135. + // that hash to the same value.
  3136. + lzfse_history_set history_table[LZFSE_ENCODE_HASH_VALUES];
  3137. +} lzfse_encoder_state;
  3138. +
  3139. +/*! @abstract Decoder state object for lzfse compressed blocks. */
  3140. +typedef struct {
  3141. + // Number of matches remaining in the block.
  3142. + uint32_t n_matches;
  3143. + // Number of bytes used to encode L, M, D triplets for the block.
  3144. + uint32_t n_lmd_payload_bytes;
  3145. + // Pointer to the next literal to emit.
  3146. + const uint8_t *current_literal;
  3147. + // L, M, D triplet for the match currently being emitted. This is used only
  3148. + // if we need to restart after reaching the end of the destination buffer in
  3149. + // the middle of a literal or match.
  3150. + int32_t l_value, m_value, d_value;
  3151. + // FSE stream object.
  3152. + fse_in_stream lmd_in_stream;
  3153. + // Offset of L,M,D encoding in the input buffer. Because we read through an
  3154. + // FSE stream *backwards* while decoding, this is decremented as we move
  3155. + // through a block.
  3156. + uint32_t lmd_in_buf;
  3157. + // The current state of the L, M, and D FSE decoders.
  3158. + uint16_t l_state, m_state, d_state;
  3159. + // Internal FSE decoder tables for the current block. These have
  3160. + // alignment forced to 8 bytes to guarantee that a single state's
  3161. + // entry cannot span two cachelines.
  3162. + fse_value_decoder_entry l_decoder[LZFSE_ENCODE_L_STATES] __attribute__((__aligned__(8)));
  3163. + fse_value_decoder_entry m_decoder[LZFSE_ENCODE_M_STATES] __attribute__((__aligned__(8)));
  3164. + fse_value_decoder_entry d_decoder[LZFSE_ENCODE_D_STATES] __attribute__((__aligned__(8)));
  3165. + int32_t literal_decoder[LZFSE_ENCODE_LITERAL_STATES];
  3166. + // The literal stream for the block, plus padding to allow for faster copy
  3167. + // operations.
  3168. + uint8_t literals[LZFSE_LITERALS_PER_BLOCK + 64];
  3169. +} lzfse_compressed_block_decoder_state;
  3170. +
  3171. +// Decoder state object for uncompressed blocks.
  3172. +typedef struct { uint32_t n_raw_bytes; } uncompressed_block_decoder_state;
  3173. +
  3174. +/*! @abstract Decoder state object for lzvn-compressed blocks. */
  3175. +typedef struct {
  3176. + uint32_t n_raw_bytes;
  3177. + uint32_t n_payload_bytes;
  3178. + uint32_t d_prev;
  3179. +} lzvn_compressed_block_decoder_state;
  3180. +
  3181. +/*! @abstract Decoder state object. */
  3182. +typedef struct {
  3183. + // Pointer to next byte to read from source buffer (this is advanced as we
  3184. + // decode; src_begin describe the buffer and do not change).
  3185. + const uint8_t *src;
  3186. + // Pointer to first byte of source buffer.
  3187. + const uint8_t *src_begin;
  3188. + // Pointer to one byte past the end of the source buffer.
  3189. + const uint8_t *src_end;
  3190. + // Pointer to the next byte to write to destination buffer (this is advanced
  3191. + // as we decode; dst_begin and dst_end describe the buffer and do not change).
  3192. + uint8_t *dst;
  3193. + // Pointer to first byte of destination buffer.
  3194. + uint8_t *dst_begin;
  3195. + // Pointer to one byte past the end of the destination buffer.
  3196. + uint8_t *dst_end;
  3197. + // 1 if we have reached the end of the stream, 0 otherwise.
  3198. + int end_of_stream;
  3199. + // magic number of the current block if we are within a block,
  3200. + // LZFSE_NO_BLOCK_MAGIC otherwise.
  3201. + uint32_t block_magic;
  3202. + lzfse_compressed_block_decoder_state compressed_lzfse_block_state;
  3203. + lzvn_compressed_block_decoder_state compressed_lzvn_block_state;
  3204. + uncompressed_block_decoder_state uncompressed_block_state;
  3205. +} lzfse_decoder_state;
  3206. +
  3207. +// MARK: - Block header objects
  3208. +
  3209. +#define LZFSE_NO_BLOCK_MAGIC 0x00000000 // 0 (invalid)
  3210. +#define LZFSE_ENDOFSTREAM_BLOCK_MAGIC 0x24787662 // bvx$ (end of stream)
  3211. +#define LZFSE_UNCOMPRESSED_BLOCK_MAGIC 0x2d787662 // bvx- (raw data)
  3212. +#define LZFSE_COMPRESSEDV1_BLOCK_MAGIC 0x31787662 // bvx1 (lzfse compressed, uncompressed tables)
  3213. +#define LZFSE_COMPRESSEDV2_BLOCK_MAGIC 0x32787662 // bvx2 (lzfse compressed, compressed tables)
  3214. +#define LZFSE_COMPRESSEDLZVN_BLOCK_MAGIC 0x6e787662 // bvxn (lzvn compressed)
  3215. +
  3216. +/*! @abstract Uncompressed block header in encoder stream. */
  3217. +typedef struct {
  3218. + // Magic number, always LZFSE_UNCOMPRESSED_BLOCK_MAGIC.
  3219. + uint32_t magic;
  3220. + // Number of raw bytes in block.
  3221. + uint32_t n_raw_bytes;
  3222. +} uncompressed_block_header;
  3223. +
  3224. +/*! @abstract Compressed block header with uncompressed tables. */
  3225. +typedef struct {
  3226. + // Magic number, always LZFSE_COMPRESSEDV1_BLOCK_MAGIC.
  3227. + uint32_t magic;
  3228. + // Number of decoded (output) bytes in block.
  3229. + uint32_t n_raw_bytes;
  3230. + // Number of encoded (source) bytes in block.
  3231. + uint32_t n_payload_bytes;
  3232. + // Number of literal bytes output by block (*not* the number of literals).
  3233. + uint32_t n_literals;
  3234. + // Number of matches in block (which is also the number of literals).
  3235. + uint32_t n_matches;
  3236. + // Number of bytes used to encode literals.
  3237. + uint32_t n_literal_payload_bytes;
  3238. + // Number of bytes used to encode matches.
  3239. + uint32_t n_lmd_payload_bytes;
  3240. +
  3241. + // Final encoder states for the block, which will be the initial states for
  3242. + // the decoder:
  3243. + // Final accum_nbits for literals stream.
  3244. + int32_t literal_bits;
  3245. + // There are four interleaved streams of literals, so there are four final
  3246. + // states.
  3247. + uint16_t literal_state[4];
  3248. + // accum_nbits for the l, m, d stream.
  3249. + int32_t lmd_bits;
  3250. + // Final L (literal length) state.
  3251. + uint16_t l_state;
  3252. + // Final M (match length) state.
  3253. + uint16_t m_state;
  3254. + // Final D (match distance) state.
  3255. + uint16_t d_state;
  3256. +
  3257. + // Normalized frequency tables for each stream. Sum of values in each
  3258. + // array is the number of states.
  3259. + uint16_t l_freq[LZFSE_ENCODE_L_SYMBOLS];
  3260. + uint16_t m_freq[LZFSE_ENCODE_M_SYMBOLS];
  3261. + uint16_t d_freq[LZFSE_ENCODE_D_SYMBOLS];
  3262. + uint16_t literal_freq[LZFSE_ENCODE_LITERAL_SYMBOLS];
  3263. +} lzfse_compressed_block_header_v1;
  3264. +
  3265. +/*! @abstract Compressed block header with compressed tables. Note that because
  3266. + * freq[] is compressed, the structure-as-stored-in-the-stream is *truncated*;
  3267. + * we only store the used bytes of freq[]. This means that some extra care must
  3268. + * be taken when reading one of these headers from the stream. */
  3269. +typedef struct {
  3270. + // Magic number, always LZFSE_COMPRESSEDV2_BLOCK_MAGIC.
  3271. + uint32_t magic;
  3272. + // Number of decoded (output) bytes in block.
  3273. + uint32_t n_raw_bytes;
  3274. + // The fields n_payload_bytes ... d_state from the
  3275. + // lzfse_compressed_block_header_v1 object are packed into three 64-bit
  3276. + // fields in the compressed header, as follows:
  3277. + //
  3278. + // offset bits value
  3279. + // 0 20 n_literals
  3280. + // 20 20 n_literal_payload_bytes
  3281. + // 40 20 n_matches
  3282. + // 60 3 literal_bits
  3283. + // 63 1 --- unused ---
  3284. + //
  3285. + // 0 10 literal_state[0]
  3286. + // 10 10 literal_state[1]
  3287. + // 20 10 literal_state[2]
  3288. + // 30 10 literal_state[3]
  3289. + // 40 20 n_lmd_payload_bytes
  3290. + // 60 3 lmd_bits
  3291. + // 63 1 --- unused ---
  3292. + //
  3293. + // 0 32 header_size (total header size in bytes; this does not
  3294. + // correspond to a field in the uncompressed header version,
  3295. + // but is required; we wouldn't know the size of the
  3296. + // compresssed header otherwise.
  3297. + // 32 10 l_state
  3298. + // 42 10 m_state
  3299. + // 52 10 d_state
  3300. + // 62 2 --- unused ---
  3301. + uint64_t packed_fields[3];
  3302. + // Variable size freq tables, using a Huffman-style fixed encoding.
  3303. + // Size allocated here is an upper bound (all values stored on 16 bits).
  3304. + uint8_t freq[2 * (LZFSE_ENCODE_L_SYMBOLS + LZFSE_ENCODE_M_SYMBOLS +
  3305. + LZFSE_ENCODE_D_SYMBOLS + LZFSE_ENCODE_LITERAL_SYMBOLS)];
  3306. +} __attribute__((__packed__, __aligned__(1)))
  3307. +lzfse_compressed_block_header_v2;
  3308. +
  3309. +/*! @abstract LZVN compressed block header. */
  3310. +typedef struct {
  3311. + // Magic number, always LZFSE_COMPRESSEDLZVN_BLOCK_MAGIC.
  3312. + uint32_t magic;
  3313. + // Number of decoded (output) bytes.
  3314. + uint32_t n_raw_bytes;
  3315. + // Number of encoded (source) bytes.
  3316. + uint32_t n_payload_bytes;
  3317. +} lzvn_compressed_block_header;
  3318. +
  3319. +// MARK: - LZFSE encode/decode interfaces
  3320. +
  3321. +int lzfse_encode_init(lzfse_encoder_state *s);
  3322. +int lzfse_encode_translate(lzfse_encoder_state *s, lzfse_offset delta);
  3323. +int lzfse_encode_base(lzfse_encoder_state *s);
  3324. +int lzfse_encode_finish(lzfse_encoder_state *s);
  3325. +int lzfse_decode(lzfse_decoder_state *s);
  3326. +
  3327. +// MARK: - LZVN encode/decode interfaces
  3328. +
  3329. +// Minimum source buffer size for compression. Smaller buffers will not be
  3330. +// compressed; the lzvn encoder will simply return.
  3331. +#define LZVN_ENCODE_MIN_SRC_SIZE ((size_t)8)
  3332. +
  3333. +// Maximum source buffer size for compression. Larger buffers will be
  3334. +// compressed partially.
  3335. +#define LZVN_ENCODE_MAX_SRC_SIZE ((size_t)0xffffffffU)
  3336. +
  3337. +// Minimum destination buffer size for compression. No compression will take
  3338. +// place if smaller.
  3339. +#define LZVN_ENCODE_MIN_DST_SIZE ((size_t)8)
  3340. +
  3341. +size_t lzvn_decode_scratch_size(void);
  3342. +size_t lzvn_encode_scratch_size(void);
  3343. +size_t lzvn_encode_buffer(void *__restrict dst, size_t dst_size,
  3344. + const void *__restrict src, size_t src_size,
  3345. + void *__restrict work);
  3346. +size_t lzvn_decode_buffer(void *__restrict dst, size_t dst_size,
  3347. + const void *__restrict src, size_t src_size);
  3348. +
  3349. +/*! @abstract Signed offset in buffers, stored on either 32 or 64 bits. */
  3350. +#if defined(_M_AMD64) || defined(__x86_64__) || defined(__arm64__)
  3351. +typedef int64_t lzvn_offset;
  3352. +#else
  3353. +typedef int32_t lzvn_offset;
  3354. +#endif
  3355. +
  3356. +// MARK: - LZFSE utility functions
  3357. +
  3358. +/*! @abstract Load bytes from memory location SRC. */
  3359. +LZFSE_INLINE uint16_t load2(const void *ptr) {
  3360. + uint16_t data;
  3361. + memcpy(&data, ptr, sizeof data);
  3362. + return data;
  3363. +}
  3364. +
  3365. +LZFSE_INLINE uint32_t load4(const void *ptr) {
  3366. + uint32_t data;
  3367. + memcpy(&data, ptr, sizeof data);
  3368. + return data;
  3369. +}
  3370. +
  3371. +LZFSE_INLINE uint64_t load8(const void *ptr) {
  3372. + uint64_t data;
  3373. + memcpy(&data, ptr, sizeof data);
  3374. + return data;
  3375. +}
  3376. +
  3377. +/*! @abstract Store bytes to memory location DST. */
  3378. +LZFSE_INLINE void store2(void *ptr, uint16_t data) {
  3379. + memcpy(ptr, &data, sizeof data);
  3380. +}
  3381. +
  3382. +LZFSE_INLINE void store4(void *ptr, uint32_t data) {
  3383. + memcpy(ptr, &data, sizeof data);
  3384. +}
  3385. +
  3386. +LZFSE_INLINE void store8(void *ptr, uint64_t data) {
  3387. + memcpy(ptr, &data, sizeof data);
  3388. +}
  3389. +
  3390. +/*! @abstract Load+store bytes from locations SRC to DST. Not intended for use
  3391. + * with overlapping buffers. Note that for LZ-style compression, you need
  3392. + * copies to behave like naive memcpy( ) implementations do, splatting the
  3393. + * leading sequence if the buffers overlap. This copy does not do that, so
  3394. + * should not be used with overlapping buffers. */
  3395. +LZFSE_INLINE void copy8(void *dst, const void *src) { store8(dst, load8(src)); }
  3396. +LZFSE_INLINE void copy16(void *dst, const void *src) {
  3397. + uint64_t m0 = load8(src);
  3398. + uint64_t m1 = load8((const unsigned char *)src + 8);
  3399. + store8(dst, m0);
  3400. + store8((unsigned char *)dst + 8, m1);
  3401. +}
  3402. +
  3403. +// ===============================================================
  3404. +// Bitfield Operations
  3405. +
  3406. +/*! @abstract Extracts \p width bits from \p container, starting with \p lsb; if
  3407. + * we view \p container as a bit array, we extract \c container[lsb:lsb+width]. */
  3408. +LZFSE_INLINE uintmax_t extract(uintmax_t container, unsigned lsb,
  3409. + unsigned width) {
  3410. + static const size_t container_width = sizeof container * 8;
  3411. + assert(lsb < container_width);
  3412. + assert(width > 0 && width <= container_width);
  3413. + assert(lsb + width <= container_width);
  3414. + if (width == container_width)
  3415. + return container;
  3416. + return (container >> lsb) & (((uintmax_t)1 << width) - 1);
  3417. +}
  3418. +
  3419. +/*! @abstract Inserts \p width bits from \p data into \p container, starting with \p lsb.
  3420. + * Viewed as bit arrays, the operations is:
  3421. + * @code
  3422. + * container[:lsb] is unchanged
  3423. + * container[lsb:lsb+width] <-- data[0:width]
  3424. + * container[lsb+width:] is unchanged
  3425. + * @endcode
  3426. + */
  3427. +LZFSE_INLINE uintmax_t insert(uintmax_t container, uintmax_t data, unsigned lsb,
  3428. + unsigned width) {
  3429. + static const size_t container_width = sizeof container * 8;
  3430. + assert(lsb < container_width);
  3431. + assert(width > 0 && width <= container_width);
  3432. + assert(lsb + width <= container_width);
  3433. + if (width == container_width)
  3434. + return container;
  3435. + uintmax_t mask = ((uintmax_t)1 << width) - 1;
  3436. + return (container & ~(mask << lsb)) | (data & mask) << lsb;
  3437. +}
  3438. +
  3439. +/*! @abstract Perform sanity checks on the values of lzfse_compressed_block_header_v1.
  3440. + * Test that the field values are in the allowed limits, verify that the
  3441. + * frequency tables sum to value less than total number of states.
  3442. + * @return 0 if all tests passed.
  3443. + * @return negative error code with 1 bit set for each failed test. */
  3444. +LZFSE_INLINE int lzfse_check_block_header_v1(
  3445. + const lzfse_compressed_block_header_v1 *header) {
  3446. + int tests_results = 0;
  3447. + tests_results =
  3448. + tests_results |
  3449. + ((header->magic == LZFSE_COMPRESSEDV1_BLOCK_MAGIC) ? 0 : (1 << 0));
  3450. + tests_results =
  3451. + tests_results |
  3452. + ((header->n_literals <= LZFSE_LITERALS_PER_BLOCK) ? 0 : (1 << 1));
  3453. + tests_results =
  3454. + tests_results |
  3455. + ((header->n_matches <= LZFSE_MATCHES_PER_BLOCK) ? 0 : (1 << 2));
  3456. +
  3457. + uint16_t literal_state[4];
  3458. + memcpy(literal_state, header->literal_state, sizeof(uint16_t) * 4);
  3459. +
  3460. + tests_results =
  3461. + tests_results |
  3462. + ((literal_state[0] < LZFSE_ENCODE_LITERAL_STATES) ? 0 : (1 << 3));
  3463. + tests_results =
  3464. + tests_results |
  3465. + ((literal_state[1] < LZFSE_ENCODE_LITERAL_STATES) ? 0 : (1 << 4));
  3466. + tests_results =
  3467. + tests_results |
  3468. + ((literal_state[2] < LZFSE_ENCODE_LITERAL_STATES) ? 0 : (1 << 5));
  3469. + tests_results =
  3470. + tests_results |
  3471. + ((literal_state[3] < LZFSE_ENCODE_LITERAL_STATES) ? 0 : (1 << 6));
  3472. +
  3473. + tests_results = tests_results |
  3474. + ((header->l_state < LZFSE_ENCODE_L_STATES) ? 0 : (1 << 7));
  3475. + tests_results = tests_results |
  3476. + ((header->m_state < LZFSE_ENCODE_M_STATES) ? 0 : (1 << 8));
  3477. + tests_results = tests_results |
  3478. + ((header->d_state < LZFSE_ENCODE_D_STATES) ? 0 : (1 << 9));
  3479. +
  3480. + int res;
  3481. + res = fse_check_freq(header->l_freq, LZFSE_ENCODE_L_SYMBOLS,
  3482. + LZFSE_ENCODE_L_STATES);
  3483. + tests_results = tests_results | ((res == 0) ? 0 : (1 << 10));
  3484. + res = fse_check_freq(header->m_freq, LZFSE_ENCODE_M_SYMBOLS,
  3485. + LZFSE_ENCODE_M_STATES);
  3486. + tests_results = tests_results | ((res == 0) ? 0 : (1 << 11));
  3487. + res = fse_check_freq(header->d_freq, LZFSE_ENCODE_D_SYMBOLS,
  3488. + LZFSE_ENCODE_D_STATES);
  3489. + tests_results = tests_results | ((res == 0) ? 0 : (1 << 12));
  3490. + res = fse_check_freq(header->literal_freq, LZFSE_ENCODE_LITERAL_SYMBOLS,
  3491. + LZFSE_ENCODE_LITERAL_STATES);
  3492. + tests_results = tests_results | ((res == 0) ? 0 : (1 << 13));
  3493. +
  3494. + if (tests_results) {
  3495. + return tests_results | 0x80000000; // each 1 bit is a test that failed
  3496. + // (except for the sign bit)
  3497. + }
  3498. +
  3499. + return 0; // OK
  3500. +}
  3501. +
  3502. +// MARK: - L, M, D encoding constants for LZFSE
  3503. +
  3504. +// Largest encodable L (literal length), M (match length) and D (match
  3505. +// distance) values.
  3506. +#define LZFSE_ENCODE_MAX_L_VALUE 315
  3507. +#define LZFSE_ENCODE_MAX_M_VALUE 2359
  3508. +#define LZFSE_ENCODE_MAX_D_VALUE 262139
  3509. +
  3510. +/*! @abstract The L, M, D data streams are all encoded as a "base" value, which is
  3511. + * FSE-encoded, and an "extra bits" value, which is the difference between
  3512. + * value and base, and is simply represented as a raw bit value (because it
  3513. + * is the low-order bits of a larger number, not much entropy can be
  3514. + * extracted from these bits by more complex encoding schemes). The following
  3515. + * tables represent the number of low-order bits to encode separately and the
  3516. + * base values for each of L, M, and D.
  3517. + *
  3518. + * @note The inverse tables for mapping the other way are significantly larger.
  3519. + * Those tables have been split out to lzfse_encode_tables.h in order to keep
  3520. + * this file relatively small. */
  3521. +static const uint8_t l_extra_bits[LZFSE_ENCODE_L_SYMBOLS] = {
  3522. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 5, 8
  3523. +};
  3524. +static const int32_t l_base_value[LZFSE_ENCODE_L_SYMBOLS] = {
  3525. + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 20, 28, 60
  3526. +};
  3527. +static const uint8_t m_extra_bits[LZFSE_ENCODE_M_SYMBOLS] = {
  3528. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 8, 11
  3529. +};
  3530. +static const int32_t m_base_value[LZFSE_ENCODE_M_SYMBOLS] = {
  3531. + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 24, 56, 312
  3532. +};
  3533. +static const uint8_t d_extra_bits[LZFSE_ENCODE_D_SYMBOLS] = {
  3534. + 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3,
  3535. + 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7,
  3536. + 8, 8, 8, 8, 9, 9, 9, 9, 10, 10, 10, 10, 11, 11, 11, 11,
  3537. + 12, 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 14, 15, 15, 15, 15
  3538. +};
  3539. +static const int32_t d_base_value[LZFSE_ENCODE_D_SYMBOLS] = {
  3540. + 0, 1, 2, 3, 4, 6, 8, 10, 12, 16,
  3541. + 20, 24, 28, 36, 44, 52, 60, 76, 92, 108,
  3542. + 124, 156, 188, 220, 252, 316, 380, 444, 508, 636,
  3543. + 764, 892, 1020, 1276, 1532, 1788, 2044, 2556, 3068, 3580,
  3544. + 4092, 5116, 6140, 7164, 8188, 10236, 12284, 14332, 16380, 20476,
  3545. + 24572, 28668, 32764, 40956, 49148, 57340, 65532, 81916, 98300, 114684,
  3546. + 131068, 163836, 196604, 229372
  3547. +};
  3548. +
  3549. +#endif // LZFSE_INTERNAL_H
  3550. diff --git a/lib/lzfse/lzfse_tunables.h b/lib/lzfse/lzfse_tunables.h
  3551. new file mode 100644
  3552. index 0000000..4485a80
  3553. --- /dev/null
  3554. +++ b/lib/lzfse/lzfse_tunables.h
  3555. @@ -0,0 +1,60 @@
  3556. +/*
  3557. +Copyright (c) 2015-2016, Apple Inc. All rights reserved.
  3558. +
  3559. +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
  3560. +
  3561. +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
  3562. +
  3563. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
  3564. + in the documentation and/or other materials provided with the distribution.
  3565. +
  3566. +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
  3567. + from this software without specific prior written permission.
  3568. +
  3569. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  3570. +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
  3571. +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  3572. +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  3573. +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  3574. +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  3575. +*/
  3576. +
  3577. +#ifndef LZFSE_TUNABLES_H
  3578. +#define LZFSE_TUNABLES_H
  3579. +
  3580. +// Parameters controlling details of the LZ-style match search. These values
  3581. +// may be modified to fine tune compression ratio vs. encoding speed, while
  3582. +// keeping the compressed format compatible with LZFSE. Note that
  3583. +// modifying them will also change the amount of work space required by
  3584. +// the encoder. The values here are those used in the compression library
  3585. +// on iOS and OS X.
  3586. +
  3587. +// Number of bits for hash function to produce. Should be in the range
  3588. +// [10, 16]. Larger values reduce the number of false-positive found during
  3589. +// the match search, and expand the history table, which may allow additional
  3590. +// matches to be found, generally improving the achieved compression ratio.
  3591. +// Larger values also increase the workspace size, and make it less likely
  3592. +// that the history table will be present in cache, which reduces performance.
  3593. +#define LZFSE_ENCODE_HASH_BITS 14
  3594. +
  3595. +// Number of positions to store for each line in the history table. May
  3596. +// be either 4 or 8. Using 8 doubles the size of the history table, which
  3597. +// increases the chance of finding matches (thus improving compression ratio),
  3598. +// but also increases the workspace size.
  3599. +#define LZFSE_ENCODE_HASH_WIDTH 4
  3600. +
  3601. +// Match length in bytes to cause immediate emission. Generally speaking,
  3602. +// LZFSE maintains multiple candidate matches and waits to decide which match
  3603. +// to emit until more information is available. When a match exceeds this
  3604. +// threshold, it is emitted immediately. Thus, smaller values may give
  3605. +// somewhat better performance, and larger values may give somewhat better
  3606. +// compression ratios.
  3607. +#define LZFSE_ENCODE_GOOD_MATCH 40
  3608. +
  3609. +// When the source buffer is very small, LZFSE doesn't compress as well as
  3610. +// some simpler algorithms. To maintain reasonable compression for these
  3611. +// cases, we transition to use LZVN instead if the size of the source buffer
  3612. +// is below this threshold.
  3613. +#define LZFSE_ENCODE_LZVN_THRESHOLD 4096
  3614. +
  3615. +#endif // LZFSE_TUNABLES_H
  3616. diff --git a/lib/lzfse/lzvn_decode_base.c b/lib/lzfse/lzvn_decode_base.c
  3617. new file mode 100644
  3618. index 0000000..c30180e
  3619. --- /dev/null
  3620. +++ b/lib/lzfse/lzvn_decode_base.c
  3621. @@ -0,0 +1,711 @@
  3622. +/*
  3623. +Copyright (c) 2015-2016, Apple Inc. All rights reserved.
  3624. +
  3625. +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
  3626. +
  3627. +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
  3628. +
  3629. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
  3630. + in the documentation and/or other materials provided with the distribution.
  3631. +
  3632. +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
  3633. + from this software without specific prior written permission.
  3634. +
  3635. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  3636. +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
  3637. +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  3638. +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  3639. +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  3640. +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  3641. +*/
  3642. +
  3643. +// LZVN low-level decoder
  3644. +
  3645. +#include "lzvn_decode_base.h"
  3646. +
  3647. +#if !defined(HAVE_LABELS_AS_VALUES)
  3648. +# if defined(__GNUC__) || defined(__clang__)
  3649. +# define HAVE_LABELS_AS_VALUES 1
  3650. +# else
  3651. +# define HAVE_LABELS_AS_VALUES 0
  3652. +# endif
  3653. +#endif
  3654. +
  3655. +// Both the source and destination buffers are represented by a pointer and
  3656. +// a length; they are *always* updated in concert using this macro; however
  3657. +// many bytes the pointer is advanced, the length is decremented by the same
  3658. +// amount. Thus, pointer + length always points to the byte one past the end
  3659. +// of the buffer.
  3660. +#define PTR_LEN_INC(_pointer, _length, _increment) \
  3661. + (_pointer += _increment, _length -= _increment)
  3662. +
  3663. +// Update state with current positions and distance, corresponding to the
  3664. +// beginning of an instruction in both streams
  3665. +#define UPDATE_GOOD \
  3666. + (state->src = src_ptr, state->dst = dst_ptr, state->d_prev = D)
  3667. +
  3668. +void lzvn_decode(lzvn_decoder_state *state) {
  3669. +#if HAVE_LABELS_AS_VALUES
  3670. + // Jump table for all instructions
  3671. + static const void *opc_tbl[256] = {
  3672. + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&eos, &&lrg_d,
  3673. + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&nop, &&lrg_d,
  3674. + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&nop, &&lrg_d,
  3675. + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&udef, &&lrg_d,
  3676. + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&udef, &&lrg_d,
  3677. + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&udef, &&lrg_d,
  3678. + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&udef, &&lrg_d,
  3679. + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&udef, &&lrg_d,
  3680. + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d,
  3681. + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d,
  3682. + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d,
  3683. + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d,
  3684. + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d,
  3685. + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d,
  3686. + &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef,
  3687. + &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef,
  3688. + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d,
  3689. + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d,
  3690. + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d,
  3691. + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d,
  3692. + &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d,
  3693. + &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d,
  3694. + &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d,
  3695. + &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d,
  3696. + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d,
  3697. + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d,
  3698. + &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef,
  3699. + &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef,
  3700. + &&lrg_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l,
  3701. + &&sml_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l,
  3702. + &&lrg_m, &&sml_m, &&sml_m, &&sml_m, &&sml_m, &&sml_m, &&sml_m, &&sml_m,
  3703. + &&sml_m, &&sml_m, &&sml_m, &&sml_m, &&sml_m, &&sml_m, &&sml_m, &&sml_m};
  3704. +#endif
  3705. + size_t src_len = state->src_end - state->src;
  3706. + size_t dst_len = state->dst_end - state->dst;
  3707. + if (src_len == 0 || dst_len == 0)
  3708. + return; // empty buffer
  3709. +
  3710. + const unsigned char *src_ptr = state->src;
  3711. + unsigned char *dst_ptr = state->dst;
  3712. + size_t D = state->d_prev;
  3713. + size_t M;
  3714. + size_t L;
  3715. + size_t opc_len;
  3716. +
  3717. + // Do we have a partially expanded match saved in state?
  3718. + if (state->L != 0 || state->M != 0) {
  3719. + L = state->L;
  3720. + M = state->M;
  3721. + D = state->D;
  3722. + opc_len = 0; // we already skipped the op
  3723. + state->L = state->M = state->D = 0;
  3724. + if (M == 0)
  3725. + goto copy_literal;
  3726. + if (L == 0)
  3727. + goto copy_match;
  3728. + goto copy_literal_and_match;
  3729. + }
  3730. +
  3731. + unsigned char opc = src_ptr[0];
  3732. +
  3733. +#if HAVE_LABELS_AS_VALUES
  3734. + goto *opc_tbl[opc];
  3735. +#else
  3736. + for (;;) {
  3737. + switch (opc) {
  3738. +#endif
  3739. +// ===============================================================
  3740. +// These four opcodes (sml_d, med_d, lrg_d, and pre_d) encode both a
  3741. +// literal and a match. The bulk of their implementations are shared;
  3742. +// each label here only does the work of setting the opcode length (not
  3743. +// including any literal bytes), and extracting the literal length, match
  3744. +// length, and match distance (except in pre_d). They then jump into the
  3745. +// shared implementation to actually output the literal and match bytes.
  3746. +//
  3747. +// No error checking happens in the first stage, except for ensuring that
  3748. +// the source has enough length to represent the full opcode before
  3749. +// reading past the first byte.
  3750. +sml_d:
  3751. +#if !HAVE_LABELS_AS_VALUES
  3752. + case 0:
  3753. + case 1:
  3754. + case 2:
  3755. + case 3:
  3756. + case 4:
  3757. + case 5:
  3758. + case 8:
  3759. + case 9:
  3760. + case 10:
  3761. + case 11:
  3762. + case 12:
  3763. + case 13:
  3764. + case 16:
  3765. + case 17:
  3766. + case 18:
  3767. + case 19:
  3768. + case 20:
  3769. + case 21:
  3770. + case 24:
  3771. + case 25:
  3772. + case 26:
  3773. + case 27:
  3774. + case 28:
  3775. + case 29:
  3776. + case 32:
  3777. + case 33:
  3778. + case 34:
  3779. + case 35:
  3780. + case 36:
  3781. + case 37:
  3782. + case 40:
  3783. + case 41:
  3784. + case 42:
  3785. + case 43:
  3786. + case 44:
  3787. + case 45:
  3788. + case 48:
  3789. + case 49:
  3790. + case 50:
  3791. + case 51:
  3792. + case 52:
  3793. + case 53:
  3794. + case 56:
  3795. + case 57:
  3796. + case 58:
  3797. + case 59:
  3798. + case 60:
  3799. + case 61:
  3800. + case 64:
  3801. + case 65:
  3802. + case 66:
  3803. + case 67:
  3804. + case 68:
  3805. + case 69:
  3806. + case 72:
  3807. + case 73:
  3808. + case 74:
  3809. + case 75:
  3810. + case 76:
  3811. + case 77:
  3812. + case 80:
  3813. + case 81:
  3814. + case 82:
  3815. + case 83:
  3816. + case 84:
  3817. + case 85:
  3818. + case 88:
  3819. + case 89:
  3820. + case 90:
  3821. + case 91:
  3822. + case 92:
  3823. + case 93:
  3824. + case 96:
  3825. + case 97:
  3826. + case 98:
  3827. + case 99:
  3828. + case 100:
  3829. + case 101:
  3830. + case 104:
  3831. + case 105:
  3832. + case 106:
  3833. + case 107:
  3834. + case 108:
  3835. + case 109:
  3836. + case 128:
  3837. + case 129:
  3838. + case 130:
  3839. + case 131:
  3840. + case 132:
  3841. + case 133:
  3842. + case 136:
  3843. + case 137:
  3844. + case 138:
  3845. + case 139:
  3846. + case 140:
  3847. + case 141:
  3848. + case 144:
  3849. + case 145:
  3850. + case 146:
  3851. + case 147:
  3852. + case 148:
  3853. + case 149:
  3854. + case 152:
  3855. + case 153:
  3856. + case 154:
  3857. + case 155:
  3858. + case 156:
  3859. + case 157:
  3860. + case 192:
  3861. + case 193:
  3862. + case 194:
  3863. + case 195:
  3864. + case 196:
  3865. + case 197:
  3866. + case 200:
  3867. + case 201:
  3868. + case 202:
  3869. + case 203:
  3870. + case 204:
  3871. + case 205:
  3872. +#endif
  3873. + UPDATE_GOOD;
  3874. + // "small distance": This opcode has the structure LLMMMDDD DDDDDDDD LITERAL
  3875. + // where the length of literal (0-3 bytes) is encoded by the high 2 bits of
  3876. + // the first byte. We first extract the literal length so we know how long
  3877. + // the opcode is, then check that the source can hold both this opcode and
  3878. + // at least one byte of the next (because any valid input stream must be
  3879. + // terminated with an eos token).
  3880. + opc_len = 2;
  3881. + L = (size_t)extract(opc, 6, 2);
  3882. + M = (size_t)extract(opc, 3, 3) + 3;
  3883. + // We need to ensure that the source buffer is long enough that we can
  3884. + // safely read this entire opcode, the literal that follows, and the first
  3885. + // byte of the next opcode. Once we satisfy this requirement, we can
  3886. + // safely unpack the match distance. A check similar to this one is
  3887. + // present in all the opcode implementations.
  3888. + if (src_len <= opc_len + L)
  3889. + return; // source truncated
  3890. + D = (size_t)extract(opc, 0, 3) << 8 | src_ptr[1];
  3891. + goto copy_literal_and_match;
  3892. +
  3893. +med_d:
  3894. +#if !HAVE_LABELS_AS_VALUES
  3895. + case 160:
  3896. + case 161:
  3897. + case 162:
  3898. + case 163:
  3899. + case 164:
  3900. + case 165:
  3901. + case 166:
  3902. + case 167:
  3903. + case 168:
  3904. + case 169:
  3905. + case 170:
  3906. + case 171:
  3907. + case 172:
  3908. + case 173:
  3909. + case 174:
  3910. + case 175:
  3911. + case 176:
  3912. + case 177:
  3913. + case 178:
  3914. + case 179:
  3915. + case 180:
  3916. + case 181:
  3917. + case 182:
  3918. + case 183:
  3919. + case 184:
  3920. + case 185:
  3921. + case 186:
  3922. + case 187:
  3923. + case 188:
  3924. + case 189:
  3925. + case 190:
  3926. + case 191:
  3927. +#endif
  3928. + UPDATE_GOOD;
  3929. + // "medium distance": This is a minor variant of the "small distance"
  3930. + // encoding, where we will now use two extra bytes instead of one to encode
  3931. + // the restof the match length and distance. This allows an extra two bits
  3932. + // for the match length, and an extra three bits for the match distance. The
  3933. + // full structure of the opcode is 101LLMMM DDDDDDMM DDDDDDDD LITERAL.
  3934. + opc_len = 3;
  3935. + L = (size_t)extract(opc, 3, 2);
  3936. + if (src_len <= opc_len + L)
  3937. + return; // source truncated
  3938. + uint16_t opc23 = load2(&src_ptr[1]);
  3939. + M = (size_t)((extract(opc, 0, 3) << 2 | extract(opc23, 0, 2)) + 3);
  3940. + D = (size_t)extract(opc23, 2, 14);
  3941. + goto copy_literal_and_match;
  3942. +
  3943. +lrg_d:
  3944. +#if !HAVE_LABELS_AS_VALUES
  3945. + case 7:
  3946. + case 15:
  3947. + case 23:
  3948. + case 31:
  3949. + case 39:
  3950. + case 47:
  3951. + case 55:
  3952. + case 63:
  3953. + case 71:
  3954. + case 79:
  3955. + case 87:
  3956. + case 95:
  3957. + case 103:
  3958. + case 111:
  3959. + case 135:
  3960. + case 143:
  3961. + case 151:
  3962. + case 159:
  3963. + case 199:
  3964. + case 207:
  3965. +#endif
  3966. + UPDATE_GOOD;
  3967. + // "large distance": This is another variant of the "small distance"
  3968. + // encoding, where we will now use two extra bytes to encode the match
  3969. + // distance, which allows distances up to 65535 to be represented. The full
  3970. + // structure of the opcode is LLMMM111 DDDDDDDD DDDDDDDD LITERAL.
  3971. + opc_len = 3;
  3972. + L = (size_t)extract(opc, 6, 2);
  3973. + M = (size_t)extract(opc, 3, 3) + 3;
  3974. + if (src_len <= opc_len + L)
  3975. + return; // source truncated
  3976. + D = load2(&src_ptr[1]);
  3977. + goto copy_literal_and_match;
  3978. +
  3979. +pre_d:
  3980. +#if !HAVE_LABELS_AS_VALUES
  3981. + case 70:
  3982. + case 78:
  3983. + case 86:
  3984. + case 94:
  3985. + case 102:
  3986. + case 110:
  3987. + case 134:
  3988. + case 142:
  3989. + case 150:
  3990. + case 158:
  3991. + case 198:
  3992. + case 206:
  3993. +#endif
  3994. + UPDATE_GOOD;
  3995. + // "previous distance": This opcode has the structure LLMMM110, where the
  3996. + // length of the literal (0-3 bytes) is encoded by the high 2 bits of the
  3997. + // first byte. We first extract the literal length so we know how long
  3998. + // the opcode is, then check that the source can hold both this opcode and
  3999. + // at least one byte of the next (because any valid input stream must be
  4000. + // terminated with an eos token).
  4001. + opc_len = 1;
  4002. + L = (size_t)extract(opc, 6, 2);
  4003. + M = (size_t)extract(opc, 3, 3) + 3;
  4004. + if (src_len <= opc_len + L)
  4005. + return; // source truncated
  4006. + goto copy_literal_and_match;
  4007. +
  4008. +copy_literal_and_match:
  4009. + // Common implementation of writing data for opcodes that have both a
  4010. + // literal and a match. We begin by advancing the source pointer past
  4011. + // the opcode, so that it points at the first literal byte (if L
  4012. + // is non-zero; otherwise it points at the next opcode).
  4013. + PTR_LEN_INC(src_ptr, src_len, opc_len);
  4014. + // Now we copy the literal from the source pointer to the destination.
  4015. + if (__builtin_expect(dst_len >= 4 && src_len >= 4, 1)) {
  4016. + // The literal is 0-3 bytes; if we are not near the end of the buffer,
  4017. + // we can safely just do a 4 byte copy (which is guaranteed to cover
  4018. + // the complete literal, and may include some other bytes as well).
  4019. + store4(dst_ptr, load4(src_ptr));
  4020. + } else if (L <= dst_len) {
  4021. + // We are too close to the end of either the input or output stream
  4022. + // to be able to safely use a four-byte copy, but we will not exhaust
  4023. + // either stream (we already know that the source will not be
  4024. + // exhausted from checks in the individual opcode implementations,
  4025. + // and we just tested that dst_len > L). Thus, we need to do a
  4026. + // byte-by-byte copy of the literal. This is slow, but it can only ever
  4027. + // happen near the very end of a buffer, so it is not an important case to
  4028. + // optimize.
  4029. + for (size_t i = 0; i < L; ++i)
  4030. + dst_ptr[i] = src_ptr[i];
  4031. + } else {
  4032. + // Destination truncated: fill DST, and store partial match
  4033. +
  4034. + // Copy partial literal
  4035. + for (size_t i = 0; i < dst_len; ++i)
  4036. + dst_ptr[i] = src_ptr[i];
  4037. + // Save state
  4038. + state->src = src_ptr + dst_len;
  4039. + state->dst = dst_ptr + dst_len;
  4040. + state->L = L - dst_len;
  4041. + state->M = M;
  4042. + state->D = D;
  4043. + return; // destination truncated
  4044. + }
  4045. + // Having completed the copy of the literal, we advance both the source
  4046. + // and destination pointers by the number of literal bytes.
  4047. + PTR_LEN_INC(dst_ptr, dst_len, L);
  4048. + PTR_LEN_INC(src_ptr, src_len, L);
  4049. + // Check if the match distance is valid; matches must not reference
  4050. + // bytes that preceed the start of the output buffer, nor can the match
  4051. + // distance be zero.
  4052. + if (D > dst_ptr - state->dst_begin || D == 0)
  4053. + goto invalid_match_distance;
  4054. +copy_match:
  4055. + // Now we copy the match from dst_ptr - D to dst_ptr. It is important to keep
  4056. + // in mind that we may have D < M, in which case the source and destination
  4057. + // windows overlap in the copy. The semantics of the match copy are *not*
  4058. + // those of memmove( ); if the buffers overlap it needs to behave as though
  4059. + // we were copying byte-by-byte in increasing address order. If, for example,
  4060. + // D is 1, the copy operation is equivalent to:
  4061. + //
  4062. + // memset(dst_ptr, dst_ptr[-1], M);
  4063. + //
  4064. + // i.e. it splats the previous byte. This means that we need to be very
  4065. + // careful about using wide loads or stores to perform the copy operation.
  4066. + if (__builtin_expect(dst_len >= M + 7 && D >= 8, 1)) {
  4067. + // We are not near the end of the buffer, and the match distance
  4068. + // is at least eight. Thus, we can safely loop using eight byte
  4069. + // copies. The last of these may slop over the intended end of
  4070. + // the match, but this is OK because we know we have a safety bound
  4071. + // away from the end of the destination buffer.
  4072. + for (size_t i = 0; i < M; i += 8)
  4073. + store8(&dst_ptr[i], load8(&dst_ptr[i - D]));
  4074. + } else if (M <= dst_len) {
  4075. + // Either the match distance is too small, or we are too close to
  4076. + // the end of the buffer to safely use eight byte copies. Fall back
  4077. + // on a simple byte-by-byte implementation.
  4078. + for (size_t i = 0; i < M; ++i)
  4079. + dst_ptr[i] = dst_ptr[i - D];
  4080. + } else {
  4081. + // Destination truncated: fill DST, and store partial match
  4082. +
  4083. + // Copy partial match
  4084. + for (size_t i = 0; i < dst_len; ++i)
  4085. + dst_ptr[i] = dst_ptr[i - D];
  4086. + // Save state
  4087. + state->src = src_ptr;
  4088. + state->dst = dst_ptr + dst_len;
  4089. + state->L = 0;
  4090. + state->M = M - dst_len;
  4091. + state->D = D;
  4092. + return; // destination truncated
  4093. + }
  4094. + // Update the destination pointer and length to account for the bytes
  4095. + // written by the match, then load the next opcode byte and branch to
  4096. + // the appropriate implementation.
  4097. + PTR_LEN_INC(dst_ptr, dst_len, M);
  4098. + opc = src_ptr[0];
  4099. +#if HAVE_LABELS_AS_VALUES
  4100. + goto *opc_tbl[opc];
  4101. +#else
  4102. + break;
  4103. +#endif
  4104. +
  4105. +// ===============================================================
  4106. +// Opcodes representing only a match (no literal).
  4107. +// These two opcodes (lrg_m and sml_m) encode only a match. The match
  4108. +// distance is carried over from the previous opcode, so all they need
  4109. +// to encode is the match length. We are able to reuse the match copy
  4110. +// sequence from the literal and match opcodes to perform the actual
  4111. +// copy implementation.
  4112. +sml_m:
  4113. +#if !HAVE_LABELS_AS_VALUES
  4114. + case 241:
  4115. + case 242:
  4116. + case 243:
  4117. + case 244:
  4118. + case 245:
  4119. + case 246:
  4120. + case 247:
  4121. + case 248:
  4122. + case 249:
  4123. + case 250:
  4124. + case 251:
  4125. + case 252:
  4126. + case 253:
  4127. + case 254:
  4128. + case 255:
  4129. +#endif
  4130. + UPDATE_GOOD;
  4131. + // "small match": This opcode has no literal, and uses the previous match
  4132. + // distance (i.e. it encodes only the match length), in a single byte as
  4133. + // 1111MMMM.
  4134. + opc_len = 1;
  4135. + if (src_len <= opc_len)
  4136. + return; // source truncated
  4137. + M = (size_t)extract(opc, 0, 4);
  4138. + PTR_LEN_INC(src_ptr, src_len, opc_len);
  4139. + goto copy_match;
  4140. +
  4141. +lrg_m:
  4142. +#if !HAVE_LABELS_AS_VALUES
  4143. + case 240:
  4144. +#endif
  4145. + UPDATE_GOOD;
  4146. + // "large match": This opcode has no literal, and uses the previous match
  4147. + // distance (i.e. it encodes only the match length). It is encoded in two
  4148. + // bytes as 11110000 MMMMMMMM. Because matches smaller than 16 bytes can
  4149. + // be represented by sml_m, there is an implicit bias of 16 on the match
  4150. + // length; the representable values are [16,271].
  4151. + opc_len = 2;
  4152. + if (src_len <= opc_len)
  4153. + return; // source truncated
  4154. + M = src_ptr[1] + 16;
  4155. + PTR_LEN_INC(src_ptr, src_len, opc_len);
  4156. + goto copy_match;
  4157. +
  4158. +// ===============================================================
  4159. +// Opcodes representing only a literal (no match).
  4160. +// These two opcodes (lrg_l and sml_l) encode only a literal. There is no
  4161. +// match length or match distance to worry about (but we need to *not*
  4162. +// touch D, as it must be preserved between opcodes).
  4163. +sml_l:
  4164. +#if !HAVE_LABELS_AS_VALUES
  4165. + case 225:
  4166. + case 226:
  4167. + case 227:
  4168. + case 228:
  4169. + case 229:
  4170. + case 230:
  4171. + case 231:
  4172. + case 232:
  4173. + case 233:
  4174. + case 234:
  4175. + case 235:
  4176. + case 236:
  4177. + case 237:
  4178. + case 238:
  4179. + case 239:
  4180. +#endif
  4181. + UPDATE_GOOD;
  4182. + // "small literal": This opcode has no match, and encodes only a literal
  4183. + // of length up to 15 bytes. The format is 1110LLLL LITERAL.
  4184. + opc_len = 1;
  4185. + L = (size_t)extract(opc, 0, 4);
  4186. + goto copy_literal;
  4187. +
  4188. +lrg_l:
  4189. +#if !HAVE_LABELS_AS_VALUES
  4190. + case 224:
  4191. +#endif
  4192. + UPDATE_GOOD;
  4193. + // "large literal": This opcode has no match, and uses the previous match
  4194. + // distance (i.e. it encodes only the match length). It is encoded in two
  4195. + // bytes as 11100000 LLLLLLLL LITERAL. Because literals smaller than 16
  4196. + // bytes can be represented by sml_l, there is an implicit bias of 16 on
  4197. + // the literal length; the representable values are [16,271].
  4198. + opc_len = 2;
  4199. + if (src_len <= 2)
  4200. + return; // source truncated
  4201. + L = src_ptr[1] + 16;
  4202. + goto copy_literal;
  4203. +
  4204. +copy_literal:
  4205. + // Check that the source buffer is large enough to hold the complete
  4206. + // literal and at least the first byte of the next opcode. If so, advance
  4207. + // the source pointer to point to the first byte of the literal and adjust
  4208. + // the source length accordingly.
  4209. + if (src_len <= opc_len + L)
  4210. + return; // source truncated
  4211. + PTR_LEN_INC(src_ptr, src_len, opc_len);
  4212. + // Now we copy the literal from the source pointer to the destination.
  4213. + if (dst_len >= L + 7 && src_len >= L + 7) {
  4214. + // We are not near the end of the source or destination buffers; thus
  4215. + // we can safely copy the literal using wide copies, without worrying
  4216. + // about reading or writing past the end of either buffer.
  4217. + for (size_t i = 0; i < L; i += 8)
  4218. + store8(&dst_ptr[i], load8(&src_ptr[i]));
  4219. + } else if (L <= dst_len) {
  4220. + // We are too close to the end of either the input or output stream
  4221. + // to be able to safely use an eight-byte copy. Instead we copy the
  4222. + // literal byte-by-byte.
  4223. + for (size_t i = 0; i < L; ++i)
  4224. + dst_ptr[i] = src_ptr[i];
  4225. + } else {
  4226. + // Destination truncated: fill DST, and store partial match
  4227. +
  4228. + // Copy partial literal
  4229. + for (size_t i = 0; i < dst_len; ++i)
  4230. + dst_ptr[i] = src_ptr[i];
  4231. + // Save state
  4232. + state->src = src_ptr + dst_len;
  4233. + state->dst = dst_ptr + dst_len;
  4234. + state->L = L - dst_len;
  4235. + state->M = 0;
  4236. + state->D = D;
  4237. + return; // destination truncated
  4238. + }
  4239. + // Having completed the copy of the literal, we advance both the source
  4240. + // and destination pointers by the number of literal bytes.
  4241. + PTR_LEN_INC(dst_ptr, dst_len, L);
  4242. + PTR_LEN_INC(src_ptr, src_len, L);
  4243. + // Load the first byte of the next opcode, and jump to its implementation.
  4244. + opc = src_ptr[0];
  4245. +#if HAVE_LABELS_AS_VALUES
  4246. + goto *opc_tbl[opc];
  4247. +#else
  4248. + break;
  4249. +#endif
  4250. +
  4251. +// ===============================================================
  4252. +// Other opcodes
  4253. +nop:
  4254. +#if !HAVE_LABELS_AS_VALUES
  4255. + case 14:
  4256. + case 22:
  4257. +#endif
  4258. + UPDATE_GOOD;
  4259. + opc_len = 1;
  4260. + if (src_len <= opc_len)
  4261. + return; // source truncated
  4262. + PTR_LEN_INC(src_ptr, src_len, opc_len);
  4263. + opc = src_ptr[0];
  4264. +#if HAVE_LABELS_AS_VALUES
  4265. + goto *opc_tbl[opc];
  4266. +#else
  4267. + break;
  4268. +#endif
  4269. +
  4270. +eos:
  4271. +#if !HAVE_LABELS_AS_VALUES
  4272. + case 6:
  4273. +#endif
  4274. + opc_len = 8;
  4275. + if (src_len < opc_len)
  4276. + return; // source truncated (here we don't need an extra byte for next op
  4277. + // code)
  4278. + PTR_LEN_INC(src_ptr, src_len, opc_len);
  4279. + state->end_of_stream = 1;
  4280. + UPDATE_GOOD;
  4281. + return; // end-of-stream
  4282. +
  4283. +// ===============================================================
  4284. +// Return on error
  4285. +udef:
  4286. +#if !HAVE_LABELS_AS_VALUES
  4287. + case 30:
  4288. + case 38:
  4289. + case 46:
  4290. + case 54:
  4291. + case 62:
  4292. + case 112:
  4293. + case 113:
  4294. + case 114:
  4295. + case 115:
  4296. + case 116:
  4297. + case 117:
  4298. + case 118:
  4299. + case 119:
  4300. + case 120:
  4301. + case 121:
  4302. + case 122:
  4303. + case 123:
  4304. + case 124:
  4305. + case 125:
  4306. + case 126:
  4307. + case 127:
  4308. + case 208:
  4309. + case 209:
  4310. + case 210:
  4311. + case 211:
  4312. + case 212:
  4313. + case 213:
  4314. + case 214:
  4315. + case 215:
  4316. + case 216:
  4317. + case 217:
  4318. + case 218:
  4319. + case 219:
  4320. + case 220:
  4321. + case 221:
  4322. + case 222:
  4323. + case 223:
  4324. +#endif
  4325. +invalid_match_distance:
  4326. +
  4327. + return; // we already updated state
  4328. +#if !HAVE_LABELS_AS_VALUES
  4329. + }
  4330. + }
  4331. +#endif
  4332. +}
  4333. diff --git a/lib/lzfse/lzvn_decode_base.h b/lib/lzfse/lzvn_decode_base.h
  4334. new file mode 100644
  4335. index 0000000..b4307a5
  4336. --- /dev/null
  4337. +++ b/lib/lzfse/lzvn_decode_base.h
  4338. @@ -0,0 +1,68 @@
  4339. +/*
  4340. +Copyright (c) 2015-2016, Apple Inc. All rights reserved.
  4341. +
  4342. +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
  4343. +
  4344. +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
  4345. +
  4346. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
  4347. + in the documentation and/or other materials provided with the distribution.
  4348. +
  4349. +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
  4350. + from this software without specific prior written permission.
  4351. +
  4352. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  4353. +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
  4354. +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  4355. +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  4356. +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  4357. +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  4358. +*/
  4359. +
  4360. +// LZVN low-level decoder (v2)
  4361. +// Functions in the low-level API should switch to these at some point.
  4362. +// Apr 2014
  4363. +
  4364. +#ifndef LZVN_DECODE_BASE_H
  4365. +#define LZVN_DECODE_BASE_H
  4366. +
  4367. +#include "lzfse_internal.h"
  4368. +
  4369. +/*! @abstract Base decoder state. */
  4370. +typedef struct {
  4371. +
  4372. + // Decoder I/O
  4373. +
  4374. + // Next byte to read in source buffer
  4375. + const unsigned char *src;
  4376. + // Next byte after source buffer
  4377. + const unsigned char *src_end;
  4378. +
  4379. + // Next byte to write in destination buffer (by decoder)
  4380. + unsigned char *dst;
  4381. + // Valid range for destination buffer is [dst_begin, dst_end - 1]
  4382. + unsigned char *dst_begin;
  4383. + unsigned char *dst_end;
  4384. + // Next byte to read in destination buffer (modified by caller)
  4385. + unsigned char *dst_current;
  4386. +
  4387. + // Decoder state
  4388. +
  4389. + // Partially expanded match, or 0,0,0.
  4390. + // In that case, src points to the next literal to copy, or the next op-code
  4391. + // if L==0.
  4392. + size_t L, M, D;
  4393. +
  4394. + // Distance for last emitted match, or 0
  4395. + lzvn_offset d_prev;
  4396. +
  4397. + // Did we decode end-of-stream?
  4398. + int end_of_stream;
  4399. +
  4400. +} lzvn_decoder_state;
  4401. +
  4402. +/*! @abstract Decode source to destination.
  4403. + * Updates \p state (src,dst,d_prev). */
  4404. +void lzvn_decode(lzvn_decoder_state *state);
  4405. +
  4406. +#endif // LZVN_DECODE_BASE_H
  4407. diff --git a/lib/lzfse/lzvn_encode_base.c b/lib/lzfse/lzvn_encode_base.c
  4408. new file mode 100644
  4409. index 0000000..c86b551
  4410. --- /dev/null
  4411. +++ b/lib/lzfse/lzvn_encode_base.c
  4412. @@ -0,0 +1,593 @@
  4413. +/*
  4414. +Copyright (c) 2015-2016, Apple Inc. All rights reserved.
  4415. +
  4416. +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
  4417. +
  4418. +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
  4419. +
  4420. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
  4421. + in the documentation and/or other materials provided with the distribution.
  4422. +
  4423. +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
  4424. + from this software without specific prior written permission.
  4425. +
  4426. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  4427. +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
  4428. +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  4429. +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  4430. +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  4431. +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  4432. +*/
  4433. +
  4434. +// LZVN low-level encoder
  4435. +
  4436. +#include "lzvn_encode_base.h"
  4437. +
  4438. +#if defined(_MSC_VER) && !defined(__clang__)
  4439. +# define restrict __restrict
  4440. +#endif
  4441. +
  4442. +// ===============================================================
  4443. +// Coarse/fine copy, non overlapping buffers
  4444. +
  4445. +/*! @abstract Copy at least \p nbytes bytes from \p src to \p dst, by blocks
  4446. + * of 8 bytes (may go beyond range). No overlap.
  4447. + * @return \p dst + \p nbytes. */
  4448. +static inline unsigned char *lzvn_copy64(unsigned char *restrict dst,
  4449. + const unsigned char *restrict src,
  4450. + size_t nbytes) {
  4451. + for (size_t i = 0; i < nbytes; i += 8)
  4452. + store8(dst + i, load8(src + i));
  4453. + return dst + nbytes;
  4454. +}
  4455. +
  4456. +/*! @abstract Copy exactly \p nbytes bytes from \p src to \p dst (respects range).
  4457. + * No overlap.
  4458. + * @return \p dst + \p nbytes. */
  4459. +static inline unsigned char *lzvn_copy8(unsigned char *restrict dst,
  4460. + const unsigned char *restrict src,
  4461. + size_t nbytes) {
  4462. + for (size_t i = 0; i < nbytes; i++)
  4463. + dst[i] = src[i];
  4464. + return dst + nbytes;
  4465. +}
  4466. +
  4467. +/*! @abstract Emit (L,0,0) instructions (final literal).
  4468. + * We read at most \p L bytes from \p p.
  4469. + * @param p input stream
  4470. + * @param q1 the first byte after the output buffer.
  4471. + * @return pointer to the next output, <= \p q1.
  4472. + * @return \p q1 if output is full. In that case, output will be partially invalid.
  4473. + */
  4474. +static inline unsigned char *emit_literal(const unsigned char *p,
  4475. + unsigned char *q, unsigned char *q1,
  4476. + size_t L) {
  4477. + size_t x;
  4478. + while (L > 15) {
  4479. + x = L < 271 ? L : 271;
  4480. + if (q + x + 10 >= q1)
  4481. + goto OUT_FULL;
  4482. + store2(q, 0xE0 + ((x - 16) << 8));
  4483. + q += 2;
  4484. + L -= x;
  4485. + q = lzvn_copy8(q, p, x);
  4486. + p += x;
  4487. + }
  4488. + if (L > 0) {
  4489. + if (q + L + 10 >= q1)
  4490. + goto OUT_FULL;
  4491. + *q++ = 0xE0 + L; // 1110LLLL
  4492. + q = lzvn_copy8(q, p, L);
  4493. + }
  4494. + return q;
  4495. +
  4496. +OUT_FULL:
  4497. + return q1;
  4498. +}
  4499. +
  4500. +/*! @abstract Emit (L,M,D) instructions. M>=3.
  4501. + * @param p input stream pointing to the beginning of the literal. We read at
  4502. + * most \p L+4 bytes from \p p.
  4503. + * @param q1 the first byte after the output buffer.
  4504. + * @return pointer to the next output, <= \p q1.
  4505. + * @return \p q1 if output is full. In that case, output will be partially invalid.
  4506. + */
  4507. +static inline unsigned char *emit(const unsigned char *p, unsigned char *q,
  4508. + unsigned char *q1, size_t L, size_t M,
  4509. + size_t D, size_t D_prev) {
  4510. + size_t x;
  4511. + while (L > 15) {
  4512. + x = L < 271 ? L : 271;
  4513. + if (q + x + 10 >= q1)
  4514. + goto OUT_FULL;
  4515. + store2(q, 0xE0 + ((x - 16) << 8));
  4516. + q += 2;
  4517. + L -= x;
  4518. + q = lzvn_copy64(q, p, x);
  4519. + p += x;
  4520. + }
  4521. + if (L > 3) {
  4522. + if (q + L + 10 >= q1)
  4523. + goto OUT_FULL;
  4524. + *q++ = 0xE0 + L; // 1110LLLL
  4525. + q = lzvn_copy64(q, p, L);
  4526. + p += L;
  4527. + L = 0;
  4528. + }
  4529. + x = M <= 10 - 2 * L ? M : 10 - 2 * L; // x = min(10-2*L,M)
  4530. + M -= x;
  4531. + x -= 3; // M = (x+3) + M' max value for x is 7-2*L
  4532. +
  4533. + // Here L<4 literals remaining, we read them here
  4534. + uint32_t literal = load4(p);
  4535. + // P is not accessed after this point
  4536. +
  4537. + // Relaxed capacity test covering all cases
  4538. + if (q + 8 >= q1)
  4539. + goto OUT_FULL;
  4540. +
  4541. + if (D == D_prev) {
  4542. + if (L == 0) {
  4543. + *q++ = 0xF0 + (x + 3); // XM!
  4544. + } else {
  4545. + *q++ = (L << 6) + (x << 3) + 6; // LLxxx110
  4546. + }
  4547. + store4(q, literal);
  4548. + q += L;
  4549. + } else if (D < 2048 - 2 * 256) {
  4550. + // Short dist D>>8 in 0..5
  4551. + *q++ = (D >> 8) + (L << 6) + (x << 3); // LLxxxDDD
  4552. + *q++ = D & 0xFF;
  4553. + store4(q, literal);
  4554. + q += L;
  4555. + } else if (D >= (1 << 14) || M == 0 || (x + 3) + M > 34) {
  4556. + // Long dist
  4557. + *q++ = (L << 6) + (x << 3) + 7;
  4558. + store2(q, D);
  4559. + q += 2;
  4560. + store4(q, literal);
  4561. + q += L;
  4562. + } else {
  4563. + // Medium distance
  4564. + x += M;
  4565. + M = 0;
  4566. + *q++ = 0xA0 + (x >> 2) + (L << 3);
  4567. + store2(q, D << 2 | (x & 3));
  4568. + q += 2;
  4569. + store4(q, literal);
  4570. + q += L;
  4571. + }
  4572. +
  4573. + // Issue remaining match
  4574. + while (M > 15) {
  4575. + if (q + 2 >= q1)
  4576. + goto OUT_FULL;
  4577. + x = M < 271 ? M : 271;
  4578. + store2(q, 0xf0 + ((x - 16) << 8));
  4579. + q += 2;
  4580. + M -= x;
  4581. + }
  4582. + if (M > 0) {
  4583. + if (q + 1 >= q1)
  4584. + goto OUT_FULL;
  4585. + *q++ = 0xF0 + M; // M = 0..15
  4586. + }
  4587. +
  4588. + return q;
  4589. +
  4590. +OUT_FULL:
  4591. + return q1;
  4592. +}
  4593. +
  4594. +// ===============================================================
  4595. +// Conversions
  4596. +
  4597. +/*! @abstract Return 32-bit value to store for offset x. */
  4598. +static inline int32_t offset_to_s32(lzvn_offset x) { return (int32_t)x; }
  4599. +
  4600. +/*! @abstract Get offset from 32-bit stored value x. */
  4601. +static inline lzvn_offset offset_from_s32(int32_t x) { return (lzvn_offset)x; }
  4602. +
  4603. +// ===============================================================
  4604. +// Hash and Matching
  4605. +
  4606. +/*! @abstract Get hash in range \c [0,LZVN_ENCODE_HASH_VALUES-1] from 3 bytes in i. */
  4607. +static inline uint32_t hash3i(uint32_t i) {
  4608. + i &= 0xffffff; // truncate to 24-bit input (slightly increases compression ratio)
  4609. + uint32_t h = (i * (1 + (1 << 6) + (1 << 12))) >> 12;
  4610. + return h & (LZVN_ENCODE_HASH_VALUES - 1);
  4611. +}
  4612. +
  4613. +/*! @abstract Return the number [0, 4] of zero bytes in \p x, starting from the
  4614. + * least significant byte. */
  4615. +static inline lzvn_offset trailing_zero_bytes(uint32_t x) {
  4616. + return (x == 0) ? 4 : (__builtin_ctzl(x) >> 3);
  4617. +}
  4618. +
  4619. +/*! @abstract Return the number [0, 4] of matching chars between values at
  4620. + * \p src+i and \p src+j, starting from the least significant byte.
  4621. + * Assumes we can read 4 chars from each position. */
  4622. +static inline lzvn_offset nmatch4(const unsigned char *src, lzvn_offset i,
  4623. + lzvn_offset j) {
  4624. + uint32_t vi = load4(src + i);
  4625. + uint32_t vj = load4(src + j);
  4626. + return trailing_zero_bytes(vi ^ vj);
  4627. +}
  4628. +
  4629. +/*! @abstract Check if l_begin, m_begin, m0_begin (m0_begin < m_begin) can be
  4630. + * expanded to a match of length at least 3.
  4631. + * @param m_begin new string to match.
  4632. + * @param m0_begin candidate old string.
  4633. + * @param src source buffer, with valid indices src_begin <= i < src_end.
  4634. + * (src_begin may be <0)
  4635. + * @return If a match can be found, return 1 and set all \p match fields,
  4636. + * otherwise return 0.
  4637. + * @note \p *match should be 0 before the call. */
  4638. +static inline int lzvn_find_match(const unsigned char *src,
  4639. + lzvn_offset src_begin,
  4640. + lzvn_offset src_end, lzvn_offset l_begin,
  4641. + lzvn_offset m0_begin, lzvn_offset m_begin,
  4642. + lzvn_match_info *match) {
  4643. + lzvn_offset n = nmatch4(src, m_begin, m0_begin);
  4644. + if (n < 3)
  4645. + return 0; // no match
  4646. +
  4647. + lzvn_offset D = m_begin - m0_begin; // actual distance
  4648. + if (D <= 0 || D > LZVN_ENCODE_MAX_DISTANCE)
  4649. + return 0; // distance out of range
  4650. +
  4651. + // Expand forward
  4652. + lzvn_offset m_end = m_begin + n;
  4653. + while (n == 4 && m_end + 4 < src_end) {
  4654. + n = nmatch4(src, m_end, m_end - D);
  4655. + m_end += n;
  4656. + }
  4657. +
  4658. + // Expand backwards over literal
  4659. + while (m0_begin > src_begin && m_begin > l_begin &&
  4660. + src[m_begin - 1] == src[m0_begin - 1]) {
  4661. + m0_begin--;
  4662. + m_begin--;
  4663. + }
  4664. +
  4665. + // OK, we keep it, update MATCH
  4666. + lzvn_offset M = m_end - m_begin; // match length
  4667. + match->m_begin = m_begin;
  4668. + match->m_end = m_end;
  4669. + match->K = M - ((D < 0x600) ? 2 : 3);
  4670. + match->M = M;
  4671. + match->D = D;
  4672. +
  4673. + return 1; // OK
  4674. +}
  4675. +
  4676. +/*! @abstract Same as lzvn_find_match, but we already know that N bytes do
  4677. + * match (N<=4). */
  4678. +static inline int lzvn_find_matchN(const unsigned char *src,
  4679. + lzvn_offset src_begin,
  4680. + lzvn_offset src_end, lzvn_offset l_begin,
  4681. + lzvn_offset m0_begin, lzvn_offset m_begin,
  4682. + lzvn_offset n, lzvn_match_info *match) {
  4683. + // We can skip the first comparison on 4 bytes
  4684. + if (n < 3)
  4685. + return 0; // no match
  4686. +
  4687. + lzvn_offset D = m_begin - m0_begin; // actual distance
  4688. + if (D <= 0 || D > LZVN_ENCODE_MAX_DISTANCE)
  4689. + return 0; // distance out of range
  4690. +
  4691. + // Expand forward
  4692. + lzvn_offset m_end = m_begin + n;
  4693. + while (n == 4 && m_end + 4 < src_end) {
  4694. + n = nmatch4(src, m_end, m_end - D);
  4695. + m_end += n;
  4696. + }
  4697. +
  4698. + // Expand backwards over literal
  4699. + while (m0_begin > src_begin && m_begin > l_begin &&
  4700. + src[m_begin - 1] == src[m0_begin - 1]) {
  4701. + m0_begin--;
  4702. + m_begin--;
  4703. + }
  4704. +
  4705. + // OK, we keep it, update MATCH
  4706. + lzvn_offset M = m_end - m_begin; // match length
  4707. + match->m_begin = m_begin;
  4708. + match->m_end = m_end;
  4709. + match->K = M - ((D < 0x600) ? 2 : 3);
  4710. + match->M = M;
  4711. + match->D = D;
  4712. +
  4713. + return 1; // OK
  4714. +}
  4715. +
  4716. +// ===============================================================
  4717. +// Encoder Backend
  4718. +
  4719. +/*! @abstract Emit a match and update state.
  4720. + * @return number of bytes written to \p dst. May be 0 if there is no more space
  4721. + * in \p dst to emit the match. */
  4722. +static inline lzvn_offset lzvn_emit_match(lzvn_encoder_state *state,
  4723. + lzvn_match_info match) {
  4724. + size_t L = (size_t)(match.m_begin - state->src_literal); // literal count
  4725. + size_t M = (size_t)match.M; // match length
  4726. + size_t D = (size_t)match.D; // match distance
  4727. + size_t D_prev = (size_t)state->d_prev; // previously emitted match distance
  4728. + unsigned char *dst = emit(state->src + state->src_literal, state->dst,
  4729. + state->dst_end, L, M, D, D_prev);
  4730. + // Check if DST is full
  4731. + if (dst >= state->dst_end) {
  4732. + return 0; // FULL
  4733. + }
  4734. +
  4735. + // Update state
  4736. + lzvn_offset dst_used = dst - state->dst;
  4737. + state->d_prev = match.D;
  4738. + state->dst = dst;
  4739. + state->src_literal = match.m_end;
  4740. + return dst_used;
  4741. +}
  4742. +
  4743. +/*! @abstract Emit a n-bytes literal and update state.
  4744. + * @return number of bytes written to \p dst. May be 0 if there is no more space
  4745. + * in \p dst to emit the literal. */
  4746. +static inline lzvn_offset lzvn_emit_literal(lzvn_encoder_state *state,
  4747. + lzvn_offset n) {
  4748. + size_t L = (size_t)n;
  4749. + unsigned char *dst = emit_literal(state->src + state->src_literal, state->dst,
  4750. + state->dst_end, L);
  4751. + // Check if DST is full
  4752. + if (dst >= state->dst_end)
  4753. + return 0; // FULL
  4754. +
  4755. + // Update state
  4756. + lzvn_offset dst_used = dst - state->dst;
  4757. + state->dst = dst;
  4758. + state->src_literal += n;
  4759. + return dst_used;
  4760. +}
  4761. +
  4762. +/*! @abstract Emit end-of-stream and update state.
  4763. + * @return number of bytes written to \p dst. May be 0 if there is no more space
  4764. + * in \p dst to emit the instruction. */
  4765. +static inline lzvn_offset lzvn_emit_end_of_stream(lzvn_encoder_state *state) {
  4766. + // Do we have 8 byte in dst?
  4767. + if (state->dst_end < state->dst + 8)
  4768. + return 0; // FULL
  4769. +
  4770. + // Insert end marker and update state
  4771. + store8(state->dst, 0x06); // end-of-stream command
  4772. + state->dst += 8;
  4773. + return 8; // dst_used
  4774. +}
  4775. +
  4776. +// ===============================================================
  4777. +// Encoder Functions
  4778. +
  4779. +/*! @abstract Initialize encoder table in \p state, uses current I/O parameters. */
  4780. +static inline void lzvn_init_table(lzvn_encoder_state *state) {
  4781. + lzvn_offset index = -LZVN_ENCODE_MAX_DISTANCE; // max match distance
  4782. + if (index < state->src_begin)
  4783. + index = state->src_begin;
  4784. + uint32_t value = load4(state->src + index);
  4785. +
  4786. + lzvn_encode_entry_type e;
  4787. + for (int i = 0; i < 4; i++) {
  4788. + e.indices[i] = offset_to_s32(index);
  4789. + e.values[i] = value;
  4790. + }
  4791. + for (int u = 0; u < LZVN_ENCODE_HASH_VALUES; u++)
  4792. + state->table[u] = e; // fill entire table
  4793. +}
  4794. +
  4795. +void lzvn_encode(lzvn_encoder_state *state) {
  4796. + const lzvn_match_info NO_MATCH = {0};
  4797. +
  4798. + for (; state->src_current < state->src_current_end; state->src_current++) {
  4799. + // Get 4 bytes at src_current
  4800. + uint32_t vi = load4(state->src + state->src_current);
  4801. +
  4802. + // Compute new hash H at position I, and push value into position table
  4803. + int h = hash3i(vi); // index of first entry
  4804. +
  4805. + // Read table entries for H
  4806. + lzvn_encode_entry_type e = state->table[h];
  4807. +
  4808. + // Update entry with index=current and value=vi
  4809. + lzvn_encode_entry_type updated_e; // rotate values, so we will replace the oldest
  4810. + updated_e.indices[0] = offset_to_s32(state->src_current);
  4811. + updated_e.indices[1] = e.indices[0];
  4812. + updated_e.indices[2] = e.indices[1];
  4813. + updated_e.indices[3] = e.indices[2];
  4814. + updated_e.values[0] = vi;
  4815. + updated_e.values[1] = e.values[0];
  4816. + updated_e.values[2] = e.values[1];
  4817. + updated_e.values[3] = e.values[2];
  4818. +
  4819. + // Do not check matches if still in previously emitted match
  4820. + if (state->src_current < state->src_literal)
  4821. + goto after_emit;
  4822. +
  4823. +// Update best with candidate if better
  4824. +#define UPDATE(best, candidate) \
  4825. + do { \
  4826. + if (candidate.K > best.K || \
  4827. + ((candidate.K == best.K) && (candidate.m_end > best.m_end + 1))) { \
  4828. + best = candidate; \
  4829. + } \
  4830. + } while (0)
  4831. +// Check candidate. Keep if better.
  4832. +#define CHECK_CANDIDATE(ik, nk) \
  4833. + do { \
  4834. + lzvn_match_info m1; \
  4835. + if (lzvn_find_matchN(state->src, state->src_begin, state->src_end, \
  4836. + state->src_literal, ik, state->src_current, nk, &m1)) { \
  4837. + UPDATE(incoming, m1); \
  4838. + } \
  4839. + } while (0)
  4840. +// Emit match M. Return if we don't have enough space in the destination buffer
  4841. +#define EMIT_MATCH(m) \
  4842. + do { \
  4843. + if (lzvn_emit_match(state, m) == 0) \
  4844. + return; \
  4845. + } while (0)
  4846. +// Emit literal of length L. Return if we don't have enough space in the
  4847. +// destination buffer
  4848. +#define EMIT_LITERAL(l) \
  4849. + do { \
  4850. + if (lzvn_emit_literal(state, l) == 0) \
  4851. + return; \
  4852. + } while (0)
  4853. +
  4854. + lzvn_match_info incoming = NO_MATCH;
  4855. +
  4856. + // Check candidates in order (closest first)
  4857. + uint32_t diffs[4];
  4858. + for (int k = 0; k < 4; k++)
  4859. + diffs[k] = e.values[k] ^ vi; // XOR, 0 if equal
  4860. + lzvn_offset ik; // index
  4861. + lzvn_offset nk; // match byte count
  4862. +
  4863. + // The values stored in e.xyzw are 32-bit signed indices, extended to signed
  4864. + // type lzvn_offset
  4865. + ik = offset_from_s32(e.indices[0]);
  4866. + nk = trailing_zero_bytes(diffs[0]);
  4867. + CHECK_CANDIDATE(ik, nk);
  4868. + ik = offset_from_s32(e.indices[1]);
  4869. + nk = trailing_zero_bytes(diffs[1]);
  4870. + CHECK_CANDIDATE(ik, nk);
  4871. + ik = offset_from_s32(e.indices[2]);
  4872. + nk = trailing_zero_bytes(diffs[2]);
  4873. + CHECK_CANDIDATE(ik, nk);
  4874. + ik = offset_from_s32(e.indices[3]);
  4875. + nk = trailing_zero_bytes(diffs[3]);
  4876. + CHECK_CANDIDATE(ik, nk);
  4877. +
  4878. + // Check candidate at previous distance
  4879. + if (state->d_prev != 0) {
  4880. + lzvn_match_info m1;
  4881. + if (lzvn_find_match(state->src, state->src_begin, state->src_end,
  4882. + state->src_literal, state->src_current - state->d_prev,
  4883. + state->src_current, &m1)) {
  4884. + m1.K = m1.M - 1; // fix K for D_prev
  4885. + UPDATE(incoming, m1);
  4886. + }
  4887. + }
  4888. +
  4889. + // Here we have the best candidate in incoming, may be NO_MATCH
  4890. +
  4891. + // If no incoming match, and literal backlog becomes too high, emit pending
  4892. + // match, or literals if there is no pending match
  4893. + if (incoming.M == 0) {
  4894. + if (state->src_current - state->src_literal >=
  4895. + LZVN_ENCODE_MAX_LITERAL_BACKLOG) // at this point, we always have
  4896. + // current >= literal
  4897. + {
  4898. + if (state->pending.M != 0) {
  4899. + EMIT_MATCH(state->pending);
  4900. + state->pending = NO_MATCH;
  4901. + } else {
  4902. + EMIT_LITERAL(271); // emit long literal (271 is the longest literal size we allow)
  4903. + }
  4904. + }
  4905. + goto after_emit;
  4906. + }
  4907. +
  4908. + if (state->pending.M == 0) {
  4909. + // NOTE. Here, we can also emit incoming right away. It will make the
  4910. + // encoder 1.5x faster, at a cost of ~10% lower compression ratio:
  4911. + // EMIT_MATCH(incoming);
  4912. + // state->pending = NO_MATCH;
  4913. +
  4914. + // No pending match, emit nothing, keep incoming
  4915. + state->pending = incoming;
  4916. + } else {
  4917. + // Here we have both incoming and pending
  4918. + if (state->pending.m_end <= incoming.m_begin) {
  4919. + // No overlap: emit pending, keep incoming
  4920. + EMIT_MATCH(state->pending);
  4921. + state->pending = incoming;
  4922. + } else {
  4923. + // If pending is better, emit pending and discard incoming.
  4924. + // Otherwise, emit incoming and discard pending.
  4925. + if (incoming.K > state->pending.K)
  4926. + state->pending = incoming;
  4927. + EMIT_MATCH(state->pending);
  4928. + state->pending = NO_MATCH;
  4929. + }
  4930. + }
  4931. +
  4932. + after_emit:
  4933. +
  4934. + // We commit state changes only after we tried to emit instructions, so we
  4935. + // can restart in the same state in case dst was full and we quit the loop.
  4936. + state->table[h] = updated_e;
  4937. +
  4938. + } // i loop
  4939. +
  4940. + // Do not emit pending match here. We do it only at the end of stream.
  4941. +}
  4942. +
  4943. +// ===============================================================
  4944. +// API entry points
  4945. +
  4946. +size_t lzvn_encode_scratch_size(void) { return LZVN_ENCODE_WORK_SIZE; }
  4947. +
  4948. +static size_t lzvn_encode_partial(void *__restrict dst, size_t dst_size,
  4949. + const void *__restrict src, size_t src_size,
  4950. + size_t *src_used, void *__restrict work) {
  4951. + // Min size checks to avoid accessing memory outside buffers.
  4952. + if (dst_size < LZVN_ENCODE_MIN_DST_SIZE) {
  4953. + *src_used = 0;
  4954. + return 0;
  4955. + }
  4956. + // Max input size check (limit to offsets on uint32_t).
  4957. + if (src_size > LZVN_ENCODE_MAX_SRC_SIZE) {
  4958. + src_size = LZVN_ENCODE_MAX_SRC_SIZE;
  4959. + }
  4960. +
  4961. + // Setup encoder state
  4962. + lzvn_encoder_state state;
  4963. + memset(&state, 0, sizeof(state));
  4964. +
  4965. + state.src = src;
  4966. + state.src_begin = 0;
  4967. + state.src_end = (lzvn_offset)src_size;
  4968. + state.src_literal = 0;
  4969. + state.src_current = 0;
  4970. + state.dst = dst;
  4971. + state.dst_begin = dst;
  4972. + state.dst_end = (unsigned char *)dst + dst_size - 8; // reserve 8 bytes for end-of-stream
  4973. + state.table = work;
  4974. +
  4975. + // Do not encode if the input buffer is too small. We'll emit a literal instead.
  4976. + if (src_size >= LZVN_ENCODE_MIN_SRC_SIZE) {
  4977. +
  4978. + state.src_current_end = (lzvn_offset)src_size - LZVN_ENCODE_MIN_MARGIN;
  4979. + lzvn_init_table(&state);
  4980. + lzvn_encode(&state);
  4981. +
  4982. + }
  4983. +
  4984. + // No need to test the return value: src_literal will not be updated on failure,
  4985. + // and we will fail later.
  4986. + lzvn_emit_literal(&state, state.src_end - state.src_literal);
  4987. +
  4988. + // Restore original size, so end-of-stream always succeeds, and emit it
  4989. + state.dst_end = (unsigned char *)dst + dst_size;
  4990. + lzvn_emit_end_of_stream(&state);
  4991. +
  4992. + *src_used = state.src_literal;
  4993. + return (size_t)(state.dst - state.dst_begin);
  4994. +}
  4995. +
  4996. +size_t lzvn_encode_buffer(void *__restrict dst, size_t dst_size,
  4997. + const void *__restrict src, size_t src_size,
  4998. + void *__restrict work) {
  4999. + size_t src_used = 0;
  5000. + size_t dst_used =
  5001. + lzvn_encode_partial(dst, dst_size, src, src_size, &src_used, work);
  5002. + if (src_used != src_size)
  5003. + return 0; // could not encode entire input stream = fail
  5004. + return dst_used; // return encoded size
  5005. +}
  5006. diff --git a/lib/lzfse/lzvn_encode_base.h b/lib/lzfse/lzvn_encode_base.h
  5007. new file mode 100644
  5008. index 0000000..1b93d0e
  5009. --- /dev/null
  5010. +++ b/lib/lzfse/lzvn_encode_base.h
  5011. @@ -0,0 +1,116 @@
  5012. +/*
  5013. +Copyright (c) 2015-2016, Apple Inc. All rights reserved.
  5014. +
  5015. +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
  5016. +
  5017. +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
  5018. +
  5019. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
  5020. + in the documentation and/or other materials provided with the distribution.
  5021. +
  5022. +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
  5023. + from this software without specific prior written permission.
  5024. +
  5025. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  5026. +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
  5027. +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  5028. +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  5029. +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  5030. +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  5031. +*/
  5032. +
  5033. +// LZVN low-level encoder
  5034. +
  5035. +#ifndef LZVN_ENCODE_BASE_H
  5036. +#define LZVN_ENCODE_BASE_H
  5037. +
  5038. +#include "lzfse_internal.h"
  5039. +
  5040. +// ===============================================================
  5041. +// Types and Constants
  5042. +
  5043. +#define LZVN_ENCODE_HASH_BITS \
  5044. + 14 // number of bits returned by the hash function [10, 16]
  5045. +#define LZVN_ENCODE_OFFSETS_PER_HASH \
  5046. + 4 // stored offsets stack for each hash value, MUST be 4
  5047. +#define LZVN_ENCODE_HASH_VALUES \
  5048. + (1 << LZVN_ENCODE_HASH_BITS) // number of entries in hash table
  5049. +#define LZVN_ENCODE_MAX_DISTANCE \
  5050. + 0xffff // max match distance we can represent with LZVN encoding, MUST be
  5051. + // 0xFFFF
  5052. +#define LZVN_ENCODE_MIN_MARGIN \
  5053. + 8 // min number of bytes required between current and end during encoding,
  5054. + // MUST be >= 8
  5055. +#define LZVN_ENCODE_MAX_LITERAL_BACKLOG \
  5056. + 400 // if the number of pending literals exceeds this size, emit a long
  5057. + // literal, MUST be >= 271
  5058. +
  5059. +/*! @abstract Type of table entry. */
  5060. +typedef struct {
  5061. + int32_t indices[4]; // signed indices in source buffer
  5062. + uint32_t values[4]; // corresponding 32-bit values
  5063. +} lzvn_encode_entry_type;
  5064. +
  5065. +// Work size
  5066. +#define LZVN_ENCODE_WORK_SIZE \
  5067. + (LZVN_ENCODE_HASH_VALUES * sizeof(lzvn_encode_entry_type))
  5068. +
  5069. +/*! @abstract Match */
  5070. +typedef struct {
  5071. + lzvn_offset m_begin; // beginning of match, current position
  5072. + lzvn_offset m_end; // end of match, this is where the next literal would begin
  5073. + // if we emit the entire match
  5074. + lzvn_offset M; // match length M: m_end - m_begin
  5075. + lzvn_offset D; // match distance D
  5076. + lzvn_offset K; // match gain: M - distance storage (L not included)
  5077. +} lzvn_match_info;
  5078. +
  5079. +// ===============================================================
  5080. +// Internal encoder state
  5081. +
  5082. +/*! @abstract Base encoder state and I/O. */
  5083. +typedef struct {
  5084. +
  5085. + // Encoder I/O
  5086. +
  5087. + // Source buffer
  5088. + const unsigned char *src;
  5089. + // Valid range in source buffer: we can access src[i] for src_begin <= i <
  5090. + // src_end. src_begin may be negative.
  5091. + lzvn_offset src_begin;
  5092. + lzvn_offset src_end;
  5093. + // Next byte to process in source buffer
  5094. + lzvn_offset src_current;
  5095. + // Next byte after the last byte to process in source buffer. We MUST have:
  5096. + // src_current_end + 8 <= src_end.
  5097. + lzvn_offset src_current_end;
  5098. + // Next byte to encode in source buffer, may be before or after src_current.
  5099. + lzvn_offset src_literal;
  5100. +
  5101. + // Next byte to write in destination buffer
  5102. + unsigned char *dst;
  5103. + // Valid range in destination buffer: [dst_begin, dst_end - 1]
  5104. + unsigned char *dst_begin;
  5105. + unsigned char *dst_end;
  5106. +
  5107. + // Encoder state
  5108. +
  5109. + // Pending match
  5110. + lzvn_match_info pending;
  5111. +
  5112. + // Distance for last emitted match, or 0
  5113. + lzvn_offset d_prev;
  5114. +
  5115. + // Hash table used to find matches. Stores LZVN_ENCODE_OFFSETS_PER_HASH 32-bit
  5116. + // signed indices in the source buffer, and the corresponding 4-byte values.
  5117. + // The number of entries in the table is LZVN_ENCODE_HASH_VALUES.
  5118. + lzvn_encode_entry_type *table;
  5119. +
  5120. +} lzvn_encoder_state;
  5121. +
  5122. +/*! @abstract Encode source to destination.
  5123. + * Update \p state.
  5124. + * The call ensures \c src_literal is never left too far behind \c src_current. */
  5125. +void lzvn_encode(lzvn_encoder_state *state);
  5126. +
  5127. +#endif // LZVN_ENCODE_BASE_H
  5128. --
  5129. 2.9.3
  5130.  
  5131.  
  5132. From f4ad7b1aff5529a5270e3fdb45395416c601145f Mon Sep 17 00:00:00 2001
  5133. From: Valery Tolstov <testautomail@mail.ru>
  5134. Date: Fri, 6 Jan 2017 23:48:58 +0300
  5135. Subject: [PATCH 2/5] Fix include issues
  5136.  
  5137. ---
  5138. include/linux/lzfse.h | 2 ++
  5139. lib/lzfse/lzfse_fse.h | 14 ++++++++------
  5140. lib/lzfse/lzfse_internal.h | 8 +++++---
  5141. 3 files changed, 15 insertions(+), 9 deletions(-)
  5142.  
  5143. diff --git a/include/linux/lzfse.h b/include/linux/lzfse.h
  5144. index 7cfb138..671a6c7 100644
  5145. --- a/include/linux/lzfse.h
  5146. +++ b/include/linux/lzfse.h
  5147. @@ -22,6 +22,8 @@ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSI
  5148. #ifndef LZFSE_H
  5149. #define LZFSE_H
  5150.  
  5151. +#include <linux/types.h>
  5152. +
  5153. /*! @abstract Get the required scratch buffer size to compress using LZFSE. */
  5154. size_t lzfse_encode_scratch_size(void);
  5155.  
  5156. diff --git a/lib/lzfse/lzfse_fse.h b/lib/lzfse/lzfse_fse.h
  5157. index 8846012..590af33 100644
  5158. --- a/lib/lzfse/lzfse_fse.h
  5159. +++ b/lib/lzfse/lzfse_fse.h
  5160. @@ -23,13 +23,13 @@ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSI
  5161. // This is an implementation of the tANS algorithm described by Jarek Duda,
  5162. // we use the more descriptive name "Finite State Entropy".
  5163.  
  5164. -#pragma once
  5165. +#ifndef LZFSE_FSE_H
  5166. +#define LZFSE_FSE_H
  5167.  
  5168. -#include <assert.h>
  5169. -#include <stddef.h>
  5170. -#include <stdint.h>
  5171. -#include <stdlib.h>
  5172. -#include <string.h>
  5173. +#include <linux/kernel.h>
  5174. +#include <linux/string.h>
  5175. +
  5176. +#define assert(x) ({ if (!(x)) printk("failed assertion\n"); })
  5177.  
  5178. // Select between 32/64-bit I/O streams for FSE. Note that the FSE stream
  5179. // size need not match the word size of the machine, but in practice you
  5180. @@ -629,3 +629,5 @@ void fse_init_value_decoder_table(int nstates, int nsymbols,
  5181. * \c freq[nsymbols]. */
  5182. void fse_normalize_freq(int nstates, int nsymbols, const uint32_t *__restrict t,
  5183. uint16_t *__restrict freq);
  5184. +
  5185. +#endif
  5186. diff --git a/lib/lzfse/lzfse_internal.h b/lib/lzfse/lzfse_internal.h
  5187. index 7ea5a30..1347fb8 100644
  5188. --- a/lib/lzfse/lzfse_internal.h
  5189. +++ b/lib/lzfse/lzfse_internal.h
  5190. @@ -29,9 +29,11 @@ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSI
  5191.  
  5192. #include "lzfse_fse.h"
  5193. #include "lzfse_tunables.h"
  5194. -#include <limits.h>
  5195. -#include <stddef.h>
  5196. -#include <stdint.h>
  5197. +
  5198. +// TODO Temporary solution
  5199. +#define uintmax_t uint64_t
  5200. +
  5201. +#define INT32_MAX S32_MAX
  5202.  
  5203. #if defined(_MSC_VER) && !defined(__clang__)
  5204. # define LZFSE_INLINE __forceinline
  5205. --
  5206. 2.9.3
  5207.  
  5208.  
  5209. From 44805480037afd57d10cb9a5299cb810710d285d Mon Sep 17 00:00:00 2001
  5210. From: Valery Tolstov <testautomail@mail.ru>
  5211. Date: Sat, 7 Jan 2017 00:13:59 +0300
  5212. Subject: [PATCH 3/5] Fix critical build issues
  5213.  
  5214. ---
  5215. lib/lzfse/lzfse_decode_base.c | 20 +++++++++++++-------
  5216. lib/lzfse/lzfse_encode_base.c | 34 ++++++++++++++++++++--------------
  5217. lib/lzfse/lzfse_fse.c | 23 ++++++++++++++---------
  5218. lib/lzfse/lzfse_fse.h | 3 ++-
  5219. lib/lzfse/lzvn_decode_base.c | 18 ++++++++++--------
  5220. lib/lzfse/lzvn_encode_base.c | 22 +++++++++++++---------
  5221. 6 files changed, 72 insertions(+), 48 deletions(-)
  5222.  
  5223. diff --git a/lib/lzfse/lzfse_decode_base.c b/lib/lzfse/lzfse_decode_base.c
  5224. index 31fe859..60ebe86 100644
  5225. --- a/lib/lzfse/lzfse_decode_base.c
  5226. +++ b/lib/lzfse/lzfse_decode_base.c
  5227. @@ -71,6 +71,8 @@ lzfse_decode_v2_header_size(const lzfse_compressed_block_header_v2 *in) {
  5228. * @return -1 on failure. */
  5229. static inline int lzfse_decode_v1(lzfse_compressed_block_header_v1 *out,
  5230. const lzfse_compressed_block_header_v2 *in) {
  5231. + int i;
  5232. +
  5233. // Clear all fields
  5234. memset(out, 0x00, sizeof(lzfse_compressed_block_header_v1));
  5235.  
  5236. @@ -114,7 +116,7 @@ static inline int lzfse_decode_v1(lzfse_compressed_block_header_v1 *out,
  5237. if (src_end == src)
  5238. return 0; // OK, freq tables were omitted
  5239.  
  5240. - for (int i = 0; i < LZFSE_ENCODE_L_SYMBOLS + LZFSE_ENCODE_M_SYMBOLS +
  5241. + for (i = 0; i < LZFSE_ENCODE_L_SYMBOLS + LZFSE_ENCODE_M_SYMBOLS +
  5242. LZFSE_ENCODE_D_SYMBOLS + LZFSE_ENCODE_LITERAL_SYMBOLS;
  5243. i++) {
  5244. // Refill accum, one byte at a time, until we reach end of header, or accum
  5245. @@ -168,6 +170,8 @@ static int lzfse_decode_lmd(lzfse_decoder_state *s) {
  5246. int32_t M = bs->m_value;
  5247. int32_t D = bs->d_value;
  5248.  
  5249. + size_t i;
  5250. +
  5251. assert(l_state < LZFSE_ENCODE_L_STATES);
  5252. assert(m_state < LZFSE_ENCODE_M_STATES);
  5253. assert(d_state < LZFSE_ENCODE_D_STATES);
  5254. @@ -237,7 +241,7 @@ static int lzfse_decode_lmd(lzfse_decoder_state *s) {
  5255. if (D >= 8 || D >= M)
  5256. copy(dst, dst - D, M);
  5257. else
  5258. - for (size_t i = 0; i < M; i++)
  5259. + for (i = 0; i < M; i++)
  5260. dst[i] = dst[i - D];
  5261. dst += M;
  5262. }
  5263. @@ -253,7 +257,7 @@ static int lzfse_decode_lmd(lzfse_decoder_state *s) {
  5264. // or there isn't; if there is, we copy the whole thing and
  5265. // update all the pointers and lengths to reflect the copy.
  5266. if (L <= remaining_bytes) {
  5267. - for (size_t i = 0; i < L; i++)
  5268. + for (i = 0; i < L; i++)
  5269. dst[i] = lit[i];
  5270. dst += L;
  5271. lit += L;
  5272. @@ -265,7 +269,7 @@ static int lzfse_decode_lmd(lzfse_decoder_state *s) {
  5273. // L, and report that the destination buffer is full. Note that
  5274. // we always write right up to the end of the destination buffer.
  5275. else {
  5276. - for (size_t i = 0; i < remaining_bytes; i++)
  5277. + for (i = 0; i < remaining_bytes; i++)
  5278. dst[i] = lit[i];
  5279. dst += remaining_bytes;
  5280. lit += remaining_bytes;
  5281. @@ -277,7 +281,7 @@ static int lzfse_decode_lmd(lzfse_decoder_state *s) {
  5282. // before finishing, we return to the caller indicating that
  5283. // the buffer is full.
  5284. if (M <= remaining_bytes) {
  5285. - for (size_t i = 0; i < M; i++)
  5286. + for (i = 0; i < M; i++)
  5287. dst[i] = dst[i - D];
  5288. dst += M;
  5289. remaining_bytes -= M;
  5290. @@ -291,7 +295,7 @@ static int lzfse_decode_lmd(lzfse_decoder_state *s) {
  5291. //
  5292. // But we still set M = 0, to maintain the post-condition.
  5293. } else {
  5294. - for (size_t i = 0; i < remaining_bytes; i++)
  5295. + for (i = 0; i < remaining_bytes; i++)
  5296. dst[i] = dst[i - D];
  5297. dst += remaining_bytes;
  5298. M -= remaining_bytes;
  5299. @@ -330,6 +334,8 @@ static int lzfse_decode_lmd(lzfse_decoder_state *s) {
  5300. }
  5301.  
  5302. int lzfse_decode(lzfse_decoder_state *s) {
  5303. + uint32_t i;
  5304. +
  5305. while (1) {
  5306. // Are we inside a block?
  5307. switch (s->block_magic) {
  5308. @@ -451,7 +457,7 @@ int lzfse_decode(lzfse_decoder_state *s) {
  5309. fse_state state2 = header1.literal_state[2];
  5310. fse_state state3 = header1.literal_state[3];
  5311.  
  5312. - for (uint32_t i = 0; i < header1.n_literals; i += 4) // n_literals is multiple of 4
  5313. + for (i = 0; i < header1.n_literals; i += 4) // n_literals is multiple of 4
  5314. {
  5315. #if FSE_IOSTREAM_64
  5316. if (fse_in_flush(&in, &buf, buf_start) != 0)
  5317. diff --git a/lib/lzfse/lzfse_encode_base.c b/lib/lzfse/lzfse_encode_base.c
  5318. index 367d7a2..c5fafd0 100644
  5319. --- a/lib/lzfse/lzfse_encode_base.c
  5320. +++ b/lib/lzfse/lzfse_encode_base.c
  5321. @@ -116,11 +116,12 @@ lzfse_encode_v1_freq_table(lzfse_compressed_block_header_v2 *out,
  5322. const lzfse_compressed_block_header_v1 *in) {
  5323. uint32_t accum = 0;
  5324. int accum_nbits = 0;
  5325. + int i;
  5326. const uint16_t *src = &(in->l_freq[0]); // first value of first table (struct
  5327. // will not be modified, so this code
  5328. // will remain valid)
  5329. uint8_t *dst = &(out->freq[0]);
  5330. - for (int i = 0; i < LZFSE_ENCODE_L_SYMBOLS + LZFSE_ENCODE_M_SYMBOLS +
  5331. + for (i = 0; i < LZFSE_ENCODE_L_SYMBOLS + LZFSE_ENCODE_M_SYMBOLS +
  5332. LZFSE_ENCODE_D_SYMBOLS + LZFSE_ENCODE_LITERAL_SYMBOLS;
  5333. i++) {
  5334. // Encode one value to accum
  5335. @@ -188,6 +189,8 @@ static int lzfse_encode_matches(lzfse_encoder_state *s) {
  5336. uint8_t *dst0 = s->dst;
  5337. uint32_t n_literals0 = s->n_literals;
  5338.  
  5339. + uint32_t i;
  5340. +
  5341. // Add 0x00 literals until n_literals multiple of 4, since we encode 4
  5342. // interleaved literal streams.
  5343. while (s->n_literals & 3) {
  5344. @@ -197,7 +200,7 @@ static int lzfse_encode_matches(lzfse_encoder_state *s) {
  5345.  
  5346. // Encode previous distance
  5347. uint32_t d_prev = 0;
  5348. - for (uint32_t i = 0; i < s->n_matches; i++) {
  5349. + for (i = 0; i < s->n_matches; i++) {
  5350. uint32_t d = s->d_values[i];
  5351. if (d == d_prev)
  5352. s->d_values[i] = 0;
  5353. @@ -214,19 +217,19 @@ static int lzfse_encode_matches(lzfse_encoder_state *s) {
  5354. // Update occurrence tables in all 4 streams (L,M,D,literals)
  5355. uint32_t l_sum = 0;
  5356. uint32_t m_sum = 0;
  5357. - for (uint32_t i = 0; i < s->n_matches; i++) {
  5358. + for (i = 0; i < s->n_matches; i++) {
  5359. uint32_t l = s->l_values[i];
  5360. l_sum += l;
  5361. l_occ[l_base_from_value(l)]++;
  5362. }
  5363. - for (uint32_t i = 0; i < s->n_matches; i++) {
  5364. + for (i = 0; i < s->n_matches; i++) {
  5365. uint32_t m = s->m_values[i];
  5366. m_sum += m;
  5367. m_occ[m_base_from_value(m)]++;
  5368. }
  5369. - for (uint32_t i = 0; i < s->n_matches; i++)
  5370. + for (i = 0; i < s->n_matches; i++)
  5371. d_occ[d_base_from_value(s->d_values[i])]++;
  5372. - for (uint32_t i = 0; i < s->n_literals; i++)
  5373. + for (i = 0; i < s->n_literals; i++)
  5374. literal_occ[s->literals[i]]++;
  5375.  
  5376. // Make sure we have enough room for a _full_ V2 header
  5377. @@ -398,7 +401,7 @@ END:
  5378.  
  5379. // Revert the d_prev encoding
  5380. uint32_t d_prev = 0;
  5381. - for (uint32_t i = 0; i < s->n_matches; i++) {
  5382. + for (i = 0; i < s->n_matches; i++) {
  5383. uint32_t d = s->d_values[i];
  5384. if (d == 0)
  5385. s->d_values[i] = d_prev;
  5386. @@ -590,12 +593,13 @@ static int lzfse_backend_end_of_stream(lzfse_encoder_state *s) {
  5387. int lzfse_encode_init(lzfse_encoder_state *s) {
  5388. const lzfse_match NO_MATCH = {0};
  5389. lzfse_history_set line;
  5390. - for (int i = 0; i < LZFSE_ENCODE_HASH_WIDTH; i++) {
  5391. + int i;
  5392. + for (i = 0; i < LZFSE_ENCODE_HASH_WIDTH; i++) {
  5393. line.pos[i] = -4 * LZFSE_ENCODE_MAX_D_VALUE; // invalid pos
  5394. line.value[i] = 0;
  5395. }
  5396. // Fill table
  5397. - for (int i = 0; i < LZFSE_ENCODE_HASH_VALUES; i++)
  5398. + for (i = 0; i < LZFSE_ENCODE_HASH_VALUES; i++)
  5399. s->history_table[i] = line;
  5400. s->pending = NO_MATCH;
  5401. s->src_literal = 0;
  5402. @@ -607,6 +611,7 @@ int lzfse_encode_init(lzfse_encoder_state *s) {
  5403. * Offsets in \p src are updated backwards to point to the same positions.
  5404. * @return LZFSE_STATUS_OK */
  5405. int lzfse_encode_translate(lzfse_encoder_state *s, lzfse_offset delta) {
  5406. + int i, j;
  5407. assert(delta >= 0);
  5408. if (delta == 0)
  5409. return LZFSE_STATUS_OK; // OK
  5410. @@ -626,9 +631,9 @@ int lzfse_encode_translate(lzfse_encoder_state *s, lzfse_offset delta) {
  5411.  
  5412. // history_table positions, translated, and clamped to invalid pos
  5413. int32_t invalidPos = -4 * LZFSE_ENCODE_MAX_D_VALUE;
  5414. - for (int i = 0; i < LZFSE_ENCODE_HASH_VALUES; i++) {
  5415. + for (i = 0; i < LZFSE_ENCODE_HASH_VALUES; i++) {
  5416. int32_t *p = &(s->history_table[i].pos[0]);
  5417. - for (int j = 0; j < LZFSE_ENCODE_HASH_WIDTH; j++) {
  5418. + for (j = 0; j < LZFSE_ENCODE_HASH_WIDTH; j++) {
  5419. lzfse_offset newPos = p[j] - delta; // translate
  5420. p[j] = (int32_t)((newPos < invalidPos) ? invalidPos : newPos); // clamp
  5421. }
  5422. @@ -646,6 +651,7 @@ int lzfse_encode_base(lzfse_encoder_state *s) {
  5423. lzfse_history_set newH;
  5424. const lzfse_match NO_MATCH = {0};
  5425. int ok = 1;
  5426. + int k;
  5427.  
  5428. memset(&newH, 0x00, sizeof(newH));
  5429.  
  5430. @@ -663,10 +669,10 @@ int lzfse_encode_base(lzfse_encoder_state *s) {
  5431. // entries (stored later)
  5432. {
  5433. newH.pos[0] = (int32_t)pos;
  5434. - for (int k = 0; k < LZFSE_ENCODE_HASH_WIDTH - 1; k++)
  5435. + for (k = 0; k < LZFSE_ENCODE_HASH_WIDTH - 1; k++)
  5436. newH.pos[k + 1] = h.pos[k];
  5437. newH.value[0] = x;
  5438. - for (int k = 0; k < LZFSE_ENCODE_HASH_WIDTH - 1; k++)
  5439. + for (k = 0; k < LZFSE_ENCODE_HASH_WIDTH - 1; k++)
  5440. newH.value[k + 1] = h.value[k];
  5441. }
  5442.  
  5443. @@ -678,7 +684,7 @@ int lzfse_encode_base(lzfse_encoder_state *s) {
  5444. lzfse_match incoming = {.pos = pos, .ref = 0, .length = 0};
  5445.  
  5446. // Check for matches. We consider matches of length >= 4 only.
  5447. - for (int k = 0; k < LZFSE_ENCODE_HASH_WIDTH; k++) {
  5448. + for (k = 0; k < LZFSE_ENCODE_HASH_WIDTH; k++) {
  5449. uint32_t d = h.value[k] ^ x;
  5450. if (d)
  5451. continue; // no 4 byte match
  5452. diff --git a/lib/lzfse/lzfse_fse.c b/lib/lzfse/lzfse_fse.c
  5453. index 631ada0..59766d2 100644
  5454. --- a/lib/lzfse/lzfse_fse.c
  5455. +++ b/lib/lzfse/lzfse_fse.c
  5456. @@ -33,7 +33,8 @@ void fse_init_encoder_table(int nstates, int nsymbols,
  5457. fse_encoder_entry *__restrict t) {
  5458. int offset = 0; // current offset
  5459. int n_clz = __builtin_clz(nstates);
  5460. - for (int i = 0; i < nsymbols; i++) {
  5461. + int i;
  5462. + for (i = 0; i < nsymbols; i++) {
  5463. int f = (int)freq[i];
  5464. if (f == 0)
  5465. continue; // skip this symbol, no occurrences
  5466. @@ -61,7 +62,8 @@ int fse_init_decoder_table(int nstates, int nsymbols,
  5467. assert(fse_check_freq(freq, nsymbols, nstates) == 0);
  5468. int n_clz = __builtin_clz(nstates);
  5469. int sum_of_freq = 0;
  5470. - for (int i = 0; i < nsymbols; i++) {
  5471. + int i, j;
  5472. + for (i = 0; i < nsymbols; i++) {
  5473. int f = (int)freq[i];
  5474. if (f == 0)
  5475. continue; // skip this symbol, no occurrences
  5476. @@ -77,7 +79,7 @@ int fse_init_decoder_table(int nstates, int nsymbols,
  5477. int j0 = ((2 * nstates) >> k) - f;
  5478.  
  5479. // Initialize all states S reached by this symbol: OFFSET <= S < OFFSET + F
  5480. - for (int j = 0; j < f; j++) {
  5481. + for (j = 0; j < f; j++) {
  5482. fse_decoder_entry e;
  5483.  
  5484. e.symbol = (uint8_t)i;
  5485. @@ -115,7 +117,8 @@ void fse_init_value_decoder_table(int nstates, int nsymbols,
  5486. assert(fse_check_freq(freq, nsymbols, nstates) == 0);
  5487.  
  5488. int n_clz = __builtin_clz(nstates);
  5489. - for (int i = 0; i < nsymbols; i++) {
  5490. + int i, j;
  5491. + for (i = 0; i < nsymbols; i++) {
  5492. int f = (int)freq[i];
  5493. if (f == 0)
  5494. continue; // skip this symbol, no occurrences
  5495. @@ -129,7 +132,7 @@ void fse_init_value_decoder_table(int nstates, int nsymbols,
  5496. ei.vbase = symbol_vbase[i];
  5497.  
  5498. // Initialize all states S reached by this symbol: OFFSET <= S < OFFSET + F
  5499. - for (int j = 0; j < f; j++) {
  5500. + for (j = 0; j < f; j++) {
  5501. fse_value_decoder_entry e = ei;
  5502.  
  5503. if (j < j0) {
  5504. @@ -148,8 +151,9 @@ void fse_init_value_decoder_table(int nstates, int nsymbols,
  5505.  
  5506. // Remove states from symbols until the correct number of states is used.
  5507. static void fse_adjust_freqs(uint16_t *freq, int overrun, int nsymbols) {
  5508. - for (int shift = 3; overrun != 0; shift--) {
  5509. - for (int sym = 0; sym < nsymbols; sym++) {
  5510. + int shift, sym;
  5511. + for (shift = 3; overrun != 0; shift--) {
  5512. + for (sym = 0; sym < nsymbols; sym++) {
  5513. if (freq[sym] > 1) {
  5514. int n = (freq[sym] - 1) >> shift;
  5515. if (n > overrun)
  5516. @@ -172,9 +176,10 @@ void fse_normalize_freq(int nstates, int nsymbols, const uint32_t *__restrict t,
  5517. int max_freq_sym = 0;
  5518. int shift = __builtin_clz(nstates) - 1;
  5519. uint32_t highprec_step;
  5520. + int i;
  5521.  
  5522. // Compute the total number of symbol occurrences
  5523. - for (int i = 0; i < nsymbols; i++)
  5524. + for (i = 0; i < nsymbols; i++)
  5525. s_count += t[i];
  5526.  
  5527. if (s_count == 0)
  5528. @@ -182,7 +187,7 @@ void fse_normalize_freq(int nstates, int nsymbols, const uint32_t *__restrict t,
  5529. else
  5530. highprec_step = ((uint32_t)1 << 31) / s_count;
  5531.  
  5532. - for (int i = 0; i < nsymbols; i++) {
  5533. + for (i = 0; i < nsymbols; i++) {
  5534.  
  5535. // Rescale the occurrence count to get the normalized frequency.
  5536. // Round up if the fractional part is >= 0.5; otherwise round down.
  5537. diff --git a/lib/lzfse/lzfse_fse.h b/lib/lzfse/lzfse_fse.h
  5538. index 590af33..5aca431 100644
  5539. --- a/lib/lzfse/lzfse_fse.h
  5540. +++ b/lib/lzfse/lzfse_fse.h
  5541. @@ -560,8 +560,9 @@ fse_value_decode(fse_state *__restrict pstate,
  5542. FSE_INLINE int fse_check_freq(const uint16_t *freq_table,
  5543. const size_t table_size,
  5544. const size_t number_of_states) {
  5545. + int i;
  5546. size_t sum_of_freq = 0;
  5547. - for (int i = 0; i < table_size; i++) {
  5548. + for (i = 0; i < table_size; i++) {
  5549. sum_of_freq += freq_table[i];
  5550. }
  5551. return (sum_of_freq > number_of_states) ? -1 : 0;
  5552. diff --git a/lib/lzfse/lzvn_decode_base.c b/lib/lzfse/lzvn_decode_base.c
  5553. index c30180e..85c59e8 100644
  5554. --- a/lib/lzfse/lzvn_decode_base.c
  5555. +++ b/lib/lzfse/lzvn_decode_base.c
  5556. @@ -93,6 +93,8 @@ void lzvn_decode(lzvn_decoder_state *state) {
  5557. size_t L;
  5558. size_t opc_len;
  5559.  
  5560. + size_t i;
  5561. +
  5562. // Do we have a partially expanded match saved in state?
  5563. if (state->L != 0 || state->M != 0) {
  5564. L = state->L;
  5565. @@ -405,13 +407,13 @@ copy_literal_and_match:
  5566. // byte-by-byte copy of the literal. This is slow, but it can only ever
  5567. // happen near the very end of a buffer, so it is not an important case to
  5568. // optimize.
  5569. - for (size_t i = 0; i < L; ++i)
  5570. + for (i = 0; i < L; ++i)
  5571. dst_ptr[i] = src_ptr[i];
  5572. } else {
  5573. // Destination truncated: fill DST, and store partial match
  5574.  
  5575. // Copy partial literal
  5576. - for (size_t i = 0; i < dst_len; ++i)
  5577. + for (i = 0; i < dst_len; ++i)
  5578. dst_ptr[i] = src_ptr[i];
  5579. // Save state
  5580. state->src = src_ptr + dst_len;
  5581. @@ -448,19 +450,19 @@ copy_match:
  5582. // copies. The last of these may slop over the intended end of
  5583. // the match, but this is OK because we know we have a safety bound
  5584. // away from the end of the destination buffer.
  5585. - for (size_t i = 0; i < M; i += 8)
  5586. + for (i = 0; i < M; i += 8)
  5587. store8(&dst_ptr[i], load8(&dst_ptr[i - D]));
  5588. } else if (M <= dst_len) {
  5589. // Either the match distance is too small, or we are too close to
  5590. // the end of the buffer to safely use eight byte copies. Fall back
  5591. // on a simple byte-by-byte implementation.
  5592. - for (size_t i = 0; i < M; ++i)
  5593. + for (i = 0; i < M; ++i)
  5594. dst_ptr[i] = dst_ptr[i - D];
  5595. } else {
  5596. // Destination truncated: fill DST, and store partial match
  5597.  
  5598. // Copy partial match
  5599. - for (size_t i = 0; i < dst_len; ++i)
  5600. + for (i = 0; i < dst_len; ++i)
  5601. dst_ptr[i] = dst_ptr[i - D];
  5602. // Save state
  5603. state->src = src_ptr;
  5604. @@ -593,19 +595,19 @@ copy_literal:
  5605. // We are not near the end of the source or destination buffers; thus
  5606. // we can safely copy the literal using wide copies, without worrying
  5607. // about reading or writing past the end of either buffer.
  5608. - for (size_t i = 0; i < L; i += 8)
  5609. + for (i = 0; i < L; i += 8)
  5610. store8(&dst_ptr[i], load8(&src_ptr[i]));
  5611. } else if (L <= dst_len) {
  5612. // We are too close to the end of either the input or output stream
  5613. // to be able to safely use an eight-byte copy. Instead we copy the
  5614. // literal byte-by-byte.
  5615. - for (size_t i = 0; i < L; ++i)
  5616. + for (i = 0; i < L; ++i)
  5617. dst_ptr[i] = src_ptr[i];
  5618. } else {
  5619. // Destination truncated: fill DST, and store partial match
  5620.  
  5621. // Copy partial literal
  5622. - for (size_t i = 0; i < dst_len; ++i)
  5623. + for (i = 0; i < dst_len; ++i)
  5624. dst_ptr[i] = src_ptr[i];
  5625. // Save state
  5626. state->src = src_ptr + dst_len;
  5627. diff --git a/lib/lzfse/lzvn_encode_base.c b/lib/lzfse/lzvn_encode_base.c
  5628. index c86b551..96a1b1c 100644
  5629. --- a/lib/lzfse/lzvn_encode_base.c
  5630. +++ b/lib/lzfse/lzvn_encode_base.c
  5631. @@ -33,10 +33,11 @@ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSI
  5632. /*! @abstract Copy at least \p nbytes bytes from \p src to \p dst, by blocks
  5633. * of 8 bytes (may go beyond range). No overlap.
  5634. * @return \p dst + \p nbytes. */
  5635. -static inline unsigned char *lzvn_copy64(unsigned char *restrict dst,
  5636. - const unsigned char *restrict src,
  5637. +static inline unsigned char *lzvn_copy64(unsigned char * /*restrict*/ dst,
  5638. + const unsigned char * /*restrict*/ src,
  5639. size_t nbytes) {
  5640. - for (size_t i = 0; i < nbytes; i += 8)
  5641. + size_t i;
  5642. + for (i = 0; i < nbytes; i += 8)
  5643. store8(dst + i, load8(src + i));
  5644. return dst + nbytes;
  5645. }
  5646. @@ -44,10 +45,11 @@ static inline unsigned char *lzvn_copy64(unsigned char *restrict dst,
  5647. /*! @abstract Copy exactly \p nbytes bytes from \p src to \p dst (respects range).
  5648. * No overlap.
  5649. * @return \p dst + \p nbytes. */
  5650. -static inline unsigned char *lzvn_copy8(unsigned char *restrict dst,
  5651. - const unsigned char *restrict src,
  5652. +static inline unsigned char *lzvn_copy8(unsigned char * /*restrict*/ dst,
  5653. + const unsigned char * /*restrict*/ src,
  5654. size_t nbytes) {
  5655. - for (size_t i = 0; i < nbytes; i++)
  5656. + size_t i;
  5657. + for (i = 0; i < nbytes; i++)
  5658. dst[i] = src[i];
  5659. return dst + nbytes;
  5660. }
  5661. @@ -366,21 +368,23 @@ static inline lzvn_offset lzvn_emit_end_of_stream(lzvn_encoder_state *state) {
  5662.  
  5663. /*! @abstract Initialize encoder table in \p state, uses current I/O parameters. */
  5664. static inline void lzvn_init_table(lzvn_encoder_state *state) {
  5665. + int i, u;
  5666. lzvn_offset index = -LZVN_ENCODE_MAX_DISTANCE; // max match distance
  5667. if (index < state->src_begin)
  5668. index = state->src_begin;
  5669. uint32_t value = load4(state->src + index);
  5670.  
  5671. lzvn_encode_entry_type e;
  5672. - for (int i = 0; i < 4; i++) {
  5673. + for (i = 0; i < 4; i++) {
  5674. e.indices[i] = offset_to_s32(index);
  5675. e.values[i] = value;
  5676. }
  5677. - for (int u = 0; u < LZVN_ENCODE_HASH_VALUES; u++)
  5678. + for (u = 0; u < LZVN_ENCODE_HASH_VALUES; u++)
  5679. state->table[u] = e; // fill entire table
  5680. }
  5681.  
  5682. void lzvn_encode(lzvn_encoder_state *state) {
  5683. + int k;
  5684. const lzvn_match_info NO_MATCH = {0};
  5685.  
  5686. for (; state->src_current < state->src_current_end; state->src_current++) {
  5687. @@ -443,7 +447,7 @@ void lzvn_encode(lzvn_encoder_state *state) {
  5688.  
  5689. // Check candidates in order (closest first)
  5690. uint32_t diffs[4];
  5691. - for (int k = 0; k < 4; k++)
  5692. + for (k = 0; k < 4; k++)
  5693. diffs[k] = e.values[k] ^ vi; // XOR, 0 if equal
  5694. lzvn_offset ik; // index
  5695. lzvn_offset nk; // match byte count
  5696. --
  5697. 2.9.3
  5698.  
  5699.  
  5700. From 1f987fd6e5e6b7d715e76eb5753388fe521ce873 Mon Sep 17 00:00:00 2001
  5701. From: Valery Tolstov <me@vtolstov.org>
  5702. Date: Wed, 18 Jan 2017 23:10:54 +0300
  5703. Subject: [PATCH 4/5] Add lsfse wrapper for btrfs
  5704.  
  5705. ---
  5706. fs/btrfs/Makefile | 2 +-
  5707. fs/btrfs/compression.c | 1 +
  5708. fs/btrfs/compression.h | 1 +
  5709. fs/btrfs/ctree.h | 14 +-
  5710. fs/btrfs/disk-io.c | 2 +
  5711. fs/btrfs/ioctl.c | 3 +
  5712. fs/btrfs/lzfse.c | 451 +++++++++++++++++++++++++++++++++++++++++++++++++
  5713. fs/btrfs/super.c | 13 +-
  5714. 8 files changed, 476 insertions(+), 11 deletions(-)
  5715. create mode 100644 fs/btrfs/lzfse.c
  5716.  
  5717. diff --git a/fs/btrfs/Makefile b/fs/btrfs/Makefile
  5718. index 6d1d0b9..e9497fe 100644
  5719. --- a/fs/btrfs/Makefile
  5720. +++ b/fs/btrfs/Makefile
  5721. @@ -6,7 +6,7 @@ btrfs-y += super.o ctree.o extent-tree.o print-tree.o root-tree.o dir-item.o \
  5722. transaction.o inode.o file.o tree-defrag.o \
  5723. extent_map.o sysfs.o struct-funcs.o xattr.o ordered-data.o \
  5724. extent_io.o volumes.o async-thread.o ioctl.o locking.o orphan.o \
  5725. - export.o tree-log.o free-space-cache.o zlib.o lzo.o \
  5726. + export.o tree-log.o free-space-cache.o zlib.o lzo.o lzfse.o \
  5727. compression.o delayed-ref.o relocation.o delayed-inode.o scrub.o \
  5728. reada.o backref.o ulist.o qgroup.o send.o dev-replace.o raid56.o \
  5729. uuid-tree.o props.o hash.o
  5730. diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c
  5731. index c473c42..c3f8f8c 100644
  5732. --- a/fs/btrfs/compression.c
  5733. +++ b/fs/btrfs/compression.c
  5734. @@ -755,6 +755,7 @@ static struct {
  5735. static const struct btrfs_compress_op * const btrfs_compress_op[] = {
  5736. &btrfs_zlib_compress,
  5737. &btrfs_lzo_compress,
  5738. + &btrfs_lzfse_compress,
  5739. };
  5740.  
  5741. void __init btrfs_init_compress(void)
  5742. diff --git a/fs/btrfs/compression.h b/fs/btrfs/compression.h
  5743. index 13a4dc0..3042670 100644
  5744. --- a/fs/btrfs/compression.h
  5745. +++ b/fs/btrfs/compression.h
  5746. @@ -79,5 +79,6 @@ struct btrfs_compress_op {
  5747.  
  5748. extern const struct btrfs_compress_op btrfs_zlib_compress;
  5749. extern const struct btrfs_compress_op btrfs_lzo_compress;
  5750. +extern const struct btrfs_compress_op btrfs_lzfse_compress;
  5751.  
  5752. #endif
  5753. diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
  5754. index 35489e7..21b13ae 100644
  5755. --- a/fs/btrfs/ctree.h
  5756. +++ b/fs/btrfs/ctree.h
  5757. @@ -504,13 +504,7 @@ struct btrfs_super_block {
  5758. #define BTRFS_FEATURE_INCOMPAT_DEFAULT_SUBVOL (1ULL << 1)
  5759. #define BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS (1ULL << 2)
  5760. #define BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO (1ULL << 3)
  5761. -/*
  5762. - * some patches floated around with a second compression method
  5763. - * lets save that incompat here for when they do get in
  5764. - * Note we don't actually support it, we're just reserving the
  5765. - * number
  5766. - */
  5767. -#define BTRFS_FEATURE_INCOMPAT_COMPRESS_LZOv2 (1ULL << 4)
  5768. +#define BTRFS_FEATURE_INCOMPAT_COMPRESS_LZFSE (1ULL << 4)
  5769.  
  5770. /*
  5771. * older kernels tried to do bigger metadata blocks, but the
  5772. @@ -539,6 +533,7 @@ struct btrfs_super_block {
  5773. BTRFS_FEATURE_INCOMPAT_RAID56 | \
  5774. BTRFS_FEATURE_INCOMPAT_EXTENDED_IREF | \
  5775. BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA | \
  5776. + BTRFS_FEATURE_INCOMPAT_COMPRESS_LZFSE | \
  5777. BTRFS_FEATURE_INCOMPAT_NO_HOLES)
  5778.  
  5779. #define BTRFS_FEATURE_INCOMPAT_SAFE_SET \
  5780. @@ -709,8 +704,9 @@ enum btrfs_compression_type {
  5781. BTRFS_COMPRESS_NONE = 0,
  5782. BTRFS_COMPRESS_ZLIB = 1,
  5783. BTRFS_COMPRESS_LZO = 2,
  5784. - BTRFS_COMPRESS_TYPES = 2,
  5785. - BTRFS_COMPRESS_LAST = 3,
  5786. + BTRFS_COMPRESS_LZFSE = 3,
  5787. + BTRFS_COMPRESS_TYPES = 3,
  5788. + BTRFS_COMPRESS_LAST = 4,
  5789. };
  5790.  
  5791. struct btrfs_inode_item {
  5792. diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
  5793. index 974be09..6e8bee4 100644
  5794. --- a/fs/btrfs/disk-io.c
  5795. +++ b/fs/btrfs/disk-io.c
  5796. @@ -2751,6 +2751,8 @@ int open_ctree(struct super_block *sb,
  5797. features |= BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF;
  5798. if (tree_root->fs_info->compress_type == BTRFS_COMPRESS_LZO)
  5799. features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO;
  5800. + if (tree_root->fs_info->compress_type == BTRFS_COMPRESS_LZFSE)
  5801. + features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_LZFSE;
  5802.  
  5803. if (features & BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA)
  5804. printk(KERN_INFO "BTRFS: has skinny extents\n");
  5805. diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c
  5806. index da94138..703fcff 100644
  5807. --- a/fs/btrfs/ioctl.c
  5808. +++ b/fs/btrfs/ioctl.c
  5809. @@ -1448,6 +1448,9 @@ int btrfs_defrag_file(struct inode *inode, struct file *file,
  5810. if (range->compress_type == BTRFS_COMPRESS_LZO) {
  5811. btrfs_set_fs_incompat(root->fs_info, COMPRESS_LZO);
  5812. }
  5813. + if (range->compress_type == BTRFS_COMPRESS_LZFSE) {
  5814. + btrfs_set_fs_incompat(root->fs_info, COMPRESS_LZFSE);
  5815. + }
  5816.  
  5817. ret = defrag_count;
  5818.  
  5819. diff --git a/fs/btrfs/lzfse.c b/fs/btrfs/lzfse.c
  5820. new file mode 100644
  5821. index 0000000..8789d44
  5822. --- /dev/null
  5823. +++ b/fs/btrfs/lzfse.c
  5824. @@ -0,0 +1,451 @@
  5825. +/*
  5826. + * Copyright (C) 2008 Oracle. All rights reserved.
  5827. + *
  5828. + * This program is free software; you can redistribute it and/or
  5829. + * modify it under the terms of the GNU General Public
  5830. + * License v2 as published by the Free Software Foundation.
  5831. + *
  5832. + * This program is distributed in the hope that it will be useful,
  5833. + * but WITHOUT ANY WARRANTY; without even the implied warranty of
  5834. + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  5835. + * General Public License for more details.
  5836. + *
  5837. + * You should have received a copy of the GNU General Public
  5838. + * License along with this program; if not, write to the
  5839. + * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  5840. + * Boston, MA 021110-1307, USA.
  5841. + */
  5842. +
  5843. +#include <linux/kernel.h>
  5844. +#include <linux/slab.h>
  5845. +#include <linux/vmalloc.h>
  5846. +#include <linux/init.h>
  5847. +#include <linux/err.h>
  5848. +#include <linux/sched.h>
  5849. +#include <linux/pagemap.h>
  5850. +#include <linux/bio.h>
  5851. +#include <linux/lzfse.h>
  5852. +#include "compression.h"
  5853. +
  5854. +#define LZFSE_LEN 4
  5855. +#define LZFSE_BUF 8192
  5856. +
  5857. +struct workspace {
  5858. + void *mem;
  5859. + void *buf; /* where decompressed data goes */
  5860. + void *cbuf; /* where compressed data goes */
  5861. + struct list_head list;
  5862. +};
  5863. +
  5864. +static void lzfse_free_workspace(struct list_head *ws)
  5865. +{
  5866. + struct workspace *workspace = list_entry(ws, struct workspace, list);
  5867. +
  5868. + vfree(workspace->buf);
  5869. + vfree(workspace->cbuf);
  5870. + vfree(workspace->mem);
  5871. + kfree(workspace);
  5872. +}
  5873. +
  5874. +static struct list_head *lzfse_alloc_workspace(void)
  5875. +{
  5876. + struct workspace *workspace;
  5877. +
  5878. + workspace = kzalloc(sizeof(*workspace), GFP_NOFS);
  5879. + if (!workspace)
  5880. + return ERR_PTR(-ENOMEM);
  5881. +
  5882. + workspace->mem = vmalloc(max(lzfse_encode_scratch_size(),
  5883. + lzfse_decode_scratch_size()));
  5884. + workspace->buf = vmalloc(LZFSE_BUF);
  5885. + workspace->cbuf = vmalloc(LZFSE_BUF);
  5886. + if (!workspace->mem || !workspace->buf || !workspace->cbuf)
  5887. + goto fail;
  5888. +
  5889. + INIT_LIST_HEAD(&workspace->list);
  5890. +
  5891. + return &workspace->list;
  5892. +fail:
  5893. + lzo_free_workspace(&workspace->list);
  5894. + return ERR_PTR(-ENOMEM);
  5895. +}
  5896. +
  5897. +static inline void write_compress_length(char *buf, size_t len)
  5898. +{
  5899. + __le32 dlen;
  5900. +
  5901. + dlen = cpu_to_le32(len);
  5902. + memcpy(buf, &dlen, LZFSE_LEN);
  5903. +}
  5904. +
  5905. +static inline size_t read_compress_length(char *buf)
  5906. +{
  5907. + __le32 dlen;
  5908. +
  5909. + memcpy(&dlen, buf, LZFSE_LEN);
  5910. + return le32_to_cpu(dlen);
  5911. +}
  5912. +
  5913. +static int lzfse_compress_pages(struct list_head *ws,
  5914. + struct address_space *mapping,
  5915. + u64 start, unsigned long len,
  5916. + struct page **pages,
  5917. + unsigned long nr_dest_pages,
  5918. + unsigned long *out_pages,
  5919. + unsigned long *total_in,
  5920. + unsigned long *total_out,
  5921. + unsigned long max_out)
  5922. +{
  5923. + struct workspace *workspace = list_entry(ws, struct workspace, list);
  5924. + size_t ret = 0;
  5925. + char *data_in;
  5926. + char *cpage_out;
  5927. + int nr_pages = 0;
  5928. + struct page *in_page = NULL;
  5929. + struct page *out_page = NULL;
  5930. + unsigned long bytes_left;
  5931. +
  5932. + size_t in_len;
  5933. + size_t out_len;
  5934. + char *buf;
  5935. + unsigned long tot_in = 0;
  5936. + unsigned long tot_out = 0;
  5937. + unsigned long pg_bytes_left;
  5938. + unsigned long out_offset;
  5939. + unsigned long bytes;
  5940. +
  5941. + *out_pages = 0;
  5942. + *total_out = 0;
  5943. + *total_in = 0;
  5944. +
  5945. + in_page = find_get_page(mapping, start >> PAGE_CACHE_SHIFT);
  5946. + data_in = kmap(in_page);
  5947. +
  5948. + /*
  5949. + * store the size of all chunks of compressed data in
  5950. + * the first 4 bytes
  5951. + */
  5952. + out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
  5953. + if (out_page == NULL) {
  5954. + ret = -ENOMEM;
  5955. + goto out;
  5956. + }
  5957. + cpage_out = kmap(out_page);
  5958. + out_offset = LZFSE_LEN;
  5959. + tot_out = LZFSE_LEN;
  5960. + pages[0] = out_page;
  5961. + nr_pages = 1;
  5962. + pg_bytes_left = PAGE_CACHE_SIZE - LZFSE_LEN;
  5963. +
  5964. + /* compress at most one page of data each time */
  5965. + in_len = min(len, PAGE_CACHE_SIZE);
  5966. + while (tot_in < len) {
  5967. + /*
  5968. + ret = (data_in, in_len, workspace->cbuf,
  5969. + &out_len, workspace->mem); */
  5970. + ret = lzfse_encode_buffer(workspace->cbuf, LZFSE_BUF,
  5971. + data_in, in_len, workspace->mem);
  5972. + if (ret == 0) {
  5973. + printk(KERN_DEBUG "BTRFS: deflate in loop returned %d\n",
  5974. + ret);
  5975. + ret = -EIO;
  5976. + goto out;
  5977. + }
  5978. + out_len = ret;
  5979. +
  5980. + /* store the size of this chunk of compressed data */
  5981. + write_compress_length(cpage_out + out_offset, out_len);
  5982. + tot_out += LZFSE_LEN;
  5983. + out_offset += LZFSE_LEN;
  5984. + pg_bytes_left -= LZFSE_LEN;
  5985. +
  5986. + tot_in += in_len;
  5987. + tot_out += out_len;
  5988. +
  5989. + /* copy bytes from the working buffer into the pages */
  5990. + buf = workspace->cbuf;
  5991. + while (out_len) {
  5992. + bytes = min_t(unsigned long, pg_bytes_left, out_len);
  5993. +
  5994. + memcpy(cpage_out + out_offset, buf, bytes);
  5995. +
  5996. + out_len -= bytes;
  5997. + pg_bytes_left -= bytes;
  5998. + buf += bytes;
  5999. + out_offset += bytes;
  6000. +
  6001. + /*
  6002. + * we need another page for writing out.
  6003. + *
  6004. + * Note if there's less than 4 bytes left, we just
  6005. + * skip to a new page.
  6006. + */
  6007. + if ((out_len == 0 && pg_bytes_left < LZFSE_LEN) ||
  6008. + pg_bytes_left == 0) {
  6009. + if (pg_bytes_left) {
  6010. + memset(cpage_out + out_offset, 0,
  6011. + pg_bytes_left);
  6012. + tot_out += pg_bytes_left;
  6013. + }
  6014. +
  6015. + /* we're done, don't allocate new page */
  6016. + if (out_len == 0 && tot_in >= len)
  6017. + break;
  6018. +
  6019. + kunmap(out_page);
  6020. + if (nr_pages == nr_dest_pages) {
  6021. + out_page = NULL;
  6022. + ret = -E2BIG;
  6023. + goto out;
  6024. + }
  6025. +
  6026. + out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
  6027. + if (out_page == NULL) {
  6028. + ret = -ENOMEM;
  6029. + goto out;
  6030. + }
  6031. + cpage_out = kmap(out_page);
  6032. + pages[nr_pages++] = out_page;
  6033. +
  6034. + pg_bytes_left = PAGE_CACHE_SIZE;
  6035. + out_offset = 0;
  6036. + }
  6037. + }
  6038. +
  6039. + /* we're making it bigger, give up */
  6040. + if (tot_in > 8192 && tot_in < tot_out) {
  6041. + ret = -E2BIG;
  6042. + goto out;
  6043. + }
  6044. +
  6045. + /* we're all done */
  6046. + if (tot_in >= len)
  6047. + break;
  6048. +
  6049. + if (tot_out > max_out)
  6050. + break;
  6051. +
  6052. + bytes_left = len - tot_in;
  6053. + kunmap(in_page);
  6054. + page_cache_release(in_page);
  6055. +
  6056. + start += PAGE_CACHE_SIZE;
  6057. + in_page = find_get_page(mapping, start >> PAGE_CACHE_SHIFT);
  6058. + data_in = kmap(in_page);
  6059. + in_len = min(bytes_left, PAGE_CACHE_SIZE);
  6060. + }
  6061. +
  6062. + if (tot_out > tot_in)
  6063. + goto out;
  6064. +
  6065. + /* store the size of all chunks of compressed data */
  6066. + cpage_out = kmap(pages[0]);
  6067. + write_compress_length(cpage_out, tot_out);
  6068. +
  6069. + kunmap(pages[0]);
  6070. +
  6071. + ret = 0;
  6072. + *total_out = tot_out;
  6073. + *total_in = tot_in;
  6074. +out:
  6075. + *out_pages = nr_pages;
  6076. + if (out_page)
  6077. + kunmap(out_page);
  6078. +
  6079. + if (in_page) {
  6080. + kunmap(in_page);
  6081. + page_cache_release(in_page);
  6082. + }
  6083. +
  6084. + return ret;
  6085. +}
  6086. +
  6087. +static int lzfse_decompress_biovec(struct list_head *ws,
  6088. + struct page **pages_in,
  6089. + u64 disk_start,
  6090. + struct bio_vec *bvec,
  6091. + int vcnt,
  6092. + size_t srclen)
  6093. +{
  6094. + struct workspace *workspace = list_entry(ws, struct workspace, list);
  6095. + size_t ret;
  6096. + int ret2;
  6097. + char *data_in;
  6098. + unsigned long page_in_index = 0;
  6099. + unsigned long page_out_index = 0;
  6100. + unsigned long total_pages_in = DIV_ROUND_UP(srclen, PAGE_CACHE_SIZE);
  6101. + unsigned long buf_start;
  6102. + unsigned long buf_offset = 0;
  6103. + unsigned long bytes;
  6104. + unsigned long working_bytes;
  6105. + unsigned long pg_offset;
  6106. +
  6107. + size_t in_len;
  6108. + size_t out_len;
  6109. + unsigned long in_offset;
  6110. + unsigned long in_page_bytes_left;
  6111. + unsigned long tot_in;
  6112. + unsigned long tot_out;
  6113. + unsigned long tot_len;
  6114. + char *buf;
  6115. + bool may_late_unmap, need_unmap;
  6116. +
  6117. + data_in = kmap(pages_in[0]);
  6118. + tot_len = read_compress_length(data_in);
  6119. +
  6120. + tot_in = LZFSE_LEN;
  6121. + in_offset = LZFSE_LEN;
  6122. + tot_len = min_t(size_t, srclen, tot_len);
  6123. + in_page_bytes_left = PAGE_CACHE_SIZE - LZFSE_LEN;
  6124. +
  6125. + tot_out = 0;
  6126. + pg_offset = 0;
  6127. +
  6128. + while (tot_in < tot_len) {
  6129. + in_len = read_compress_length(data_in + in_offset);
  6130. + in_page_bytes_left -= LZFSE_LEN;
  6131. + in_offset += LZFSE_LEN;
  6132. + tot_in += LZFSE_LEN;
  6133. +
  6134. + tot_in += in_len;
  6135. + working_bytes = in_len;
  6136. + may_late_unmap = need_unmap = false;
  6137. +
  6138. + /* fast path: avoid using the working buffer */
  6139. + if (in_page_bytes_left >= in_len) {
  6140. + buf = data_in + in_offset;
  6141. + bytes = in_len;
  6142. + may_late_unmap = true;
  6143. + goto cont;
  6144. + }
  6145. +
  6146. + /* copy bytes from the pages into the working buffer */
  6147. + buf = workspace->cbuf;
  6148. + buf_offset = 0;
  6149. + while (working_bytes) {
  6150. + bytes = min(working_bytes, in_page_bytes_left);
  6151. +
  6152. + memcpy(buf + buf_offset, data_in + in_offset, bytes);
  6153. + buf_offset += bytes;
  6154. +cont:
  6155. + working_bytes -= bytes;
  6156. + in_page_bytes_left -= bytes;
  6157. + in_offset += bytes;
  6158. +
  6159. + /* check if we need to pick another page */
  6160. + if ((working_bytes == 0 && in_page_bytes_left < LZFSE_LEN)
  6161. + || in_page_bytes_left == 0) {
  6162. + tot_in += in_page_bytes_left;
  6163. +
  6164. + if (working_bytes == 0 && tot_in >= tot_len)
  6165. + break;
  6166. +
  6167. + if (page_in_index + 1 >= total_pages_in) {
  6168. + ret = -EIO;
  6169. + goto done;
  6170. + }
  6171. +
  6172. + if (may_late_unmap)
  6173. + need_unmap = true;
  6174. + else
  6175. + kunmap(pages_in[page_in_index]);
  6176. +
  6177. + data_in = kmap(pages_in[++page_in_index]);
  6178. +
  6179. + in_page_bytes_left = PAGE_CACHE_SIZE;
  6180. + in_offset = 0;
  6181. + }
  6182. + }
  6183. +
  6184. + ret = lzfse_decode_buffer(workspace->buf, LZFSE_BUF, buf,
  6185. + in_len, workspace->mem);
  6186. + if (need_unmap)
  6187. + kunmap(pages_in[page_in_index - 1]);
  6188. + if (ret == 0) {
  6189. + printk(KERN_WARNING "BTRFS: decompress failed\n");
  6190. + ret = -EIO;
  6191. + break;
  6192. + }
  6193. + out_len = ret;
  6194. +
  6195. + buf_start = tot_out;
  6196. + tot_out += out_len;
  6197. +
  6198. + ret2 = btrfs_decompress_buf2page(workspace->buf, buf_start,
  6199. + tot_out, disk_start,
  6200. + bvec, vcnt,
  6201. + &page_out_index, &pg_offset);
  6202. + if (ret2 == 0)
  6203. + break;
  6204. + }
  6205. +done:
  6206. + kunmap(pages_in[page_in_index]);
  6207. + if (!ret)
  6208. + btrfs_clear_biovec_end(bvec, vcnt, page_out_index, pg_offset);
  6209. + return ret;
  6210. +}
  6211. +
  6212. +static int lzfse_decompress(struct list_head *ws, unsigned char *data_in,
  6213. + struct page *dest_page,
  6214. + unsigned long start_byte,
  6215. + size_t srclen, size_t destlen)
  6216. +{
  6217. + struct workspace *workspace = list_entry(ws, struct workspace, list);
  6218. + size_t in_len;
  6219. + size_t out_len;
  6220. + size_t tot_len;
  6221. + int ret = 0;
  6222. + char *kaddr;
  6223. + unsigned long bytes;
  6224. +
  6225. + BUG_ON(srclen < LZFSE_LEN);
  6226. +
  6227. + tot_len = read_compress_length(data_in);
  6228. + data_in += LZFSE_LEN;
  6229. +
  6230. + in_len = read_compress_length(data_in);
  6231. + data_in += LZFSE_LEN;
  6232. +
  6233. + ret = lzfse_decode_buffer(workspace->buf, LZFSE_BUF, buf,
  6234. + in_len, workspace->mem);
  6235. + if (ret == 0) {
  6236. + printk(KERN_WARNING "BTRFS: decompress failed!\n");
  6237. + ret = -EIO;
  6238. + goto out;
  6239. + }
  6240. + out_len = ret;
  6241. +
  6242. + if (out_len < start_byte) {
  6243. + ret = -EIO;
  6244. + goto out;
  6245. + }
  6246. +
  6247. + /*
  6248. + * the caller is already checking against PAGE_SIZE, but lets
  6249. + * move this check closer to the memcpy/memset
  6250. + */
  6251. + destlen = min_t(unsigned long, destlen, PAGE_SIZE);
  6252. + bytes = min_t(unsigned long, destlen, out_len - start_byte);
  6253. +
  6254. + kaddr = kmap_atomic(dest_page);
  6255. + memcpy(kaddr, workspace->buf + start_byte, bytes);
  6256. +
  6257. + /*
  6258. + * btrfs_getblock is doing a zero on the tail of the page too,
  6259. + * but this will cover anything missing from the decompressed
  6260. + * data.
  6261. + */
  6262. + if (bytes < destlen)
  6263. + memset(kaddr+bytes, 0, destlen-bytes);
  6264. + kunmap_atomic(kaddr);
  6265. +out:
  6266. + return ret;
  6267. +}
  6268. +
  6269. +const struct btrfs_compress_op btrfs_lzfse_compress = {
  6270. + .alloc_workspace = lzfse_alloc_workspace,
  6271. + .free_workspace = lzfse_free_workspace,
  6272. + .compress_pages = lzfse_compress_pages,
  6273. + .decompress_biovec = lzfse_decompress_biovec,
  6274. + .decompress = lzfse_decompress,
  6275. +};
  6276. diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c
  6277. index 24154e4..c198e3d 100644
  6278. --- a/fs/btrfs/super.c
  6279. +++ b/fs/btrfs/super.c
  6280. @@ -473,6 +473,13 @@ int btrfs_parse_options(struct btrfs_root *root, char *options)
  6281. btrfs_clear_opt(info->mount_opt, NODATACOW);
  6282. btrfs_clear_opt(info->mount_opt, NODATASUM);
  6283. btrfs_set_fs_incompat(info, COMPRESS_LZO);
  6284. + } else if (strcmp(args[0].from, "lzfse") == 0) {
  6285. + compress_type = "lzo";
  6286. + info->compress_type = BTRFS_COMPRESS_LZO;
  6287. + btrfs_set_opt(info->mount_opt, COMPRESS);
  6288. + btrfs_clear_opt(info->mount_opt, NODATACOW);
  6289. + btrfs_clear_opt(info->mount_opt, NODATASUM);
  6290. + btrfs_set_fs_incompat(info, COMPRESS_LZFSE);
  6291. } else if (strncmp(args[0].from, "no", 2) == 0) {
  6292. compress_type = "no";
  6293. btrfs_clear_opt(info->mount_opt, COMPRESS);
  6294. @@ -1139,8 +1146,12 @@ static int btrfs_show_options(struct seq_file *seq, struct dentry *dentry)
  6295. if (btrfs_test_opt(root, COMPRESS)) {
  6296. if (info->compress_type == BTRFS_COMPRESS_ZLIB)
  6297. compress_type = "zlib";
  6298. - else
  6299. + else if (info->compress_type == BTRFS_COMPRESS_LZO)
  6300. compress_type = "lzo";
  6301. + else if (info->compress_type == BTRFS_COMPRESS_LZFSE)
  6302. + compress_type = "lzfse";
  6303. + else
  6304. + compress_type = "no";
  6305. if (btrfs_test_opt(root, FORCE_COMPRESS))
  6306. seq_printf(seq, ",compress-force=%s", compress_type);
  6307. else
  6308. --
  6309. 2.9.3
  6310.  
  6311.  
  6312. From d4e00394a3b27c41bd8544a944c25cd30001d5c0 Mon Sep 17 00:00:00 2001
  6313. From: Valery Tolstov <me@vtolstov.org>
  6314. Date: Thu, 19 Jan 2017 00:50:31 +0300
  6315. Subject: [PATCH 5/5] Remove lzo mention
  6316.  
  6317. ---
  6318. fs/btrfs/lzfse.c | 6 +++---
  6319. 1 file changed, 3 insertions(+), 3 deletions(-)
  6320.  
  6321. diff --git a/fs/btrfs/lzfse.c b/fs/btrfs/lzfse.c
  6322. index 8789d44..57a6b94 100644
  6323. --- a/fs/btrfs/lzfse.c
  6324. +++ b/fs/btrfs/lzfse.c
  6325. @@ -66,7 +66,7 @@ static struct list_head *lzfse_alloc_workspace(void)
  6326.  
  6327. return &workspace->list;
  6328. fail:
  6329. - lzo_free_workspace(&workspace->list);
  6330. + lzfse_free_workspace(&workspace->list);
  6331. return ERR_PTR(-ENOMEM);
  6332. }
  6333.  
  6334. @@ -146,7 +146,7 @@ static int lzfse_compress_pages(struct list_head *ws,
  6335. ret = lzfse_encode_buffer(workspace->cbuf, LZFSE_BUF,
  6336. data_in, in_len, workspace->mem);
  6337. if (ret == 0) {
  6338. - printk(KERN_DEBUG "BTRFS: deflate in loop returned %d\n",
  6339. + printk(KERN_DEBUG "BTRFS: deflate in loop returned %zu\n",
  6340. ret);
  6341. ret = -EIO;
  6342. goto out;
  6343. @@ -406,7 +406,7 @@ static int lzfse_decompress(struct list_head *ws, unsigned char *data_in,
  6344. in_len = read_compress_length(data_in);
  6345. data_in += LZFSE_LEN;
  6346.  
  6347. - ret = lzfse_decode_buffer(workspace->buf, LZFSE_BUF, buf,
  6348. + ret = lzfse_decode_buffer(workspace->buf, LZFSE_BUF, data_in,
  6349. in_len, workspace->mem);
  6350. if (ret == 0) {
  6351. printk(KERN_WARNING "BTRFS: decompress failed!\n");
  6352. --
  6353. 2.9.3
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!