Guest User

diff -B redis_hll.c wy_hll.c > hll.diff

a guest
Mar 15th, 2021
267
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Diff 20.84 KB | None | 0 0
  1. 0a1,17
  2. > #include <assert.h>
  3. > #include <math.h>
  4. > #include <stddef.h>
  5. > #include <stdint.h>
  6. > #include <stdio.h>
  7. > #include <stdlib.h>
  8. > #include <string.h>
  9. >
  10. > #include "hyperloglog.h"
  11. >
  12. > #ifdef DEBUG
  13. > #define xprintf(fmt, ...)                                                  \
  14. >   printf("FILE:<%s>,FUN:<%s>,LINE:<%d>," fmt "\n", __FILE__, __FUNCTION__, \
  15. >          __LINE__, ##__VA_ARGS__)
  16. > #else
  17. > #define xprintf(fmt, ...)
  18. > #endif
  19. 1a19,23
  20. > #define xassert(X)                     \
  21. >   if (!(X)) {                          \
  22. >     xprintf(" (%s) assert fail!", #X); \
  23. >     assert(0);                         \
  24. >   }
  25. 3c25
  26. < #include "server.h"
  27. ---
  28. > enum { _Little_endian = 0, _Big_endian };
  29. 5,6c27,126
  30. < #include <math.h>
  31. < #include <stdint.h>
  32. ---
  33. > inline static int byte_order() {
  34. >   union {
  35. >     short a;
  36. >     char b;
  37. >   } w;
  38. >
  39. >   w.a = 1;
  40. >
  41. >   return (w.b == 1) ? _Little_endian : _Big_endian;
  42. > }
  43. >
  44. > const uint16_t hll_sparse_max_bytes = 3000;
  45. >
  46. > typedef struct sdshdr sdshdr;
  47. > typedef char *sds;
  48. >
  49. > struct __attribute__((__packed__)) sdshdr {
  50. >   uint16_t len;
  51. >   uint16_t alloc;
  52. >   char buf[];
  53. > };
  54. >
  55. > #define sdsHdrSize sizeof(sdshdr)
  56. >
  57. > static inline size_t sdsavail(const sds s) {
  58. >   sdshdr *sh = (sdshdr *)(s - sdsHdrSize);
  59. >   return (sh->alloc - sh->len);
  60. > }
  61. >
  62. > static inline void sdssetlen(void *ptr, size_t len) {
  63. >   sds s = (sds)ptr;
  64. >   sdshdr *sh = (sdshdr *)(s - sdsHdrSize);
  65. >   sh->len = len;
  66. > }
  67. >
  68. > static inline void sdssetalloc(void *ptr, size_t len) {
  69. >   sds s = (sds)ptr;
  70. >   sdshdr *sh = (sdshdr *)(s - sdsHdrSize);
  71. >   sh->alloc = len;
  72. > }
  73. >
  74. > static inline size_t sdslen(const void *ptr) {
  75. >   sds s = (sds)ptr;
  76. >   sdshdr *sh = (sdshdr *)(s - sdsHdrSize);
  77. >   return sh->len;
  78. > }
  79. >
  80. > static inline void sdsIncrLen(void *ptr, int inc) {
  81. >   sds s = (sds)ptr;
  82. >   sdshdr *sh = (sdshdr *)(s - sdsHdrSize);
  83. >   sh->len += inc;
  84. > }
  85. >
  86. > sds sdsnewlen(const void *init, size_t initlen) {
  87. >   sdshdr *sh = (sdshdr *)calloc(sdsHdrSize + initlen + 1, 1);
  88. >   sds s = (char *)sh + sdsHdrSize;
  89. >   if (init) {
  90. >     memcpy(s, init, initlen);
  91. >   }
  92. >   sh->alloc = initlen;
  93. >   sh->len = initlen;
  94. >   s[initlen] = '\0';
  95. >   return s;
  96. > }
  97. >
  98. > void sdsfree(void *s) {
  99. >   if (s == NULL) return;
  100. >   free(((char *)s - sdsHdrSize));
  101. > }
  102. >
  103. > sds sdsMakeRoomFor(void *ptr, size_t addlen) {
  104. >   void *sh, *newsh;
  105. >   sds s = (sds)ptr;
  106. >   size_t avail = sdsavail(s);
  107. >   size_t len = sdslen(s);
  108. >   size_t newlen = len + addlen;
  109. >
  110. >   if (avail >= addlen || len >= hll_sparse_max_bytes) {
  111. >     return s;
  112. >   }
  113. >
  114. >   if (newlen * 2 < (hll_sparse_max_bytes / 4 * 3)) {
  115. >     newlen *= 2;
  116. >   } else {
  117. >     newlen = hll_sparse_max_bytes;
  118. >   }
  119. >
  120. >   sh = (char *)s - sdsHdrSize;
  121. >   newsh = realloc(sh, sdsHdrSize + newlen + 1);
  122. >
  123. >   if (!newsh) {
  124. >     return NULL;
  125. >   }
  126. >
  127. >   s = (char *)newsh + sdsHdrSize;
  128. >   sdssetlen(s, len);
  129. >   sdssetalloc(s, newlen);
  130. >
  131. >   return ((char *)newsh + sdsHdrSize);
  132. > }
  133. 32c154
  134. < static char *invalid_hll_err = "-INVALIDOBJ Corrupted HLL object detected";
  135. ---
  136. > static char *invalid_hll_err = "-INVALIDOBJ Corrupted HLL object detected\r\n";
  137. 97,112c234,245
  138. < #if (BYTE_ORDER == LITTLE_ENDIAN)
  139. < #ifdef USE_ALIGNED_ACCESS
  140. <     memcpy(&k, data, sizeof(uint64_t));
  141. < #else
  142. <     k = *((uint64_t *)data);
  143. < #endif
  144. < #else
  145. <     k = (uint64_t)data[0];
  146. <     k |= (uint64_t)data[1] << 8;
  147. <     k |= (uint64_t)data[2] << 16;
  148. <     k |= (uint64_t)data[3] << 24;
  149. <     k |= (uint64_t)data[4] << 32;
  150. <     k |= (uint64_t)data[5] << 40;
  151. <     k |= (uint64_t)data[6] << 48;
  152. <     k |= (uint64_t)data[7] << 56;
  153. < #endif
  154. ---
  155. >     if (byte_order() == _Little_endian) {
  156. >       k = *((uint64_t *)data);
  157. >     } else {
  158. >       k = (uint64_t)data[0];
  159. >       k |= (uint64_t)data[1] << 8;
  160. >       k |= (uint64_t)data[2] << 16;
  161. >       k |= (uint64_t)data[3] << 24;
  162. >       k |= (uint64_t)data[4] << 32;
  163. >       k |= (uint64_t)data[5] << 40;
  164. >       k |= (uint64_t)data[6] << 48;
  165. >       k |= (uint64_t)data[7] << 56;
  166. >     }
  167. 146c279
  168. < int hllPatLen(unsigned char *ele, size_t elesize, long *regp) {
  169. ---
  170. > int hllPatLen(const unsigned char *ele, size_t elesize, long *regp) {
  171. 176c309
  172. < int hllDenseAdd(uint8_t *registers, unsigned char *ele, size_t elesize) {
  173. ---
  174. > int hllDenseAdd(uint8_t *registers, const unsigned char *ele, size_t elesize) {
  175. 236,238c369,375
  176. < int hllSparseToDense(robj *o) {
  177. <   sds sparse = o->ptr, dense;
  178. <   struct hllhdr *hdr, *oldhdr = (struct hllhdr *)sparse;
  179. ---
  180. > hllhdr *hllSparseToDense(hllhdr *ptr) {
  181. >   if (!ptr) {
  182. >     return NULL;
  183. >   }
  184. >
  185. >   sds sparse = (sds)ptr;
  186. >   hllhdr *hdr = NULL;
  187. 242,243c379
  188. <   hdr = (struct hllhdr *)sparse;
  189. <   if (hdr->encoding == HLL_DENSE) return C_OK;
  190. ---
  191. >   if (ptr->encoding == HLL_DENSE) return ptr;
  192. 245,247c381,383
  193. <   dense = sdsnewlen(NULL, HLL_DENSE_SIZE);
  194. <   hdr = (struct hllhdr *)dense;
  195. <   *hdr = *oldhdr;
  196. ---
  197. >   hdr = (hllhdr *)sdsnewlen(NULL, HLL_DENSE_SIZE);
  198. >
  199. >   *hdr = *ptr;
  200. 260c396,398
  201. <     } else {
  202. ---
  203. >     }
  204. >
  205. >     else {
  206. 273,274c410,411
  207. <     sdsfree(dense);
  208. <     return C_ERR;
  209. ---
  210. >     sdsfree(hdr);
  211. >     return NULL;
  212. 277,279c414,415
  213. <   sdsfree(o->ptr);
  214. <   o->ptr = dense;
  215. <   return C_OK;
  216. ---
  217. >   sdsfree(ptr);
  218. >   return hdr;
  219. 282,283c418,419
  220. < int hllSparseSet(robj *o, long index, uint8_t count) {
  221. <   struct hllhdr *hdr;
  222. ---
  223. > int hllSparseSet(hllhdr **ptr, long index, uint8_t count) {
  224. >   hllhdr *hdr = *ptr;
  225. 290c426
  226. <   o->ptr = sdsMakeRoomFor(o->ptr, 3);
  227. ---
  228. >   hdr = (hllhdr *)sdsMakeRoomFor(hdr, 3);
  229. 292,293c428,434
  230. <   sparse = p = ((uint8_t *)o->ptr) + HLL_HDR_SIZE;
  231. <   end = p + sdslen(o->ptr) - HLL_HDR_SIZE;
  232. ---
  233. >   if (!hdr) {
  234. >     xprintf("memory rellac error: %s!", sterrror(errno));
  235. >     return -1;
  236. >   }
  237. >
  238. >   sparse = p = ((uint8_t *)hdr) + HLL_HDR_SIZE;
  239. >   end = p + sdslen(hdr) - HLL_HDR_SIZE;
  240. 317c458
  241. <   if (span == 0 || p >= end) return -1;
  242. ---
  243. >   if (span == 0) return -1;
  244. 319c460
  245. <   next = HLL_SPARSE_IS_XZERO(p) ? p + 2 : p + 1;
  246. ---
  247. >   next = HLL_SPARSE_IS_XZERO(p) ? (p + 2) : (p + 1);
  248. 397c537
  249. <   if (deltalen > 0 && sdslen(o->ptr) + deltalen > server.hll_sparse_max_bytes)
  250. ---
  251. >   if (deltalen > 0 && sdslen(hdr) + deltalen > hll_sparse_max_bytes)
  252. 400c540
  253. <   sdsIncrLen(o->ptr, deltalen);
  254. ---
  255. >   sdsIncrLen(hdr, deltalen);
  256. 425c564
  257. <           sdsIncrLen(o->ptr, -1);
  258. ---
  259. >           sdsIncrLen(hdr, -1);
  260. 435d572
  261. <   hdr = o->ptr;
  262. 436a574
  263. >   *ptr = hdr;
  264. 440,441c578
  265. <   if (hllSparseToDense(o) == C_ERR) return -1;
  266. <   hdr = o->ptr;
  267. ---
  268. >   if ((hdr = hllSparseToDense(hdr)) == NULL) return -1;
  269. 444c581,582
  270. <   serverAssert(dense_retval == 1);
  271. ---
  272. >   assert(dense_retval == 1);
  273. >   *ptr = hdr;
  274. 448c586
  275. < int hllSparseAdd(robj *o, unsigned char *ele, size_t elesize) {
  276. ---
  277. > int hllSparseAdd(hllhdr **ptr, const unsigned char *ele, size_t elesize) {
  278. 451,452c589
  279. <
  280. <   return hllSparseSet(o, index, count);
  281. ---
  282. >   return hllSparseSet(ptr, index, count);
  283. 548c685,686
  284. <     serverPanic("Unknown HyperLogLog encoding in hllCount()");
  285. ---
  286. >     *invalid = 1;
  287. >     return 0;
  288. 562,568c700,711
  289. < int hllAdd(robj *o, unsigned char *ele, size_t elesize) {
  290. <   struct hllhdr *hdr = o->ptr;
  291. <   switch (hdr->encoding) {
  292. <     case HLL_DENSE:
  293. <       return hllDenseAdd(hdr->registers, ele, elesize);
  294. <     case HLL_SPARSE:
  295. <       return hllSparseAdd(o, ele, elesize);
  296. ---
  297. > int hllAdd(hllhdr **ptr, const unsigned char *ele, size_t elesize) {
  298. >   switch ((*ptr)->encoding) {
  299. >     case HLL_DENSE: {
  300. >       if (hllDenseAdd((*ptr)->registers, ele, elesize)) {
  301. >         HLL_INVALIDATE_CACHE(*ptr);
  302. >         return 1;
  303. >       }
  304. >       return 0;
  305. >     }
  306. >     case HLL_SPARSE: {
  307. >       return hllSparseAdd(ptr, ele, elesize);
  308. >     }
  309. 574,575c717
  310. < int hllMerge(uint8_t *max, robj *hll) {
  311. <   struct hllhdr *hdr = hll->ptr;
  312. ---
  313. > int hllMerge(uint8_t *max, hllhdr *ptr) {
  314. 578c720
  315. <   if (hdr->encoding == HLL_DENSE) {
  316. ---
  317. >   if (ptr->encoding == HLL_DENSE) {
  318. 582c724
  319. <       HLL_DENSE_GET_REGISTER(val, hdr->registers, i);
  320. ---
  321. >       HLL_DENSE_GET_REGISTER(val, ptr->registers, i);
  322. 586c728
  323. <     uint8_t *p = hll->ptr, *end = p + sdslen(hll->ptr);
  324. ---
  325. >     uint8_t *p = (uint8_t *)ptr, *end = p + sdslen(ptr);
  326. 611c753
  327. <     if (i != HLL_REGISTERS) return C_ERR;
  328. ---
  329. >     if (i != HLL_REGISTERS) return -1;
  330. 613c755
  331. <   return C_OK;
  332. ---
  333. >   return 0;
  334. 616,617c758,760
  335. < robj *createHLLObject(void) {
  336. <   robj *o;
  337. ---
  338. > void deleteHLLObject(hllhdr *hdr) { sdsfree(hdr); }
  339. >
  340. > hllhdr *createHLLObject(void) {
  341. 628a773
  342. >   hdr = (hllhdr *)s;
  343. 637c782
  344. <   serverAssert((p - (uint8_t *)s) == sparselen);
  345. ---
  346. >   assert((p - (uint8_t *)s) == sparselen);
  347. 639,641c784
  348. <   o = createObject(OBJ_STRING, s);
  349. <   hdr = o->ptr;
  350. <   memcpy(hdr->magic, "HYLL", 4);
  351. ---
  352. >   memcpy(hdr->magic, hll_magic_char, 4);
  353. 643c786
  354. <   return o;
  355. ---
  356. >   return hdr;
  357. 646c789
  358. < int isHLLObjectOrReply(client *c, robj *o) {
  359. ---
  360. > int isHLLObjectOrReply(void *ptr) {
  361. 649,653c792
  362. <   if (checkType(c, o, OBJ_STRING)) return C_ERR;
  363. <
  364. <   if (!sdsEncodedObject(o)) goto invalid;
  365. <   if (stringObjectLen(o) < sizeof(*hdr)) goto invalid;
  366. <   hdr = o->ptr;
  367. ---
  368. >   hdr = ptr;
  369. 661,662c800
  370. <   if (hdr->encoding == HLL_DENSE && stringObjectLen(o) != HLL_DENSE_SIZE)
  371. <     goto invalid;
  372. ---
  373. >   if (hdr->encoding == HLL_DENSE && sdslen(ptr) != HLL_DENSE_SIZE) goto invalid;
  374. 664c802
  375. <   return C_OK;
  376. ---
  377. >   return 0;
  378. 667,670c805
  379. <   addReplyError(c,
  380. <                 "-WRONGTYPE Key is not a valid "
  381. <                 "HyperLogLog string value.");
  382. <   return C_ERR;
  383. ---
  384. >   return -1;
  385. 673,684c808,810
  386. < void pfaddCommand(client *c) {
  387. <   robj *o = lookupKeyWrite(c->db, c->argv[1]);
  388. <   struct hllhdr *hdr;
  389. <   int updated = 0, j;
  390. <
  391. <   if (o == NULL) {
  392. <     o = createHLLObject();
  393. <     dbAdd(c->db, c->argv[1], o);
  394. <     updated++;
  395. <   } else {
  396. <     if (isHLLObjectOrReply(c, o) != C_OK) return;
  397. <     o = dbUnshareStringValue(c->db, c->argv[1], o);
  398. ---
  399. > int pfaddCommand(hllhdr **ptr, const unsigned char *key, int len) {
  400. >   if (!ptr || !*ptr || !key) {
  401. >     return -1;
  402. 687,706c813,814
  403. <   for (j = 2; j < c->argc; j++) {
  404. <     int retval =
  405. <         hllAdd(o, (unsigned char *)c->argv[j]->ptr, sdslen(c->argv[j]->ptr));
  406. <     switch (retval) {
  407. <       case 1:
  408. <         updated++;
  409. <         break;
  410. <       case -1:
  411. <         addReplyError(c, invalid_hll_err);
  412. <         return;
  413. <     }
  414. <   }
  415. <   hdr = o->ptr;
  416. <   if (updated) {
  417. <     signalModifiedKey(c, c->db, c->argv[1]);
  418. <     notifyKeyspaceEvent(NOTIFY_STRING, "pfadd", c->argv[1], c->db->id);
  419. <     server.dirty += updated;
  420. <     HLL_INVALIDATE_CACHE(hdr);
  421. <   }
  422. <   addReply(c, updated ? shared.cone : shared.czero);
  423. ---
  424. >   len = (len > 0) ? len : strlen(key);
  425. >   return hllAdd(ptr, key, len);
  426. 709,711c817
  427. < void pfcountCommand(client *c) {
  428. <   robj *o;
  429. <   struct hllhdr *hdr;
  430. ---
  431. > int pfcountCommand(hllhdr *hdr, uint64_t *ret) {
  432. 713,739c819,827
  433. <
  434. <   if (c->argc > 2) {
  435. <     uint8_t max[HLL_HDR_SIZE + HLL_REGISTERS], *registers;
  436. <     int j;
  437. <
  438. <     memset(max, 0, sizeof(max));
  439. <     hdr = (struct hllhdr *)max;
  440. <     hdr->encoding = HLL_RAW;
  441. <     registers = max + HLL_HDR_SIZE;
  442. <     for (j = 1; j < c->argc; j++) {
  443. <       robj *o = lookupKeyRead(c->db, c->argv[j]);
  444. <       if (o == NULL) continue;
  445. <       if (isHLLObjectOrReply(c, o) != C_OK) return;
  446. <
  447. <       if (hllMerge(registers, o) == C_ERR) {
  448. <         addReplyError(c, invalid_hll_err);
  449. <         return;
  450. <       }
  451. <     }
  452. <
  453. <     addReplyLongLong(c, hllCount(hdr, NULL));
  454. <     return;
  455. <   }
  456. <
  457. <   o = lookupKeyWrite(c->db, c->argv[1]);
  458. <   if (o == NULL) {
  459. <     addReply(c, shared.czero);
  460. ---
  461. >   if (HLL_VALID_CACHE(hdr)) {
  462. >     card = (uint64_t)hdr->card[0];
  463. >     card |= (uint64_t)hdr->card[1] << 8;
  464. >     card |= (uint64_t)hdr->card[2] << 16;
  465. >     card |= (uint64_t)hdr->card[3] << 24;
  466. >     card |= (uint64_t)hdr->card[4] << 32;
  467. >     card |= (uint64_t)hdr->card[5] << 40;
  468. >     card |= (uint64_t)hdr->card[6] << 48;
  469. >     card |= (uint64_t)hdr->card[7] << 56;
  470. 741,772c829,832
  471. <     if (isHLLObjectOrReply(c, o) != C_OK) return;
  472. <     o = dbUnshareStringValue(c->db, c->argv[1], o);
  473. <
  474. <     hdr = o->ptr;
  475. <     if (HLL_VALID_CACHE(hdr)) {
  476. <       card = (uint64_t)hdr->card[0];
  477. <       card |= (uint64_t)hdr->card[1] << 8;
  478. <       card |= (uint64_t)hdr->card[2] << 16;
  479. <       card |= (uint64_t)hdr->card[3] << 24;
  480. <       card |= (uint64_t)hdr->card[4] << 32;
  481. <       card |= (uint64_t)hdr->card[5] << 40;
  482. <       card |= (uint64_t)hdr->card[6] << 48;
  483. <       card |= (uint64_t)hdr->card[7] << 56;
  484. <     } else {
  485. <       int invalid = 0;
  486. <
  487. <       card = hllCount(hdr, &invalid);
  488. <       if (invalid) {
  489. <         addReplyError(c, invalid_hll_err);
  490. <         return;
  491. <       }
  492. <       hdr->card[0] = card & 0xff;
  493. <       hdr->card[1] = (card >> 8) & 0xff;
  494. <       hdr->card[2] = (card >> 16) & 0xff;
  495. <       hdr->card[3] = (card >> 24) & 0xff;
  496. <       hdr->card[4] = (card >> 32) & 0xff;
  497. <       hdr->card[5] = (card >> 40) & 0xff;
  498. <       hdr->card[6] = (card >> 48) & 0xff;
  499. <       hdr->card[7] = (card >> 56) & 0xff;
  500. <
  501. <       signalModifiedKey(c, c->db, c->argv[1]);
  502. <       server.dirty++;
  503. ---
  504. >     int invalid = 0;
  505. >     card = hllCount(hdr, &invalid);
  506. >     if (invalid) {
  507. >       return -1;
  508. 774c834,841
  509. <     addReplyLongLong(c, card);
  510. ---
  511. >     hdr->card[0] = card & 0xff;
  512. >     hdr->card[1] = (card >> 8) & 0xff;
  513. >     hdr->card[2] = (card >> 16) & 0xff;
  514. >     hdr->card[3] = (card >> 24) & 0xff;
  515. >     hdr->card[4] = (card >> 32) & 0xff;
  516. >     hdr->card[5] = (card >> 40) & 0xff;
  517. >     hdr->card[6] = (card >> 48) & 0xff;
  518. >     hdr->card[7] = (card >> 56) & 0xff;
  519. 775a843,845
  520. >
  521. >   *ret = card;
  522. >   return 0;
  523. 778c848
  524. < void pfmergeCommand(client *c) {
  525. ---
  526. > int pfmergeCommand(hllhdr **hdr, hllhdr *src) {
  527. 780d849
  528. <   struct hllhdr *hdr;
  529. 784,822c853,854
  530. <   memset(max, 0, sizeof(max));
  531. <   for (j = 1; j < c->argc; j++) {
  532. <     robj *o = lookupKeyRead(c->db, c->argv[j]);
  533. <     if (o == NULL) continue;
  534. <     if (isHLLObjectOrReply(c, o) != C_OK) return;
  535. <
  536. <     hdr = o->ptr;
  537. <     if (hdr->encoding == HLL_DENSE) use_dense = 1;
  538. <
  539. <     if (hllMerge(max, o) == C_ERR) {
  540. <       addReplyError(c, invalid_hll_err);
  541. <       return;
  542. <     }
  543. <   }
  544. <
  545. <   robj *o = lookupKeyWrite(c->db, c->argv[1]);
  546. <   if (o == NULL) {
  547. <     o = createHLLObject();
  548. <     dbAdd(c->db, c->argv[1], o);
  549. <   } else {
  550. <     o = dbUnshareStringValue(c->db, c->argv[1], o);
  551. <   }
  552. <
  553. <   if (use_dense && hllSparseToDense(o) == C_ERR) {
  554. <     addReplyError(c, invalid_hll_err);
  555. <     return;
  556. <   }
  557. <
  558. <   for (j = 0; j < HLL_REGISTERS; j++) {
  559. <     if (max[j] == 0) continue;
  560. <     hdr = o->ptr;
  561. <     switch (hdr->encoding) {
  562. <       case HLL_DENSE:
  563. <         hllDenseSet(hdr->registers, j, max[j]);
  564. <         break;
  565. <       case HLL_SPARSE:
  566. <         hllSparseSet(o, j, max[j]);
  567. <         break;
  568. <     }
  569. ---
  570. >   if (!hdr || !*hdr || !src) {
  571. >     return -1;
  572. 824,825d855
  573. <   hdr = o->ptr;
  574. <   HLL_INVALIDATE_CACHE(hdr);
  575. 827,980c857
  576. <   signalModifiedKey(c, c->db, c->argv[1]);
  577. <
  578. <   notifyKeyspaceEvent(NOTIFY_STRING, "pfadd", c->argv[1], c->db->id);
  579. <   server.dirty++;
  580. <   addReply(c, shared.ok);
  581. < }
  582. <
  583. < #define HLL_TEST_CYCLES 1000
  584. < void pfselftestCommand(client *c) {
  585. <   unsigned int j, i;
  586. <   sds bitcounters = sdsnewlen(NULL, HLL_DENSE_SIZE);
  587. <   struct hllhdr *hdr = (struct hllhdr *)bitcounters, *hdr2;
  588. <   robj *o = NULL;
  589. <   uint8_t bytecounters[HLL_REGISTERS];
  590. <
  591. <   for (j = 0; j < HLL_TEST_CYCLES; j++) {
  592. <     for (i = 0; i < HLL_REGISTERS; i++) {
  593. <       unsigned int r = rand() & HLL_REGISTER_MAX;
  594. <
  595. <       bytecounters[i] = r;
  596. <       HLL_DENSE_SET_REGISTER(hdr->registers, i, r);
  597. <     }
  598. <
  599. <     for (i = 0; i < HLL_REGISTERS; i++) {
  600. <       unsigned int val;
  601. <
  602. <       HLL_DENSE_GET_REGISTER(val, hdr->registers, i);
  603. <       if (val != bytecounters[i]) {
  604. <         addReplyErrorFormat(c, "TESTFAILED Register %d should be %d but is %d",
  605. <                             i, (int)bytecounters[i], (int)val);
  606. <         goto cleanup;
  607. <       }
  608. <     }
  609. <   }
  610. <
  611. <   memset(hdr->registers, 0, HLL_DENSE_SIZE - HLL_HDR_SIZE);
  612. <   o = createHLLObject();
  613. <   double relerr = 1.04 / sqrt(HLL_REGISTERS);
  614. <   int64_t checkpoint = 1;
  615. <   uint64_t seed = (uint64_t)rand() | (uint64_t)rand() << 32;
  616. <   uint64_t ele;
  617. <   for (j = 1; j <= 10000000; j++) {
  618. <     ele = j ^ seed;
  619. <     hllDenseAdd(hdr->registers, (unsigned char *)&ele, sizeof(ele));
  620. <     hllAdd(o, (unsigned char *)&ele, sizeof(ele));
  621. <
  622. <     if (j == checkpoint && j < server.hll_sparse_max_bytes / 2) {
  623. <       hdr2 = o->ptr;
  624. <       if (hdr2->encoding != HLL_SPARSE) {
  625. <         addReplyError(c, "TESTFAILED sparse encoding not used");
  626. <         goto cleanup;
  627. <       }
  628. <     }
  629. <
  630. <     if (j == checkpoint && hllCount(hdr, NULL) != hllCount(o->ptr, NULL)) {
  631. <       addReplyError(c, "TESTFAILED dense/sparse disagree");
  632. <       goto cleanup;
  633. <     }
  634. <
  635. <     if (j == checkpoint) {
  636. <       int64_t abserr = checkpoint - (int64_t)hllCount(hdr, NULL);
  637. <       uint64_t maxerr = ceil(relerr * 6 * checkpoint);
  638. <
  639. <       if (j == 10) maxerr = 1;
  640. <
  641. <       if (abserr < 0) abserr = -abserr;
  642. <       if (abserr > (int64_t)maxerr) {
  643. <         addReplyErrorFormat(
  644. <             c, "TESTFAILED Too big error. card:%llu abserr:%llu",
  645. <             (unsigned long long)checkpoint, (unsigned long long)abserr);
  646. <         goto cleanup;
  647. <       }
  648. <       checkpoint *= 10;
  649. <     }
  650. <   }
  651. <
  652. <   addReply(c, shared.ok);
  653. <
  654. < cleanup:
  655. <   sdsfree(bitcounters);
  656. <   if (o) decrRefCount(o);
  657. < }
  658. <
  659. < void pfdebugCommand(client *c) {
  660. <   char *cmd = c->argv[1]->ptr;
  661. <   struct hllhdr *hdr;
  662. <   robj *o;
  663. <   int j;
  664. <
  665. <   o = lookupKeyWrite(c->db, c->argv[2]);
  666. <   if (o == NULL) {
  667. <     addReplyError(c, "The specified key does not exist");
  668. <     return;
  669. <   }
  670. <   if (isHLLObjectOrReply(c, o) != C_OK) return;
  671. <   o = dbUnshareStringValue(c->db, c->argv[2], o);
  672. <   hdr = o->ptr;
  673. <
  674. <   if (!strcasecmp(cmd, "getreg")) {
  675. <     if (c->argc != 3) goto arityerr;
  676. <
  677. <     if (hdr->encoding == HLL_SPARSE) {
  678. <       if (hllSparseToDense(o) == C_ERR) {
  679. <         addReplyError(c, invalid_hll_err);
  680. <         return;
  681. <       }
  682. <       server.dirty++;
  683. <     }
  684. <
  685. <     hdr = o->ptr;
  686. <     addReplyArrayLen(c, HLL_REGISTERS);
  687. <     for (j = 0; j < HLL_REGISTERS; j++) {
  688. <       uint8_t val;
  689. <
  690. <       HLL_DENSE_GET_REGISTER(val, hdr->registers, j);
  691. <       addReplyLongLong(c, val);
  692. <     }
  693. <   }
  694. <
  695. <   else if (!strcasecmp(cmd, "decode")) {
  696. <     if (c->argc != 3) goto arityerr;
  697. <
  698. <     uint8_t *p = o->ptr, *end = p + sdslen(o->ptr);
  699. <     sds decoded = sdsempty();
  700. <
  701. <     if (hdr->encoding != HLL_SPARSE) {
  702. <       sdsfree(decoded);
  703. <       addReplyError(c, "HLL encoding is not sparse");
  704. <       return;
  705. <     }
  706. <
  707. <     p += HLL_HDR_SIZE;
  708. <     while (p < end) {
  709. <       int runlen, regval;
  710. <
  711. <       if (HLL_SPARSE_IS_ZERO(p)) {
  712. <         runlen = HLL_SPARSE_ZERO_LEN(p);
  713. <         p++;
  714. <         decoded = sdscatprintf(decoded, "z:%d ", runlen);
  715. <       } else if (HLL_SPARSE_IS_XZERO(p)) {
  716. <         runlen = HLL_SPARSE_XZERO_LEN(p);
  717. <         p += 2;
  718. <         decoded = sdscatprintf(decoded, "Z:%d ", runlen);
  719. <       } else {
  720. <         runlen = HLL_SPARSE_VAL_LEN(p);
  721. <         regval = HLL_SPARSE_VAL_VALUE(p);
  722. <         p++;
  723. <         decoded = sdscatprintf(decoded, "v:%d,%d ", regval, runlen);
  724. <       }
  725. <     }
  726. <     decoded = sdstrim(decoded, " ");
  727. <     addReplyBulkCBuffer(c, decoded, sdslen(decoded));
  728. <     sdsfree(decoded);
  729. <   }
  730. ---
  731. >   memset(max, 0, sizeof(max));
  732. 982,984c859
  733. <   else if (!strcasecmp(cmd, "encoding")) {
  734. <     char *encodingstr[2] = {"dense", "sparse"};
  735. <     if (c->argc != 3) goto arityerr;
  736. ---
  737. >   if ((*hdr)->encoding == HLL_DENSE) use_dense = 1;
  738. 986c861,862
  739. <     addReplyStatus(c, encodingstr[hdr->encoding]);
  740. ---
  741. >   if (hllMerge(max, src)) {
  742. >     return -1;
  743. 989,999c865,869
  744. <   else if (!strcasecmp(cmd, "todense")) {
  745. <     int conv = 0;
  746. <     if (c->argc != 3) goto arityerr;
  747. <
  748. <     if (hdr->encoding == HLL_SPARSE) {
  749. <       if (hllSparseToDense(o) == C_ERR) {
  750. <         addReplyError(c, invalid_hll_err);
  751. <         return;
  752. <       }
  753. <       conv = 1;
  754. <       server.dirty++;
  755. ---
  756. >   for (j = 0; j < HLL_REGISTERS; j++) {
  757. >     if (use_dense) {
  758. >       hllDenseSet((*hdr)->registers, j, max[j]);
  759. >     } else {
  760. >       hllSparseSet(hdr, j, max[j]);
  761. 1001,1003d870
  762. <     addReply(c, conv ? shared.cone : shared.czero);
  763. <   } else {
  764. <     addReplyErrorFormat(c, "Unknown PFDEBUG subcommand '%s'", cmd);
  765. 1005d871
  766. <   return;
  767. 1007,1009c873,874
  768. < arityerr:
  769. <   addReplyErrorFormat(c, "Wrong number of arguments for the '%s' subcommand",
  770. <                       cmd);
  771. ---
  772. >   HLL_INVALIDATE_CACHE(*hdr);
  773. >   return 0;
  774.  
Advertisement
Add Comment
Please, Sign In to add comment