API tools faq
paste
Advertisement
Guest User

squashfs-tools-4.2+20130409 to squashfs-tools-4.3+2014

a guest
Jul 24th, 2015
850
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Diff 350.04 KB | None | 0 0
raw download clone embed print report
  1. diff -Nru squashfs-tools-4.2+20130409/action.c squashfs-tools-4.3+20140919/action.c
  2. --- squashfs-tools-4.2+20130409/action.c    2013-05-09 10:39:11.000000000 +0200
  3. +++ squashfs-tools-4.3+20140919/action.c    2015-07-20 21:03:05.000000000 +0200
  4. @@ -2,7 +2,7 @@
  5.   * Create a squashfs filesystem.  This is a highly compressed read only
  6.   * filesystem.
  7.   *
  8. - * Copyright (c) 2011, 2012, 2013
  9. + * Copyright (c) 2011, 2012, 2013, 2014
  10.   * Phillip Lougher <phillip@squashfs.org.uk>
  11.   *
  12.   * This program is free software; you can redistribute it and/or
  13. @@ -53,12 +53,15 @@
  14.  static struct action *exclude_spec = NULL;
  15.  static struct action *empty_spec = NULL;
  16.  static struct action *move_spec = NULL;
  17. +static struct action *prune_spec = NULL;
  18.  static struct action *other_spec = NULL;
  19.  static int fragment_count = 0;
  20.  static int exclude_count = 0;
  21.  static int empty_count = 0;
  22.  static int move_count = 0;
  23. +static int prune_count = 0;
  24.  static int other_count = 0;
  25. +static struct action_entry *parsing_action;
  26.  
  27.  static struct file_buffer *def_fragment = NULL;
  28.  
  29. @@ -198,12 +201,34 @@
  30.  /*
  31.   * Expression parser
  32.   */
  33. +static void free_parse_tree(struct expr *expr)
  34. +{
  35. +   if(expr->type == ATOM_TYPE) {
  36. +       int i;
  37. +
  38. +       for(i = 0; i < expr->atom.test->args; i++)
  39. +           free(expr->atom.argv[i]);
  40. +
  41. +       free(expr->atom.argv);
  42. +   } else if (expr->type == UNARY_TYPE)
  43. +       free_parse_tree(expr->unary_op.expr);
  44. +   else {
  45. +       free_parse_tree(expr->expr_op.lhs);
  46. +       free_parse_tree(expr->expr_op.rhs);
  47. +   }
  48. +
  49. +   free(expr);
  50. +}
  51. +
  52. +
  53.  static struct expr *create_expr(struct expr *lhs, int op, struct expr *rhs)
  54.  {
  55.     struct expr *expr;
  56.  
  57. -   if (rhs == NULL)
  58. +   if (rhs == NULL) {
  59. +       free_parse_tree(lhs);
  60.         return NULL;
  61. +   }
  62.  
  63.     expr = malloc(sizeof(*expr));
  64.     if (expr == NULL)
  65. @@ -239,8 +264,8 @@
  66.  
  67.  static struct expr *parse_test(char *name)
  68.  {
  69. -   char *string;
  70. -   int token;
  71. +   char *string, **argv = NULL;
  72. +   int token, args = 0;
  73.     int i;
  74.     struct test_entry *test;
  75.     struct expr *expr;
  76. @@ -249,12 +274,20 @@
  77.         if (strcmp(name, test_table[i].name) == 0)
  78.             break;
  79.  
  80. -   if (test_table[i].args == -1) {
  81. +   test = &test_table[i];
  82. +
  83. +   if (test->args == -1) {
  84.         SYNTAX_ERROR("Non-existent test \"%s\"\n", name);
  85.         return NULL;
  86.     }
  87.  
  88. -   test = &test_table[i];
  89. +   if(parsing_action->type == EXCLUDE_ACTION && !test->exclude_ok) {
  90. +       fprintf(stderr, "Failed to parse action \"%s\"\n", source);
  91. +       fprintf(stderr, "Test \"%s\" cannot be used in exclude "
  92. +                           "actions\n", name);
  93. +       fprintf(stderr, "Use prune action instead ...\n");
  94. +       return NULL;
  95. +   }
  96.  
  97.     expr = malloc(sizeof(*expr));
  98.     if (expr == NULL)
  99. @@ -262,55 +295,68 @@
  100.  
  101.     expr->type = ATOM_TYPE;
  102.  
  103. -   expr->atom.argv = malloc(test->args * sizeof(char *));
  104. -   if (expr->atom.argv == NULL)
  105. -       MEM_ERROR();
  106. -
  107.     expr->atom.test = test;
  108.     expr->atom.data = NULL;
  109.  
  110.     /*
  111. -    * If the test has no arguments, allow it to be typed
  112. -    *  without brackets
  113. +    * If the test has no arguments, then go straight to checking if there's
  114. +    * enough arguments
  115.      */
  116. -   if (test->args == 0) {
  117. -       token = peek_token(&string);
  118. +   token = peek_token(&string);
  119.  
  120. -       if (token != TOK_OPEN_BRACKET)
  121. +   if (token != TOK_OPEN_BRACKET)
  122.             goto skip_args;
  123. -   }
  124. -
  125. -   token = get_token(&string);
  126.  
  127. -   if (token != TOK_OPEN_BRACKET) {
  128. -       SYNTAX_ERROR("Unexpected token \"%s\", expected \"(\"\n",
  129. -                       TOK_TO_STR(token, string));
  130. -       goto failed;
  131. -   }
  132. +   get_token(&string);
  133.  
  134. -   for (i = 0; i < test->args; i++) {
  135. -       token = get_token(&string);
  136. +   /*
  137. +    * speculatively read all the arguments, and then see if the
  138. +    * number of arguments read is the number expected, this handles
  139. +    * tests with a variable number of arguments
  140. +    */
  141. +   token = get_token(&string);
  142. +   if (token == TOK_CLOSE_BRACKET)
  143. +       goto skip_args;
  144.  
  145. +   while(1) {
  146.         if (token != TOK_STRING) {
  147.             SYNTAX_ERROR("Unexpected token \"%s\", expected "
  148.                 "argument\n", TOK_TO_STR(token, string));
  149.             goto failed;
  150.         }
  151.  
  152. -       expr->atom.argv[i] = strdup(string);
  153. +       argv = realloc(argv, (args + 1) * sizeof(char *));
  154. +       if (argv == NULL)
  155. +           MEM_ERROR();
  156. +
  157. +       argv[args ++ ] = strdup(string);
  158. +
  159. +       token = get_token(&string);
  160.  
  161. -       if (i + 1 < test->args) {
  162. -           token = get_token(&string);
  163. +       if (token == TOK_CLOSE_BRACKET)
  164. +           break;
  165.  
  166. -           if (token != TOK_COMMA) {
  167. -               SYNTAX_ERROR("Unexpected token \"%s\", "
  168. -                   "expected \",\"\n",
  169. -                   TOK_TO_STR(token, string));
  170. +       if (token != TOK_COMMA) {
  171. +           SYNTAX_ERROR("Unexpected token \"%s\", expected "
  172. +               "\",\" or \")\"\n", TOK_TO_STR(token, string));
  173.             goto failed;
  174. -           }
  175.         }
  176. +       token = get_token(&string);
  177. +   }
  178. +
  179. +skip_args:
  180. +   /*
  181. +    * expected number of arguments?
  182. +    */
  183. +   if(test->args != -2 && args != test->args) {
  184. +       SYNTAX_ERROR("Unexpected number of arguments, expected %d, "
  185. +           "got %d\n", test->args, args);
  186. +       goto failed;
  187.     }
  188.  
  189. +   expr->atom.args = args;
  190. +   expr->atom.argv = argv;
  191. +
  192.     if (test->parse_args) {
  193.         int res = test->parse_args(test, &expr->atom);
  194.  
  195. @@ -318,19 +364,10 @@
  196.             goto failed;
  197.     }
  198.  
  199. -   token = get_token(&string);
  200. -
  201. -   if (token != TOK_CLOSE_BRACKET) {
  202. -       SYNTAX_ERROR("Unexpected token \"%s\", expected \")\"\n",
  203. -                       TOK_TO_STR(token, string));
  204. -       goto failed;
  205. -   }
  206. -
  207. -skip_args:
  208.     return expr;
  209.  
  210.  failed:
  211. -   free(expr->atom.argv);
  212. +   free(argv);
  213.     free(expr);
  214.     return NULL;
  215.  }
  216. @@ -367,6 +404,7 @@
  217.  
  218.         if (op == TOK_EOF) {
  219.             if (subexp) {
  220. +               free_parse_tree(expr);
  221.                 SYNTAX_ERROR("Expected \"&&\", \"||\" or "
  222.                         "\")\", got EOF\n");
  223.                 return NULL;
  224. @@ -376,6 +414,7 @@
  225.  
  226.         if (op == TOK_CLOSE_BRACKET) {
  227.             if (!subexp) {
  228. +               free_parse_tree(expr);
  229.                 SYNTAX_ERROR("Unexpected \")\", expected "
  230.                         "\"&&\", \"!!\" or EOF\n");
  231.                 return NULL;
  232. @@ -384,6 +423,7 @@
  233.         }
  234.        
  235.         if (op != TOK_AND && op != TOK_OR) {
  236. +           free_parse_tree(expr);
  237.             SYNTAX_ERROR("Unexpected token \"%s\", expected "
  238.                 "\"&&\" or \"||\"\n", TOK_TO_STR(op, string));
  239.             return NULL;
  240. @@ -399,7 +439,7 @@
  241.  /*
  242.   * Action parser
  243.   */
  244. -int parse_action(char *s)
  245. +int parse_action(char *s, int verbose)
  246.  {
  247.     char *string, **argv = NULL;
  248.     int i, token, args = 0;
  249. @@ -501,6 +541,7 @@
  250.         goto failed;
  251.     }
  252.    
  253. +   parsing_action = action;
  254.     expr = parse_expr(0);
  255.  
  256.     if (expr == NULL)
  257. @@ -526,6 +567,10 @@
  258.         spec_count = move_count ++;
  259.         spec_list = &move_spec;
  260.         break;
  261. +   case PRUNE_ACTION:
  262. +       spec_count = prune_count ++;
  263. +       spec_list = &prune_spec;
  264. +       break;
  265.     default:
  266.         spec_count = other_count ++;
  267.         spec_list = &other_spec;
  268. @@ -542,6 +587,7 @@
  269.     (*spec_list)[spec_count].argv = argv;
  270.     (*spec_list)[spec_count].expr = expr;
  271.     (*spec_list)[spec_count].data = data;
  272. +   (*spec_list)[spec_count].verbose = verbose;
  273.  
  274.     return 1;
  275.  
  276. @@ -551,68 +597,133 @@
  277.  }
  278.  
  279.  
  280. -static void dump_parse_tree(struct expr *expr)
  281. -{
  282. -   if(expr->type == ATOM_TYPE) {
  283. -       int i;
  284. +/*
  285. + * Evaluate expressions
  286. + */
  287.  
  288. -       printf("%s(", expr->atom.test->name);
  289. -       for(i = 0; i < expr->atom.test->args; i++) {
  290. -           printf("%s", expr->atom.argv[i]);
  291. -           if (i + 1 < expr->atom.test->args)
  292. -               printf(",");
  293. -       }
  294. -       printf(")");
  295. -   } else if (expr->type == UNARY_TYPE) {
  296. -       printf("%s", token_table[expr->unary_op.op].string);
  297. -       dump_parse_tree(expr->unary_op.expr);
  298. -   } else {
  299. -       printf("(");
  300. -       dump_parse_tree(expr->expr_op.lhs);
  301. -       printf("%s", token_table[expr->expr_op.op].string);
  302. -       dump_parse_tree(expr->expr_op.rhs);
  303. -       printf(")");
  304. +#define ALLOC_SZ 128
  305. +
  306. +#define LOG_ENABLE 0
  307. +#define LOG_DISABLE    1
  308. +#define LOG_PRINT  2
  309. +#define LOG_ENABLED    3
  310. +
  311. +char *_expr_log(char *string, int cmnd)
  312. +{
  313. +   static char *expr_msg = NULL;
  314. +   static int cur_size = 0, alloc_size = 0;
  315. +   int size;
  316. +
  317. +   switch(cmnd) {
  318. +   case LOG_ENABLE:
  319. +       expr_msg = malloc(ALLOC_SZ);
  320. +       alloc_size = ALLOC_SZ;
  321. +       cur_size = 0;
  322. +       return expr_msg;
  323. +   case LOG_DISABLE:
  324. +       free(expr_msg);
  325. +       alloc_size = cur_size = 0;
  326. +       return expr_msg = NULL;
  327. +   case LOG_ENABLED:
  328. +       return expr_msg;
  329. +   default:
  330. +       if(expr_msg == NULL)
  331. +           return NULL;
  332. +       break;
  333. +   }
  334. +
  335. +   /* if string is empty append '\0' */
  336. +   size = strlen(string) ? : 1;
  337. +
  338. +   if(alloc_size - cur_size < size) {
  339. +       /* buffer too small, expand */
  340. +       alloc_size = (cur_size + size + ALLOC_SZ - 1) & ~(ALLOC_SZ - 1);
  341. +
  342. +       expr_msg = realloc(expr_msg, alloc_size);
  343. +       if(expr_msg == NULL)
  344. +           MEM_ERROR();
  345.     }
  346. +
  347. +   memcpy(expr_msg + cur_size, string, size);
  348. +   cur_size += size;
  349. +
  350. +   return expr_msg;
  351.  }
  352.  
  353.  
  354. -void dump_action_list(struct action *spec_list, int spec_count)
  355. +char *expr_log_cmnd(int cmnd)
  356. +{
  357. +   return _expr_log(NULL, cmnd);
  358. +}
  359. +
  360. +
  361. +char *expr_log(char *string)
  362. +{
  363. +   return _expr_log(string, LOG_PRINT);
  364. +}
  365. +
  366. +
  367. +void expr_log_atom(struct atom *atom)
  368.  {
  369.     int i;
  370.  
  371. -   for (i = 0; i < spec_count; i++) {
  372. -       printf("%s", spec_list[i].action->name);
  373. -       if (spec_list[i].action->args) {
  374. -           int n;
  375. +   if(atom->test->handle_logging)
  376. +       return;
  377.  
  378. -           printf("(");
  379. -           for (n = 0; n < spec_list[i].action->args; n++) {
  380. -               printf("%s", spec_list[i].argv[n]);
  381. -               if (n + 1 < spec_list[i].action->args)
  382. -                   printf(",");
  383. -           }
  384. -           printf(")");
  385. +   expr_log(atom->test->name);
  386. +
  387. +   if(atom->args) {
  388. +       expr_log("(");
  389. +       for(i = 0; i < atom->args; i++) {
  390. +           expr_log(atom->argv[i]);
  391. +           if (i + 1 < atom->args)
  392. +               expr_log(",");
  393.         }
  394. -       printf("=");
  395. -       dump_parse_tree(spec_list[i].expr);
  396. -       printf("\n");
  397. +       expr_log(")");
  398.     }
  399.  }
  400.  
  401.  
  402. -void dump_actions()
  403. +void expr_log_match(int match)
  404.  {
  405. -   dump_action_list(exclude_spec, exclude_count);
  406. -   dump_action_list(fragment_spec, fragment_count);
  407. -   dump_action_list(other_spec, other_count);
  408. -   dump_action_list(move_spec, move_count);
  409. -   dump_action_list(empty_spec, empty_count);
  410. +   if(match)
  411. +       expr_log("=True");
  412. +   else
  413. +       expr_log("=False");
  414. +}
  415. +
  416. +
  417. +static int eval_expr_log(struct expr *expr, struct action_data *action_data)
  418. +{
  419. +   int match;
  420. +
  421. +   switch (expr->type) {
  422. +   case ATOM_TYPE:
  423. +       expr_log_atom(&expr->atom);
  424. +       match = expr->atom.test->fn(&expr->atom, action_data);
  425. +       expr_log_match(match);
  426. +       break;
  427. +   case UNARY_TYPE:
  428. +       expr_log("!");
  429. +       match = !eval_expr_log(expr->unary_op.expr, action_data);
  430. +       break;
  431. +   default:
  432. +       expr_log("(");
  433. +       match = eval_expr_log(expr->expr_op.lhs, action_data);
  434. +
  435. +       if ((expr->expr_op.op == TOK_AND && match) ||
  436. +               (expr->expr_op.op == TOK_OR && !match)) {
  437. +           expr_log(token_table[expr->expr_op.op].string);
  438. +           match = eval_expr_log(expr->expr_op.rhs, action_data);
  439. +       }
  440. +       expr_log(")");
  441. +       break;
  442. +   }
  443. +
  444. +   return match;
  445.  }
  446.  
  447.  
  448. -/*
  449. - * Evaluate expressions
  450. - */
  451.  static int eval_expr(struct expr *expr, struct action_data *action_data)
  452.  {
  453.     int match;
  454. @@ -637,6 +748,49 @@
  455.  }
  456.  
  457.  
  458. +static int eval_expr_top(struct action *action, struct action_data *action_data)
  459. +{
  460. +   if(action->verbose) {
  461. +       int match, n;
  462. +
  463. +       expr_log_cmnd(LOG_ENABLE);
  464. +
  465. +       if(action_data->subpath)
  466. +           expr_log(action_data->subpath);
  467. +
  468. +       expr_log("=");
  469. +       expr_log(action->action->name);
  470. +
  471. +       if(action->args) {
  472. +           expr_log("(");
  473. +           for (n = 0; n < action->args; n++) {
  474. +               expr_log(action->argv[n]);
  475. +               if(n + 1 < action->args)
  476. +                   expr_log(",");
  477. +           }
  478. +           expr_log(")");
  479. +       }
  480. +
  481. +       expr_log("@");
  482. +
  483. +       match = eval_expr_log(action->expr, action_data);
  484. +
  485. +       /*
  486. +        * Print the evaluated expression log, if the
  487. +        * result matches the logging specified
  488. +        */
  489. +       if((match && (action->verbose & ACTION_LOG_TRUE)) || (!match
  490. +               && (action->verbose & ACTION_LOG_FALSE)))
  491. +           progressbar_info("%s\n", expr_log(""));
  492. +
  493. +       expr_log_cmnd(LOG_DISABLE);
  494. +
  495. +       return match;
  496. +   } else
  497. +       return eval_expr(action->expr, action_data);
  498. +}
  499. +
  500. +
  501.  /*
  502.   * Read action file, passing each line to parse_action() for
  503.   * parsing.
  504. @@ -648,9 +802,65 @@
  505.   *
  506.   * Blank lines and comment lines indicated by # are supported.
  507.   */
  508. -int read_action_file(char *filename)
  509. +int parse_action_true(char *s)
  510. +{
  511. +   return parse_action(s, ACTION_LOG_TRUE);
  512. +}
  513. +
  514. +
  515. +int parse_action_false(char *s)
  516. +{
  517. +   return parse_action(s, ACTION_LOG_FALSE);
  518. +}
  519. +
  520. +
  521. +int parse_action_verbose(char *s)
  522. +{
  523. +   return parse_action(s, ACTION_LOG_VERBOSE);
  524. +}
  525. +
  526. +
  527. +int parse_action_nonverbose(char *s)
  528. +{
  529. +   return parse_action(s, ACTION_LOG_NONE);
  530. +}
  531. +
  532. +
  533. +int read_action_file(char *filename, int verbose)
  534. +{
  535. +   switch(verbose) {
  536. +   case ACTION_LOG_TRUE:
  537. +       return read_file(filename, "action", parse_action_true);
  538. +   case ACTION_LOG_FALSE:
  539. +       return read_file(filename, "action", parse_action_false);
  540. +   case ACTION_LOG_VERBOSE:
  541. +       return read_file(filename, "action", parse_action_verbose);
  542. +   default:
  543. +       return read_file(filename, "action", parse_action_nonverbose);
  544. +   }
  545. +}
  546. +
  547. +
  548. +/*
  549. + * helper to evaluate whether action/test acts on this file type
  550. + */
  551. +static int file_type_match(int st_mode, int type)
  552.  {
  553. -   return read_file(filename, "action", parse_action);
  554. +   switch(type) {
  555. +   case ACTION_DIR:
  556. +       return S_ISDIR(st_mode);
  557. +   case ACTION_REG:
  558. +       return S_ISREG(st_mode);
  559. +   case ACTION_ALL:
  560. +       return S_ISREG(st_mode) || S_ISDIR(st_mode) ||
  561. +           S_ISCHR(st_mode) || S_ISBLK(st_mode) ||
  562. +           S_ISFIFO(st_mode) || S_ISSOCK(st_mode);
  563. +   case ACTION_LNK:
  564. +       return S_ISLNK(st_mode);
  565. +   case ACTION_ALL_LNK:
  566. +   default:
  567. +       return 1;
  568. +   }
  569.  }
  570.  
  571.  
  572. @@ -663,53 +873,65 @@
  573.  }
  574.  
  575.  
  576. -void eval_actions(struct dir_ent *dir_ent)
  577. +void eval_actions(struct dir_info *root, struct dir_ent *dir_ent)
  578.  {
  579.     int i, match;
  580.     struct action_data action_data;
  581. -   int file_type = dir_ent->inode->buf.st_mode & S_IFMT;
  582. +   int st_mode = dir_ent->inode->buf.st_mode;
  583.  
  584.     action_data.name = dir_ent->name;
  585. -   action_data.pathname = pathname(dir_ent);
  586. -   action_data.subpath = subpathname(dir_ent);
  587. +   action_data.pathname = strdup(pathname(dir_ent));
  588. +   action_data.subpath = strdup(subpathname(dir_ent));
  589.     action_data.buf = &dir_ent->inode->buf;
  590.     action_data.depth = dir_ent->our_dir->depth;
  591. +   action_data.dir_ent = dir_ent;
  592. +   action_data.root = root;
  593.  
  594.     for (i = 0; i < other_count; i++) {
  595.         struct action *action = &other_spec[i];
  596.  
  597. -       if ((action->action->file_types & file_type) == 0)
  598. +       if (!file_type_match(st_mode, action->action->file_types))
  599.             /* action does not operate on this file type */
  600.             continue;
  601.  
  602. -       match = eval_expr(action->expr, &action_data);
  603. +       match = eval_expr_top(action, &action_data);
  604.  
  605.         if (match)
  606.             action->action->run_action(action, dir_ent);
  607.     }
  608. +
  609. +   free(action_data.pathname);
  610. +   free(action_data.subpath);
  611.  }
  612.  
  613.  
  614.  /*
  615.   * Fragment specific action code
  616.   */
  617. -void *eval_frag_actions(struct dir_ent *dir_ent)
  618. +void *eval_frag_actions(struct dir_info *root, struct dir_ent *dir_ent)
  619.  {
  620.     int i, match;
  621.     struct action_data action_data;
  622.  
  623.     action_data.name = dir_ent->name;
  624. -   action_data.pathname = pathname(dir_ent);
  625. -   action_data.subpath = subpathname(dir_ent);
  626. +   action_data.pathname = strdup(pathname(dir_ent));
  627. +   action_data.subpath = strdup(subpathname(dir_ent));
  628.     action_data.buf = &dir_ent->inode->buf;
  629.     action_data.depth = dir_ent->our_dir->depth;
  630. +   action_data.dir_ent = dir_ent;
  631. +   action_data.root = root;
  632.  
  633.     for (i = 0; i < fragment_count; i++) {
  634. -       match = eval_expr(fragment_spec[i].expr, &action_data);
  635. -       if (match)
  636. +       match = eval_expr_top(&fragment_spec[i], &action_data);
  637. +       if (match) {
  638. +           free(action_data.pathname);
  639. +           free(action_data.subpath);
  640.             return &fragment_spec[i].data;
  641. +       }
  642.     }
  643.  
  644. +   free(action_data.pathname);
  645. +   free(action_data.subpath);
  646.     return &def_fragment;
  647.  }
  648.  
  649. @@ -747,7 +969,7 @@
  650.  
  651.  
  652.  int eval_exclude_actions(char *name, char *pathname, char *subpath,
  653. -   struct stat *buf, int depth)
  654. +   struct stat *buf, int depth, struct dir_ent *dir_ent)
  655.  {
  656.     int i, match = 0;
  657.     struct action_data action_data;
  658. @@ -757,9 +979,10 @@
  659.     action_data.subpath = subpath;
  660.     action_data.buf = buf;
  661.     action_data.depth = depth;
  662. +   action_data.dir_ent = dir_ent;
  663.  
  664.     for (i = 0; i < exclude_count && !match; i++)
  665. -       match = eval_expr(exclude_spec[i].expr, &action_data);
  666. +       match = eval_expr_top(&exclude_spec[i], &action_data);
  667.  
  668.     return match;
  669.  }
  670. @@ -823,8 +1046,8 @@
  671.     long long uid = strtoll(arg, &b, 10);
  672.  
  673.     if (*b == '\0') {
  674. -       if (uid < 0 || uid >= (1LL < 32)) {
  675. -           SYNTAX_ERROR("action: uid out of range\n");
  676. +       if (uid < 0 || uid >= (1LL << 32)) {
  677. +           SYNTAX_ERROR("Uid out of range\n");
  678.             return -1;
  679.         }
  680.     } else {
  681. @@ -833,7 +1056,7 @@
  682.         if (passwd)
  683.             uid = passwd->pw_uid;
  684.         else {
  685. -           SYNTAX_ERROR("action: invalid uid or unknown user\n");
  686. +           SYNTAX_ERROR("Invalid uid or unknown user\n");
  687.             return -1;
  688.         }
  689.     }
  690. @@ -847,8 +1070,8 @@
  691.     long long gid = strtoll(arg, &b, 10);
  692.  
  693.     if (*b == '\0') {
  694. -       if (gid < 0 || gid >= (1LL < 32)) {
  695. -           SYNTAX_ERROR("action: gid out of range\n");
  696. +       if (gid < 0 || gid >= (1LL << 32)) {
  697. +           SYNTAX_ERROR("Gid out of range\n");
  698.             return -1;
  699.         }
  700.     } else {
  701. @@ -857,7 +1080,7 @@
  702.         if (group)
  703.             gid = group->gr_gid;
  704.         else {
  705. -           SYNTAX_ERROR("action: invalid gid or unknown user\n");
  706. +           SYNTAX_ERROR("Invalid gid or unknown group\n");
  707.             return -1;
  708.         }
  709.     }
  710. @@ -964,29 +1187,37 @@
  711.  /*
  712.   * Mode specific action code
  713.   */
  714. -static int parse_octal_mode_args(unsigned int mode, int bytes, int args,
  715. -                   char **argv, void **data)
  716. +static int parse_octal_mode_args(int args, char **argv,
  717. +           void **data)
  718.  {
  719. +   int n, bytes;
  720. +   unsigned int mode;
  721.     struct mode_data *mode_data;
  722.  
  723. +   /* octal mode number? */
  724. +   n = sscanf(argv[0], "%o%n", &mode, &bytes);
  725. +   if (n == 0)
  726. +       return -1; /* not an octal number arg */
  727. +
  728. +
  729.     /* check there's no trailing junk */
  730.     if (argv[0][bytes] != '\0') {
  731.         SYNTAX_ERROR("Unexpected trailing bytes after octal "
  732.             "mode number\n");
  733. -       return 0;
  734. +       return 0; /* bad octal number arg */
  735.     }
  736.  
  737.     /* check there's only one argument */
  738.     if (args > 1) {
  739.         SYNTAX_ERROR("Octal mode number is first argument, "
  740.             "expected one argument, got %d\n", args);
  741. -       return 0;
  742. +       return 0; /* bad octal number arg */
  743.     }
  744.  
  745.     /*  check mode is within range */
  746.     if (mode > 07777) {
  747.         SYNTAX_ERROR("Octal mode %o is out of range\n", mode);
  748. -       return 0;
  749. +       return 0; /* bad octal number arg */
  750.     }
  751.  
  752.     mode_data = malloc(sizeof(struct mode_data));
  753. @@ -1003,19 +1234,17 @@
  754.  
  755.  
  756.  /*
  757. - * Parse symbolic mode of format [ugoa]+[+-=]PERMS
  758. + * Parse symbolic mode of format [ugoa]*[[+-=]PERMS]+
  759.   * PERMS = [rwxXst]+ or [ugo]
  760.   */
  761. -static struct mode_data *parse_sym_mode_arg(char *arg)
  762. +static int parse_sym_mode_arg(char *arg, struct mode_data **head,
  763. +   struct mode_data **cur)
  764.  {
  765. -   struct mode_data *mode_data = malloc(sizeof(*mode_data));
  766. -   int mode = 0;
  767. +   struct mode_data *mode_data;
  768. +   int mode;
  769.     int mask = 0;
  770.     int op;
  771. -   char X = 0;
  772. -
  773. -   if (mode_data == NULL)
  774. -       MEM_ERROR();
  775. +   char X;
  776.  
  777.     if (arg[0] != 'u' && arg[0] != 'g' && arg[0] != 'o' && arg[0] != 'a') {
  778.         /* no ownership specifiers, default to a */
  779. @@ -1045,130 +1274,138 @@
  780.     }
  781.  
  782.  parse_operation:
  783. -   switch(*arg) {
  784. -   case '+':
  785. -       op = ACTION_MODE_ADD;
  786. -       break;
  787. -   case '-':
  788. -       op = ACTION_MODE_REM;
  789. -       break;
  790. -   case '=':
  791. -       op = ACTION_MODE_SET;
  792. -       break;
  793. -   default:
  794. -       SYNTAX_ERROR("Action mode: Expected one of '+', '-' or '=', "
  795. -           "got '%c'\n", *arg);
  796. +   /* trap a symbolic mode with just an ownership specification */
  797. +   if(*arg == '\0') {
  798. +       SYNTAX_ERROR("Expected one of '+', '-' or '=', got EOF\n");
  799.         goto failed;
  800.     }
  801.  
  802. -   arg ++;
  803. +   while(*arg != '\0') {
  804. +       mode = 0;
  805. +       X = 0;
  806.  
  807. -   /* Parse PERMS */
  808. -   if (*arg == 'u' || *arg == 'g' || *arg == 'o') {
  809. -       /* PERMS = [ugo] */
  810. -       mode = - *arg;
  811. -       if (*++arg != '\0') {
  812. -           SYNTAX_ERROR("Action mode: permission 'u', 'g' or 'o' "
  813. -               "has trailing characters\n");
  814. +       switch(*arg) {
  815. +       case '+':
  816. +           op = ACTION_MODE_ADD;
  817. +           break;
  818. +       case '-':
  819. +           op = ACTION_MODE_REM;
  820. +           break;
  821. +       case '=':
  822. +           op = ACTION_MODE_SET;
  823. +           break;
  824. +       default:
  825. +           SYNTAX_ERROR("Expected one of '+', '-' or '=', got "
  826. +               "'%c'\n", *arg);
  827.             goto failed;
  828.         }
  829. -   } else {
  830. -       /* PERMS = [rwxXst]+ */
  831. -       while(*arg != '\0') {
  832. -           switch(*arg) {
  833. -           case 'r':
  834. -               mode |= 0444;
  835. -               break;
  836. -           case 'w':
  837. -               mode |= 0222;
  838. -               break;
  839. -           case 'x':
  840. -               mode |= 0111;
  841. -               break;
  842. -           case 's':
  843. -               mode |= 06000;
  844. -               break;
  845. -           case 't':
  846. -               mode |= 01000;
  847. -               break;
  848. -           case 'X':
  849. -               X = 1;
  850. -               break;
  851. -           default:
  852. -               SYNTAX_ERROR("Action mode: unrecognised "
  853. -                       "permission '%c'\n", *arg);
  854. -               goto failed;
  855. -           }
  856. -
  857. +  
  858. +       arg ++;
  859. +  
  860. +       /* Parse PERMS */
  861. +       if (*arg == 'u' || *arg == 'g' || *arg == 'o') {
  862. +           /* PERMS = [ugo] */
  863. +           mode = - *arg;
  864.             arg ++;
  865. +       } else {
  866. +           /* PERMS = [rwxXst]* */
  867. +           while(1) {
  868. +               switch(*arg) {
  869. +               case 'r':
  870. +                   mode |= 0444;
  871. +                   break;
  872. +               case 'w':
  873. +                   mode |= 0222;
  874. +                   break;
  875. +               case 'x':
  876. +                   mode |= 0111;
  877. +                   break;
  878. +               case 's':
  879. +                   mode |= 06000;
  880. +                   break;
  881. +               case 't':
  882. +                   mode |= 01000;
  883. +                   break;
  884. +               case 'X':
  885. +                   X = 1;
  886. +                   break;
  887. +               case '+':
  888. +               case '-':
  889. +               case '=':
  890. +               case '\0':
  891. +                   mode &= mask;
  892. +                   goto perms_parsed;
  893. +               default:
  894. +                   SYNTAX_ERROR("Unrecognised permission "
  895. +                               "'%c'\n", *arg);
  896. +                   goto failed;
  897. +               }
  898. +  
  899. +               arg ++;
  900. +           }
  901.         }
  902. -       mode &= mask;
  903. -   }
  904. +  
  905. +perms_parsed:
  906. +       mode_data = malloc(sizeof(*mode_data));
  907. +       if (mode_data == NULL)
  908. +           MEM_ERROR();
  909.  
  910. -   mode_data->operation = op;
  911. -   mode_data->mode = mode;
  912. -   mode_data->mask = mask;
  913. -   mode_data->X = X;
  914. -   mode_data->next = NULL;
  915. +       mode_data->operation = op;
  916. +       mode_data->mode = mode;
  917. +       mode_data->mask = mask;
  918. +       mode_data->X = X;
  919. +       mode_data->next = NULL;
  920. +
  921. +       if (*cur) {
  922. +           (*cur)->next = mode_data;
  923. +           *cur = mode_data;
  924. +       } else
  925. +           *head = *cur = mode_data;
  926. +   }
  927.  
  928. -   return mode_data;
  929. +   return 1;
  930.  
  931.  failed:
  932. -   free(mode_data);
  933. -   return NULL;
  934. +   return 0;
  935.  }
  936.  
  937.  
  938.  static int parse_sym_mode_args(struct action_entry *action, int args,
  939.                     char **argv, void **data)
  940.  {
  941. -   int i;
  942. +   int i, res = 1;
  943.     struct mode_data *head = NULL, *cur = NULL;
  944.  
  945. -   for (i = 0; i < args; i++) {
  946. -       struct mode_data *entry = parse_sym_mode_arg(argv[i]);
  947. -
  948. -       if (entry == NULL)
  949. -           return 0;
  950. -
  951. -       if (cur) {
  952. -           cur->next = entry;
  953. -           cur = entry;
  954. -       } else
  955. -           head = cur = entry;
  956. -   }
  957. +   for (i = 0; i < args && res; i++)
  958. +       res = parse_sym_mode_arg(argv[i], &head, &cur);
  959.  
  960.     *data = head;
  961.  
  962. -   return 1;
  963. +   return res;
  964.  }
  965.  
  966.  
  967.  static int parse_mode_args(struct action_entry *action, int args,
  968.                     char **argv, void **data)
  969.  {
  970. -   int n, bytes;
  971. -   unsigned int mode;
  972. +   int res;
  973.  
  974.     if (args == 0) {
  975.         SYNTAX_ERROR("Mode action expects one or more arguments\n");
  976.         return 0;
  977.     }
  978.  
  979. -   /* octal mode number? */
  980. -   n = sscanf(argv[0], "%o%n", &mode, &bytes);
  981. -
  982. -   if(n >= 1)
  983. -       return parse_octal_mode_args(mode, bytes, args, argv, data);
  984. -   else
  985. +   res = parse_octal_mode_args(args, argv, data);
  986. +   if(res >= 0)
  987. +       /* Got an octal mode argument */
  988. +       return res;
  989. +   else  /* not an octal mode argument */
  990.         return parse_sym_mode_args(action, args, argv, data);
  991.  }
  992.  
  993.  
  994. -static void mode_action(struct action *action, struct dir_ent *dir_ent)
  995. +static int mode_execute(struct mode_data *mode_data, int st_mode)
  996.  {
  997. -   struct stat *buf = &dir_ent->inode->buf;
  998. -   struct mode_data *mode_data = action->data;
  999.     int mode = 0;
  1000.  
  1001.     for (;mode_data; mode_data = mode_data->next) {
  1002. @@ -1176,20 +1413,20 @@
  1003.             /* 'u', 'g' or 'o' */
  1004.             switch(-mode_data->mode) {
  1005.             case 'u':
  1006. -               mode = (buf->st_mode >> 6) & 07;
  1007. +               mode = (st_mode >> 6) & 07;
  1008.                 break;
  1009.             case 'g':
  1010. -               mode = (buf->st_mode >> 3) & 07;
  1011. +               mode = (st_mode >> 3) & 07;
  1012.                 break;
  1013.             case 'o':
  1014. -               mode = buf->st_mode & 07;
  1015. +               mode = st_mode & 07;
  1016.                 break;
  1017.             }
  1018.             mode = ((mode << 6) | (mode << 3) | mode) &
  1019.                 mode_data->mask;
  1020.         } else if (mode_data->X &&
  1021. -               ((buf->st_mode & S_IFMT) == S_IFDIR ||
  1022. -               (buf->st_mode & 0111)))
  1023. +               ((st_mode & S_IFMT) == S_IFDIR ||
  1024. +               (st_mode & 0111)))
  1025.             /* X permission, only takes effect if inode is a
  1026.              * directory or x is set for some owner */
  1027.             mode = mode_data->mode | (0111 & mode_data->mask);
  1028. @@ -1198,18 +1435,27 @@
  1029.  
  1030.         switch(mode_data->operation) {
  1031.         case ACTION_MODE_OCT:
  1032. -           buf->st_mode = (buf->st_mode & ~S_IFMT) | mode;
  1033. +           st_mode = (st_mode & S_IFMT) | mode;
  1034.             break;
  1035.         case ACTION_MODE_SET:
  1036. -           buf->st_mode = (buf->st_mode & ~mode_data->mask) | mode;
  1037. +           st_mode = (st_mode & ~mode_data->mask) | mode;
  1038.             break;
  1039.         case ACTION_MODE_ADD:
  1040. -           buf->st_mode |= mode;
  1041. +           st_mode |= mode;
  1042.             break;
  1043.         case ACTION_MODE_REM:
  1044. -           buf->st_mode &= ~mode;
  1045. +           st_mode &= ~mode;
  1046.         }
  1047.     }
  1048. +
  1049. +   return st_mode;
  1050. +}
  1051. +
  1052. +
  1053. +static void mode_action(struct action *action, struct dir_ent *dir_ent)
  1054. +{
  1055. +   dir_ent->inode->buf.st_mode = mode_execute(action->data,
  1056. +                   dir_ent->inode->buf.st_mode);
  1057.  }
  1058.  
  1059.  
  1060. @@ -1257,7 +1503,7 @@
  1061.  }
  1062.  
  1063.  
  1064. -int eval_empty_actions(struct dir_ent *dir_ent)
  1065. +int eval_empty_actions(struct dir_info *root, struct dir_ent *dir_ent)
  1066.  {
  1067.     int i, match = 0;
  1068.     struct action_data action_data;
  1069. @@ -1271,10 +1517,12 @@
  1070.         return 0;
  1071.  
  1072.     action_data.name = dir_ent->name;
  1073. -   action_data.pathname = pathname(dir_ent);
  1074. -   action_data.subpath = subpathname(dir_ent);
  1075. +   action_data.pathname = strdup(pathname(dir_ent));
  1076. +   action_data.subpath = strdup(subpathname(dir_ent));
  1077.     action_data.buf = &dir_ent->inode->buf;
  1078.     action_data.depth = dir_ent->our_dir->depth;
  1079. +   action_data.dir_ent = dir_ent;
  1080. +   action_data.root = root;
  1081.  
  1082.     for (i = 0; i < empty_count && !match; i++) {
  1083.         data = empty_spec[i].data;
  1084. @@ -1297,9 +1545,12 @@
  1085.                 (data->val == EMPTY_SOURCE && dir->excluded))
  1086.             continue;
  1087.        
  1088. -       match = eval_expr(empty_spec[i].expr, &action_data);
  1089. +       match = eval_expr_top(&empty_spec[i], &action_data);
  1090.     }
  1091.  
  1092. +   free(action_data.pathname);
  1093. +   free(action_data.subpath);
  1094. +
  1095.     return match;
  1096.  }
  1097.  
  1098. @@ -1510,10 +1761,12 @@
  1099.     struct move_ent *move = NULL;
  1100.  
  1101.     action_data.name = dir_ent->name;
  1102. -   action_data.pathname = pathname(dir_ent);
  1103. -   action_data.subpath = subpathname(dir_ent);
  1104. +   action_data.pathname = strdup(pathname(dir_ent));
  1105. +   action_data.subpath = strdup(subpathname(dir_ent));
  1106.     action_data.buf = &dir_ent->inode->buf;
  1107.     action_data.depth = dir_ent->our_dir->depth;
  1108. +   action_data.dir_ent = dir_ent;
  1109. +   action_data.root = root;
  1110.  
  1111.     /*
  1112.      * Evaluate each move action against the current file.  For any
  1113. @@ -1526,7 +1779,7 @@
  1114.      */
  1115.     for (i = 0; i < move_count; i++) {
  1116.         struct action *action = &move_spec[i];
  1117. -       int match = eval_expr(action->expr, &action_data);
  1118. +       int match = eval_expr_top(action, &action_data);
  1119.  
  1120.         if(match) {
  1121.             if(move == NULL) {
  1122. @@ -1553,7 +1806,7 @@
  1123.          */
  1124.         if(move->ops == 0) {
  1125.             free(move);
  1126. -           return;
  1127. +           goto finish;
  1128.         }
  1129.  
  1130.         dest = (move->ops & ACTION_MOVE_MOVE) ?
  1131. @@ -1568,7 +1821,7 @@
  1132.                 action_data.subpath, conf_path);
  1133.             free(conf_path);
  1134.             free(move);
  1135. -           return;
  1136. +           goto finish;
  1137.         }
  1138.  
  1139.         /*
  1140. @@ -1582,11 +1835,15 @@
  1141.                 action_data.subpath, conf_path);
  1142.             free(conf_path);
  1143.             free(move);
  1144. -           return;
  1145. +           goto finish;
  1146.         }
  1147.         move->next = move_list;
  1148.         move_list = move;
  1149.     }
  1150. +
  1151. +finish:
  1152. +   free(action_data.pathname);
  1153. +   free(action_data.subpath);
  1154.  }
  1155.  
  1156.  
  1157. @@ -1714,6 +1971,46 @@
  1158.  
  1159.  
  1160.  /*
  1161. + * Prune specific action code
  1162. + */
  1163. +int prune_actions()
  1164. +{
  1165. +   return prune_count;
  1166. +}
  1167. +
  1168. +
  1169. +int eval_prune_actions(struct dir_info *root, struct dir_ent *dir_ent)
  1170. +{
  1171. +   int i, match = 0;
  1172. +   struct action_data action_data;
  1173. +
  1174. +   action_data.name = dir_ent->name;
  1175. +   action_data.pathname = strdup(pathname(dir_ent));
  1176. +   action_data.subpath = strdup(subpathname(dir_ent));
  1177. +   action_data.buf = &dir_ent->inode->buf;
  1178. +   action_data.depth = dir_ent->our_dir->depth;
  1179. +   action_data.dir_ent = dir_ent;
  1180. +   action_data.root = root;
  1181. +
  1182. +   for (i = 0; i < prune_count && !match; i++)
  1183. +       match = eval_expr_top(&prune_spec[i], &action_data);
  1184. +
  1185. +   free(action_data.pathname);
  1186. +   free(action_data.subpath);
  1187. +
  1188. +   return match;
  1189. +}
  1190. +
  1191. +
  1192. +/*
  1193. + * Noop specific action code
  1194. + */
  1195. +static void noop_action(struct action *action, struct dir_ent *dir_ent)
  1196. +{
  1197. +}
  1198. +
  1199. +
  1200. +/*
  1201.   * General test evaluation code
  1202.   */
  1203.  
  1204. @@ -1768,13 +2065,13 @@
  1205.     switch (end[0]) {
  1206.     case 'g':
  1207.     case 'G':
  1208. -       *size *= 1024;
  1209. +       number *= 1024;
  1210.     case 'm':
  1211.     case 'M':
  1212. -       *size *= 1024;
  1213. +       number *= 1024;
  1214.     case 'k':
  1215.     case 'K':
  1216. -       *size *= 1024;
  1217. +       number *= 1024;
  1218.  
  1219.         if (end[1] != '\0') {
  1220.             *error = "Trailing junk after size specifier";
  1221. @@ -1873,7 +2170,7 @@
  1222.  static int NAME##_fn(struct atom *atom, struct action_data *action_data) \
  1223.  { \
  1224.     /* test operates on MATCH file types only */ \
  1225. -   if (!(action_data->buf->st_mode & MATCH)) \
  1226. +   if (!file_type_match(action_data->buf->st_mode, MATCH)) \
  1227.         return 0; \
  1228.   \
  1229.     CODE \
  1230. @@ -1992,6 +2289,12 @@
  1231.         FNM_PATHNAME|FNM_PERIOD|FNM_EXTMATCH) == 0;
  1232.  }
  1233.  
  1234. +/*
  1235. + * Inode attribute test operations using generic
  1236. + * TEST_VAR_FN(test name, file scope, attribute name) macro.
  1237. + * This is for tests that do not need to be specially handled in any way.
  1238. + * They just take a variable and compare it against a number.
  1239. + */
  1240.  TEST_VAR_FN(filesize, ACTION_REG, action_data->buf->st_size)
  1241.  
  1242.  TEST_VAR_FN(dirsize, ACTION_DIR, action_data->buf->st_size)
  1243. @@ -2008,9 +2311,7 @@
  1244.  
  1245.  TEST_VAR_FN(blocks, ACTION_ALL_LNK, action_data->buf->st_blocks)
  1246.  
  1247. -TEST_VAR_FN(gid, ACTION_ALL_LNK, action_data->buf->st_gid)
  1248. -
  1249. -TEST_VAR_FN(uid, ACTION_ALL_LNK, action_data->buf->st_uid)
  1250. +TEST_VAR_FN(dircount, ACTION_DIR, action_data->dir_ent->dir->count)
  1251.  
  1252.  TEST_VAR_FN(depth, ACTION_ALL_LNK, action_data->depth)
  1253.  
  1254. @@ -2036,6 +2337,98 @@
  1255.  
  1256.  TEST_VAR_RANGE_FN(depth, ACTION_ALL_LNK, action_data->depth)
  1257.  
  1258. +TEST_VAR_RANGE_FN(dircount, ACTION_DIR, action_data->dir_ent->dir->count)
  1259. +
  1260. +/*
  1261. + * uid specific test code
  1262. + */
  1263. +TEST_VAR_FN(uid, ACTION_ALL_LNK, action_data->buf->st_uid)
  1264. +
  1265. +static int parse_uid_arg(struct test_entry *test, struct atom *atom)
  1266. +{
  1267. +   struct test_number_arg *number;
  1268. +   long long size;
  1269. +   int range;
  1270. +   char *error;
  1271. +
  1272. +   if(parse_number(atom->argv[0], &size, &range, &error)) {
  1273. +       /* managed to fully parse argument as a number */
  1274. +       if(size < 0 || size > (((long long) 1 << 32) - 1)) {
  1275. +           TEST_SYNTAX_ERROR(test, 1, "Numeric uid out of "
  1276. +                               "range\n");
  1277. +           return 0;
  1278. +       }
  1279. +   } else {
  1280. +       /* couldn't parse (fully) as a number, is it a user name? */
  1281. +       struct passwd *uid = getpwnam(atom->argv[0]);
  1282. +       if(uid) {
  1283. +           size = uid->pw_uid;
  1284. +           range = NUM_EQ;
  1285. +       } else {
  1286. +           TEST_SYNTAX_ERROR(test, 1, "Invalid uid or unknown "
  1287. +                               "user\n");
  1288. +           return 0;
  1289. +       }
  1290. +   }
  1291. +
  1292. +   number = malloc(sizeof(*number));
  1293. +   if(number == NULL)
  1294. +       MEM_ERROR();
  1295. +
  1296. +   number->range = range;
  1297. +   number->size= size;
  1298. +
  1299. +   atom->data = number;
  1300. +
  1301. +   return 1;
  1302. +}
  1303. +
  1304. +
  1305. +/*
  1306. + * gid specific test code
  1307. + */
  1308. +TEST_VAR_FN(gid, ACTION_ALL_LNK, action_data->buf->st_gid)
  1309. +
  1310. +static int parse_gid_arg(struct test_entry *test, struct atom *atom)
  1311. +{
  1312. +   struct test_number_arg *number;
  1313. +   long long size;
  1314. +   int range;
  1315. +   char *error;
  1316. +
  1317. +   if(parse_number(atom->argv[0], &size, &range, &error)) {
  1318. +       /* managed to fully parse argument as a number */
  1319. +       if(size < 0 || size > (((long long) 1 << 32) - 1)) {
  1320. +           TEST_SYNTAX_ERROR(test, 1, "Numeric gid out of "
  1321. +                               "range\n");
  1322. +           return 0;
  1323. +       }
  1324. +   } else {
  1325. +       /* couldn't parse (fully) as a number, is it a group name? */
  1326. +       struct group *gid = getgrnam(atom->argv[0]);
  1327. +       if(gid) {
  1328. +           size = gid->gr_gid;
  1329. +           range = NUM_EQ;
  1330. +       } else {
  1331. +           TEST_SYNTAX_ERROR(test, 1, "Invalid gid or unknown "
  1332. +                               "group\n");
  1333. +           return 0;
  1334. +       }
  1335. +   }
  1336. +
  1337. +   number = malloc(sizeof(*number));
  1338. +   if(number == NULL)
  1339. +       MEM_ERROR();
  1340. +
  1341. +   number->range = range;
  1342. +   number->size= size;
  1343. +
  1344. +   atom->data = number;
  1345. +
  1346. +   return 1;
  1347. +}
  1348. +
  1349. +
  1350.  /*
  1351.   * Type test specific code
  1352.   */
  1353. @@ -2116,6 +2509,7 @@
  1354.         char str[1024]; /* overflow safe */
  1355.  
  1356.         regerror(res, preg, str, 1024);
  1357. +       free(preg);
  1358.         TEST_SYNTAX_ERROR(test, 0, "invalid regex \"%s\" because "
  1359.             "\"%s\"\n", atom->argv[0], str);
  1360.         return 0;
  1361. @@ -2265,37 +2659,583 @@
  1362.  }
  1363.  
  1364.  
  1365. +/*
  1366. + * Symbolic link specific test code
  1367. + */
  1368. +
  1369. +/*
  1370. + * Walk the supplied pathname and return the directory entry corresponding
  1371. + * to the pathname.  If any symlinks are encountered whilst walking the
  1372. + * pathname, then recursively walk these, to obtain the fully
  1373. + * dereferenced canonicalised directory entry.
  1374. + *
  1375. + * If follow_path fails to walk a pathname either because a component
  1376. + * doesn't exist, it is a non directory component when a directory
  1377. + * component is expected, a symlink with an absolute path is encountered,
  1378. + * or a symlink is encountered which cannot be recursively walked due to
  1379. + * the above failures, then return NULL.
  1380. + */
  1381. +static struct dir_ent *follow_path(struct dir_info *dir, char *pathname)
  1382. +{
  1383. +   char *comp, *path = pathname;
  1384. +   struct dir_ent *dir_ent = NULL;
  1385. +
  1386. +   /* We cannot follow absolute paths */
  1387. +   if(pathname[0] == '/')
  1388. +       return NULL;
  1389. +
  1390. +   for(comp = get_comp(&path); comp; free(comp), comp = get_comp(&path)) {
  1391. +       if(strcmp(comp, ".") == 0)
  1392. +           continue;
  1393. +
  1394. +       if(strcmp(comp, "..") == 0) {
  1395. +           /* Move to parent if we're not in the root directory */
  1396. +           if(dir->depth > 1) {
  1397. +               dir = dir->dir_ent->our_dir;
  1398. +               dir_ent = NULL; /* lazily eval at loop exit */
  1399. +               continue;
  1400. +           } else
  1401. +               /* Failed to walk pathname */
  1402. +               return NULL;
  1403. +       }
  1404. +
  1405. +       /* Lookup comp in current directory */
  1406. +       dir_ent = lookup_comp(comp, dir);
  1407. +       if(dir_ent == NULL)
  1408. +           /* Doesn't exist, failed to walk pathname */
  1409. +           return NULL;
  1410. +
  1411. +       if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFLNK) {
  1412. +           /* Symbolic link, try to walk it */
  1413. +           dir_ent = follow_path(dir, dir_ent->inode->symlink);
  1414. +           if(dir_ent == NULL)
  1415. +               /* Failed to follow symlink */
  1416. +               return NULL;
  1417. +       }
  1418. +
  1419. +       if((dir_ent->inode->buf.st_mode & S_IFMT) != S_IFDIR)
  1420. +           /* Cannot walk further */
  1421. +           break;
  1422. +
  1423. +       dir = dir_ent->dir;
  1424. +   }
  1425. +
  1426. +   /* We will have exited the loop either because we've processed
  1427. +    * all the components, which means we've successfully walked the
  1428. +    * pathname, or because we've hit a non-directory, in which case
  1429. +    * it's success if this is the leaf component */
  1430. +   if(comp) {
  1431. +       free(comp);
  1432. +       comp = get_comp(&path);
  1433. +       free(comp);
  1434. +       if(comp != NULL)
  1435. +           /* Not a leaf component */
  1436. +           return NULL;
  1437. +   } else {
  1438. +       /* Fully walked pathname, dir_ent contains correct value unless
  1439. +        * we've walked to the parent ("..") in which case we need
  1440. +        * to resolve it here */
  1441. +       if(!dir_ent)
  1442. +           dir_ent = dir->dir_ent;
  1443. +   }
  1444. +
  1445. +   return dir_ent;
  1446. +}
  1447. +
  1448. +
  1449. +static int exists_fn(struct atom *atom, struct action_data *action_data)
  1450. +{
  1451. +   /*
  1452. +    * Test if a symlink exists within the output filesystem, that is,
  1453. +    * the symlink has a relative path, and the relative path refers
  1454. +    * to an entry within the output filesystem.
  1455. +    *
  1456. +    * This test function evaluates the path for symlinks - that is it
  1457. +    * follows any symlinks in the path (and any symlinks that it contains
  1458. +    * etc.), to discover the fully dereferenced canonicalised relative
  1459. +    * path.
  1460. +    *
  1461. +    * If any symlinks within the path do not exist or are absolute
  1462. +    * then the symlink is considered to not exist, as it cannot be
  1463. +    * fully dereferenced.
  1464. +    *
  1465. +    * exists operates on symlinks only, other files by definition
  1466. +    * exist
  1467. +    */
  1468. +   if (!file_type_match(action_data->buf->st_mode, ACTION_LNK))
  1469. +       return 1;
  1470. +
  1471. +   /* dereference the symlink, and return TRUE if it exists */
  1472. +   return follow_path(action_data->dir_ent->our_dir,
  1473. +           action_data->dir_ent->inode->symlink) ? 1 : 0;
  1474. +}
  1475. +
  1476. +
  1477. +static int absolute_fn(struct atom *atom, struct action_data *action_data)
  1478. +{
  1479. +   /*
  1480. +    * Test if a symlink has an absolute path, which by definition
  1481. +    * means the symbolic link may be broken (even if the absolute path
  1482. +    * does point into the filesystem being squashed, because the resultant
  1483. +    * filesystem can be mounted/unsquashed anywhere, it is unlikely the
  1484. +    * absolute path will still point to the right place).  If you know that
  1485. +    * an absolute symlink will point to the right place then you don't need
  1486. +    * to use this function, and/or these symlinks can be excluded by
  1487. +    * use of other test operators.
  1488. +    *
  1489. +    * absolute operates on symlinks only, other files by definition
  1490. +    * don't have problems
  1491. +    */
  1492. +   if (!file_type_match(action_data->buf->st_mode, ACTION_LNK))
  1493. +       return 0;
  1494. +
  1495. +   return action_data->dir_ent->inode->symlink[0] == '/';
  1496. +}
  1497. +
  1498. +
  1499. +static int parse_expr_argX(struct test_entry *test, struct atom *atom,
  1500. +   int argno)
  1501. +{
  1502. +   /* Call parse_expr to parse argument, which should be an expression */
  1503. +
  1504. +    /* save the current parser state */
  1505. +   char *save_cur_ptr = cur_ptr;
  1506. +   char *save_source = source;
  1507. +
  1508. +   cur_ptr = source = atom->argv[argno];
  1509. +   atom->data = parse_expr(0);
  1510. +
  1511. +   cur_ptr = save_cur_ptr;
  1512. +   source = save_source;
  1513. +
  1514. +   if(atom->data == NULL) {
  1515. +       /* parse_expr(0) will have reported the exact syntax error,
  1516. +        * but, because we recursively evaluated the expression, it
  1517. +        * will have been reported without the context of the stat
  1518. +        * test().  So here additionally report our failure to parse
  1519. +        * the expression in the stat() test to give context */
  1520. +       TEST_SYNTAX_ERROR(test, 0, "Failed to parse expression\n");
  1521. +       return 0;
  1522. +   }
  1523. +
  1524. +   return 1;
  1525. +}
  1526. +
  1527. +
  1528. +static int parse_expr_arg0(struct test_entry *test, struct atom *atom)
  1529. +{
  1530. +   return parse_expr_argX(test, atom, 0);
  1531. +}
  1532. +
  1533. +
  1534. +static int parse_expr_arg1(struct test_entry *test, struct atom *atom)
  1535. +{
  1536. +   return parse_expr_argX(test, atom, 1);
  1537. +}
  1538. +
  1539. +
  1540. +static int stat_fn(struct atom *atom, struct action_data *action_data)
  1541. +{
  1542. +   struct stat buf;
  1543. +   struct action_data eval_action;
  1544. +   int match, res;
  1545. +
  1546. +   /* evaluate the expression using the context of the inode
  1547. +    * pointed to by the symlink.  This allows the inode attributes
  1548. +    * of the file pointed to by the symlink to be evaluated, rather
  1549. +    * than the symlink itself.
  1550. +    *
  1551. +    * Note, stat() deliberately does not evaluate the pathname, name or
  1552. +    * depth of the symlink, these are left with the symlink values.
  1553. +    * This allows stat() to be used on any symlink, rather than
  1554. +    * just symlinks which are contained (if the symlink is *not*
  1555. +    * contained then pathname, name and depth are meaningless as they
  1556. +    * are relative to the filesystem being squashed). */
  1557. +
  1558. +   /* if this isn't a symlink then stat will just return the current
  1559. +    * information, i.e. stat(expr) == expr.  This is harmless and
  1560. +    * is better than returning TRUE or FALSE in a non symlink case */
  1561. +   res = stat(action_data->pathname, &buf);
  1562. +   if(res == -1) {
  1563. +       if(expr_log_cmnd(LOG_ENABLED)) {
  1564. +           expr_log(atom->test->name);
  1565. +           expr_log("(");
  1566. +           expr_log_match(0);
  1567. +           expr_log(")");
  1568. +       }
  1569. +       return 0;
  1570. +   }
  1571. +
  1572. +   /* fill in the inode values of the file pointed to by the
  1573. +    * symlink, but, leave everything else the same */
  1574. +   memcpy(&eval_action, action_data, sizeof(struct action_data));
  1575. +   eval_action.buf = &buf;
  1576. +
  1577. +   if(expr_log_cmnd(LOG_ENABLED)) {
  1578. +       expr_log(atom->test->name);
  1579. +       expr_log("(");
  1580. +       match = eval_expr_log(atom->data, &eval_action);
  1581. +       expr_log(")");
  1582. +   } else
  1583. +       match = eval_expr(atom->data, &eval_action);
  1584. +
  1585. +   return match;
  1586. +}
  1587. +
  1588. +
  1589. +static int readlink_fn(struct atom *atom, struct action_data *action_data)
  1590. +{
  1591. +   int match = 0;
  1592. +   struct dir_ent *dir_ent;
  1593. +   struct action_data eval_action;
  1594. +
  1595. +   /* Dereference the symlink and evaluate the expression in the
  1596. +    * context of the file pointed to by the symlink.
  1597. +    * All attributes are updated to refer to the file that is pointed to.
  1598. +    * Thus the inode attributes, pathname, name and depth all refer to
  1599. +    * the dereferenced file, and not the symlink.
  1600. +    *
  1601. +    * If the symlink cannot be dereferenced because it doesn't exist in
  1602. +    * the output filesystem, or due to some other failure to
  1603. +    * walk the pathname (see follow_path above), then FALSE is returned.
  1604. +    *
  1605. +    * If you wish to evaluate the inode attributes of symlinks which
  1606. +    * exist in the source filestem (but not in the output filesystem then
  1607. +    * use stat instead (see above).
  1608. +    *
  1609. +    * readlink operates on symlinks only */
  1610. +   if (!file_type_match(action_data->buf->st_mode, ACTION_LNK))
  1611. +       goto finish;
  1612. +
  1613. +   /* dereference the symlink, and get the directory entry it points to */
  1614. +   dir_ent = follow_path(action_data->dir_ent->our_dir,
  1615. +           action_data->dir_ent->inode->symlink);
  1616. +   if(dir_ent == NULL)
  1617. +       goto finish;
  1618. +
  1619. +   eval_action.name = dir_ent->name;
  1620. +   eval_action.pathname = strdup(pathname(dir_ent));
  1621. +   eval_action.subpath = strdup(subpathname(dir_ent));
  1622. +   eval_action.buf = &dir_ent->inode->buf;
  1623. +   eval_action.depth = dir_ent->our_dir->depth;
  1624. +   eval_action.dir_ent = dir_ent;
  1625. +   eval_action.root = action_data->root;
  1626. +
  1627. +   if(expr_log_cmnd(LOG_ENABLED)) {
  1628. +       expr_log(atom->test->name);
  1629. +       expr_log("(");
  1630. +       match = eval_expr_log(atom->data, &eval_action);
  1631. +       expr_log(")");
  1632. +   } else
  1633. +       match = eval_expr(atom->data, &eval_action);
  1634. +
  1635. +   free(eval_action.pathname);
  1636. +   free(eval_action.subpath);
  1637. +
  1638. +   return match;
  1639. +
  1640. +finish:
  1641. +   if(expr_log_cmnd(LOG_ENABLED)) {
  1642. +       expr_log(atom->test->name);
  1643. +       expr_log("(");
  1644. +       expr_log_match(0);
  1645. +       expr_log(")");
  1646. +   }
  1647. +
  1648. +   return 0;
  1649. +}
  1650. +
  1651. +
  1652. +static int eval_fn(struct atom *atom, struct action_data *action_data)
  1653. +{
  1654. +   int match;
  1655. +   char *path = atom->argv[0];
  1656. +   struct dir_ent *dir_ent = action_data->dir_ent;
  1657. +   struct stat *buf = action_data->buf;
  1658. +   struct action_data eval_action;
  1659. +
  1660. +   /* Follow path (arg1) and evaluate the expression (arg2)
  1661. +    * in the context of the file discovered.  All attributes are updated
  1662. +    * to refer to the file that is pointed to.
  1663. +    *
  1664. +    * This test operation allows you to add additional context to the
  1665. +    * evaluation of the file being scanned, such as "if current file is
  1666. +    * XXX and the parent is YYY, then ..."  Often times you need or
  1667. +    * want to test a combination of file status
  1668. +    *
  1669. +    * If the file referenced by the path does not exist in
  1670. +    * the output filesystem, or some other failure is experienced in
  1671. +    * walking the path (see follow_path above), then FALSE is returned.
  1672. +    *
  1673. +    * If you wish to evaluate the inode attributes of files which
  1674. +    * exist in the source filestem (but not in the output filesystem then
  1675. +    * use stat instead (see above). */
  1676. +
  1677. +   /* try to follow path, and get the directory entry it points to */
  1678. +   if(path[0] == '/') {
  1679. +       /* absolute, walk from root - first skip the leading / */
  1680. +       while(path[0] == '/')
  1681. +           path ++;
  1682. +       if(path[0] == '\0')
  1683. +           dir_ent = action_data->root->dir_ent;
  1684. +       else
  1685. +           dir_ent = follow_path(action_data->root, path);
  1686. +   } else {
  1687. +       /* relative, if first component is ".." walk from parent,
  1688. +        * otherwise walk from dir_ent.
  1689. +        * Note: this has to be handled here because follow_path
  1690. +        * will quite correctly refuse to execute ".." on anything
  1691. +        * which isn't a directory */
  1692. +       if(strncmp(path, "..", 2) == 0 && (path[2] == '\0' ||
  1693. +                           path[2] == '/')) {
  1694. +           /* walk from parent */
  1695. +           path += 2;
  1696. +           while(path[0] == '/')
  1697. +               path ++;
  1698. +           if(path[0] == '\0')
  1699. +               dir_ent = dir_ent->our_dir->dir_ent;
  1700. +           else
  1701. +               dir_ent = follow_path(dir_ent->our_dir, path);
  1702. +       } else if(!file_type_match(buf->st_mode, ACTION_DIR))
  1703. +           dir_ent = NULL;
  1704. +       else
  1705. +           dir_ent = follow_path(dir_ent->dir, path);
  1706. +   }
  1707. +
  1708. +   if(dir_ent == NULL) {
  1709. +       if(expr_log_cmnd(LOG_ENABLED)) {
  1710. +           expr_log(atom->test->name);
  1711. +           expr_log("(");
  1712. +           expr_log(atom->argv[0]);
  1713. +           expr_log(",");
  1714. +           expr_log_match(0);
  1715. +           expr_log(")");
  1716. +       }
  1717. +
  1718. +       return 0;
  1719. +   }
  1720. +
  1721. +   eval_action.name = dir_ent->name;
  1722. +   eval_action.pathname = strdup(pathname(dir_ent));
  1723. +   eval_action.subpath = strdup(subpathname(dir_ent));
  1724. +   eval_action.buf = &dir_ent->inode->buf;
  1725. +   eval_action.depth = dir_ent->our_dir->depth;
  1726. +   eval_action.dir_ent = dir_ent;
  1727. +   eval_action.root = action_data->root;
  1728. +
  1729. +   if(expr_log_cmnd(LOG_ENABLED)) {
  1730. +       expr_log(atom->test->name);
  1731. +       expr_log("(");
  1732. +       expr_log(eval_action.subpath);
  1733. +       expr_log(",");
  1734. +       match = eval_expr_log(atom->data, &eval_action);
  1735. +       expr_log(")");
  1736. +   } else
  1737. +       match = eval_expr(atom->data, &eval_action);
  1738. +
  1739. +   free(eval_action.pathname);
  1740. +   free(eval_action.subpath);
  1741. +
  1742. +   return match;
  1743. +}
  1744. +
  1745. +
  1746. +/*
  1747. + * Perm specific test code
  1748. + */
  1749. +static int parse_perm_args(struct test_entry *test, struct atom *atom)
  1750. +{
  1751. +   int res = 1, mode, op, i;
  1752. +   char *arg;
  1753. +   struct mode_data *head = NULL, *cur = NULL;
  1754. +   struct perm_data *perm_data;
  1755. +
  1756. +   if(atom->args == 0) {
  1757. +       TEST_SYNTAX_ERROR(test, 0, "One or more arguments expected\n");
  1758. +       return 0;
  1759. +   }
  1760. +
  1761. +   switch(atom->argv[0][0]) {
  1762. +   case '-':
  1763. +       op = PERM_ALL;
  1764. +       arg = atom->argv[0] + 1;
  1765. +       break;
  1766. +   case '/':
  1767. +       op = PERM_ANY;
  1768. +       arg = atom->argv[0] + 1;
  1769. +       break;
  1770. +   default:
  1771. +       op = PERM_EXACT;
  1772. +       arg = atom->argv[0];
  1773. +       break;
  1774. +   }
  1775. +
  1776. +   /* try to parse as an octal number */
  1777. +   res = parse_octal_mode_args(atom->args, atom->argv, (void **) &head);
  1778. +   if(res == -1) {
  1779. +       /* parse as sym mode argument */
  1780. +       for(i = 0; i < atom->args && res; i++, arg = atom->argv[i])
  1781. +           res = parse_sym_mode_arg(arg, &head, &cur);
  1782. +   }
  1783. +
  1784. +   if (res == 0)
  1785. +       goto finish;
  1786. +
  1787. +   /*
  1788. +    * Evaluate the symbolic mode against a permission of 0000 octal
  1789. +    */
  1790. +   mode = mode_execute(head, 0);
  1791. +
  1792. +   perm_data = malloc(sizeof(struct perm_data));
  1793. +   if (perm_data == NULL)
  1794. +       MEM_ERROR();
  1795. +
  1796. +   perm_data->op = op;
  1797. +   perm_data->mode = mode;
  1798. +
  1799. +   atom->data = perm_data;
  1800. +  
  1801. +finish:
  1802. +   while(head) {
  1803. +       struct mode_data *tmp = head;
  1804. +       head = head->next;
  1805. +       free(tmp);
  1806. +   }
  1807. +
  1808. +   return res;
  1809. +}
  1810. +
  1811. +
  1812. +static int perm_fn(struct atom *atom, struct action_data *action_data)
  1813. +{
  1814. +   struct perm_data *perm_data = atom->data;
  1815. +   struct stat *buf = action_data->buf;
  1816. +
  1817. +   switch(perm_data->op) {
  1818. +   case PERM_EXACT:
  1819. +       return (buf->st_mode & ~S_IFMT) == perm_data->mode;
  1820. +   case PERM_ALL:
  1821. +       return (buf->st_mode & perm_data->mode) == perm_data->mode;
  1822. +   case PERM_ANY:
  1823. +   default:
  1824. +       /*
  1825. +        * if no permission bits are set in perm_data->mode match
  1826. +        * on any file, this is to be consistent with find, which
  1827. +        * does this to be consistent with the behaviour of
  1828. +        * -perm -000
  1829. +        */
  1830. +       return perm_data->mode == 0 || (buf->st_mode & perm_data->mode);
  1831. +   }
  1832. +}
  1833. +
  1834. +
  1835. +#ifdef SQUASHFS_TRACE
  1836. +static void dump_parse_tree(struct expr *expr)
  1837. +{
  1838. +   int i;
  1839. +
  1840. +   if(expr->type == ATOM_TYPE) {
  1841. +       printf("%s", expr->atom.test->name);
  1842. +       if(expr->atom.args) {
  1843. +           printf("(");
  1844. +           for(i = 0; i < expr->atom.args; i++) {
  1845. +               printf("%s", expr->atom.argv[i]);
  1846. +               if (i + 1 < expr->atom.args)
  1847. +                   printf(",");
  1848. +           }
  1849. +           printf(")");
  1850. +       }
  1851. +   } else if (expr->type == UNARY_TYPE) {
  1852. +       printf("%s", token_table[expr->unary_op.op].string);
  1853. +       dump_parse_tree(expr->unary_op.expr);
  1854. +   } else {
  1855. +       printf("(");
  1856. +       dump_parse_tree(expr->expr_op.lhs);
  1857. +       printf("%s", token_table[expr->expr_op.op].string);
  1858. +       dump_parse_tree(expr->expr_op.rhs);
  1859. +       printf(")");
  1860. +   }
  1861. +}
  1862. +
  1863. +
  1864. +void dump_action_list(struct action *spec_list, int spec_count)
  1865. +{
  1866. +   int i;
  1867. +
  1868. +   for (i = 0; i < spec_count; i++) {
  1869. +       printf("%s", spec_list[i].action->name);
  1870. +       if (spec_list[i].args) {
  1871. +           int n;
  1872. +
  1873. +           printf("(");
  1874. +           for (n = 0; n < spec_list[i].args; n++) {
  1875. +               printf("%s", spec_list[i].argv[n]);
  1876. +               if (n + 1 < spec_list[i].args)
  1877. +                   printf(",");
  1878. +           }
  1879. +           printf(")");
  1880. +       }
  1881. +       printf("=");
  1882. +       dump_parse_tree(spec_list[i].expr);
  1883. +       printf("\n");
  1884. +   }
  1885. +}
  1886. +
  1887. +
  1888. +void dump_actions()
  1889. +{
  1890. +   dump_action_list(exclude_spec, exclude_count);
  1891. +   dump_action_list(fragment_spec, fragment_count);
  1892. +   dump_action_list(other_spec, other_count);
  1893. +   dump_action_list(move_spec, move_count);
  1894. +   dump_action_list(empty_spec, empty_count);
  1895. +}
  1896. +#else
  1897. +void dump_actions()
  1898. +{
  1899. +}
  1900. +#endif
  1901. +
  1902. +
  1903.  static struct test_entry test_table[] = {
  1904. -   { "name", 1, name_fn},
  1905. -   { "pathname", 1, pathname_fn, check_pathname},
  1906. -   { "subpathname", 1, subpathname_fn, check_pathname},
  1907. -   { "filesize", 1, filesize_fn, parse_number_arg},
  1908. -   { "dirsize", 1, dirsize_fn, parse_number_arg},
  1909. -   { "size", 1, size_fn, parse_number_arg},
  1910. -   { "inode", 1, inode_fn, parse_number_arg},
  1911. -   { "nlink", 1, nlink_fn, parse_number_arg},
  1912. -   { "fileblocks", 1, fileblocks_fn, parse_number_arg},
  1913. -   { "dirblocks", 1, dirblocks_fn, parse_number_arg},
  1914. -   { "blocks", 1, blocks_fn, parse_number_arg},
  1915. -   { "gid", 1, gid_fn, parse_number_arg},
  1916. -   { "uid", 1, uid_fn, parse_number_arg},
  1917. -   { "depth", 1, depth_fn, parse_number_arg},
  1918. -   { "filesize_range", 2, filesize_range_fn, parse_range_args},
  1919. -   { "dirsize_range", 2, dirsize_range_fn, parse_range_args},
  1920. -   { "size_range", 2, size_range_fn, parse_range_args},
  1921. -   { "inode_range", 2, inode_range_fn, parse_range_args},
  1922. -   { "nlink_range", 2, nlink_range_fn, parse_range_args},
  1923. -   { "fileblocks_range", 2, fileblocks_range_fn, parse_range_args},
  1924. -   { "dirblocks_range", 2, dirblocks_range_fn, parse_range_args},
  1925. -   { "blocks_range", 2, blocks_range_fn, parse_range_args},
  1926. -   { "gid_range", 2, gid_range_fn, parse_range_args},
  1927. -   { "uid_range", 2, uid_range_fn, parse_range_args},
  1928. -   { "depth_range", 2, depth_range_fn, parse_range_args},
  1929. -   { "type", 1, type_fn, parse_type_arg},
  1930. -   { "true", 0, true_fn, NULL},
  1931. -   { "false", 0, false_fn, NULL},
  1932. -   { "file", 1, file_fn, parse_file_arg},
  1933. -   { "exec", 1, exec_fn, NULL},
  1934. +   { "name", 1, name_fn, NULL, 1},
  1935. +   { "pathname", 1, pathname_fn, check_pathname, 1, 0},
  1936. +   { "subpathname", 1, subpathname_fn, check_pathname, 1, 0},
  1937. +   { "filesize", 1, filesize_fn, parse_number_arg, 1, 0},
  1938. +   { "dirsize", 1, dirsize_fn, parse_number_arg, 1, 0},
  1939. +   { "size", 1, size_fn, parse_number_arg, 1, 0},
  1940. +   { "inode", 1, inode_fn, parse_number_arg, 1, 0},
  1941. +   { "nlink", 1, nlink_fn, parse_number_arg, 1, 0},
  1942. +   { "fileblocks", 1, fileblocks_fn, parse_number_arg, 1, 0},
  1943. +   { "dirblocks", 1, dirblocks_fn, parse_number_arg, 1, 0},
  1944. +   { "blocks", 1, blocks_fn, parse_number_arg, 1, 0},
  1945. +   { "gid", 1, gid_fn, parse_gid_arg, 1, 0},
  1946. +   { "uid", 1, uid_fn, parse_uid_arg, 1, 0},
  1947. +   { "depth", 1, depth_fn, parse_number_arg, 1, 0},
  1948. +   { "dircount", 1, dircount_fn, parse_number_arg, 0, 0},
  1949. +   { "filesize_range", 2, filesize_range_fn, parse_range_args, 1, 0},
  1950. +   { "dirsize_range", 2, dirsize_range_fn, parse_range_args, 1, 0},
  1951. +   { "size_range", 2, size_range_fn, parse_range_args, 1, 0},
  1952. +   { "inode_range", 2, inode_range_fn, parse_range_args, 1, 0},
  1953. +   { "nlink_range", 2, nlink_range_fn, parse_range_args, 1, 0},
  1954. +   { "fileblocks_range", 2, fileblocks_range_fn, parse_range_args, 1, 0},
  1955. +   { "dirblocks_range", 2, dirblocks_range_fn, parse_range_args, 1, 0},
  1956. +   { "blocks_range", 2, blocks_range_fn, parse_range_args, 1, 0},
  1957. +   { "gid_range", 2, gid_range_fn, parse_range_args, 1, 0},
  1958. +   { "uid_range", 2, uid_range_fn, parse_range_args, 1, 0},
  1959. +   { "depth_range", 2, depth_range_fn, parse_range_args, 1, 0},
  1960. +   { "dircount_range", 2, dircount_range_fn, parse_range_args, 0, 0},
  1961. +   { "type", 1, type_fn, parse_type_arg, 1, 0},
  1962. +   { "true", 0, true_fn, NULL, 1, 0},
  1963. +   { "false", 0, false_fn, NULL, 1, 0},
  1964. +   { "file", 1, file_fn, parse_file_arg, 1, 0},
  1965. +   { "exec", 1, exec_fn, NULL, 1, 0},
  1966. +   { "exists", 0, exists_fn, NULL, 0, 0},
  1967. +   { "absolute", 0, absolute_fn, NULL, 0, 0},
  1968. +   { "stat", 1, stat_fn, parse_expr_arg0, 1, 1},
  1969. +   { "readlink", 1, readlink_fn, parse_expr_arg0, 0, 1},
  1970. +   { "eval", 2, eval_fn, parse_expr_arg1, 0, 1},
  1971. +   { "perm", -2, perm_fn, parse_perm_args, 1, 0},
  1972.     { "", -1 }
  1973.  };
  1974.  
  1975. @@ -2318,6 +3258,9 @@
  1976.     { "guid", GUID_ACTION, 2, ACTION_ALL_LNK, parse_guid_args, guid_action},
  1977.     { "mode", MODE_ACTION, -2, ACTION_ALL, parse_mode_args, mode_action },
  1978.     { "empty", EMPTY_ACTION, -2, ACTION_DIR, parse_empty_args, NULL},
  1979. -   { "move", MOVE_ACTION, -2, ACTION_ALL_LNK, NULL, NULL},
  1980. +   { "move", MOVE_ACTION, 1, ACTION_ALL_LNK, NULL, NULL},
  1981. +   { "prune", PRUNE_ACTION, 0, ACTION_ALL_LNK, NULL, NULL},
  1982. +   { "chmod", MODE_ACTION, -2, ACTION_ALL, parse_mode_args, mode_action },
  1983. +   { "noop", NOOP_ACTION, 0, ACTION_ALL, NULL, noop_action },
  1984.     { "", 0, -1, 0, NULL, NULL}
  1985.  };
  1986. diff -Nru squashfs-tools-4.2+20130409/action.h squashfs-tools-4.3+20140919/action.h
  1987. --- squashfs-tools-4.2+20130409/action.h    2013-05-09 10:39:11.000000000 +0200
  1988. +++ squashfs-tools-4.3+20140919/action.h    2015-07-20 21:03:05.000000000 +0200
  1989. @@ -1,8 +1,10 @@
  1990. +#ifndef ACTION_H
  1991. +#define ACTION_H
  1992.  /*
  1993.   * Create a squashfs filesystem.  This is a highly compressed read only
  1994.   * filesystem.
  1995.   *
  1996. - * Copyright (c) 2011, 2012, 2013
  1997. + * Copyright (c) 2011, 2012, 2013, 2014
  1998.   * Phillip Lougher <phillip@squashfs.org.uk>
  1999.   *
  2000.   * This program is free software; you can redistribute it and/or
  2001. @@ -72,6 +74,7 @@
  2002.     fprintf(stderr, "Failed to parse action \"%s\"\n", source); \
  2003.     fprintf(stderr, "Syntax error: "S, ##ARGS); \
  2004.     fprintf(stderr, "Got here \"%s\"\n", src); \
  2005. +   free(src); \
  2006.  }
  2007.  
  2008.  #define TEST_SYNTAX_ERROR(TEST, ARG, S, ARGS...) { \
  2009. @@ -81,6 +84,7 @@
  2010.     fprintf(stderr, "Syntax error in \"%s()\", arg %d: "S, TEST->name, \
  2011.              ARG, ##ARGS); \
  2012.     fprintf(stderr, "Got here \"%s\"\n", src); \
  2013. +   free(src); \
  2014.  }
  2015.  
  2016.  struct expr;
  2017. @@ -94,6 +98,7 @@
  2018.  
  2019.  struct atom {
  2020.     struct test_entry *test;
  2021. +   int args;
  2022.     char **argv;
  2023.     void *data;
  2024.  };
  2025. @@ -139,6 +144,8 @@
  2026.     int args;
  2027.     int (*fn)(struct atom *, struct action_data *);
  2028.     int (*parse_args)(struct test_entry *, struct atom *);
  2029. +   int exclude_ok;
  2030. +   int handle_logging;
  2031.  };
  2032.  
  2033.  
  2034. @@ -168,15 +175,28 @@
  2035.  #define MODE_ACTION 11
  2036.  #define EMPTY_ACTION 12
  2037.  #define MOVE_ACTION 13
  2038. +#define PRUNE_ACTION 14
  2039. +#define NOOP_ACTION 15
  2040.  
  2041.  /*
  2042.   * Define what file types each action operates over
  2043.   */
  2044. -#define ACTION_DIR S_IFDIR
  2045. -#define ACTION_REG S_IFREG
  2046. -#define ACTION_ALL_LNK (S_IFDIR | S_IFREG | S_IFBLK | S_IFCHR | S_IFSOCK | \
  2047. -           S_IFIFO | S_IFLNK)
  2048. -#define ACTION_ALL (S_IFDIR | S_IFREG | S_IFBLK | S_IFCHR | S_IFSOCK | S_IFIFO)
  2049. +#define ACTION_DIR 1
  2050. +#define ACTION_REG 2
  2051. +#define ACTION_ALL_LNK 3
  2052. +#define ACTION_ALL 4
  2053. +#define ACTION_LNK 5
  2054. +
  2055. +
  2056. +/*
  2057. + * Action logging requested, specified by the various
  2058. + * -action, -true-action, -false-action and -verbose-action
  2059. + * options
  2060. + */
  2061. +#define ACTION_LOG_NONE    0
  2062. +#define ACTION_LOG_TRUE 1
  2063. +#define ACTION_LOG_FALSE 2
  2064. +#define ACTION_LOG_VERBOSE ACTION_LOG_TRUE | ACTION_LOG_FALSE
  2065.  
  2066.  struct action_entry {
  2067.     char *name;
  2068. @@ -194,6 +214,8 @@
  2069.     char *pathname;
  2070.     char *subpath;
  2071.     struct stat *buf;
  2072. +   struct dir_ent *dir_ent;
  2073. +   struct dir_info *root;
  2074.  };
  2075.  
  2076.  
  2077. @@ -204,6 +226,7 @@
  2078.     char **argv;
  2079.     struct expr *expr;
  2080.     void *data;
  2081. +   int verbose;
  2082.  };
  2083.  
  2084.  
  2085. @@ -269,20 +292,37 @@
  2086.  
  2087.  
  2088.  /*
  2089. + * Perm test function specific definitions
  2090. + */
  2091. +#define PERM_ALL 1
  2092. +#define PERM_ANY 2
  2093. +#define PERM_EXACT 3
  2094. +
  2095. +struct perm_data {
  2096. +   int op;
  2097. +   int mode;
  2098. +};
  2099. +
  2100. +
  2101. +/*
  2102.   * External function definitions
  2103.   */
  2104. -extern int parse_action(char *);
  2105. +extern int parse_action(char *, int verbose);
  2106.  extern void dump_actions();
  2107. -extern void *eval_frag_actions(struct dir_ent *);
  2108. +extern void *eval_frag_actions(struct dir_info *, struct dir_ent *);
  2109.  extern void *get_frag_action(void *);
  2110. -extern int eval_exclude_actions(char *, char *, char *, struct stat *, int);
  2111. -extern void eval_actions(struct dir_ent *);
  2112. -extern int eval_empty_actions(struct dir_ent *dir_ent);
  2113. +extern int eval_exclude_actions(char *, char *, char *, struct stat *, int,
  2114. +                           struct dir_ent *);
  2115. +extern void eval_actions(struct dir_info *, struct dir_ent *);
  2116. +extern int eval_empty_actions(struct dir_info *, struct dir_ent *dir_ent);
  2117.  extern void eval_move_actions(struct dir_info *, struct dir_ent *);
  2118. +extern int eval_prune_actions(struct dir_info *, struct dir_ent *);
  2119.  extern void do_move_actions();
  2120.  extern int read_bytes(int, void *, int);
  2121.  extern int actions();
  2122.  extern int move_actions();
  2123.  extern int empty_actions();
  2124. -extern int read_action_file(char *);
  2125. +extern int read_action_file(char *, int);
  2126.  extern int exclude_actions();
  2127. +extern int prune_actions();
  2128. +#endif
  2129. diff -Nru squashfs-tools-4.2+20130409/caches-queues-lists.c squashfs-tools-4.3+20140919/caches-queues-lists.c
  2130. --- squashfs-tools-4.2+20130409/caches-queues-lists.c   2013-05-09 10:39:11.000000000 +0200
  2131. +++ squashfs-tools-4.3+20140919/caches-queues-lists.c   2015-07-20 21:03:05.000000000 +0200
  2132. @@ -2,7 +2,7 @@
  2133.   * Create a squashfs filesystem.  This is a highly compressed read only
  2134.   * filesystem.
  2135.   *
  2136. - * Copyright (c) 2013
  2137. + * Copyright (c) 2013, 2014
  2138.   * Phillip Lougher <phillip@squashfs.org.uk>
  2139.   *
  2140.   * This program is free software; you can redistribute it and/or
  2141. @@ -25,12 +25,11 @@
  2142.  #include <pthread.h>
  2143.  #include <stdlib.h>
  2144.  #include <string.h>
  2145. +#include <stdio.h>
  2146.  
  2147.  #include "error.h"
  2148.  #include "caches-queues-lists.h"
  2149.  
  2150. -int first_freelist = 1;
  2151. -
  2152.  extern int add_overflow(int, int);
  2153.  extern int multiply_overflow(int, int);
  2154.  
  2155. @@ -98,37 +97,182 @@
  2156.  }
  2157.  
  2158.  
  2159. -#define CALCULATE_HASH(start)  (start & 0xffff) \
  2160. +int queue_empty(struct queue *queue)
  2161. +{
  2162. +   int empty;
  2163.  
  2164. +   pthread_cleanup_push((void *) pthread_mutex_unlock, &queue->mutex);
  2165. +   pthread_mutex_lock(&queue->mutex);
  2166.  
  2167. -/* Called with the cache mutex held */
  2168. -void insert_hash_table(struct cache *cache, struct file_buffer *entry)
  2169. +   empty = queue->readp == queue->writep;
  2170. +
  2171. +   pthread_cleanup_pop(1);
  2172. +
  2173. +   return empty;
  2174. +}
  2175. +
  2176. +
  2177. +void queue_flush(struct queue *queue)
  2178. +{
  2179. +   pthread_cleanup_push((void *) pthread_mutex_unlock, &queue->mutex);
  2180. +   pthread_mutex_lock(&queue->mutex);
  2181. +
  2182. +   queue->readp = queue->writep;
  2183. +
  2184. +   pthread_cleanup_pop(1);
  2185. +}
  2186. +
  2187. +
  2188. +void dump_queue(struct queue *queue)
  2189.  {
  2190. -   int hash = CALCULATE_HASH(entry->index);
  2191. +   pthread_cleanup_push((void *) pthread_mutex_unlock, &queue->mutex);
  2192. +   pthread_mutex_lock(&queue->mutex);
  2193. +
  2194. +   printf("\tMax size %d, size %d%s\n", queue->size - 1,  
  2195. +       queue->readp <= queue->writep ? queue->writep - queue->readp :
  2196. +           queue->size - queue->readp + queue->writep,
  2197. +       queue->readp == queue->writep ? " (EMPTY)" :
  2198. +           ((queue->writep + 1) % queue->size) == queue->readp ?
  2199. +           " (FULL)" : "");
  2200.  
  2201. -   entry->hash_next = cache->hash_table[hash];
  2202. -   cache->hash_table[hash] = entry;
  2203. -   entry->hash_prev = NULL;
  2204. -   if(entry->hash_next)
  2205. -       entry->hash_next->hash_prev = entry;
  2206. +   pthread_cleanup_pop(1);
  2207.  }
  2208.  
  2209.  
  2210. +/* define seq queue hash tables */
  2211. +#define CALCULATE_SEQ_HASH(N) CALCULATE_HASH(N)
  2212. +
  2213. +/* Called with the seq queue mutex held */
  2214. +INSERT_HASH_TABLE(seq, struct seq_queue, CALCULATE_SEQ_HASH, sequence, seq)
  2215. +
  2216.  /* Called with the cache mutex held */
  2217. -void remove_hash_table(struct cache *cache, struct file_buffer *entry)
  2218. +REMOVE_HASH_TABLE(seq, struct seq_queue, CALCULATE_SEQ_HASH, sequence, seq);
  2219. +
  2220. +static unsigned int sequence = 0;
  2221. +
  2222. +
  2223. +struct seq_queue *seq_queue_init()
  2224. +{
  2225. +   struct seq_queue *queue = malloc(sizeof(struct seq_queue));
  2226. +   if(queue == NULL)
  2227. +       MEM_ERROR();
  2228. +
  2229. +   memset(queue, 0, sizeof(struct seq_queue));
  2230. +
  2231. +   pthread_mutex_init(&queue->mutex, NULL);
  2232. +   pthread_cond_init(&queue->wait, NULL);
  2233. +
  2234. +   return queue;
  2235. +}
  2236. +
  2237. +
  2238. +void seq_queue_put(struct seq_queue *queue, struct file_buffer *entry)
  2239.  {
  2240. -   if(entry->hash_prev)
  2241. -       entry->hash_prev->hash_next = entry->hash_next;
  2242. +   pthread_cleanup_push((void *) pthread_mutex_unlock, &queue->mutex);
  2243. +   pthread_mutex_lock(&queue->mutex);
  2244. +
  2245. +   insert_seq_hash_table(queue, entry);
  2246. +
  2247. +   if(entry->fragment)
  2248. +       queue->fragment_count ++;
  2249.     else
  2250. -       cache->hash_table[CALCULATE_HASH(entry->index)] =
  2251. -           entry->hash_next;
  2252. -   if(entry->hash_next)
  2253. -       entry->hash_next->hash_prev = entry->hash_prev;
  2254. +       queue->block_count ++;
  2255. +
  2256. +   if(entry->sequence == sequence)
  2257. +       pthread_cond_signal(&queue->wait);
  2258.  
  2259. -   entry->hash_prev = entry->hash_next = NULL;
  2260. +   pthread_cleanup_pop(1);
  2261.  }
  2262.  
  2263.  
  2264. +struct file_buffer *seq_queue_get(struct seq_queue *queue)
  2265. +{
  2266. +   /*
  2267. +    * Look-up buffer matching sequence in the queue, if found return
  2268. +    * it, otherwise wait until it arrives
  2269. +    */
  2270. +   int hash = CALCULATE_SEQ_HASH(sequence);
  2271. +   struct file_buffer *entry;
  2272. +
  2273. +   pthread_cleanup_push((void *) pthread_mutex_unlock, &queue->mutex);
  2274. +   pthread_mutex_lock(&queue->mutex);
  2275. +
  2276. +   while(1) {
  2277. +       for(entry = queue->hash_table[hash]; entry;
  2278. +                       entry = entry->seq_next)
  2279. +           if(entry->sequence == sequence)
  2280. +               break;
  2281. +
  2282. +       if(entry) {
  2283. +           /*
  2284. +            * found the buffer in the queue, decrement the
  2285. +            * appropriate count, and remove from hash list
  2286. +            */
  2287. +           if(entry->fragment)
  2288. +               queue->fragment_count --;
  2289. +           else
  2290. +               queue->block_count --;
  2291. +
  2292. +           remove_seq_hash_table(queue, entry);
  2293. +
  2294. +           sequence ++;
  2295. +
  2296. +           break;
  2297. +       }
  2298. +
  2299. +       /* entry not found, wait for it to arrive */   
  2300. +       pthread_cond_wait(&queue->wait, &queue->mutex);
  2301. +   }
  2302. +
  2303. +   pthread_cleanup_pop(1);
  2304. +
  2305. +   return entry;
  2306. +}
  2307. +
  2308. +
  2309. +void seq_queue_flush(struct seq_queue *queue)
  2310. +{
  2311. +   int i;
  2312. +
  2313. +   pthread_cleanup_push((void *) pthread_mutex_unlock, &queue->mutex);
  2314. +   pthread_mutex_lock(&queue->mutex);
  2315. +
  2316. +   for(i = 0; i < HASH_SIZE; i++)
  2317. +       queue->hash_table[i] = NULL;
  2318. +
  2319. +   queue->fragment_count = queue->block_count = 0;
  2320. +
  2321. +   pthread_cleanup_pop(1);
  2322. +}
  2323. +
  2324. +
  2325. +void dump_seq_queue(struct seq_queue *queue, int fragment_queue)
  2326. +{
  2327. +   int size;
  2328. +
  2329. +   pthread_cleanup_push((void *) pthread_mutex_unlock, &queue->mutex);
  2330. +   pthread_mutex_lock(&queue->mutex);
  2331. +
  2332. +   size = fragment_queue ? queue->fragment_count : queue->block_count;
  2333. +
  2334. +   printf("\tMax size unlimited, size %d%s\n", size,
  2335. +                       size == 0 ? " (EMPTY)" : "");
  2336. +
  2337. +   pthread_cleanup_pop(1);
  2338. +}
  2339. +
  2340. +
  2341. +/* define cache hash tables */
  2342. +#define CALCULATE_CACHE_HASH(N) CALCULATE_HASH(llabs(N))
  2343. +
  2344. +/* Called with the cache mutex held */
  2345. +INSERT_HASH_TABLE(cache, struct cache, CALCULATE_CACHE_HASH, index, hash)
  2346. +
  2347. +/* Called with the cache mutex held */
  2348. +REMOVE_HASH_TABLE(cache, struct cache, CALCULATE_CACHE_HASH, index, hash);
  2349. +
  2350. +/* define cache free list */
  2351. +
  2352.  /* Called with the cache mutex held */
  2353.  INSERT_LIST(free, struct file_buffer)
  2354.  
  2355. @@ -136,7 +280,8 @@
  2356.  REMOVE_LIST(free, struct file_buffer)
  2357.  
  2358.  
  2359. -struct cache *cache_init(int buffer_size, int max_buffers)
  2360. +struct cache *cache_init(int buffer_size, int max_buffers, int noshrink_lookup,
  2361. +   int first_freelist)
  2362.  {
  2363.     struct cache *cache = malloc(sizeof(struct cache));
  2364.  
  2365. @@ -146,10 +291,38 @@
  2366.     cache->max_buffers = max_buffers;
  2367.     cache->buffer_size = buffer_size;
  2368.     cache->count = 0;
  2369. +   cache->used = 0;
  2370.     cache->free_list = NULL;
  2371. +
  2372. +   /*
  2373. +    * The cache will grow up to max_buffers in size in response to
  2374. +    * an increase in readhead/number of buffers in flight.  But
  2375. +    * once the outstanding buffers gets returned, we can either elect
  2376. +    * to shrink the cache, or to put the freed blocks onto a free list.
  2377. +    *
  2378. +    * For the caches where we want to do lookup (fragment/writer),
  2379. +    * a don't shrink policy is best, for the reader cache it
  2380. +    * makes no sense to keep buffers around longer than necessary as
  2381. +    * we don't do any lookup on those blocks.
  2382. +    */
  2383. +   cache->noshrink_lookup = noshrink_lookup;
  2384. +
  2385. +   /*
  2386. +    * The default use freelist before growing cache policy behaves
  2387. +    * poorly with appending - with many duplicates the caches
  2388. +    * do not grow due to the fact that large queues of outstanding
  2389. +    * fragments/writer blocks do not occur, leading to small caches
  2390. +    * and un-uncessary performance loss to frequent cache
  2391. +    * replacement in the small caches.  Therefore with appending
  2392. +    * change the policy to grow the caches before reusing blocks
  2393. +    * from the freelist
  2394. +    */
  2395. +   cache->first_freelist = first_freelist;
  2396. +
  2397.     memset(cache->hash_table, 0, sizeof(struct file_buffer *) * 65536);
  2398.     pthread_mutex_init(&cache->mutex, NULL);
  2399.     pthread_cond_init(&cache->wait_for_free, NULL);
  2400. +   pthread_cond_init(&cache->wait_for_unlock, NULL);
  2401.  
  2402.     return cache;
  2403.  }
  2404. @@ -159,7 +332,7 @@
  2405.  {
  2406.     /* Lookup block in the cache, if found return with usage count
  2407.      * incremented, if not found return NULL */
  2408. -   int hash = CALCULATE_HASH(index);
  2409. +   int hash = CALCULATE_CACHE_HASH(index);
  2410.     struct file_buffer *entry;
  2411.  
  2412.     pthread_cleanup_push((void *) pthread_mutex_unlock, &cache->mutex);
  2413. @@ -173,8 +346,11 @@
  2414.         /* found the block in the cache, increment used count and
  2415.          * if necessary remove from free list so it won't disappear
  2416.          */
  2417. +       if(entry->used == 0) {
  2418. +           remove_free_list(&cache->free_list, entry);
  2419. +           cache->used ++;
  2420. +       }
  2421.         entry->used ++;
  2422. -       remove_free_list(&cache->free_list, entry);
  2423.     }
  2424.  
  2425.     pthread_cleanup_pop(1);
  2426. @@ -183,50 +359,76 @@
  2427.  }
  2428.  
  2429.  
  2430. -struct file_buffer *cache_get(struct cache *cache, long long index, int keep)
  2431. +static struct file_buffer *cache_freelist(struct cache *cache)
  2432.  {
  2433. -   /* Get a free block out of the cache indexed on index. */
  2434. -   struct file_buffer *entry;
  2435. +   struct file_buffer *entry = cache->free_list;
  2436. +
  2437. +   remove_free_list(&cache->free_list, entry);
  2438. +
  2439. +   /* a block on the free_list is hashed */
  2440. +   remove_cache_hash_table(cache, entry);
  2441.  
  2442. +   cache->used ++;
  2443. +   return entry;
  2444. +}
  2445. +
  2446. +
  2447. +static struct file_buffer *cache_alloc(struct cache *cache)
  2448. +{
  2449. +   struct file_buffer *entry = malloc(sizeof(struct file_buffer) +
  2450. +                           cache->buffer_size);
  2451. +   if(entry == NULL)
  2452. +           MEM_ERROR();
  2453. +
  2454. +   entry->cache = cache;
  2455. +   entry->free_prev = entry->free_next = NULL;
  2456. +   cache->count ++;
  2457. +   return entry;
  2458. +}
  2459. +
  2460. +
  2461. +static struct file_buffer *_cache_get(struct cache *cache, long long index,
  2462. +   int hash)
  2463. +{
  2464. +   /* Get a free block out of the cache indexed on index. */
  2465. +   struct file_buffer *entry = NULL;
  2466. +
  2467.     pthread_cleanup_push((void *) pthread_mutex_unlock, &cache->mutex);
  2468.     pthread_mutex_lock(&cache->mutex);
  2469.  
  2470.     while(1) {
  2471. -       /* first try to get a block from the free list */
  2472. -       if(first_freelist && cache->free_list) {
  2473. -           /* a block on the free_list is a "keep" block */
  2474. -           entry = cache->free_list;
  2475. -           remove_free_list(&cache->free_list, entry);
  2476. -           remove_hash_table(cache, entry);
  2477. -           break;
  2478. -       } else if(cache->count < cache->max_buffers) {
  2479. -           /* next try to allocate new block */
  2480. -           entry = malloc(sizeof(struct file_buffer) +
  2481. -               cache->buffer_size);
  2482. -           if(entry == NULL)
  2483. -               MEM_ERROR();
  2484. -           entry->cache = cache;
  2485. -           entry->free_prev = entry->free_next = NULL;
  2486. -           cache->count ++;
  2487. -           break;
  2488. -       } else if(!first_freelist && cache->free_list) {
  2489. -           /* a block on the free_list is a "keep" block */
  2490. -           entry = cache->free_list;
  2491. -           remove_free_list(&cache->free_list, entry);
  2492. -           remove_hash_table(cache, entry);
  2493. +       if(cache->noshrink_lookup) {   
  2494. +           /* first try to get a block from the free list */
  2495. +           if(cache->first_freelist && cache->free_list)
  2496. +               entry = cache_freelist(cache);
  2497. +           else if(cache->count < cache->max_buffers) {
  2498. +               entry = cache_alloc(cache);
  2499. +               cache->used ++;
  2500. +           } else if(!cache->first_freelist && cache->free_list)
  2501. +               entry = cache_freelist(cache);
  2502. +       } else { /* shrinking non-lookup cache */
  2503. +           if(cache->count < cache->max_buffers) {
  2504. +               entry = cache_alloc(cache);
  2505. +               if(cache->count > cache->max_count)
  2506. +                   cache->max_count = cache->count;
  2507. +           }
  2508. +       }
  2509. +
  2510. +       if(entry)
  2511.             break;
  2512. -       } else
  2513. -           /* wait for a block */
  2514. -           pthread_cond_wait(&cache->wait_for_free, &cache->mutex);
  2515. +
  2516. +       /* wait for a block */
  2517. +       pthread_cond_wait(&cache->wait_for_free, &cache->mutex);
  2518.     }
  2519.  
  2520. -   /* initialise block and if a keep block insert into the hash table */
  2521. +   /* initialise block and if hash is set insert into the hash table */
  2522.     entry->used = 1;
  2523. +   entry->locked = FALSE;
  2524. +   entry->wait_on_unlock = FALSE;
  2525.     entry->error = FALSE;
  2526. -   entry->keep = keep;
  2527. -   if(keep) {
  2528. +   if(hash) {
  2529.         entry->index = index;
  2530. -       insert_hash_table(cache, entry);
  2531. +       insert_cache_hash_table(cache, entry);
  2532.     }
  2533.  
  2534.     pthread_cleanup_pop(1);
  2535. @@ -235,17 +437,28 @@
  2536.  }
  2537.  
  2538.  
  2539. -void cache_rehash(struct file_buffer *entry, long long index)
  2540. +struct file_buffer *cache_get(struct cache *cache, long long index)
  2541. +{
  2542. +   return _cache_get(cache, index, 1);
  2543. +}
  2544. +
  2545. +
  2546. +struct file_buffer *cache_get_nohash(struct cache *cache)
  2547. +{
  2548. +   return _cache_get(cache, 0, 0);
  2549. +}
  2550. +
  2551. +
  2552. +void cache_hash(struct file_buffer *entry, long long index)
  2553.  {
  2554.     struct cache *cache = entry->cache;
  2555.  
  2556.     pthread_cleanup_push((void *) pthread_mutex_unlock, &cache->mutex);
  2557.     pthread_mutex_lock(&cache->mutex);
  2558. -   if(entry->keep)
  2559. -       remove_hash_table(cache, entry);
  2560. -   entry->keep = TRUE;
  2561. +
  2562.     entry->index = index;
  2563. -   insert_hash_table(cache, entry);
  2564. +   insert_cache_hash_table(cache, entry);
  2565. +
  2566.     pthread_cleanup_pop(1);
  2567.  }
  2568.  
  2569. @@ -254,10 +467,16 @@
  2570.  {
  2571.     struct cache *cache;
  2572.  
  2573. -   /* finished with this cache entry, once the usage count reaches zero it
  2574. -    * can be reused and if a keep block put onto the free list.  As keep
  2575. -    * blocks remain accessible via the hash table they can be found
  2576. -    * getting a new lease of life before they are reused. */
  2577. +   /*
  2578. +    * Finished with this cache entry, once the usage count reaches zero it
  2579. +    * can be reused.
  2580. +    *
  2581. +    * If noshrink_lookup is set, put the block onto the free list.
  2582. +    * As blocks remain accessible via the hash table they can be found
  2583. +    * getting a new lease of life before they are reused.
  2584. +    *
  2585. +    * if noshrink_lookup is not set then shrink the cache.
  2586. +    */
  2587.  
  2588.     if(entry == NULL)
  2589.         return;
  2590. @@ -269,9 +488,10 @@
  2591.  
  2592.     entry->used --;
  2593.     if(entry->used == 0) {
  2594. -       if(entry->keep)
  2595. +       if(cache->noshrink_lookup) {
  2596.             insert_free_list(&cache->free_list, entry);
  2597. -       else {
  2598. +           cache->used --;
  2599. +       } else {
  2600.             free(entry);
  2601.             cache->count --;
  2602.         }
  2603. @@ -281,4 +501,144 @@
  2604.     }
  2605.  
  2606.     pthread_cleanup_pop(1);
  2607. +}
  2608. +
  2609. +
  2610. +void dump_cache(struct cache *cache)
  2611. +{
  2612. +   pthread_cleanup_push((void *) pthread_mutex_unlock, &cache->mutex);
  2613. +   pthread_mutex_lock(&cache->mutex);
  2614. +
  2615. +   if(cache->noshrink_lookup)
  2616. +       printf("\tMax buffers %d, Current size %d, Used %d,  %s\n",
  2617. +           cache->max_buffers, cache->count, cache->used,
  2618. +           cache->free_list ?  "Free buffers" : "No free buffers");
  2619. +   else
  2620. +       printf("\tMax buffers %d, Current size %d, Maximum historical "
  2621. +           "size %d\n", cache->max_buffers, cache->count,
  2622. +           cache->max_count);
  2623. +
  2624. +   pthread_cleanup_pop(1);
  2625. +}
  2626. +
  2627. +
  2628. +struct file_buffer *cache_get_nowait(struct cache *cache, long long index)
  2629. +{
  2630. +   struct file_buffer *entry = NULL;
  2631. +   /*
  2632. +    * block doesn't exist, create it, but return it with the
  2633. +    * locked flag set, so nothing tries to use it while it doesn't
  2634. +    * contain data.
  2635. +    *
  2636. +    * If there's no space in the cache then return NULL.
  2637. +    */
  2638. +
  2639. +   pthread_cleanup_push((void *) pthread_mutex_unlock, &cache->mutex);
  2640. +   pthread_mutex_lock(&cache->mutex);
  2641. +
  2642. +   /* first try to get a block from the free list */
  2643. +   if(cache->first_freelist && cache->free_list)
  2644. +       entry = cache_freelist(cache);
  2645. +   else if(cache->count < cache->max_buffers) {
  2646. +       entry = cache_alloc(cache);
  2647. +       cache->used ++;
  2648. +   } else if(!cache->first_freelist && cache->free_list)
  2649. +       entry = cache_freelist(cache);
  2650. +
  2651. +   if(entry) {
  2652. +       /* initialise block and insert into the hash table */
  2653. +       entry->used = 1;
  2654. +       entry->locked = TRUE;
  2655. +       entry->wait_on_unlock = FALSE;
  2656. +       entry->error = FALSE;
  2657. +       entry->index = index;
  2658. +       insert_cache_hash_table(cache, entry);
  2659. +   }
  2660. +
  2661. +   pthread_cleanup_pop(1);
  2662. +
  2663. +   return entry;
  2664. +}
  2665. +
  2666. +
  2667. +struct file_buffer *cache_lookup_nowait(struct cache *cache, long long index,
  2668. +   char *locked)
  2669. +{
  2670. +   /*
  2671. +    * Lookup block in the cache, if found return it with the locked flag
  2672. +    * indicating whether it is currently locked.  In both cases increment
  2673. +    * the used count.
  2674. +    *
  2675. +    * If it doesn't exist in the cache return NULL;
  2676. +    */
  2677. +   int hash = CALCULATE_CACHE_HASH(index);
  2678. +   struct file_buffer *entry;
  2679. +
  2680. +   pthread_cleanup_push((void *) pthread_mutex_unlock, &cache->mutex);
  2681. +   pthread_mutex_lock(&cache->mutex);
  2682. +
  2683. +   /* first check if the entry already exists */
  2684. +   for(entry = cache->hash_table[hash]; entry; entry = entry->hash_next)
  2685. +       if(entry->index == index)
  2686. +           break;
  2687. +
  2688. +   if(entry) {
  2689. +       if(entry->used == 0) {
  2690. +           remove_free_list(&cache->free_list, entry);
  2691. +           cache->used ++;
  2692. +       }
  2693. +       entry->used ++;
  2694. +       *locked = entry->locked;
  2695. +   }
  2696. +
  2697. +   pthread_cleanup_pop(1);
  2698. +
  2699. +   return entry;
  2700. +}
  2701. +
  2702. +
  2703. +void cache_wait_unlock(struct file_buffer *buffer)
  2704. +{
  2705. +   struct cache *cache = buffer->cache;
  2706. +
  2707. +   pthread_cleanup_push((void *) pthread_mutex_unlock, &cache->mutex);
  2708. +   pthread_mutex_lock(&cache->mutex);
  2709. +
  2710. +   while(buffer->locked) {
  2711. +       /*
  2712. +        * another thread is filling this in, wait until it
  2713. +        * becomes unlocked.  Used has been incremented to ensure it
  2714. +        * doesn't get reused.  By definition a block can't be
  2715. +        * locked and unused, and so we don't need to worry
  2716. +        * about it being on the freelist now, but, it may
  2717. +        * become unused when unlocked unless used is
  2718. +        * incremented
  2719. +        */
  2720. +       buffer->wait_on_unlock = TRUE;
  2721. +       pthread_cond_wait(&cache->wait_for_unlock, &cache->mutex);
  2722. +   }
  2723. +
  2724. +   pthread_cleanup_pop(1);
  2725. +}
  2726. +
  2727. +
  2728. +void cache_unlock(struct file_buffer *entry)
  2729. +{
  2730. +   struct cache *cache = entry->cache;
  2731. +
  2732. +   /*
  2733. +    * Unlock this locked cache entry.  If anything is waiting for this
  2734. +    * to become unlocked, wake it up.
  2735. +    */
  2736. +   pthread_cleanup_push((void *) pthread_mutex_unlock, &cache->mutex);
  2737. +   pthread_mutex_lock(&cache->mutex);
  2738. +
  2739. +   entry->locked = FALSE;
  2740. +
  2741. +   if(entry->wait_on_unlock) {
  2742. +       entry->wait_on_unlock = FALSE;
  2743. +       pthread_cond_broadcast(&cache->wait_for_unlock);
  2744. +   }
  2745. +
  2746. +   pthread_cleanup_pop(1);
  2747.  }
  2748. diff -Nru squashfs-tools-4.2+20130409/caches-queues-lists.h squashfs-tools-4.3+20140919/caches-queues-lists.h
  2749. --- squashfs-tools-4.2+20130409/caches-queues-lists.h   2013-05-09 10:39:11.000000000 +0200
  2750. +++ squashfs-tools-4.3+20140919/caches-queues-lists.h   2015-07-20 21:03:05.000000000 +0200
  2751. @@ -1,8 +1,10 @@
  2752. +#ifndef CACHES_QUEUES_LISTS_H
  2753. +#define CACHES_QUEUES_LISTS_H
  2754.  /*
  2755.   * Create a squashfs filesystem.  This is a highly compressed read only
  2756.   * filesystem.
  2757.   *
  2758. - * Copyright (c) 2013
  2759. + * Copyright (c) 2013, 2014
  2760.   * Phillip Lougher <phillip@squashfs.org.uk>
  2761.   *
  2762.   * This program is free software; you can redistribute it and/or
  2763. @@ -22,24 +24,104 @@
  2764.   * caches-queues-lists.h
  2765.   */
  2766.  
  2767. +#define INSERT_LIST(NAME, TYPE) \
  2768. +void insert_##NAME##_list(TYPE **list, TYPE *entry) { \
  2769. +   if(*list) { \
  2770. +       entry->NAME##_next = *list; \
  2771. +       entry->NAME##_prev = (*list)->NAME##_prev; \
  2772. +       (*list)->NAME##_prev->NAME##_next = entry; \
  2773. +       (*list)->NAME##_prev = entry; \
  2774. +   } else { \
  2775. +       *list = entry; \
  2776. +       entry->NAME##_prev = entry->NAME##_next = entry; \
  2777. +   } \
  2778. +}
  2779. +
  2780. +
  2781. +#define REMOVE_LIST(NAME, TYPE) \
  2782. +void remove_##NAME##_list(TYPE **list, TYPE *entry) { \
  2783. +   if(entry->NAME##_prev == entry && entry->NAME##_next == entry) { \
  2784. +       /* only this entry in the list */ \
  2785. +       *list = NULL; \
  2786. +   } else if(entry->NAME##_prev != NULL && entry->NAME##_next != NULL) { \
  2787. +       /* more than one entry in the list */ \
  2788. +       entry->NAME##_next->NAME##_prev = entry->NAME##_prev; \
  2789. +       entry->NAME##_prev->NAME##_next = entry->NAME##_next; \
  2790. +       if(*list == entry) \
  2791. +           *list = entry->NAME##_next; \
  2792. +   } \
  2793. +   entry->NAME##_prev = entry->NAME##_next = NULL; \
  2794. +}
  2795. +
  2796. +
  2797. +#define INSERT_HASH_TABLE(NAME, TYPE, HASH_FUNCTION, FIELD, LINK) \
  2798. +void insert_##NAME##_hash_table(TYPE *container, struct file_buffer *entry) \
  2799. +{ \
  2800. +   int hash = HASH_FUNCTION(entry->FIELD); \
  2801. +\
  2802. +   entry->LINK##_next = container->hash_table[hash]; \
  2803. +   container->hash_table[hash] = entry; \
  2804. +   entry->LINK##_prev = NULL; \
  2805. +   if(entry->LINK##_next) \
  2806. +       entry->LINK##_next->LINK##_prev = entry; \
  2807. +}
  2808. +
  2809. +
  2810. +#define REMOVE_HASH_TABLE(NAME, TYPE, HASH_FUNCTION, FIELD, LINK) \
  2811. +void remove_##NAME##_hash_table(TYPE *container, struct file_buffer *entry) \
  2812. +{ \
  2813. +   if(entry->LINK##_prev) \
  2814. +       entry->LINK##_prev->LINK##_next = entry->LINK##_next; \
  2815. +   else \
  2816. +       container->hash_table[HASH_FUNCTION(entry->FIELD)] = \
  2817. +           entry->LINK##_next; \
  2818. +   if(entry->LINK##_next) \
  2819. +       entry->LINK##_next->LINK##_prev = entry->LINK##_prev; \
  2820. +\
  2821. +   entry->LINK##_prev = entry->LINK##_next = NULL; \
  2822. +}
  2823. +
  2824. +#define HASH_SIZE 65536
  2825. +#define CALCULATE_HASH(n) ((n) & 0xffff)
  2826. +
  2827. +
  2828.  /* struct describing a cache entry passed between threads */
  2829.  struct file_buffer {
  2830. -   struct cache *cache;
  2831. -   struct file_buffer *hash_next;
  2832. -   struct file_buffer *hash_prev;
  2833. -   struct file_buffer *free_next;
  2834. -   struct file_buffer *free_prev;
  2835. -   struct file_buffer *next;
  2836. +   union {
  2837. +       long long index;
  2838. +       long long sequence;
  2839. +   };
  2840.     long long file_size;
  2841. -   long long index;
  2842. -   long long block;
  2843. -   long long sequence;
  2844. +   union {
  2845. +       long long block;
  2846. +       unsigned short checksum;
  2847. +   };
  2848. +   struct cache *cache;
  2849. +   union {
  2850. +       struct file_info *dupl_start;
  2851. +       struct file_buffer *hash_next;
  2852. +   };
  2853. +   union {
  2854. +       int duplicate;
  2855. +       struct file_buffer *hash_prev;
  2856. +   };
  2857. +   union {
  2858. +       struct {
  2859. +           struct file_buffer *free_next;
  2860. +           struct file_buffer *free_prev;
  2861. +       };
  2862. +       struct {
  2863. +           struct file_buffer *seq_next;
  2864. +           struct file_buffer *seq_prev;
  2865. +       };
  2866. +   };
  2867.     int size;
  2868.     int c_byte;
  2869. -   char keep;
  2870.     char used;
  2871.     char fragment;
  2872.     char error;
  2873. +   char locked;
  2874. +   char wait_on_unlock;
  2875.     char noD;
  2876.     char data[0];
  2877.  };
  2878. @@ -57,59 +139,62 @@
  2879.  };
  2880.  
  2881.  
  2882. +/*
  2883. + * struct describing seq_queues used to pass data between the read
  2884. + * thread and the deflate and main threads
  2885. + */
  2886. +struct seq_queue {
  2887. +   int         fragment_count;
  2888. +   int         block_count;
  2889. +   struct file_buffer  *hash_table[HASH_SIZE];
  2890. +   pthread_mutex_t     mutex;
  2891. +   pthread_cond_t      wait;
  2892. +};
  2893. +
  2894. +
  2895.  /* Cache status struct.  Caches are used to keep
  2896.    track of memory buffers passed between different threads */
  2897.  struct cache {
  2898.     int max_buffers;
  2899.     int count;
  2900.     int buffer_size;
  2901. +   int noshrink_lookup;
  2902. +   int first_freelist;
  2903. +   union {
  2904. +       int used;
  2905. +       int max_count;
  2906. +   };
  2907.     pthread_mutex_t mutex;
  2908.     pthread_cond_t wait_for_free;
  2909. +   pthread_cond_t wait_for_unlock;
  2910.     struct file_buffer *free_list;
  2911. -   struct file_buffer *hash_table[65536];
  2912. +   struct file_buffer *hash_table[HASH_SIZE];
  2913.  };
  2914.  
  2915.  
  2916. -#define INSERT_LIST(NAME, TYPE) \
  2917. -void insert_##NAME##_list(TYPE **list, TYPE *entry) { \
  2918. -   if(*list) { \
  2919. -       entry->NAME##_next = *list; \
  2920. -       entry->NAME##_prev = (*list)->NAME##_prev; \
  2921. -       (*list)->NAME##_prev->NAME##_next = entry; \
  2922. -       (*list)->NAME##_prev = entry; \
  2923. -   } else { \
  2924. -       *list = entry; \
  2925. -       entry->NAME##_prev = entry->NAME##_next = entry; \
  2926. -   } \
  2927. -}
  2928. -
  2929. -
  2930. -#define REMOVE_LIST(NAME, TYPE) \
  2931. -void remove_##NAME##_list(TYPE **list, TYPE *entry) { \
  2932. -   if(entry->NAME##_prev == entry && entry->NAME##_next == entry) { \
  2933. -       /* only this entry in the list */ \
  2934. -       *list = NULL; \
  2935. -   } else if(entry->NAME##_prev != NULL && entry->NAME##_next != NULL) { \
  2936. -       /* more than one entry in the list */ \
  2937. -       entry->NAME##_next->NAME##_prev = entry->NAME##_prev; \
  2938. -       entry->NAME##_prev->NAME##_next = entry->NAME##_next; \
  2939. -       if(*list == entry) \
  2940. -           *list = entry->NAME##_next; \
  2941. -   } \
  2942. -   entry->NAME##_prev = entry->NAME##_next = NULL; \
  2943. -}
  2944. -
  2945.  extern struct queue *queue_init(int);
  2946.  extern void queue_put(struct queue *, void *);
  2947.  extern void *queue_get(struct queue *);
  2948. -extern struct cache *cache_init(int, int);
  2949. +extern int queue_empty(struct queue *);
  2950. +extern void queue_flush(struct queue *);
  2951. +extern void dump_queue(struct queue *);
  2952. +extern struct seq_queue *seq_queue_init();
  2953. +extern void seq_queue_put(struct seq_queue *, struct file_buffer *);
  2954. +extern void dump_seq_queue(struct seq_queue *, int);
  2955. +extern struct file_buffer *seq_queue_get(struct seq_queue *);
  2956. +extern void seq_queue_flush(struct seq_queue *);
  2957. +extern struct cache *cache_init(int, int, int, int);
  2958.  extern struct file_buffer *cache_lookup(struct cache *, long long);
  2959. -extern struct file_buffer *cache_get(struct cache *, long long, int);
  2960. -extern void cache_rehash(struct file_buffer *, long long);
  2961. +extern struct file_buffer *cache_get(struct cache *, long long);
  2962. +extern struct file_buffer *cache_get_nohash(struct cache *);
  2963. +extern void cache_hash(struct file_buffer *, long long);
  2964.  extern void cache_block_put(struct file_buffer *);
  2965. -
  2966. -//extern void insert_free_list(struct file_buffer **, struct file_buffer *);
  2967. -//extern void remove_free_list(struct file_buffer **, struct file_buffer *);
  2968. +extern void dump_cache(struct cache *);
  2969. +extern struct file_buffer *cache_get_nowait(struct cache *, long long);
  2970. +extern struct file_buffer *cache_lookup_nowait(struct cache *, long long,
  2971. +   char *);
  2972. +extern void cache_wait_unlock(struct file_buffer *);
  2973. +extern void cache_unlock(struct file_buffer *);
  2974.  
  2975.  extern int first_freelist;
  2976. -
  2977. +#endif
  2978. diff -Nru squashfs-tools-4.2+20130409/compressor.c squashfs-tools-4.3+20140919/compressor.c
  2979. --- squashfs-tools-4.2+20130409/compressor.c    2013-05-09 10:39:11.000000000 +0200
  2980. +++ squashfs-tools-4.3+20140919/compressor.c    2015-07-20 21:03:05.000000000 +0200
  2981. @@ -49,6 +49,14 @@
  2982.  extern struct compressor lzo_comp_ops;
  2983.  #endif
  2984.  
  2985. +#ifndef LZ4_SUPPORT
  2986. +static struct compressor lz4_comp_ops = {
  2987. +   LZ4_COMPRESSION, "lz4"
  2988. +};
  2989. +#else
  2990. +extern struct compressor lz4_comp_ops;
  2991. +#endif
  2992. +
  2993.  #ifndef XZ_SUPPORT
  2994.  static struct compressor xz_comp_ops = {
  2995.     XZ_COMPRESSION, "xz"
  2996. @@ -67,6 +75,7 @@
  2997.     &gzip_comp_ops,
  2998.     &lzma_comp_ops,
  2999.     &lzo_comp_ops,
  3000. +   &lz4_comp_ops,
  3001.     &xz_comp_ops,
  3002.     &unknown_comp_ops
  3003.  };
  3004. diff -Nru squashfs-tools-4.2+20130409/compressor.h squashfs-tools-4.3+20140919/compressor.h
  3005. --- squashfs-tools-4.2+20130409/compressor.h    2013-05-09 10:39:11.000000000 +0200
  3006. +++ squashfs-tools-4.3+20140919/compressor.h    2015-07-20 21:03:05.000000000 +0200
  3007. @@ -1,6 +1,8 @@
  3008. +#ifndef COMPRESSOR_H
  3009. +#define COMPRESSOR_H
  3010.  /*
  3011.   *
  3012. - * Copyright (c) 2009, 2010, 2011, 2012, 2013
  3013. + * Copyright (c) 2009, 2010, 2011, 2012, 2013, 2014
  3014.   * Phillip Lougher <phillip@squashfs.org.uk>
  3015.   *
  3016.   * This program is free software; you can redistribute it and/or
  3017. @@ -31,6 +33,7 @@
  3018.     int (*options_post)(int);
  3019.     void *(*dump_options)(int, int *);
  3020.     int (*extract_options)(int, void *, int);
  3021. +   int (*check_options)(int, void *, int);
  3022.     void (*display_options)(void *, int);
  3023.     void (*usage)();
  3024.  };
  3025. @@ -64,9 +67,8 @@
  3026.  
  3027.  
  3028.  /*
  3029. - * For the following functions, please see xz_wrapper.c for
  3030. - * commented examples of how they are used (xz is currently
  3031. - * the only compressor that uses compression options).
  3032. + * For the following functions please see the lzo, lz4 or xz
  3033. + * compressors for commented examples of how they are used.
  3034.   */
  3035.  static inline int compressor_options(struct compressor *comp, char *argv[],
  3036.     int argc)
  3037. @@ -104,9 +106,19 @@
  3038.  }
  3039.  
  3040.  
  3041. +static inline int compressor_check_options(struct compressor *comp,
  3042. +   int block_size, void *buffer, int size)
  3043. +{
  3044. +   if(comp->check_options == NULL)
  3045. +       return 0;
  3046. +   return comp->check_options(block_size, buffer, size);
  3047. +}
  3048. +
  3049. +
  3050.  static inline void compressor_display_options(struct compressor *comp,
  3051.     void *buffer, int size)
  3052.  {
  3053.     if(comp->display_options != NULL)
  3054.         comp->display_options(buffer, size);
  3055.  }
  3056. +#endif
  3057. diff -Nru squashfs-tools-4.2+20130409/debian/changelog squashfs-tools-4.3+20140919/debian/changelog
  3058. --- squashfs-tools-4.2+20130409/debian/changelog    2013-09-18 10:22:27.000000000 +0200
  3059. +++ squashfs-tools-4.3+20140919/debian/changelog    2015-07-20 22:44:05.000000000 +0200
  3060. @@ -1,3 +1,11 @@
  3061. +squashfs-tools (1:4.3+20140919-0) unstable; urgency=low
  3062. +
  3063. +  * Merging upstream version 4.3+20140919.
  3064. +  * Fix for CVE 2015-4645/4646
  3065. +  * Updates man pages
  3066. +
  3067. + -- Romeo Papa <romeopapa@caramail.com>  Mon, 20 Jul 2015 21:28:12 +0200
  3068. +
  3069.  squashfs-tools (1:4.2+20130409-2) unstable; urgency=low
  3070.  
  3071.    * New maintainer (closes: #723600).
  3072. diff -Nru squashfs-tools-4.2+20130409/debian/manpages/mksquashfs.1 squashfs-tools-4.3+20140919/debian/manpages/mksquashfs.1
  3073. --- squashfs-tools-4.2+20130409/debian/manpages/mksquashfs.1    2013-05-09 22:22:49.000000000 +0200
  3074. +++ squashfs-tools-4.3+20140919/debian/manpages/mksquashfs.1    2015-07-20 22:44:05.000000000 +0200
  3075. @@ -1,4 +1,4 @@
  3076. -.TH MKSQUASHFS 1 "2012\-06\-30" "4.2" "create and append squashfs filesystems"
  3077. +.TH MKSQUASHFS 1 "2014\-05\-13" "4.3" "create and append squashfs filesystems"
  3078.  
  3079.  .SH NAME
  3080.  mksquashfs \- tool to create and append to squashfs filesystems
  3081. @@ -15,9 +15,9 @@
  3082.  
  3083.  .SS Filesystem build options
  3084.  .IP "\-comp \fICOMPRESSION\fR" 4
  3085. -select \fICOMPRESSION\fR compression. Compressors available: gzip (default), lzo, xz.
  3086. +select \fICOMPRESSION\fR compression. Compressors available: gzip (default), lzma (no kernel support), lzo, lz4 and xz.
  3087.  .IP "\-b \fIBLOCK_SIZE\fR"
  3088. -set data block to \fIBLOCK_SIZE\fR. Default 131072 bytes.
  3089. +set data block to \fIBLOCK_SIZE\fR. Default 131072 bytes. Optionally K or M can be used as a suffix to specify kilobytes or megabytes, respectively.
  3090.  .IP "\-no\-exports" 4
  3091.  don't make the filesystem exportable via NFS.
  3092.  .IP "\-no\-sparse" 4
  3093. @@ -74,6 +74,8 @@
  3094.  .SS Mksquashfs runtime options:
  3095.  .IP "\-version" 4
  3096.  print version, licence and copyright message.
  3097. +.IP "\-exit\-on\-error" 4
  3098. +treat normally ignored errors as fatal.
  3099.  .IP "\-recover \fINAME\fR" 4
  3100.  recover filesystem data using recovery file \fINAME\fR.
  3101.  .IP "\-no\-recovery" 4
  3102. @@ -82,14 +84,18 @@
  3103.  print files written to filesystem.
  3104.  .IP "\-no\-progress" 4
  3105.  don't display the progress bar.
  3106. +.IP "\-progress" 4
  3107. +display progress bar when using the \-info option.
  3108.  .IP "\-processors \fINUMBER\fR" 4
  3109.  Use \fINUMBER\fR processors. By default will use number of processors available.
  3110. +.IP "\-mem \fISIZE\fR" 4
  3111. +Use \fISIZE\fR physical memory. Optionally K or M can be used as a suffix for kilobytes or megabytes, respectively. Default 25% of memory.
  3112.  .IP "\-read\-queue \fISIZE\fR" 4
  3113. -Set input queue to \fISIZE\fR Mbytes. Default 64 Mbytes.
  3114. +Deprecated. Use \-mem instead.
  3115.  .IP "\-write\-queue \fISIZE\fR" 4
  3116. -Set output queue to \fISIZE\fR Mbytes. Default 512 Mbytes.
  3117. +Deprecated. Use \-mem instead.
  3118.  .IP "\-fragment\-queue \fISIZE\fR" 4
  3119. -Set fragment queue to \fISIZE\fR Mbytes. Default 64 Mbytes.
  3120. +Deprecated. Use \-mem instead.
  3121.  
  3122.  .SS Miscellaneous options
  3123.  .IP "\-root\-owned" 4
  3124. @@ -102,11 +108,27 @@
  3125.  alternative name for \-noF.
  3126.  .IP "\-noXattrCompression" 4
  3127.  alternative name for \-noX.
  3128. +.IP "\-Xhelp" 4
  3129. +print compressor options for selected compressor
  3130.  
  3131.  .SS Compressors available and compressor specific options
  3132. -.IP "gzip (no options) (default)"
  3133. -.IP "lzo (no options)"
  3134. -.IP "xz"
  3135. +.IP "gzip (default)"
  3136. +.IP "\-Xcompression-level \fIcompression\-level\fR" 4
  3137. +\fIcompression\-level\fR should be 1 .. 9 (default 9)
  3138. +.IP "\-Xwindow\-size \fIwindow\-size\fR" 4
  3139. +\fIwindow\-size\fR should be 8 .. 15 (default 15)
  3140. +.IP "\-Xstrategy strategy1,strategy2,...,strategyN" 4
  3141. +Compress using strategy1,strategy2,...,strategyN in turn and choose the best compression. Available strategies: default, filtered, huffman_only, run_length_encoded and fixed
  3142. +.IP "lzmz (no options) (no kernel support)" 4
  3143. +.IP "lzo" 4
  3144. +.IP "\-Xalgorithm \fIalgorithm\fR" 4
  3145. +Where \fIalgorithm\fR is one of: lzo1x_1, lzo1x_1_11, lzo1x_1_12, lzo1x_1_15 or lzo1x_999. (default lzo1x_999)
  3146. +.IP "\-Xcompression\-level \fIcompression\-level\fR" 4
  3147. +\fIcompression\-level\fR should be 1 .. 9 (default 8)
  3148. +.IP "lz4" 4
  3149. +.IP "\-Xhc"
  3150. +Compress using LZ4 High Compression
  3151. +.IP "xz" 4
  3152.  .IP "\-Xbcj filter1,filter2,...,filterN" 4
  3153.  Compress using filter1,filter2,...,filterN in turn (in addition to no filter), and choose the best compression. Available filters: x86, arm, armthumb, powerpc, sparc, ia64.
  3154.  .IP "\-Xdict\-size \fIDICT_SIZE\fR" 4
  3155. @@ -119,6 +141,6 @@
  3156.  More information about mksquashfs and the squashfs filesystem can be found at <\fIhttp://squashfs.sourceforge.net/\fR>.
  3157.  
  3158.  .SH AUTHOR
  3159. -squashfs was written by Phillip Lougher <\fIphillip@squashfs.org.uk\fR>.
  3160. +squashfs was written by Phillip Lougher <\fIplougher@users.sourceforge.net\fR>.
  3161.  .PP
  3162. -This manual page was written by Daniel Baumann <\fImail@daniel\-baumann.ch\fR>.
  3163. +This manual page was written by Daniel Baumann <\fIdaniel.baumann@progress\-technologies.net\fR>. With some updates for 4.3 for use with Fedora and Debian.
  3164. diff -Nru squashfs-tools-4.2+20130409/debian/manpages/unsquashfs.1 squashfs-tools-4.3+20140919/debian/manpages/unsquashfs.1
  3165. --- squashfs-tools-4.2+20130409/debian/manpages/unsquashfs.1    2013-05-09 22:22:49.000000000 +0200
  3166. +++ squashfs-tools-4.3+20140919/debian/manpages/unsquashfs.1    2015-07-20 22:44:05.000000000 +0200
  3167. @@ -1,4 +1,4 @@
  3168. -.TH UNSQUASHFS 1 "2012\-06\-30" "4.2" "uncompress squashfs filesystems"
  3169. +.TH UNSQUASHFS 1 "2014\-05\-13" "4.3" "uncompress squashfs filesystems"
  3170.  
  3171.  .SH NAME
  3172.  mksquashfs \- tool to uncompress squashfs filesystems
  3173. @@ -22,6 +22,8 @@
  3174.  don't extract xattrs in file system.
  3175.  .IP "\-x, \-xattrs" 4
  3176.  extract xattrs in file system (default).
  3177. +.IP "\-u, \-user\-xattrs" 4
  3178. +only extract user xattrs in file system. Enables extracting xattrs.
  3179.  .IP "\-p \fINUMBER\fR, \-processors \fINUMBER\fR" 4
  3180.  use \fINUMBER\fR processors. By default will use number of processors available.
  3181.  .IP "\-i, \-info" 4
  3182. @@ -38,7 +40,7 @@
  3183.  display filesystem superblock information.
  3184.  .IP "\-e \fIEXTRACT_FILE\fR, \-ef \fIEXTRACT_FILE\fR" 4
  3185.  list of directories or files to extract. One per line.
  3186. -.IP "\-da \fISIZE\fR, \-data-queue \fISIZE\fR" 4
  3187. +.IP "\-da \fISIZE\fR, \-data\-queue \fISIZE\fR" 4
  3188.  Set data queue to \fISIZE\fR Mbytes. Default 256 Mbytes.
  3189.  .IP "\-fr \fISIZE\fR, \-frag\-queue \fISIZE\fR" 4
  3190.  Set fragment queue to \fISIZE\fR Mbytes. Default 256 Mbytes.
  3191. @@ -47,7 +49,9 @@
  3192.  
  3193.  .SS Decompressors available
  3194.  .IP "gzip" 4
  3195. +.IP "lzma" 4
  3196.  .IP "lzo" 4
  3197. +.IP "lz4" 4
  3198.  .IP "xz" 4
  3199.  
  3200.  .SH SEE ALSO
  3201. @@ -57,6 +61,6 @@
  3202.  More information about unsquashfs and the squashfs filesystem can be found at <\fIhttp://squashfs.sourceforge.net/\fR>.
  3203.  
  3204.  .SH AUTHOR
  3205. -squashfs was written by Phillip Lougher <\fIphillip@squashfs.org.uk\fR>.
  3206. +squashfs was written by Phillip Lougher <\fIplougher@users.sourceforge.net\fR>.
  3207.  .PP
  3208. -This manual page was written by Daniel Baumann <\fImail@daniel\-baumann.ch\fR>.
  3209. +This manual page was written by Daniel Baumann <\fIdaniel.baumann@progress\-technologies.net\fR>. With some updates for 4.3 for use with Fedora and Debian.
  3210. diff -Nru squashfs-tools-4.2+20130409/debian/patches/0001-kfreebsd.patch squashfs-tools-4.3+20140919/debian/patches/0001-kfreebsd.patch
  3211. --- squashfs-tools-4.2+20130409/debian/patches/0001-kfreebsd.patch  2013-05-09 22:25:25.000000000 +0200
  3212. +++ squashfs-tools-4.3+20140919/debian/patches/0001-kfreebsd.patch  1970-01-01 01:00:00.000000000 +0100
  3213. @@ -1,103 +0,0 @@
  3214. -Author: Cyril Brulebois <kibi@debian.org>
  3215. -Description: Fixes FTBFS on kfreebsd (Closes: #557174).
  3216. -
  3217. -Index: squashfs-tools/mksquashfs.c
  3218. -===================================================================
  3219. ---- squashfs-tools.orig/mksquashfs.c   2013-05-09 20:25:19.000000000 +0000
  3220. -+++ squashfs-tools/mksquashfs.c    2013-05-09 20:25:19.000000000 +0000
  3221. -@@ -51,7 +51,7 @@
  3222. - #include <limits.h>
  3223. - #include <ctype.h>
  3224. -
  3225. --#ifndef linux
  3226. -+#if !defined(linux) && !defined(__GLIBC__)
  3227. - #define __BYTE_ORDER BYTE_ORDER
  3228. - #define __BIG_ENDIAN BIG_ENDIAN
  3229. - #define __LITTLE_ENDIAN LITTLE_ENDIAN
  3230. -@@ -4038,7 +4038,7 @@
  3231. -       BAD_ERROR("Failed to set signal mask in intialise_threads\n");
  3232. -
  3233. -   if(processors == -1) {
  3234. --#ifndef linux
  3235. -+#if !defined(linux) && !defined(__GLIBC__)
  3236. -       int mib[2];
  3237. -       size_t len = sizeof(processors);
  3238. -
  3239. -Index: squashfs-tools/read_fs.c
  3240. -===================================================================
  3241. ---- squashfs-tools.orig/read_fs.c  2013-05-09 20:25:19.000000000 +0000
  3242. -+++ squashfs-tools/read_fs.c   2013-05-09 20:25:19.000000000 +0000
  3243. -@@ -34,7 +34,7 @@
  3244. - #include <sys/mman.h>
  3245. - #include <limits.h>
  3246. -
  3247. --#ifndef linux
  3248. -+#if !defined(linux) && !defined(__GLIBC__)
  3249. - #define __BYTE_ORDER BYTE_ORDER
  3250. - #define __BIG_ENDIAN BIG_ENDIAN
  3251. - #define __LITTLE_ENDIAN LITTLE_ENDIAN
  3252. -Index: squashfs-tools/read_xattrs.c
  3253. -===================================================================
  3254. ---- squashfs-tools.orig/read_xattrs.c  2013-05-09 20:25:19.000000000 +0000
  3255. -+++ squashfs-tools/read_xattrs.c   2013-05-09 20:25:19.000000000 +0000
  3256. -@@ -31,7 +31,7 @@
  3257. - #include <stdio.h>
  3258. - #include <string.h>
  3259. -
  3260. --#ifndef linux
  3261. -+#if !defined(linux) && !defined(__GLIBC__)
  3262. - #define __BYTE_ORDER BYTE_ORDER
  3263. - #define __BIG_ENDIAN BIG_ENDIAN
  3264. - #define __LITTLE_ENDIAN LITTLE_ENDIAN
  3265. -Index: squashfs-tools/swap.c
  3266. -===================================================================
  3267. ---- squashfs-tools.orig/swap.c 2013-05-09 20:25:19.000000000 +0000
  3268. -+++ squashfs-tools/swap.c  2013-05-09 20:25:19.000000000 +0000
  3269. -@@ -19,7 +19,7 @@
  3270. -  * swap.c
  3271. -  */
  3272. -
  3273. --#ifndef linux
  3274. -+#if !defined(linux) && !defined(__GLIBC__)
  3275. - #define __BYTE_ORDER BYTE_ORDER
  3276. - #define __BIG_ENDIAN BIG_ENDIAN
  3277. - #define __LITTLE_ENDIAN LITTLE_ENDIAN
  3278. -Index: squashfs-tools/unsquashfs.c
  3279. -===================================================================
  3280. ---- squashfs-tools.orig/unsquashfs.c   2013-05-09 20:25:19.000000000 +0000
  3281. -+++ squashfs-tools/unsquashfs.c    2013-05-09 20:25:19.000000000 +0000
  3282. -@@ -2104,7 +2104,7 @@
  3283. -           "\n");
  3284. -
  3285. -   if(processors == -1) {
  3286. --#ifndef linux
  3287. -+#if !defined(linux) && !defined(__GLIBC__)
  3288. -       int mib[2];
  3289. -       size_t len = sizeof(processors);
  3290. -
  3291. -Index: squashfs-tools/unsquashfs.h
  3292. -===================================================================
  3293. ---- squashfs-tools.orig/unsquashfs.h   2013-05-09 20:25:19.000000000 +0000
  3294. -+++ squashfs-tools/unsquashfs.h    2013-05-09 20:25:19.000000000 +0000
  3295. -@@ -45,7 +45,7 @@
  3296. - #include <sys/ioctl.h>
  3297. - #include <sys/time.h>
  3298. -
  3299. --#ifndef linux
  3300. -+#if !defined(linux) && !defined(__GLIBC__)
  3301. - #define __BYTE_ORDER BYTE_ORDER
  3302. - #define __BIG_ENDIAN BIG_ENDIAN
  3303. - #define __LITTLE_ENDIAN LITTLE_ENDIAN
  3304. -Index: squashfs-tools/xz_wrapper.h
  3305. -===================================================================
  3306. ---- squashfs-tools.orig/xz_wrapper.h   2013-05-09 20:25:19.000000000 +0000
  3307. -+++ squashfs-tools/xz_wrapper.h    2013-05-09 20:25:19.000000000 +0000
  3308. -@@ -24,7 +24,7 @@
  3309. -  *
  3310. -  */
  3311. -
  3312. --#ifndef linux
  3313. -+#if !defined(linux) && !defined(__GLIBC__)
  3314. - #define __BYTE_ORDER BYTE_ORDER
  3315. - #define __BIG_ENDIAN BIG_ENDIAN
  3316. - #define __LITTLE_ENDIAN LITTLE_ENDIAN
  3317. diff -Nru squashfs-tools-4.2+20130409/debian/patches/cve-2015-4646 squashfs-tools-4.3+20140919/debian/patches/cve-2015-4646
  3318. --- squashfs-tools-4.2+20130409/debian/patches/cve-2015-4646    1970-01-01 01:00:00.000000000 +0100
  3319. +++ squashfs-tools-4.3+20140919/debian/patches/cve-2015-4646    2015-07-20 22:29:13.000000000 +0200
  3320. @@ -0,0 +1,47 @@
  3321. +Description: CVE-2015-4645 integer overflow in read_fragment_table_4
  3322. + Integer overflow issue was reported in squashfs-tools.
  3323. + Following code in unsquash-4.c overflows the bytes variable, so that the allocation of fragments_bytes[] has an erroneous size.
  3324. +
  3325. +   int bytes = SQUASHFS_FRAGMENT_BYTES(sBlk.s.fragments);
  3326. +   ...
  3327. +   fragment_table = malloc(bytes)
  3328. + .
  3329. + squashfs-tools (1:4.3+20140919-0) unstable; urgency=low
  3330. + .
  3331. +   * Fixes https://security-tracker.debian.org/tracker/CVE-2015-4645
  3332. +   * Fixes https://security-tracker.debian.org/tracker/CVE-2015-4646
  3333. +
  3334. +Author: Romeo Papa <romeopapa@caramail.com>
  3335. +Origin: upstream, https://github.com/gcanalesb/sasquatch/commit/6777e08cc38bc780d27c69c1d8c272867b74524f
  3336. +Bug: https://github.com/devttys0/sasquatch/pull/5
  3337. +Bug-Debian: https://security-tracker.debian.org/tracker/CVE-2015-4646
  3338. +Bug-Fedora: http://people.canonical.com/~ubuntu-security/cve/2015/CVE-2015-4646.html
  3339. +Last-Update: 2015-06-17
  3340. +
  3341. +--- squashfs-tools-4.3+20140919.orig/unsquash-4.c
  3342. ++++ squashfs-tools-4.3+20140919/unsquash-4.c
  3343. +@@ -31,9 +31,9 @@ static unsigned int *id_table;
  3344. + int read_fragment_table_4(long long *directory_table_end)
  3345. + {
  3346. +   int res, i;
  3347. +-  int bytes = SQUASHFS_FRAGMENT_BYTES(sBlk.s.fragments);
  3348. +-  int  indexes = SQUASHFS_FRAGMENT_INDEXES(sBlk.s.fragments);
  3349. +-  long long fragment_table_index[indexes];
  3350. ++  size_t bytes = SQUASHFS_FRAGMENT_BYTES(sBlk.s.fragments);
  3351. ++  size_t indexes = SQUASHFS_FRAGMENT_INDEXES(sBlk.s.fragments);
  3352. ++  long long *fragment_table_index;
  3353. +
  3354. +   TRACE("read_fragment_table: %d fragments, reading %d fragment indexes "
  3355. +       "from 0x%llx\n", sBlk.s.fragments, indexes,
  3356. +@@ -44,6 +44,11 @@ int read_fragment_table_4(long long *dir
  3357. +       return TRUE;
  3358. +   }
  3359. +
  3360. ++  fragment_table_index = malloc(indexes*sizeof(long long));
  3361. ++  if(fragment_table_index == NULL)
  3362. ++      EXIT_UNSQUASH("read_fragment_table: failed to allocate "
  3363. ++          "fragment table index\n");
  3364. ++
  3365. +   fragment_table = malloc(bytes);
  3366. +   if(fragment_table == NULL)
  3367. +       EXIT_UNSQUASH("read_fragment_table: failed to allocate "
  3368. diff -Nru squashfs-tools-4.2+20130409/debian/patches/series squashfs-tools-4.3+20140919/debian/patches/series
  3369. --- squashfs-tools-4.2+20130409/debian/patches/series   2013-05-09 22:22:49.000000000 +0200
  3370. +++ squashfs-tools-4.3+20140919/debian/patches/series   2015-07-20 22:30:50.000000000 +0200
  3371. @@ -1 +1 @@
  3372. -0001-kfreebsd.patch
  3373. +cve-2015-4646
  3374. diff -Nru squashfs-tools-4.2+20130409/error.h squashfs-tools-4.3+20140919/error.h
  3375. --- squashfs-tools-4.2+20130409/error.h 2013-05-09 10:39:11.000000000 +0200
  3376. +++ squashfs-tools-4.3+20140919/error.h 2015-07-20 21:03:05.000000000 +0200
  3377. @@ -1,8 +1,10 @@
  3378. +#ifndef ERROR_H
  3379. +#define ERROR_H
  3380.  /*
  3381.   * Create a squashfs filesystem.  This is a highly compressed read only
  3382.   * filesystem.
  3383.   *
  3384. - * Copyright (c) 2012, 2013
  3385. + * Copyright (c) 2012, 2013, 2014
  3386.   * Phillip Lougher <phillip@squashfs.org.uk>
  3387.   *
  3388.   * This program is free software; you can redistribute it and/or
  3389. @@ -22,6 +24,8 @@
  3390.   * error.h
  3391.   */
  3392.  
  3393. +extern int exit_on_error;
  3394. +
  3395.  extern void prep_exit();
  3396.  extern void progressbar_error(char *fmt, ...);
  3397.  extern void progressbar_info(char *fmt, ...);
  3398. @@ -46,6 +50,23 @@
  3399.             progressbar_error(s, ## args); \
  3400.         } while(0)
  3401.  
  3402. +#define ERROR_START(s, args...) \
  3403. +       do { \
  3404. +           disable_progress_bar(); \
  3405. +           fprintf(stderr, s, ## args); \
  3406. +       } while(0)
  3407. +
  3408. +#define ERROR_EXIT(s, args...) \
  3409. +       do {\
  3410. +           if (exit_on_error) { \
  3411. +               fprintf(stderr, "\n"); \
  3412. +               EXIT_MKSQUASHFS(); \
  3413. +           } else { \
  3414. +               fprintf(stderr, s, ## args); \
  3415. +               enable_progress_bar(); \
  3416. +           } \
  3417. +       } while(0)
  3418. +
  3419.  #define EXIT_MKSQUASHFS() \
  3420.         do {\
  3421.             prep_exit();\
  3422. @@ -66,3 +87,4 @@
  3423.                                 __func__); \
  3424.         EXIT_MKSQUASHFS();\
  3425.     } while(0)
  3426. +#endif
  3427. diff -Nru squashfs-tools-4.2+20130409/gzip_wrapper.c squashfs-tools-4.3+20140919/gzip_wrapper.c
  3428. --- squashfs-tools-4.2+20130409/gzip_wrapper.c  2013-05-09 10:39:11.000000000 +0200
  3429. +++ squashfs-tools-4.3+20140919/gzip_wrapper.c  2015-07-20 21:03:05.000000000 +0200
  3430. @@ -1,5 +1,5 @@
  3431.  /*
  3432. - * Copyright (c) 2009, 2010, 2013
  3433. + * Copyright (c) 2009, 2010, 2013, 2014
  3434.   * Phillip Lougher <phillip@squashfs.org.uk>
  3435.   *
  3436.   * This program is free software; you can redistribute it and/or
  3437. @@ -17,34 +17,376 @@
  3438.   * Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  3439.   *
  3440.   * gzip_wrapper.c
  3441. + *
  3442. + * Support for ZLIB compression http://www.zlib.net
  3443.   */
  3444.  
  3445. +#include <stdio.h>
  3446. +#include <string.h>
  3447.  #include <stdlib.h>
  3448.  #include <zlib.h>
  3449.  
  3450.  #include "squashfs_fs.h"
  3451. +#include "gzip_wrapper.h"
  3452.  #include "compressor.h"
  3453.  
  3454. -static int gzip_init(void **strm, int block_size, int flags)
  3455. +static struct strategy strategy[] = {
  3456. +   { "default", Z_DEFAULT_STRATEGY, 0 },
  3457. +   { "filtered", Z_FILTERED, 0 },
  3458. +   { "huffman_only", Z_HUFFMAN_ONLY, 0 },
  3459. +   { "run_length_encoded", Z_RLE, 0 },
  3460. +   { "fixed", Z_FIXED, 0 },
  3461. +   { NULL, 0, 0 }
  3462. +};
  3463. +
  3464. +static int strategy_count = 0;
  3465. +
  3466. +/* default compression level */
  3467. +static int compression_level = GZIP_DEFAULT_COMPRESSION_LEVEL;
  3468. +
  3469. +/* default window size */
  3470. +static int window_size = GZIP_DEFAULT_WINDOW_SIZE;
  3471. +
  3472. +/*
  3473. + * This function is called by the options parsing code in mksquashfs.c
  3474. + * to parse any -X compressor option.
  3475. + *
  3476. + * This function returns:
  3477. + * >=0 (number of additional args parsed) on success
  3478. + * -1 if the option was unrecognised, or
  3479. + * -2 if the option was recognised, but otherwise bad in
  3480. + *    some way (e.g. invalid parameter)
  3481. + *
  3482. + * Note: this function sets internal compressor state, but does not
  3483. + * pass back the results of the parsing other than success/failure.
  3484. + * The gzip_dump_options() function is called later to get the options in
  3485. + * a format suitable for writing to the filesystem.
  3486. + */
  3487. +static int gzip_options(char *argv[], int argc)
  3488.  {
  3489. -   int res;
  3490. -   z_stream *stream;
  3491. +   if(strcmp(argv[0], "-Xcompression-level") == 0) {
  3492. +       if(argc < 2) {
  3493. +           fprintf(stderr, "gzip: -Xcompression-level missing "
  3494. +               "compression level\n");
  3495. +           fprintf(stderr, "gzip: -Xcompression-level it "
  3496. +               "should be 1 >= n <= 9\n");
  3497. +           goto failed;
  3498. +       }
  3499. +
  3500. +       compression_level = atoi(argv[1]);
  3501. +       if(compression_level < 1 || compression_level > 9) {
  3502. +           fprintf(stderr, "gzip: -Xcompression-level invalid, it "
  3503. +               "should be 1 >= n <= 9\n");
  3504. +           goto failed;
  3505. +       }
  3506. +
  3507. +       return 1;
  3508. +   } else if(strcmp(argv[0], "-Xwindow-size") == 0) {
  3509. +       if(argc < 2) {
  3510. +           fprintf(stderr, "gzip: -Xwindow-size missing window "
  3511. +               "   size\n");
  3512. +           fprintf(stderr, "gzip: -Xwindow-size <window-size>\n");
  3513. +           goto failed;
  3514. +       }
  3515. +
  3516. +       window_size = atoi(argv[1]);
  3517. +       if(window_size < 8 || window_size > 15) {
  3518. +           fprintf(stderr, "gzip: -Xwindow-size invalid, it "
  3519. +               "should be 8 >= n <= 15\n");
  3520. +           goto failed;
  3521. +       }
  3522. +
  3523. +       return 1;
  3524. +   } else if(strcmp(argv[0], "-Xstrategy") == 0) {
  3525. +       char *name;
  3526. +       int i;
  3527. +
  3528. +       if(argc < 2) {
  3529. +           fprintf(stderr, "gzip: -Xstrategy missing "
  3530. +                           "strategies\n");
  3531. +           goto failed;
  3532. +       }
  3533. +
  3534. +       name = argv[1];
  3535. +       while(name[0] != '\0') {
  3536. +           for(i = 0; strategy[i].name; i++) {
  3537. +               int n = strlen(strategy[i].name);
  3538. +               if((strncmp(name, strategy[i].name, n) == 0) &&
  3539. +                       (name[n] == '\0' ||
  3540. +                        name[n] == ',')) {
  3541. +                   if(strategy[i].selected == 0) {
  3542. +                       strategy[i].selected = 1;
  3543. +                       strategy_count++;
  3544. +                   }
  3545. +                   name += name[n] == ',' ? n + 1 : n;
  3546. +                   break;
  3547. +               }
  3548. +           }
  3549. +           if(strategy[i].name == NULL) {
  3550. +               fprintf(stderr, "gzip: -Xstrategy unrecognised "
  3551. +                   "strategy\n");
  3552. +               goto failed;
  3553. +           }
  3554. +       }
  3555. +  
  3556. +       return 1;
  3557. +   }
  3558. +
  3559. +   return -1;
  3560. +
  3561. +failed:
  3562. +   return -2;
  3563. +}
  3564. +
  3565. +
  3566. +/*
  3567. + * This function is called after all options have been parsed.
  3568. + * It is used to do post-processing on the compressor options using
  3569. + * values that were not expected to be known at option parse time.
  3570. + *
  3571. + * This function returns 0 on successful post processing, or
  3572. + *         -1 on error
  3573. + */
  3574. +static int gzip_options_post(int block_size)
  3575. +{
  3576. +   if(strategy_count == 1 && strategy[0].selected) {
  3577. +       strategy_count = 0;
  3578. +       strategy[0].selected = 0;
  3579. +   }
  3580. +
  3581. +   return 0;
  3582. +}
  3583. +
  3584. +
  3585. +/*
  3586. + * This function is called by mksquashfs to dump the parsed
  3587. + * compressor options in a format suitable for writing to the
  3588. + * compressor options field in the filesystem (stored immediately
  3589. + * after the superblock).
  3590. + *
  3591. + * This function returns a pointer to the compression options structure
  3592. + * to be stored (and the size), or NULL if there are no compression
  3593. + * options
  3594. + *
  3595. + */
  3596. +static void *gzip_dump_options(int block_size, int *size)
  3597. +{
  3598. +   static struct gzip_comp_opts comp_opts;
  3599. +   int i, strategies = 0;
  3600. +
  3601. +   /*
  3602. +    * If default compression options of:
  3603. +    * compression-level: 8 and
  3604. +    * window-size: 15 and
  3605. +    * strategy_count == 0 then
  3606. +    * don't store a compression options structure (this is compatible
  3607. +    * with the legacy implementation of GZIP for Squashfs)
  3608. +    */
  3609. +   if(compression_level == GZIP_DEFAULT_COMPRESSION_LEVEL &&
  3610. +               window_size == GZIP_DEFAULT_WINDOW_SIZE &&
  3611. +               strategy_count == 0)
  3612. +       return NULL;
  3613. +
  3614. +   for(i = 0; strategy[i].name; i++)
  3615. +       strategies |= strategy[i].selected << i;
  3616. +
  3617. +   comp_opts.compression_level = compression_level;
  3618. +   comp_opts.window_size = window_size;
  3619. +   comp_opts.strategy = strategies;
  3620. +
  3621. +   SQUASHFS_INSWAP_COMP_OPTS(&comp_opts);
  3622. +
  3623. +   *size = sizeof(comp_opts);
  3624. +   return &comp_opts;
  3625. +}
  3626. +
  3627. +
  3628. +/*
  3629. + * This function is a helper specifically for the append mode of
  3630. + * mksquashfs.  Its purpose is to set the internal compressor state
  3631. + * to the stored compressor options in the passed compressor options
  3632. + * structure.
  3633. + *
  3634. + * In effect this function sets up the compressor options
  3635. + * to the same state they were when the filesystem was originally
  3636. + * generated, this is to ensure on appending, the compressor uses
  3637. + * the same compression options that were used to generate the
  3638. + * original filesystem.
  3639. + *
  3640. + * Note, even if there are no compressor options, this function is still
  3641. + * called with an empty compressor structure (size == 0), to explicitly
  3642. + * set the default options, this is to ensure any user supplied
  3643. + * -X options on the appending mksquashfs command line are over-ridden
  3644. + *
  3645. + * This function returns 0 on sucessful extraction of options, and
  3646. + *         -1 on error
  3647. + */
  3648. +static int gzip_extract_options(int block_size, void *buffer, int size)
  3649. +{
  3650. +   struct gzip_comp_opts *comp_opts = buffer;
  3651. +   int i;
  3652. +
  3653. +   if(size == 0) {
  3654. +       /* Set default values */
  3655. +       compression_level = GZIP_DEFAULT_COMPRESSION_LEVEL;
  3656. +       window_size = GZIP_DEFAULT_WINDOW_SIZE;
  3657. +       strategy_count = 0;
  3658. +       return 0;
  3659. +   }
  3660. +
  3661. +   /* we expect a comp_opts structure of sufficient size to be present */
  3662. +   if(size < sizeof(*comp_opts))
  3663. +       goto failed;
  3664. +
  3665. +   SQUASHFS_INSWAP_COMP_OPTS(comp_opts);
  3666. +
  3667. +   /* Check comp_opts structure for correctness */
  3668. +   if(comp_opts->compression_level < 1 ||
  3669. +           comp_opts->compression_level > 9) {
  3670. +       fprintf(stderr, "gzip: bad compression level in "
  3671. +           "compression options structure\n");
  3672. +       goto failed;
  3673. +   }
  3674. +   compression_level = comp_opts->compression_level;
  3675.  
  3676. -   stream = *strm = malloc(sizeof(z_stream));
  3677. -   if(stream == NULL)
  3678. +   if(comp_opts->window_size < 8 ||
  3679. +           comp_opts->window_size > 15) {
  3680. +       fprintf(stderr, "gzip: bad window size in "
  3681. +           "compression options structure\n");
  3682.         goto failed;
  3683. +   }
  3684. +   window_size = comp_opts->window_size;
  3685. +
  3686. +   strategy_count = 0;
  3687. +   for(i = 0; strategy[i].name; i++) {
  3688. +       if((comp_opts->strategy >> i) & 1) {
  3689. +           strategy[i].selected = 1;
  3690. +           strategy_count ++;
  3691. +       } else
  3692. +           strategy[i].selected = 0;
  3693. +   }
  3694. +  
  3695. +   return 0;
  3696. +
  3697. +failed:
  3698. +   fprintf(stderr, "gzip: error reading stored compressor options from "
  3699. +       "filesystem!\n");
  3700. +
  3701. +   return -1;
  3702. +}
  3703. +
  3704. +
  3705. +void gzip_display_options(void *buffer, int size)
  3706. +{
  3707. +   struct gzip_comp_opts *comp_opts = buffer;
  3708. +   int i, printed;
  3709. +
  3710. +   /* we expect a comp_opts structure of sufficient size to be present */
  3711. +   if(size < sizeof(*comp_opts))
  3712. +       goto failed;
  3713. +
  3714. +   SQUASHFS_INSWAP_COMP_OPTS(comp_opts);
  3715. +
  3716. +   /* Check comp_opts structure for correctness */
  3717. +   if(comp_opts->compression_level < 1 ||
  3718. +           comp_opts->compression_level > 9) {
  3719. +       fprintf(stderr, "gzip: bad compression level in "
  3720. +           "compression options structure\n");
  3721. +       goto failed;
  3722. +   }
  3723. +   printf("\tcompression-level %d\n", comp_opts->compression_level);
  3724. +
  3725. +   if(comp_opts->window_size < 8 ||
  3726. +           comp_opts->window_size > 15) {
  3727. +       fprintf(stderr, "gzip: bad window size in "
  3728. +           "compression options structure\n");
  3729. +       goto failed;
  3730. +   }
  3731. +   printf("\twindow-size %d\n", comp_opts->window_size);
  3732. +
  3733. +   for(i = 0, printed = 0; strategy[i].name; i++) {
  3734. +       if((comp_opts->strategy >> i) & 1) {
  3735. +           if(printed)
  3736. +               printf(", ");
  3737. +           else
  3738. +               printf("\tStrategies selected: ");
  3739. +           printf("%s", strategy[i].name);
  3740. +           printed = 1;
  3741. +       }
  3742. +   }
  3743. +
  3744. +   if(!printed)
  3745. +       printf("\tStrategies selected: default\n");
  3746. +   else
  3747. +       printf("\n");
  3748. +
  3749. +   return;
  3750. +
  3751. +failed:
  3752. +   fprintf(stderr, "gzip: error reading stored compressor options from "
  3753. +       "filesystem!\n");
  3754. +} 
  3755.  
  3756. -   stream->zalloc = Z_NULL;
  3757. -   stream->zfree = Z_NULL;
  3758. -   stream->opaque = 0;
  3759.  
  3760. -   res = deflateInit(stream, 9);
  3761. +/*
  3762. + * This function is called by mksquashfs to initialise the
  3763. + * compressor, before compress() is called.
  3764. + *
  3765. + * This function returns 0 on success, and
  3766. + *         -1 on error
  3767. + */
  3768. +static int gzip_init(void **strm, int block_size, int datablock)
  3769. +{
  3770. +   int i, j, res;
  3771. +   struct gzip_stream *stream;
  3772. +
  3773. +   if(!datablock || !strategy_count) {
  3774. +       stream = malloc(sizeof(*stream) + sizeof(struct gzip_strategy));
  3775. +       if(stream == NULL)
  3776. +           goto failed;
  3777. +
  3778. +       stream->strategies = 1;
  3779. +       stream->strategy[0].strategy = Z_DEFAULT_STRATEGY;
  3780. +   } else {
  3781. +       stream = malloc(sizeof(*stream) +
  3782. +           sizeof(struct gzip_strategy) * strategy_count);
  3783. +       if(stream == NULL)
  3784. +           goto failed;
  3785. +
  3786. +       memset(stream->strategy, 0, sizeof(struct gzip_strategy) *
  3787. +           strategy_count);
  3788. +
  3789. +       stream->strategies = strategy_count;
  3790. +
  3791. +       for(i = 0, j = 0; strategy[i].name; i++) {
  3792. +           if(!strategy[i].selected)
  3793. +               continue;
  3794. +
  3795. +           stream->strategy[j].strategy = strategy[i].strategy;
  3796. +           if(j) {
  3797. +               stream->strategy[j].buffer = malloc(block_size);
  3798. +               if(stream->strategy[j].buffer == NULL)
  3799. +                   goto failed2;
  3800. +           }
  3801. +           j++;
  3802. +       }
  3803. +   }
  3804. +      
  3805. +   stream->stream.zalloc = Z_NULL;
  3806. +   stream->stream.zfree = Z_NULL;
  3807. +   stream->stream.opaque = 0;
  3808. +
  3809. +   res = deflateInit2(&stream->stream, compression_level, Z_DEFLATED,
  3810. +       window_size, 8, stream->strategy[0].strategy);
  3811.     if(res != Z_OK)
  3812.         goto failed2;
  3813.  
  3814. +   *strm = stream;
  3815.     return 0;
  3816.  
  3817.  failed2:
  3818. +   for(i = 1; i < stream->strategies; i++)
  3819. +       free(stream->strategy[i].buffer);
  3820.     free(stream);
  3821.  failed:
  3822.     return -1;
  3823. @@ -54,29 +396,51 @@
  3824.  static int gzip_compress(void *strm, void *d, void *s, int size, int block_size,
  3825.         int *error)
  3826.  {
  3827. -   int res;
  3828. -   z_stream *stream = strm;
  3829. -
  3830. -   res = deflateReset(stream);
  3831. -   if(res != Z_OK)
  3832. -       goto failed;
  3833. -
  3834. -   stream->next_in = s;
  3835. -   stream->avail_in = size;
  3836. -   stream->next_out = d;
  3837. -   stream->avail_out = block_size;
  3838. +   int i, res;
  3839. +   struct gzip_stream *stream = strm;
  3840. +   struct gzip_strategy *selected = NULL;
  3841. +
  3842. +   stream->strategy[0].buffer = d;
  3843. +
  3844. +   for(i = 0; i < stream->strategies; i++) {
  3845. +       struct gzip_strategy *strategy = &stream->strategy[i];
  3846. +
  3847. +       res = deflateReset(&stream->stream);
  3848. +       if(res != Z_OK)
  3849. +           goto failed;
  3850. +
  3851. +       stream->stream.next_in = s;
  3852. +       stream->stream.avail_in = size;
  3853. +       stream->stream.next_out = strategy->buffer;
  3854. +       stream->stream.avail_out = block_size;
  3855. +
  3856. +       if(stream->strategies > 1) {
  3857. +           res = deflateParams(&stream->stream,
  3858. +               compression_level, strategy->strategy);
  3859. +           if(res != Z_OK)
  3860. +               goto failed;
  3861. +       }
  3862. +
  3863. +       res = deflate(&stream->stream, Z_FINISH);
  3864. +       strategy->length = stream->stream.total_out;
  3865. +       if(res == Z_STREAM_END) {
  3866. +           if(!selected || selected->length > strategy->length)
  3867. +               selected = strategy;
  3868. +       } else if(res != Z_OK)
  3869. +           goto failed;
  3870. +   }
  3871.  
  3872. -   res = deflate(stream, Z_FINISH);
  3873. -   if(res == Z_STREAM_END)
  3874. -       /*
  3875. -        * Success, return the compressed size.
  3876. -        */
  3877. -       return (int) stream->total_out;
  3878. -   if(res == Z_OK)
  3879. +   if(!selected)
  3880.         /*
  3881.          * Output buffer overflow.  Return out of buffer space
  3882.          */
  3883.         return 0;
  3884. +
  3885. +   if(selected->buffer != d)
  3886. +       memcpy(d, selected->buffer, selected->length);
  3887. +
  3888. +   return (int) selected->length;
  3889. +
  3890.  failed:
  3891.     /*
  3892.      * All other errors return failure, with the compressor
  3893. @@ -94,8 +458,29 @@
  3894.  
  3895.     res = uncompress(d, &bytes, s, size);
  3896.  
  3897. -   *error = res;
  3898. -   return res == Z_OK ? (int) bytes : -1;
  3899. +   if(res == Z_OK)
  3900. +       return (int) bytes;
  3901. +   else {
  3902. +       *error = res;
  3903. +       return -1;
  3904. +   }
  3905. +}
  3906. +
  3907. +
  3908. +void gzip_usage()
  3909. +{
  3910. +   fprintf(stderr, "\t  -Xcompression-level <compression-level>\n");
  3911. +   fprintf(stderr, "\t\t<compression-level> should be 1 .. 9 (default "
  3912. +       "%d)\n", GZIP_DEFAULT_COMPRESSION_LEVEL);
  3913. +   fprintf(stderr, "\t  -Xwindow-size <window-size>\n");
  3914. +   fprintf(stderr, "\t\t<window-size> should be 8 .. 15 (default "
  3915. +       "%d)\n", GZIP_DEFAULT_WINDOW_SIZE);
  3916. +   fprintf(stderr, "\t  -Xstrategy strategy1,strategy2,...,strategyN\n");
  3917. +   fprintf(stderr, "\t\tCompress using strategy1,strategy2,...,strategyN"
  3918. +       " in turn\n");
  3919. +   fprintf(stderr, "\t\tand choose the best compression.\n");
  3920. +   fprintf(stderr, "\t\tAvailable strategies: default, filtered, "
  3921. +       "huffman_only,\n\t\trun_length_encoded and fixed\n");
  3922.  }
  3923.  
  3924.  
  3925. @@ -103,10 +488,13 @@
  3926.     .init = gzip_init,
  3927.     .compress = gzip_compress,
  3928.     .uncompress = gzip_uncompress,
  3929. -   .options = NULL,
  3930. -   .usage = NULL,
  3931. +   .options = gzip_options,
  3932. +   .options_post = gzip_options_post,
  3933. +   .dump_options = gzip_dump_options,
  3934. +   .extract_options = gzip_extract_options,
  3935. +   .display_options = gzip_display_options,
  3936. +   .usage = gzip_usage,
  3937.     .id = ZLIB_COMPRESSION,
  3938.     .name = "gzip",
  3939.     .supported = 1
  3940.  };
  3941. -
  3942. diff -Nru squashfs-tools-4.2+20130409/gzip_wrapper.h squashfs-tools-4.3+20140919/gzip_wrapper.h
  3943. --- squashfs-tools-4.2+20130409/gzip_wrapper.h  1970-01-01 01:00:00.000000000 +0100
  3944. +++ squashfs-tools-4.3+20140919/gzip_wrapper.h  2015-07-20 21:03:05.000000000 +0200
  3945. @@ -0,0 +1,75 @@
  3946. +#ifndef GZIP_WRAPPER_H
  3947. +#define GZIP_WRAPPER_H
  3948. +/*
  3949. + * Squashfs
  3950. + *
  3951. + * Copyright (c) 2014
  3952. + * Phillip Lougher <phillip@squashfs.org.uk>
  3953. + *
  3954. + * This program is free software; you can redistribute it and/or
  3955. + * modify it under the terms of the GNU General Public License
  3956. + * as published by the Free Software Foundation; either version 2,
  3957. + * or (at your option) any later version.
  3958. + *
  3959. + * This program is distributed in the hope that it will be useful,
  3960. + * but WITHOUT ANY WARRANTY; without even the implied warranty of
  3961. + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  3962. + * GNU General Public License for more details.
  3963. + *
  3964. + * You should have received a copy of the GNU General Public License
  3965. + * along with this program; if not, write to the Free Software
  3966. + * Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  3967. + *
  3968. + * gzip_wrapper.h
  3969. + *
  3970. + */
  3971. +
  3972. +#ifndef linux
  3973. +#define __BYTE_ORDER BYTE_ORDER
  3974. +#define __BIG_ENDIAN BIG_ENDIAN
  3975. +#define __LITTLE_ENDIAN LITTLE_ENDIAN
  3976. +#else
  3977. +#include <endian.h>
  3978. +#endif
  3979. +
  3980. +#if __BYTE_ORDER == __BIG_ENDIAN
  3981. +extern unsigned int inswap_le16(unsigned short);
  3982. +extern unsigned int inswap_le32(unsigned int);
  3983. +
  3984. +#define SQUASHFS_INSWAP_COMP_OPTS(s) { \
  3985. +   (s)->compression_level = inswap_le32((s)->compression_level); \
  3986. +   (s)->window_size = inswap_le16((s)->window_size); \
  3987. +   (s)->strategy = inswap_le16((s)->strategy); \
  3988. +}
  3989. +#else
  3990. +#define SQUASHFS_INSWAP_COMP_OPTS(s)
  3991. +#endif
  3992. +
  3993. +/* Default compression */
  3994. +#define GZIP_DEFAULT_COMPRESSION_LEVEL 9
  3995. +#define GZIP_DEFAULT_WINDOW_SIZE 15
  3996. +
  3997. +struct gzip_comp_opts {
  3998. +   int compression_level;
  3999. +   short window_size;
  4000. +   short strategy;
  4001. +};
  4002. +
  4003. +struct strategy {
  4004. +   char *name;
  4005. +   int strategy;
  4006. +   int selected;
  4007. +};
  4008. +
  4009. +struct gzip_strategy {
  4010. +   int strategy;
  4011. +   int length;
  4012. +   void *buffer;
  4013. +};
  4014. +
  4015. +struct gzip_stream {
  4016. +   z_stream stream;
  4017. +   int strategies;
  4018. +   struct gzip_strategy strategy[0];
  4019. +};
  4020. +#endif
  4021. diff -Nru squashfs-tools-4.2+20130409/info.c squashfs-tools-4.3+20140919/info.c
  4022. --- squashfs-tools-4.2+20130409/info.c  2013-05-09 10:39:11.000000000 +0200
  4023. +++ squashfs-tools-4.3+20140919/info.c  2015-07-20 21:03:05.000000000 +0200
  4024. @@ -2,7 +2,7 @@
  4025.   * Create a squashfs filesystem.  This is a highly compressed read only
  4026.   * filesystem.
  4027.   *
  4028. - * Copyright (c) 2013
  4029. + * Copyright (c) 2013, 2014
  4030.   * Phillip Lougher <phillip@squashfs.org.uk>
  4031.   *
  4032.   * This program is free software; you can redistribute it and/or
  4033. @@ -35,55 +35,142 @@
  4034.  #include <dirent.h>
  4035.  #include <sys/types.h>
  4036.  #include <sys/stat.h>
  4037. +#include <string.h>
  4038.  
  4039.  #include "squashfs_fs.h"
  4040.  #include "mksquashfs.h"
  4041.  #include "error.h"
  4042. +#include "progressbar.h"
  4043. +#include "caches-queues-lists.h"
  4044.  
  4045.  static int silent = 0;
  4046. -static struct dir_ent *dir_ent = NULL;
  4047. +static struct dir_ent *ent = NULL;
  4048.  
  4049.  pthread_t info_thread;
  4050.  
  4051.  
  4052.  void disable_info()
  4053.  {
  4054. -   dir_ent = NULL;
  4055. +   ent = NULL;
  4056.  }
  4057.  
  4058.  
  4059. -void update_info(struct dir_ent *ent)
  4060. +void update_info(struct dir_ent *dir_ent)
  4061.  {
  4062. -   dir_ent = ent;
  4063. +   ent = dir_ent;
  4064. +}
  4065. +
  4066. +
  4067. +void print_filename()
  4068. +{
  4069. +   struct dir_ent *dir_ent = ent;
  4070. +
  4071. +   if(dir_ent == NULL)
  4072. +       return;
  4073. +
  4074. +   if(dir_ent->our_dir->subpath[0] != '\0')
  4075. +       INFO("%s/%s\n", dir_ent->our_dir->subpath, dir_ent->name);
  4076. +   else
  4077. +       INFO("/%s\n", dir_ent->name);
  4078. +}
  4079. +
  4080. +
  4081. +void dump_state()
  4082. +{
  4083. +   disable_progress_bar();
  4084. +
  4085. +   printf("Queue and Cache status dump\n");
  4086. +   printf("===========================\n");
  4087. +
  4088. +   printf("file buffer queue (reader thread -> deflate thread(s))\n");
  4089. +   dump_queue(to_deflate);
  4090. +
  4091. +   printf("uncompressed fragment queue (reader thread -> fragment"
  4092. +                       " thread(s))\n");
  4093. +   dump_queue(to_process_frag);
  4094. +
  4095. +   printf("processed fragment queue (fragment thread(s) -> main"
  4096. +                       " thread)\n");
  4097. +   dump_seq_queue(to_main, 1);
  4098. +
  4099. +   printf("compressed block queue (deflate thread(s) -> main thread)\n");
  4100. +   dump_seq_queue(to_main, 0);
  4101. +
  4102. +   printf("uncompressed packed fragment queue (main thread -> fragment"
  4103. +                       " deflate thread(s))\n");
  4104. +   dump_queue(to_frag);
  4105. +
  4106. +
  4107. +   printf("locked frag queue (compressed frags waiting while multi-block"
  4108. +                       " file is written)\n");
  4109. +   dump_queue(locked_fragment);
  4110. +
  4111. +   printf("compressed block queue (main & fragment deflate threads(s) ->"
  4112. +                       " writer thread)\n");
  4113. +   dump_queue(to_writer);
  4114. +
  4115. +   printf("read cache (uncompressed blocks read by reader thread)\n");
  4116. +   dump_cache(reader_buffer);
  4117. +
  4118. +   printf("block write cache (compressed blocks waiting for the writer"
  4119. +                       " thread)\n");
  4120. +   dump_cache(bwriter_buffer);
  4121. +   printf("fragment write cache (compressed fragments waiting for the"
  4122. +                       " writer thread)\n");
  4123. +   dump_cache(fwriter_buffer);
  4124. +
  4125. +   printf("fragment cache (frags waiting to be compressed by fragment"
  4126. +                       " deflate thread(s))\n");
  4127. +   dump_cache(fragment_buffer);
  4128. +
  4129. +   printf("fragment reserve cache (avoids pipeline stall if frag cache"
  4130. +                       " full in dup check)\n");
  4131. +   dump_cache(reserve_cache);
  4132. +
  4133. +   enable_progress_bar();
  4134.  }
  4135.  
  4136.  
  4137.  void *info_thrd(void *arg)
  4138.  {
  4139.     sigset_t sigmask;
  4140. -   int sig, res;
  4141. -   char *subpath;
  4142. +   struct timespec timespec = { .tv_sec = 1, .tv_nsec = 0 };
  4143. +   int sig, waiting = 0;
  4144.  
  4145.     sigemptyset(&sigmask);
  4146.     sigaddset(&sigmask, SIGQUIT);
  4147. +   sigaddset(&sigmask, SIGHUP);
  4148.  
  4149.     while(1) {
  4150. -       sigwait(&sigmask, &sig);
  4151. -
  4152. -       if(dir_ent == NULL)
  4153. -           continue;
  4154. -
  4155. -       if(dir_ent->our_dir->subpath[0] != '\0')
  4156. -           res = asprintf(&subpath, "%s/%s",
  4157. -               dir_ent->our_dir->subpath, dir_ent->name);
  4158. +       if(waiting)
  4159. +           sig = sigtimedwait(&sigmask, NULL, &timespec);
  4160.         else
  4161. -           res = asprintf(&subpath, "/%s", dir_ent->name);
  4162. -
  4163. -       if(res < 0)
  4164. -           printf("asprintf failed in info_thrd\n");
  4165. +           sig = sigwaitinfo(&sigmask, NULL);
  4166.  
  4167. -       INFO("%s\n", subpath);
  4168. -       free(subpath);
  4169. +       if(sig == -1) {
  4170. +           switch(errno) {
  4171. +           case EAGAIN:
  4172. +               /* interval timed out */
  4173. +               waiting = 0;
  4174. +               /* FALLTHROUGH */
  4175. +           case EINTR:
  4176. +               /* if waiting, the wait will be longer, but
  4177. +                  that's OK */
  4178. +               continue;
  4179. +           default:
  4180. +               BAD_ERROR("sigtimedwait/sigwaitinfo failed "
  4181. +                   "because %s\n", strerror(errno));
  4182. +           }
  4183. +       }
  4184. +
  4185. +       if(sig == SIGQUIT && !waiting) {
  4186. +           print_filename();
  4187. +
  4188. +           /* set one second interval period, if ^\ received
  4189. +              within then, dump queue and cache status */
  4190. +           waiting = 1;
  4191. +       } else
  4192. +           dump_state();
  4193.     }
  4194.  }
  4195.  
  4196. diff -Nru squashfs-tools-4.2+20130409/info.h squashfs-tools-4.3+20140919/info.h
  4197. --- squashfs-tools-4.2+20130409/info.h  2013-05-09 10:39:11.000000000 +0200
  4198. +++ squashfs-tools-4.3+20140919/info.h  2015-07-20 21:03:05.000000000 +0200
  4199. @@ -1,8 +1,10 @@
  4200. +#ifndef INFO_H
  4201. +#define INFO_H
  4202.  /*
  4203.   * Create a squashfs filesystem.  This is a highly compressed read only
  4204.   * filesystem.
  4205.   *
  4206. - * Copyright (c) 2013
  4207. + * Copyright (c) 2013, 2014
  4208.   * Phillip Lougher <phillip@squashfs.org.uk>
  4209.   *
  4210.   * This program is free software; you can redistribute it and/or
  4211. @@ -23,5 +25,6 @@
  4212.   */
  4213.  
  4214.  extern void disable_info();
  4215. -extern void update_info();
  4216. +extern void update_info(struct dir_ent *);
  4217.  extern void init_info();
  4218. +#endif
  4219. diff -Nru squashfs-tools-4.2+20130409/lz4_wrapper.c squashfs-tools-4.3+20140919/lz4_wrapper.c
  4220. --- squashfs-tools-4.2+20130409/lz4_wrapper.c   1970-01-01 01:00:00.000000000 +0100
  4221. +++ squashfs-tools-4.3+20140919/lz4_wrapper.c   2015-07-20 21:03:05.000000000 +0200
  4222. @@ -0,0 +1,283 @@
  4223. +/*
  4224. + * Copyright (c) 2013
  4225. + * Phillip Lougher <phillip@squashfs.org.uk>
  4226. + *
  4227. + * This program is free software; you can redistribute it and/or
  4228. + * modify it under the terms of the GNU General Public License
  4229. + * as published by the Free Software Foundation; either version 2,
  4230. + * or (at your option) any later version.
  4231. + *
  4232. + * This program is distributed in the hope that it will be useful,
  4233. + * but WITHOUT ANY WARRANTY; without even the implied warranty of
  4234. + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  4235. + * GNU General Public License for more details.
  4236. + *
  4237. + * You should have received a copy of the GNU General Public License
  4238. + * along with this program; if not, write to the Free Software
  4239. + * Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  4240. + *
  4241. + * lz4_wrapper.c
  4242. + *
  4243. + * Support for LZ4 compression http://fastcompression.blogspot.com/p/lz4.html
  4244. + */
  4245. +
  4246. +#include <stdio.h>
  4247. +#include <string.h>
  4248. +#include <stdlib.h>
  4249. +#include <lz4.h>
  4250. +#include <lz4hc.h>
  4251. +
  4252. +#include "squashfs_fs.h"
  4253. +#include "lz4_wrapper.h"
  4254. +#include "compressor.h"
  4255. +
  4256. +static int hc = 0;
  4257. +
  4258. +/*
  4259. + * This function is called by the options parsing code in mksquashfs.c
  4260. + * to parse any -X compressor option.
  4261. + *
  4262. + * This function returns:
  4263. + * >=0 (number of additional args parsed) on success
  4264. + * -1 if the option was unrecognised, or
  4265. + * -2 if the option was recognised, but otherwise bad in
  4266. + *    some way (e.g. invalid parameter)
  4267. + *
  4268. + * Note: this function sets internal compressor state, but does not
  4269. + * pass back the results of the parsing other than success/failure.
  4270. + * The lz4_dump_options() function is called later to get the options in
  4271. + * a format suitable for writing to the filesystem.
  4272. + */
  4273. +static int lz4_options(char *argv[], int argc)
  4274. +{
  4275. +   if(strcmp(argv[0], "-Xhc") == 0) {
  4276. +       hc = 1;
  4277. +       return 0;
  4278. +   }
  4279. +
  4280. +   return -1;
  4281. +}
  4282. +
  4283. +
  4284. +/*
  4285. + * This function is called by mksquashfs to dump the parsed
  4286. + * compressor options in a format suitable for writing to the
  4287. + * compressor options field in the filesystem (stored immediately
  4288. + * after the superblock).
  4289. + *
  4290. + * This function returns a pointer to the compression options structure
  4291. + * to be stored (and the size), or NULL if there are no compression
  4292. + * options
  4293. + *
  4294. + * Currently LZ4 always returns a comp_opts structure, with
  4295. + * the version indicating LZ4_LEGACY stream fomat.  This is to
  4296. + * easily accomodate changes in the kernel code to different
  4297. + * stream formats
  4298. + */
  4299. +static void *lz4_dump_options(int block_size, int *size)
  4300. +{
  4301. +   static struct lz4_comp_opts comp_opts;
  4302. +
  4303. +   comp_opts.version = LZ4_LEGACY;
  4304. +   comp_opts.flags = hc ? LZ4_HC : 0;
  4305. +   SQUASHFS_INSWAP_COMP_OPTS(&comp_opts);
  4306. +
  4307. +   *size = sizeof(comp_opts);
  4308. +   return &comp_opts;
  4309. +}
  4310. +
  4311. +
  4312. +/*
  4313. + * This function is a helper specifically for the append mode of
  4314. + * mksquashfs.  Its purpose is to set the internal compressor state
  4315. + * to the stored compressor options in the passed compressor options
  4316. + * structure.
  4317. + *
  4318. + * In effect this function sets up the compressor options
  4319. + * to the same state they were when the filesystem was originally
  4320. + * generated, this is to ensure on appending, the compressor uses
  4321. + * the same compression options that were used to generate the
  4322. + * original filesystem.
  4323. + *
  4324. + * Note, even if there are no compressor options, this function is still
  4325. + * called with an empty compressor structure (size == 0), to explicitly
  4326. + * set the default options, this is to ensure any user supplied
  4327. + * -X options on the appending mksquashfs command line are over-ridden
  4328. + *
  4329. + * This function returns 0 on sucessful extraction of options, and
  4330. + *         -1 on error
  4331. + */
  4332. +static int lz4_extract_options(int block_size, void *buffer, int size)
  4333. +{
  4334. +   struct lz4_comp_opts *comp_opts = buffer;
  4335. +
  4336. +   /* we expect a comp_opts structure to be present */
  4337. +   if(size < sizeof(*comp_opts))
  4338. +       goto failed;
  4339. +
  4340. +   SQUASHFS_INSWAP_COMP_OPTS(comp_opts);
  4341. +
  4342. +   /* we expect the stream format to be LZ4_LEGACY */
  4343. +   if(comp_opts->version != LZ4_LEGACY) {
  4344. +       fprintf(stderr, "lz4: unknown LZ4 version\n");
  4345. +       goto failed;
  4346. +   }
  4347. +
  4348. +   /*
  4349. +    * Check compression flags, currently only LZ4_HC ("high compression")
  4350. +    * can be set.
  4351. +    */
  4352. +   if(comp_opts->flags == LZ4_HC)
  4353. +       hc = 1;
  4354. +   else if(comp_opts->flags != 0) {
  4355. +       fprintf(stderr, "lz4: unknown LZ4 flags\n");
  4356. +       goto failed;
  4357. +   }
  4358. +
  4359. +   return 0;
  4360. +
  4361. +failed:
  4362. +   fprintf(stderr, "lz4: error reading stored compressor options from "
  4363. +       "filesystem!\n");
  4364. +
  4365. +   return -1;
  4366. +}
  4367. +
  4368. +
  4369. +/*
  4370. + * This function is a helper specifically for unsquashfs.
  4371. + * Its purpose is to check that the compression options are
  4372. + * understood by this version of LZ4.
  4373. + *
  4374. + * This is important for LZ4 because the format understood by the
  4375. + * Linux kernel may change from the already obsolete legacy format
  4376. + * currently supported.
  4377. + *
  4378. + * If this does happen, then this version of LZ4 will not be able to decode
  4379. + * the newer format.  So we need to check for this.
  4380. + *
  4381. + * This function returns 0 on sucessful checking of options, and
  4382. + *         -1 on error
  4383. + */
  4384. +static int lz4_check_options(int block_size, void *buffer, int size)
  4385. +{
  4386. +   struct lz4_comp_opts *comp_opts = buffer;
  4387. +
  4388. +   /* we expect a comp_opts structure to be present */
  4389. +   if(size < sizeof(*comp_opts))
  4390. +       goto failed;
  4391. +
  4392. +   SQUASHFS_INSWAP_COMP_OPTS(comp_opts);
  4393. +
  4394. +   /* we expect the stream format to be LZ4_LEGACY */
  4395. +   if(comp_opts->version != LZ4_LEGACY) {
  4396. +       fprintf(stderr, "lz4: unknown LZ4 version\n");
  4397. +       goto failed;
  4398. +   }
  4399. +
  4400. +   return 0;
  4401. +
  4402. +failed:
  4403. +   fprintf(stderr, "lz4: error reading stored compressor options from "
  4404. +       "filesystem!\n");
  4405. +   return -1;
  4406. +}
  4407. +
  4408. +
  4409. +void lz4_display_options(void *buffer, int size)
  4410. +{
  4411. +   struct lz4_comp_opts *comp_opts = buffer;
  4412. +
  4413. +   /* check passed comp opts struct is of the correct length */
  4414. +   if(size < sizeof(*comp_opts))
  4415. +       goto failed;
  4416. +
  4417. +   SQUASHFS_INSWAP_COMP_OPTS(comp_opts);
  4418. +
  4419. +   /* we expect the stream format to be LZ4_LEGACY */
  4420. +   if(comp_opts->version != LZ4_LEGACY) {
  4421. +       fprintf(stderr, "lz4: unknown LZ4 version\n");
  4422. +       goto failed;
  4423. +   }
  4424. +
  4425. +   /*
  4426. +    * Check compression flags, currently only LZ4_HC ("high compression")
  4427. +    * can be set.
  4428. +    */
  4429. +   if(comp_opts->flags & ~LZ4_FLAGS_MASK) {
  4430. +       fprintf(stderr, "lz4: unknown LZ4 flags\n");
  4431. +       goto failed;
  4432. +   }
  4433. +
  4434. +   if(comp_opts->flags & LZ4_HC)
  4435. +       printf("\tHigh Compression option specified (-Xhc)\n");
  4436. +
  4437. +   return;
  4438. +
  4439. +failed:
  4440. +   fprintf(stderr, "lz4: error reading stored compressor options from "
  4441. +       "filesystem!\n");
  4442. +} 
  4443. +
  4444. +
  4445. +static int lz4_compress(void *strm, void *dest, void *src,  int size,
  4446. +   int block_size, int *error)
  4447. +{
  4448. +   int res;
  4449. +
  4450. +   if(hc)
  4451. +       res = LZ4_compressHC_limitedOutput(src, dest, size, block_size);
  4452. +   else
  4453. +       res = LZ4_compress_limitedOutput(src, dest, size, block_size);
  4454. +
  4455. +   if(res == 0) {
  4456. +       /*
  4457. +        * Output buffer overflow.  Return out of buffer space
  4458. +        */
  4459. +       return 0;
  4460. +   } else if(res < 0) {
  4461. +       /*
  4462. +        * All other errors return failure, with the compressor
  4463. +        * specific error code in *error
  4464. +        */
  4465. +       *error = res;
  4466. +       return -1;
  4467. +   }
  4468. +
  4469. +   return res;
  4470. +}
  4471. +
  4472. +
  4473. +static int lz4_uncompress(void *dest, void *src, int size, int outsize,
  4474. +   int *error)
  4475. +{
  4476. +   int res = LZ4_decompress_safe(src, dest, size, outsize);
  4477. +   if(res < 0) {
  4478. +       *error = res;
  4479. +       return -1;
  4480. +   }
  4481. +
  4482. +   return res;
  4483. +}
  4484. +
  4485. +
  4486. +void lz4_usage()
  4487. +{
  4488. +   fprintf(stderr, "\t  -Xhc\n");
  4489. +   fprintf(stderr, "\t\tCompress using LZ4 High Compression\n");
  4490. +}
  4491. +
  4492. +
  4493. +struct compressor lz4_comp_ops = {
  4494. +   .compress = lz4_compress,
  4495. +   .uncompress = lz4_uncompress,
  4496. +   .options = lz4_options,
  4497. +   .dump_options = lz4_dump_options,
  4498. +   .extract_options = lz4_extract_options,
  4499. +   .check_options = lz4_check_options,
  4500. +   .display_options = lz4_display_options,
  4501. +   .usage = lz4_usage,
  4502. +   .id = LZ4_COMPRESSION,
  4503. +   .name = "lz4",
  4504. +   .supported = 1
  4505. +};
  4506. diff -Nru squashfs-tools-4.2+20130409/lz4_wrapper.h squashfs-tools-4.3+20140919/lz4_wrapper.h
  4507. --- squashfs-tools-4.2+20130409/lz4_wrapper.h   1970-01-01 01:00:00.000000000 +0100
  4508. +++ squashfs-tools-4.3+20140919/lz4_wrapper.h   2015-07-20 21:03:05.000000000 +0200
  4509. @@ -0,0 +1,61 @@
  4510. +#ifndef LZ4_WRAPPER_H
  4511. +#define LZ4_WRAPPER_H
  4512. +/*
  4513. + * Squashfs
  4514. + *
  4515. + * Copyright (c) 2013
  4516. + * Phillip Lougher <phillip@squashfs.org.uk>
  4517. + *
  4518. + * This program is free software; you can redistribute it and/or
  4519. + * modify it under the terms of the GNU General Public License
  4520. + * as published by the Free Software Foundation; either version 2,
  4521. + * or (at your option) any later version.
  4522. + *
  4523. + * This program is distributed in the hope that it will be useful,
  4524. + * but WITHOUT ANY WARRANTY; without even the implied warranty of
  4525. + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  4526. + * GNU General Public License for more details.
  4527. + *
  4528. + * You should have received a copy of the GNU General Public License
  4529. + * along with this program; if not, write to the Free Software
  4530. + * Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  4531. + *
  4532. + * lz4_wrapper.h
  4533. + *
  4534. + */
  4535. +
  4536. +#ifndef linux
  4537. +#define __BYTE_ORDER BYTE_ORDER
  4538. +#define __BIG_ENDIAN BIG_ENDIAN
  4539. +#define __LITTLE_ENDIAN LITTLE_ENDIAN
  4540. +#else
  4541. +#include <endian.h>
  4542. +#endif
  4543. +
  4544. +#if __BYTE_ORDER == __BIG_ENDIAN
  4545. +extern unsigned int inswap_le32(unsigned int);
  4546. +
  4547. +#define SQUASHFS_INSWAP_COMP_OPTS(s) { \
  4548. +   (s)->version = inswap_le32((s)->version); \
  4549. +   (s)->flags = inswap_le32((s)->flags); \
  4550. +}
  4551. +#else
  4552. +#define SQUASHFS_INSWAP_COMP_OPTS(s)
  4553. +#endif
  4554. +
  4555. +/*
  4556. + * Define the various stream formats recognised.
  4557. + * Currently omly legacy stream format is supported by the
  4558. + * kernel
  4559. + */
  4560. +#define LZ4_LEGACY 1
  4561. +#define LZ4_FLAGS_MASK 1
  4562. +
  4563. +/* Define the compression flags recognised. */
  4564. +#define LZ4_HC     1
  4565. +
  4566. +struct lz4_comp_opts {
  4567. +   int version;
  4568. +   int flags;
  4569. +};
  4570. +#endif
  4571. diff -Nru squashfs-tools-4.2+20130409/lzma_wrapper.c squashfs-tools-4.3+20140919/lzma_wrapper.c
  4572. --- squashfs-tools-4.2+20130409/lzma_wrapper.c  2013-05-09 10:39:11.000000000 +0200
  4573. +++ squashfs-tools-4.3+20140919/lzma_wrapper.c  2015-07-20 21:03:05.000000000 +0200
  4574. @@ -99,8 +99,12 @@
  4575.     res = LzmaUncompress(dest, &outlen, src + LZMA_HEADER_SIZE, &inlen, src,
  4576.         LZMA_PROPS_SIZE);
  4577.    
  4578. -   *error = res;
  4579. -   return res == SZ_OK ? outlen : -1;
  4580. +   if(res == SZ_OK)
  4581. +       return outlen;
  4582. +   else {
  4583. +       *error = res;
  4584. +       return -1;
  4585. +   }
  4586.  }
  4587.  
  4588.  
  4589. diff -Nru squashfs-tools-4.2+20130409/lzo_wrapper.c squashfs-tools-4.3+20140919/lzo_wrapper.c
  4590. --- squashfs-tools-4.2+20130409/lzo_wrapper.c   2013-05-09 10:39:11.000000000 +0200
  4591. +++ squashfs-tools-4.3+20140919/lzo_wrapper.c   2015-07-20 21:03:05.000000000 +0200
  4592. @@ -1,8 +1,5 @@
  4593.  /*
  4594. - * Copyright (c) 2010 LG Electronics
  4595. - * Chan Jeong <chan.jeong@lge.com>
  4596. - *
  4597. - * All modifications Copyright (c) 2010, 2013
  4598. + * Copyright (c) 2013, 2014
  4599.   * Phillip Lougher <phillip@squashfs.org.uk>
  4600.   *
  4601.   * This program is free software; you can redistribute it and/or
  4602. @@ -20,44 +17,319 @@
  4603.   * Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  4604.   *
  4605.   * lzo_wrapper.c
  4606. + *
  4607. + * Support for LZO compression http://www.oberhumer.com/opensource/lzo
  4608.   */
  4609.  
  4610. -#include <stdlib.h>
  4611. +#include <stdio.h>
  4612.  #include <string.h>
  4613. -
  4614. +#include <stdlib.h>
  4615.  #include <lzo/lzoconf.h>
  4616.  #include <lzo/lzo1x.h>
  4617.  
  4618.  #include "squashfs_fs.h"
  4619. +#include "lzo_wrapper.h"
  4620.  #include "compressor.h"
  4621.  
  4622. -/* worst-case expansion calculation during compression,
  4623. -   see LZO FAQ for more information */
  4624. -#define LZO_OUTPUT_BUFFER_SIZE(size)   (size + (size/16) + 64 + 3)
  4625. -
  4626. -struct lzo_stream {
  4627. -   lzo_voidp wrkmem;
  4628. -   lzo_bytep out;
  4629. +static struct lzo_algorithm lzo[] = {
  4630. +   { "lzo1x_1", LZO1X_1_MEM_COMPRESS, lzo1x_1_compress },
  4631. +   { "lzo1x_1_11", LZO1X_1_11_MEM_COMPRESS, lzo1x_1_11_compress },
  4632. +   { "lzo1x_1_12", LZO1X_1_12_MEM_COMPRESS, lzo1x_1_12_compress },
  4633. +   { "lzo1x_1_15", LZO1X_1_15_MEM_COMPRESS, lzo1x_1_15_compress },
  4634. +   { "lzo1x_999", LZO1X_999_MEM_COMPRESS, lzo1x_999_wrapper },
  4635. +   { NULL, 0, NULL }
  4636.  };
  4637.  
  4638. +/* default LZO compression algorithm and compression level */
  4639. +static int algorithm = SQUASHFS_LZO1X_999;
  4640. +static int compression_level = SQUASHFS_LZO1X_999_COMP_DEFAULT;
  4641. +
  4642. +/* user specified compression level */
  4643. +static int user_comp_level = -1;
  4644. +
  4645. +
  4646. +/*
  4647. + * This function is called by the options parsing code in mksquashfs.c
  4648. + * to parse any -X compressor option.
  4649. + *
  4650. + * This function returns:
  4651. + * >=0 (number of additional args parsed) on success
  4652. + * -1 if the option was unrecognised, or
  4653. + * -2 if the option was recognised, but otherwise bad in
  4654. + *    some way (e.g. invalid parameter)
  4655. + *
  4656. + * Note: this function sets internal compressor state, but does not
  4657. + * pass back the results of the parsing other than success/failure.
  4658. + * The lzo_dump_options() function is called later to get the options in
  4659. + * a format suitable for writing to the filesystem.
  4660. + */
  4661. +static int lzo_options(char *argv[], int argc)
  4662. +{
  4663. +   int i;
  4664. +
  4665. +   if(strcmp(argv[0], "-Xalgorithm") == 0) {
  4666. +       if(argc < 2) {
  4667. +           fprintf(stderr, "lzo: -Xalgorithm missing algorithm\n");
  4668. +           fprintf(stderr, "lzo: -Xalgorithm <algorithm>\n");
  4669. +           goto failed2;
  4670. +       }
  4671. +
  4672. +       for(i = 0; lzo[i].name; i++) {
  4673. +           if(strcmp(argv[1], lzo[i].name) == 0) {
  4674. +               algorithm = i;
  4675. +               return 1;
  4676. +           }
  4677. +       }
  4678. +
  4679. +       fprintf(stderr, "lzo: -Xalgorithm unrecognised algorithm\n");
  4680. +       goto failed2;
  4681. +   } else if(strcmp(argv[0], "-Xcompression-level") == 0) {
  4682. +       if(argc < 2) {
  4683. +           fprintf(stderr, "lzo: -Xcompression-level missing "
  4684. +               "compression level\n");
  4685. +           fprintf(stderr, "lzo: -Xcompression-level it "
  4686. +               "should be 1 >= n <= 9\n");
  4687. +           goto failed;
  4688. +       }
  4689. +
  4690. +       user_comp_level = atoi(argv[1]);
  4691. +       if(user_comp_level < 1 || user_comp_level > 9) {
  4692. +           fprintf(stderr, "lzo: -Xcompression-level invalid, it "
  4693. +               "should be 1 >= n <= 9\n");
  4694. +           goto failed;
  4695. +       }
  4696. +
  4697. +       return 1;
  4698. +   }
  4699. +
  4700. +   return -1;
  4701.  
  4702. -static int squashfs_lzo_init(void **strm, int block_size, int flags)
  4703. +failed:
  4704. +   return -2;
  4705. +
  4706. +failed2:
  4707. +   fprintf(stderr, "lzo: compression algorithm should be one of:\n");
  4708. +   for(i = 0; lzo[i].name; i++)
  4709. +       fprintf(stderr, "\t%s\n", lzo[i].name);
  4710. +   return -2;
  4711. +}
  4712. +
  4713. +
  4714. +/*
  4715. + * This function is called after all options have been parsed.
  4716. + * It is used to do post-processing on the compressor options using
  4717. + * values that were not expected to be known at option parse time.
  4718. + *
  4719. + * In this case the LZO algorithm may not be known until after the
  4720. + * compression level has been set (-Xalgorithm used after -Xcompression-level)
  4721. + *
  4722. + * This function returns 0 on successful post processing, or
  4723. + *         -1 on error
  4724. + */
  4725. +static int lzo_options_post(int block_size)
  4726. +{
  4727. +   /*
  4728. +    * Use of compression level only makes sense for
  4729. +    * LZO1X_999 algorithm
  4730. +    */
  4731. +   if(user_comp_level != -1) {
  4732. +       if(algorithm != SQUASHFS_LZO1X_999) {
  4733. +           fprintf(stderr, "lzo: -Xcompression-level not "
  4734. +               "supported by selected %s algorithm\n",
  4735. +               lzo[algorithm].name);
  4736. +           fprintf(stderr, "lzo: -Xcompression-level is only "
  4737. +               "applicable for the lzo1x_999 algorithm\n");
  4738. +           goto failed;
  4739. +       }
  4740. +       compression_level = user_comp_level;
  4741. +   }
  4742. +
  4743. +   return 0;
  4744. +
  4745. +failed:
  4746. +   return -1;
  4747. +}
  4748. +
  4749. +
  4750. +/*
  4751. + * This function is called by mksquashfs to dump the parsed
  4752. + * compressor options in a format suitable for writing to the
  4753. + * compressor options field in the filesystem (stored immediately
  4754. + * after the superblock).
  4755. + *
  4756. + * This function returns a pointer to the compression options structure
  4757. + * to be stored (and the size), or NULL if there are no compression
  4758. + * options
  4759. + *
  4760. + */
  4761. +static void *lzo_dump_options(int block_size, int *size)
  4762. +{
  4763. +   static struct lzo_comp_opts comp_opts;
  4764. +
  4765. +   /*
  4766. +    * If default compression options of SQUASHFS_LZO1X_999 and
  4767. +    * compression level of SQUASHFS_LZO1X_999_COMP_DEFAULT then
  4768. +    * don't store a compression options structure (this is compatible
  4769. +    * with the legacy implementation of LZO for Squashfs)
  4770. +    */
  4771. +   if(algorithm == SQUASHFS_LZO1X_999 &&
  4772. +           compression_level == SQUASHFS_LZO1X_999_COMP_DEFAULT)
  4773. +       return NULL;
  4774. +
  4775. +   comp_opts.algorithm = algorithm;
  4776. +   comp_opts.compression_level = algorithm == SQUASHFS_LZO1X_999 ?
  4777. +       compression_level : 0;
  4778. +
  4779. +   SQUASHFS_INSWAP_COMP_OPTS(&comp_opts);
  4780. +
  4781. +   *size = sizeof(comp_opts);
  4782. +   return &comp_opts;
  4783. +}
  4784. +
  4785. +
  4786. +/*
  4787. + * This function is a helper specifically for the append mode of
  4788. + * mksquashfs.  Its purpose is to set the internal compressor state
  4789. + * to the stored compressor options in the passed compressor options
  4790. + * structure.
  4791. + *
  4792. + * In effect this function sets up the compressor options
  4793. + * to the same state they were when the filesystem was originally
  4794. + * generated, this is to ensure on appending, the compressor uses
  4795. + * the same compression options that were used to generate the
  4796. + * original filesystem.
  4797. + *
  4798. + * Note, even if there are no compressor options, this function is still
  4799. + * called with an empty compressor structure (size == 0), to explicitly
  4800. + * set the default options, this is to ensure any user supplied
  4801. + * -X options on the appending mksquashfs command line are over-ridden
  4802. + *
  4803. + * This function returns 0 on sucessful extraction of options, and
  4804. + *         -1 on error
  4805. + */
  4806. +static int lzo_extract_options(int block_size, void *buffer, int size)
  4807. +{
  4808. +   struct lzo_comp_opts *comp_opts = buffer;
  4809. +
  4810. +   if(size == 0) {
  4811. +       /* Set default values */
  4812. +       algorithm = SQUASHFS_LZO1X_999;
  4813. +       compression_level = SQUASHFS_LZO1X_999_COMP_DEFAULT;
  4814. +       return 0;
  4815. +   }
  4816. +
  4817. +   /* we expect a comp_opts structure of sufficient size to be present */
  4818. +   if(size < sizeof(*comp_opts))
  4819. +       goto failed;
  4820. +
  4821. +   SQUASHFS_INSWAP_COMP_OPTS(comp_opts);
  4822. +
  4823. +   /* Check comp_opts structure for correctness */
  4824. +   switch(comp_opts->algorithm) {
  4825. +   case SQUASHFS_LZO1X_1:
  4826. +   case SQUASHFS_LZO1X_1_11:
  4827. +   case SQUASHFS_LZO1X_1_12:
  4828. +   case SQUASHFS_LZO1X_1_15:
  4829. +       if(comp_opts->compression_level != 0) {
  4830. +           fprintf(stderr, "lzo: bad compression level in "
  4831. +               "compression options structure\n");
  4832. +           goto failed;
  4833. +       }
  4834. +       break;
  4835. +   case SQUASHFS_LZO1X_999:
  4836. +       if(comp_opts->compression_level < 1 ||
  4837. +               comp_opts->compression_level > 9) {
  4838. +           fprintf(stderr, "lzo: bad compression level in "
  4839. +               "compression options structure\n");
  4840. +           goto failed;
  4841. +       }
  4842. +       compression_level = comp_opts->compression_level;
  4843. +       break;
  4844. +   default:
  4845. +       fprintf(stderr, "lzo: bad algorithm in compression options "
  4846. +               "structure\n");
  4847. +           goto failed;
  4848. +   }
  4849. +
  4850. +   algorithm = comp_opts->algorithm;
  4851. +
  4852. +   return 0;
  4853. +
  4854. +failed:
  4855. +   fprintf(stderr, "lzo: error reading stored compressor options from "
  4856. +       "filesystem!\n");
  4857. +
  4858. +   return -1;
  4859. +}
  4860. +
  4861. +
  4862. +void lzo_display_options(void *buffer, int size)
  4863. +{
  4864. +   struct lzo_comp_opts *comp_opts = buffer;
  4865. +
  4866. +   /* we expect a comp_opts structure of sufficient size to be present */
  4867. +   if(size < sizeof(*comp_opts))
  4868. +       goto failed;
  4869. +
  4870. +   SQUASHFS_INSWAP_COMP_OPTS(comp_opts);
  4871. +
  4872. +   /* Check comp_opts structure for correctness */
  4873. +   switch(comp_opts->algorithm) {
  4874. +   case SQUASHFS_LZO1X_1:
  4875. +   case SQUASHFS_LZO1X_1_11:
  4876. +   case SQUASHFS_LZO1X_1_12:
  4877. +   case SQUASHFS_LZO1X_1_15:
  4878. +       printf("\talgorithm %s\n", lzo[comp_opts->algorithm].name);
  4879. +       break;
  4880. +   case SQUASHFS_LZO1X_999:
  4881. +       if(comp_opts->compression_level < 1 ||
  4882. +               comp_opts->compression_level > 9) {
  4883. +           fprintf(stderr, "lzo: bad compression level in "
  4884. +               "compression options structure\n");
  4885. +           goto failed;
  4886. +       }
  4887. +       printf("\talgorithm %s\n", lzo[comp_opts->algorithm].name);
  4888. +       printf("\tcompression level %d\n",
  4889. +                       comp_opts->compression_level);
  4890. +       break;
  4891. +   default:
  4892. +       fprintf(stderr, "lzo: bad algorithm in compression options "
  4893. +               "structure\n");
  4894. +           goto failed;
  4895. +   }
  4896. +
  4897. +   return;
  4898. +
  4899. +failed:
  4900. +   fprintf(stderr, "lzo: error reading stored compressor options from "
  4901. +       "filesystem!\n");
  4902. +} 
  4903. +
  4904. +
  4905. +/*
  4906. + * This function is called by mksquashfs to initialise the
  4907. + * compressor, before compress() is called.
  4908. + *
  4909. + * This function returns 0 on success, and
  4910. + *         -1 on error
  4911. + */
  4912. +static int squashfs_lzo_init(void **strm, int block_size, int datablock)
  4913.  {
  4914.     struct lzo_stream *stream;
  4915.  
  4916. -   if((stream = *strm = malloc(sizeof(struct lzo_stream))) == NULL)
  4917. +   stream = *strm = malloc(sizeof(struct lzo_stream));
  4918. +   if(stream == NULL)
  4919.         goto failed;
  4920. -   /* work memory for compression */
  4921. -   if((stream->wrkmem = malloc(LZO1X_999_MEM_COMPRESS)) == NULL)
  4922. +
  4923. +   stream->workspace = malloc(lzo[algorithm].size);
  4924. +   if(stream->workspace == NULL)
  4925.         goto failed2;
  4926. -   /* temporal output buffer */
  4927. -   if((stream->out = malloc(LZO_OUTPUT_BUFFER_SIZE(block_size))) == NULL)
  4928. -       goto failed3;
  4929.  
  4930. -   return 0;
  4931. +   stream->buffer = malloc(LZO_MAX_EXPANSION(block_size));
  4932. +   if(stream->buffer != NULL)
  4933. +       return 0;
  4934.  
  4935. -failed3:
  4936. -   free(stream->wrkmem);
  4937. +   free(stream->workspace);
  4938.  failed2:
  4939.     free(stream);
  4940.  failed:
  4941. @@ -65,47 +337,89 @@
  4942.  }
  4943.  
  4944.  
  4945. -static int lzo_compress(void *strm, void *d, void *s, int size, int block_size,
  4946. -       int *error)
  4947. +static int lzo_compress(void *strm, void *dest, void *src,  int size,
  4948. +   int block_size, int *error)
  4949.  {
  4950.     int res;
  4951. -   lzo_uint outlen;
  4952. +   lzo_uint compsize, orig_size = size;
  4953.     struct lzo_stream *stream = strm;
  4954.  
  4955. -   res = lzo1x_999_compress(s, size, stream->out, &outlen, stream->wrkmem);
  4956. +   res = lzo[algorithm].compress(src, size, stream->buffer, &compsize,
  4957. +                           stream->workspace);
  4958.     if(res != LZO_E_OK)
  4959. -       goto failed;
  4960. -   if(outlen >= size)
  4961. -       /*
  4962. -        * Output buffer overflow. Return out of buffer space
  4963. -        */
  4964. -       return 0;
  4965. +       goto failed;   
  4966.  
  4967. -   /*
  4968. -    * Success, return the compressed size.
  4969. +   /* Successful compression, however, we need to check that
  4970. +    * the compressed size is not larger than the available
  4971. +    * buffer space.  Normally in other compressor APIs they take
  4972. +    * a destination buffer size, and overflows return an error.
  4973. +    * With LZO it lacks a destination size and so we must output
  4974. +    * to a temporary buffer large enough to accomodate any
  4975. +    * result, and explictly check here for overflow
  4976.      */
  4977. -   memcpy(d, stream->out, outlen);
  4978. -   return outlen;
  4979. +   if(compsize > block_size)
  4980. +       return 0;
  4981. +
  4982. +   res = lzo1x_optimize(stream->buffer, compsize, src, &orig_size, NULL);
  4983. +
  4984. +   if (res != LZO_E_OK || orig_size != size)
  4985. +       goto failed;
  4986. +
  4987. +   memcpy(dest, stream->buffer, compsize);
  4988. +   return compsize;
  4989.  
  4990.  failed:
  4991. -   /*
  4992. -    * All other errors return failure, with the compressor
  4993. -    * specific error code in *error
  4994. -    */
  4995. +   /* fail, compressor specific error code returned in error */
  4996.     *error = res;
  4997.     return -1;
  4998.  }
  4999.  
  5000.  
  5001. -static int lzo_uncompress(void *d, void *s, int size, int outsize, int *error)
  5002. +static int lzo_uncompress(void *dest, void *src, int size, int outsize,
  5003. +   int *error)
  5004.  {
  5005.     int res;
  5006. -   lzo_uint bytes = outsize;
  5007. +   lzo_uint outlen = outsize;
  5008.  
  5009. -   res = lzo1x_decompress_safe(s, size, d, &bytes, NULL);
  5010. +   res = lzo1x_decompress_safe(src, size, dest, &outlen, NULL);
  5011. +   if(res != LZO_E_OK) {
  5012. +       *error = res;
  5013. +       return -1;
  5014. +   }
  5015.  
  5016. -   *error = res;
  5017. -   return res == LZO_E_OK ? bytes : -1;
  5018. +   return outlen;
  5019. +}
  5020. +
  5021. +
  5022. +void lzo_usage()
  5023. +{
  5024. +   int i;
  5025. +
  5026. +   fprintf(stderr, "\t  -Xalgorithm <algorithm>\n");
  5027. +   fprintf(stderr, "\t\tWhere <algorithm> is one of:\n");
  5028. +
  5029. +   for(i = 0; lzo[i].name; i++)
  5030. +       fprintf(stderr, "\t\t\t%s%s\n", lzo[i].name,
  5031. +               i == SQUASHFS_LZO1X_999 ? " (default)" : "");
  5032. +
  5033. +   fprintf(stderr, "\t  -Xcompression-level <compression-level>\n");
  5034. +   fprintf(stderr, "\t\t<compression-level> should be 1 .. 9 (default "
  5035. +       "%d)\n", SQUASHFS_LZO1X_999_COMP_DEFAULT);
  5036. +   fprintf(stderr, "\t\tOnly applies to lzo1x_999 algorithm\n");
  5037. +}
  5038. +
  5039. +
  5040. +/*
  5041. + * Helper function for lzo1x_999 compression algorithm.
  5042. + * All other lzo1x_xxx compressors do not take a compression level,
  5043. + * so we need to wrap lzo1x_999 to pass the compression level which
  5044. + * is applicable to it
  5045. + */
  5046. +int lzo1x_999_wrapper(const lzo_bytep src, lzo_uint src_len, lzo_bytep dst,
  5047. +   lzo_uintp compsize, lzo_voidp workspace)
  5048. +{
  5049. +   return lzo1x_999_compress_level(src, src_len, dst, compsize,
  5050. +       workspace, NULL, 0, 0, compression_level);
  5051.  }
  5052.  
  5053.  
  5054. @@ -113,10 +427,13 @@
  5055.     .init = squashfs_lzo_init,
  5056.     .compress = lzo_compress,
  5057.     .uncompress = lzo_uncompress,
  5058. -   .options = NULL,
  5059. -   .usage = NULL,
  5060. +   .options = lzo_options,
  5061. +   .options_post = lzo_options_post,
  5062. +   .dump_options = lzo_dump_options,
  5063. +   .extract_options = lzo_extract_options,
  5064. +   .display_options = lzo_display_options,
  5065. +   .usage = lzo_usage,
  5066.     .id = LZO_COMPRESSION,
  5067.     .name = "lzo",
  5068.     .supported = 1
  5069.  };
  5070. -
  5071. diff -Nru squashfs-tools-4.2+20130409/lzo_wrapper.h squashfs-tools-4.3+20140919/lzo_wrapper.h
  5072. --- squashfs-tools-4.2+20130409/lzo_wrapper.h   1970-01-01 01:00:00.000000000 +0100
  5073. +++ squashfs-tools-4.3+20140919/lzo_wrapper.h   2015-07-20 21:03:05.000000000 +0200
  5074. @@ -0,0 +1,78 @@
  5075. +#ifndef LZO_WRAPPER_H
  5076. +#define LZO_WRAPPER_H
  5077. +/*
  5078. + * Squashfs
  5079. + *
  5080. + * Copyright (c) 2013
  5081. + * Phillip Lougher <phillip@squashfs.org.uk>
  5082. + *
  5083. + * This program is free software; you can redistribute it and/or
  5084. + * modify it under the terms of the GNU General Public License
  5085. + * as published by the Free Software Foundation; either version 2,
  5086. + * or (at your option) any later version.
  5087. + *
  5088. + * This program is distributed in the hope that it will be useful,
  5089. + * but WITHOUT ANY WARRANTY; without even the implied warranty of
  5090. + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  5091. + * GNU General Public License for more details.
  5092. + *
  5093. + * You should have received a copy of the GNU General Public License
  5094. + * along with this program; if not, write to the Free Software
  5095. + * Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  5096. + *
  5097. + * lzo_wrapper.h
  5098. + *
  5099. + */
  5100. +
  5101. +#ifndef linux
  5102. +#define __BYTE_ORDER BYTE_ORDER
  5103. +#define __BIG_ENDIAN BIG_ENDIAN
  5104. +#define __LITTLE_ENDIAN LITTLE_ENDIAN
  5105. +#else
  5106. +#include <endian.h>
  5107. +#endif
  5108. +
  5109. +#if __BYTE_ORDER == __BIG_ENDIAN
  5110. +extern unsigned int inswap_le32(unsigned int);
  5111. +
  5112. +#define SQUASHFS_INSWAP_COMP_OPTS(s) { \
  5113. +   (s)->algorithm = inswap_le32((s)->algorithm); \
  5114. +   (s)->compression_level = inswap_le32((s)->compression_level); \
  5115. +}
  5116. +#else
  5117. +#define SQUASHFS_INSWAP_COMP_OPTS(s)
  5118. +#endif
  5119. +
  5120. +/* Define the compression flags recognised. */
  5121. +#define SQUASHFS_LZO1X_1   0
  5122. +#define SQUASHFS_LZO1X_1_11    1
  5123. +#define SQUASHFS_LZO1X_1_12    2
  5124. +#define SQUASHFS_LZO1X_1_15    3
  5125. +#define SQUASHFS_LZO1X_999 4
  5126. +
  5127. +/* Default compression level used by SQUASHFS_LZO1X_999 */
  5128. +#define SQUASHFS_LZO1X_999_COMP_DEFAULT    8
  5129. +
  5130. +struct lzo_comp_opts {
  5131. +   int algorithm;
  5132. +   int compression_level;
  5133. +};
  5134. +
  5135. +struct lzo_algorithm {
  5136. +   char *name;
  5137. +   int size;
  5138. +   int (*compress) (const lzo_bytep, lzo_uint, lzo_bytep, lzo_uintp,
  5139. +       lzo_voidp);
  5140. +};
  5141. +
  5142. +struct lzo_stream {
  5143. +   void *workspace;
  5144. +   void *buffer;
  5145. +};
  5146. +
  5147. +#define LZO_MAX_EXPANSION(size)    (size + (size / 16) + 64 + 3)
  5148. +
  5149. +int lzo1x_999_wrapper(const lzo_bytep, lzo_uint, lzo_bytep, lzo_uintp,
  5150. +       lzo_voidp);
  5151. +
  5152. +#endif
  5153. diff -Nru squashfs-tools-4.2+20130409/Makefile squashfs-tools-4.3+20140919/Makefile
  5154. --- squashfs-tools-4.2+20130409/Makefile    2013-05-09 10:39:11.000000000 +0200
  5155. +++ squashfs-tools-4.3+20140919/Makefile    2015-07-20 21:03:05.000000000 +0200
  5156. @@ -40,6 +40,20 @@
  5157.  #LZO_SUPPORT = 1
  5158.  #LZO_DIR = /usr/local
  5159.  
  5160. +
  5161. +########### Building LZ4 support #############
  5162. +#
  5163. +# Yann Collet's LZ4 tools are supported
  5164. +# LZ4 homepage: http://fastcompression.blogspot.com/p/lz4.html
  5165. +# LZ4 source repository: http://code.google.com/p/lz4
  5166. +#
  5167. +# To build configure the tools using cmake to build shared libraries,
  5168. +# install and uncomment
  5169. +# the LZ4_SUPPORT line below.
  5170. +#
  5171. +#LZ4_SUPPORT = 1
  5172. +
  5173. +
  5174.  ########### Building LZMA support #############
  5175.  #
  5176.  # LZMA1 compression.
  5177. @@ -96,12 +110,12 @@
  5178.  INCLUDEDIR = -I.
  5179.  INSTALL_DIR = /usr/local/bin
  5180.  
  5181. -MKSQUASHFS_OBJS = mksquashfs.o read_fs.o sort.o swap.o pseudo.o compressor.o \
  5182. -   action.o progressbar.o read_file.o info.o restore.o \
  5183. +MKSQUASHFS_OBJS = mksquashfs.o read_fs.o action.o swap.o pseudo.o compressor.o \
  5184. +   sort.o progressbar.o read_file.o info.o restore.o process_fragments.o \
  5185.     caches-queues-lists.o
  5186.  
  5187.  UNSQUASHFS_OBJS = unsquashfs.o unsquash-1.o unsquash-2.o unsquash-3.o \
  5188. -   unsquash-4.o swap.o compressor.o
  5189. +   unsquash-4.o swap.o compressor.o unsquashfs_info.o
  5190.  
  5191.  CFLAGS ?= -O2
  5192.  CFLAGS += $(EXTRA_CFLAGS) $(INCLUDEDIR) -D_FILE_OFFSET_BITS=64 \
  5193. @@ -155,6 +169,14 @@
  5194.  COMPRESSORS += lzo
  5195.  endif
  5196.  
  5197. +ifeq ($(LZ4_SUPPORT),1)
  5198. +CFLAGS += -DLZ4_SUPPORT
  5199. +MKSQUASHFS_OBJS += lz4_wrapper.o
  5200. +UNSQUASHFS_OBJS += lz4_wrapper.o
  5201. +LIBS += -llz4
  5202. +COMPRESSORS += lz4
  5203. +endif
  5204. +
  5205.  ifeq ($(XATTR_SUPPORT),1)
  5206.  ifeq ($(XATTR_DEFAULT),1)
  5207.  CFLAGS += -DXATTR_SUPPORT -DXATTR_DEFAULT
  5208. @@ -187,14 +209,16 @@
  5209.  # At least one compressor must have been selected
  5210.  #
  5211.  ifndef COMPRESSORS
  5212. -$(error "No compressor selected! Select one or more of GZIP, LZMA, XZ or LZO!")
  5213. +$(error "No compressor selected! Select one or more of GZIP, LZMA, XZ, LZO or \
  5214. +   LZ4!")
  5215.  endif
  5216.  
  5217.  #
  5218.  # COMP_DEFAULT must be a selected compressor
  5219.  #
  5220.  ifeq (, $(findstring $(COMP_DEFAULT), $(COMPRESSORS)))
  5221. -$(error "COMP_DEFAULT isn't selected to be built!")
  5222. +$(error "COMP_DEFAULT is set to ${COMP_DEFAULT}, which  isn't selected to be \
  5223. +   built!")
  5224.  endif
  5225.  
  5226.  .PHONY: all
  5227. @@ -203,45 +227,53 @@
  5228.  mksquashfs: $(MKSQUASHFS_OBJS)
  5229.     $(CC) $(LDFLAGS) $(EXTRA_LDFLAGS) $(MKSQUASHFS_OBJS) $(LIBS) -o $@
  5230.  
  5231. -mksquashfs.o: mksquashfs.c squashfs_fs.h mksquashfs.h sort.h squashfs_swap.h \
  5232. -   xattr.h pseudo.h compressor.h action.h progressbar.h
  5233. +mksquashfs.o: Makefile mksquashfs.c squashfs_fs.h squashfs_swap.h mksquashfs.h \
  5234. +   sort.h pseudo.h compressor.h xattr.h action.h error.h progressbar.h \
  5235. +   info.h caches-queues-lists.h read_fs.h restore.h process_fragments.h
  5236.  
  5237. -read_fs.o: read_fs.c squashfs_fs.h read_fs.h squashfs_swap.h compressor.h \
  5238. -   xattr.h
  5239. +read_fs.o: read_fs.c squashfs_fs.h squashfs_swap.h compressor.h xattr.h \
  5240. +   error.h mksquashfs.h
  5241.  
  5242. -sort.o: sort.c squashfs_fs.h sort.h mksquashfs.h
  5243. +sort.o: sort.c squashfs_fs.h mksquashfs.h sort.h error.h progressbar.h
  5244.  
  5245.  swap.o: swap.c
  5246.  
  5247. -pseudo.o: pseudo.c pseudo.h
  5248. +pseudo.o: pseudo.c pseudo.h error.h progressbar.h
  5249.  
  5250. -compressor.o: compressor.c compressor.h squashfs_fs.h
  5251. +compressor.o: Makefile compressor.c compressor.h squashfs_fs.h
  5252.  
  5253. -xattr.o: xattr.c xattr.h squashfs_fs.h squashfs_swap.h mksquashfs.h
  5254. +xattr.o: xattr.c squashfs_fs.h squashfs_swap.h mksquashfs.h xattr.h error.h \
  5255. +   progressbar.h
  5256.  
  5257. -read_xattrs.o: read_xattrs.c xattr.h squashfs_fs.h squashfs_swap.h read_fs.h
  5258. +read_xattrs.o: read_xattrs.c squashfs_fs.h squashfs_swap.h xattr.h error.h
  5259.  
  5260. -action.o: action.h squashfs_fs.h mksquashfs.h
  5261. +action.o: action.c squashfs_fs.h mksquashfs.h action.h error.h
  5262.  
  5263. -progressbar.o: progressbar.c
  5264. +progressbar.o: progressbar.c error.h
  5265.  
  5266. -read_file.o: read_file.c
  5267. +read_file.o: read_file.c error.h
  5268.  
  5269. -info.o: info.c
  5270. +info.o: info.c squashfs_fs.h mksquashfs.h error.h progressbar.h \
  5271. +   caches-queues-lists.h
  5272.  
  5273. -restore.o: restore.c
  5274. +restore.o: restore.c caches-queues-lists.h squashfs_fs.h mksquashfs.h error.h \
  5275. +   progressbar.h info.h
  5276.  
  5277. -caches-queues-lists.o: caches-queues-lists.c
  5278. +process_fragments.o: process_fragments.c process_fragments.h
  5279.  
  5280. -gzip_wrapper.o: gzip_wrapper.c compressor.h squashfs_fs.h
  5281. +caches-queues-lists.o: caches-queues-lists.c error.h caches-queues-lists.h
  5282. +
  5283. +gzip_wrapper.o: gzip_wrapper.c squashfs_fs.h gzip_wrapper.h compressor.h
  5284.  
  5285.  lzma_wrapper.o: lzma_wrapper.c compressor.h squashfs_fs.h
  5286.  
  5287.  lzma_xz_wrapper.o: lzma_xz_wrapper.c compressor.h squashfs_fs.h
  5288.  
  5289. -lzo_wrapper.o: lzo_wrapper.c compressor.h squashfs_fs.h
  5290. +lzo_wrapper.o: lzo_wrapper.c squashfs_fs.h lzo_wrapper.h compressor.h
  5291. +
  5292. +lz4_wrapper.o: lz4_wrapper.c squashfs_fs.h lz4_wrapper.h compressor.h
  5293.  
  5294. -xz_wrapper.o: xz_wrapper.c compressor.h squashfs_fs.h
  5295. +xz_wrapper.o: xz_wrapper.c squashfs_fs.h xz_wrapper.h compressor.h
  5296.  
  5297.  unsquashfs: $(UNSQUASHFS_OBJS)
  5298.     $(CC) $(LDFLAGS) $(EXTRA_LDFLAGS) $(UNSQUASHFS_OBJS) $(LIBS) -o $@
  5299. @@ -260,6 +292,7 @@
  5300.  
  5301.  unsquashfs_xattr.o: unsquashfs_xattr.c unsquashfs.h squashfs_fs.h xattr.h
  5302.  
  5303. +unsquashfs_info.o: unsquashfs.h squashfs_fs.h
  5304.  
  5305.  .PHONY: clean
  5306.  clean:
  5307. diff -Nru squashfs-tools-4.2+20130409/mksquashfs.c squashfs-tools-4.3+20140919/mksquashfs.c
  5308. --- squashfs-tools-4.2+20130409/mksquashfs.c    2013-05-09 10:39:11.000000000 +0200
  5309. +++ squashfs-tools-4.3+20140919/mksquashfs.c    2015-07-20 21:03:05.000000000 +0200
  5310. @@ -3,7 +3,7 @@
  5311.   * filesystem.
  5312.   *
  5313.   * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011,
  5314. - * 2012, 2013
  5315. + * 2012, 2013, 2014
  5316.   * Phillip Lougher <phillip@squashfs.org.uk>
  5317.   *
  5318.   * This program is free software; you can redistribute it and/or
  5319. @@ -73,6 +73,9 @@
  5320.  #include "progressbar.h"
  5321.  #include "info.h"
  5322.  #include "caches-queues-lists.h"
  5323. +#include "read_fs.h"
  5324. +#include "restore.h"
  5325. +#include "process_fragments.h"
  5326.  
  5327.  int delete = FALSE;
  5328.  int fd;
  5329. @@ -80,17 +83,23 @@
  5330.  
  5331.  /* filesystem flags for building */
  5332.  int comp_opts = FALSE;
  5333. -int no_xattrs = XATTR_DEF, noX = 0;
  5334. -int duplicate_checking = 1, noF = 0, no_fragments = 0, always_use_fragments = 0;
  5335. -int noI = 0, noD = 0;
  5336. +int no_xattrs = XATTR_DEF;
  5337. +int noX = FALSE;
  5338. +int duplicate_checking = TRUE;
  5339. +int noF = FALSE;
  5340. +int no_fragments = FALSE;
  5341. +int always_use_fragments = FALSE;
  5342. +int noI = FALSE;
  5343. +int noD = FALSE;
  5344.  int silent = TRUE;
  5345. -long long global_uid = -1, global_gid = -1;
  5346.  int exportable = TRUE;
  5347. -int progress = TRUE;
  5348.  int sparse_files = TRUE;
  5349.  int old_exclude = TRUE;
  5350.  int use_regex = FALSE;
  5351.  int nopad = FALSE;
  5352. +int exit_on_error = FALSE;
  5353. +
  5354. +long long global_uid = -1, global_gid = -1;
  5355.  
  5356.  /* superblock attributes */
  5357.  int block_size = SQUASHFS_FILE_SIZE, block_log;
  5358. @@ -186,21 +195,15 @@
  5359.  struct pathname *stickypath = NULL;
  5360.  int excluded(char *name, struct pathnames *paths, struct pathnames **new);
  5361.  
  5362. -/* fragment block data structures */
  5363.  int fragments = 0;
  5364.  
  5365. -struct fragment {
  5366. -   unsigned int        index;
  5367. -   int         offset;
  5368. -   int         size;
  5369. -};
  5370. -
  5371.  #define FRAG_SIZE 32768
  5372. -#define FRAG_INDEX (1LL << 32)
  5373.  
  5374.  struct squashfs_fragment_entry *fragment_table = NULL;
  5375.  int fragments_outstanding = 0;
  5376.  
  5377. +int fragments_locked = FALSE;
  5378. +
  5379.  /* current inode number for directories and non directories */
  5380.  unsigned int inode_no = 1;
  5381.  unsigned int root_inode_number = 0;
  5382. @@ -217,21 +220,9 @@
  5383.  };
  5384.  struct old_root_entry_info *old_root_entry;
  5385.  
  5386. -/* in memory file info */
  5387. -struct file_info {
  5388. -   long long       file_size;
  5389. -   long long       bytes;
  5390. -   unsigned short      checksum;
  5391. -   unsigned short      fragment_checksum;
  5392. -   long long       start;
  5393. -   unsigned int        *block_list;
  5394. -   struct file_info    *next;
  5395. -   struct fragment     *fragment;
  5396. -   char            checksum_flag;
  5397. -};
  5398. -
  5399.  /* restore orignal filesystem state if appending to existing filesystem is
  5400.   * cancelled */
  5401. +int appending = FALSE;
  5402.  char *sdata_cache, *sdirectory_data_cache, *sdirectory_compressed;
  5403.  
  5404.  long long sbytes, stotal_bytes;
  5405. @@ -245,10 +236,10 @@
  5406.  int threads;
  5407.  
  5408.  /* flag whether destination file is a block device */
  5409. -int block_device = 0;
  5410. +int block_device = FALSE;
  5411.  
  5412.  /* flag indicating whether files are sorted using sort list(s) */
  5413. -int sorted = 0;
  5414. +int sorted = FALSE;
  5415.  
  5416.  /* save destination file name for deleting on error */
  5417.  char *destination_file = NULL;
  5418. @@ -257,93 +248,67 @@
  5419.  char *recovery_file = NULL;
  5420.  int recover = TRUE;
  5421.  
  5422. -/* in memory uid tables */
  5423. -#define ID_ENTRIES 256
  5424. -#define ID_HASH(id) (id & (ID_ENTRIES - 1))
  5425. -#define ISA_UID 1
  5426. -#define ISA_GID 2
  5427. -struct id {
  5428. -   unsigned int id;
  5429. -   int index;
  5430. -   char    flags;
  5431. -   struct id *next;
  5432. -};
  5433.  struct id *id_hash_table[ID_ENTRIES];
  5434.  struct id *id_table[SQUASHFS_IDS], *sid_table[SQUASHFS_IDS];
  5435.  unsigned int uid_count = 0, guid_count = 0;
  5436.  unsigned int sid_count = 0, suid_count = 0, sguid_count = 0;
  5437.  
  5438. -struct cache *reader_buffer, *writer_buffer, *fragment_buffer;
  5439. -struct queue *to_reader, *from_reader, *to_writer, *from_writer, *from_deflate,
  5440. -   *to_frag;
  5441. -pthread_t *thread, *deflator_thread, *frag_deflator_thread;
  5442. +struct cache *reader_buffer, *fragment_buffer, *reserve_cache;
  5443. +struct cache *bwriter_buffer, *fwriter_buffer;
  5444. +struct queue *to_reader, *to_deflate, *to_writer, *from_writer,
  5445. +   *to_frag, *locked_fragment, *to_process_frag;
  5446. +struct seq_queue *to_main;
  5447. +pthread_t reader_thread, writer_thread, main_thread;
  5448. +pthread_t *deflator_thread, *frag_deflator_thread, *frag_thread;
  5449.  pthread_t *restore_thread = NULL;
  5450. -pthread_mutex_t    fragment_mutex;
  5451. -pthread_cond_t fragment_waiting;
  5452. -pthread_mutex_t    pos_mutex;
  5453. +pthread_mutex_t    fragment_mutex = PTHREAD_MUTEX_INITIALIZER;
  5454. +pthread_mutex_t    pos_mutex = PTHREAD_MUTEX_INITIALIZER;
  5455. +pthread_mutex_t    dup_mutex = PTHREAD_MUTEX_INITIALIZER;
  5456.  
  5457.  /* user options that control parallelisation */
  5458.  int processors = -1;
  5459. -/* default size of output buffer in Mbytes */
  5460. -#define WRITER_BUFFER_DEFAULT 512
  5461. -/* default size of input buffer in Mbytes */
  5462. -#define READER_BUFFER_DEFAULT 64
  5463. -/* default size of fragment buffer in Mbytes */
  5464. -#define FRAGMENT_BUFFER_DEFAULT 64
  5465. -int writer_buffer_size;
  5466. +int bwriter_size;
  5467.  
  5468.  /* compression operations */
  5469. -static struct compressor *comp;
  5470. -int compressor_opts_parsed = 0;
  5471. +struct compressor *comp = NULL;
  5472. +int compressor_opt_parsed = FALSE;
  5473.  void *stream = NULL;
  5474.  
  5475.  /* xattr stats */
  5476.  unsigned int xattr_bytes = 0, total_xattr_bytes = 0;
  5477.  
  5478. -char *read_from_disk(long long start, unsigned int avail_bytes);
  5479. +/* fragment to file mapping used when appending */
  5480. +int append_fragments = 0;
  5481. +struct append_file **file_mapping;
  5482. +
  5483. +/* root of the in-core directory structure */
  5484. +struct dir_info *root_dir;
  5485. +
  5486. +static char *read_from_disk(long long start, unsigned int avail_bytes);
  5487.  void add_old_root_entry(char *name, squashfs_inode inode, int inode_number,
  5488.     int type);
  5489. -extern struct compressor  *read_super(int fd, struct squashfs_super_block *sBlk,
  5490. -   char *source);
  5491. -extern long long read_filesystem(char *root_name, int fd,
  5492. -   struct squashfs_super_block *sBlk, char **cinode_table, char **data_cache,
  5493. -   char **cdirectory_table, char **directory_data_cache,
  5494. -   unsigned int *last_directory_block, unsigned int *inode_dir_offset,
  5495. -   unsigned int *inode_dir_file_size, unsigned int *root_inode_size,
  5496. -   unsigned int *inode_dir_start_block, int *file_count, int *sym_count,
  5497. -   int *dev_count, int *dir_count, int *fifo_count, int *sock_count,
  5498. -   long long *uncompressed_file, unsigned int *uncompressed_inode,
  5499. -   unsigned int *uncompressed_directory,
  5500. -   unsigned int *inode_dir_inode_number,
  5501. -   unsigned int *inode_dir_parent_inode,
  5502. -   void (push_directory_entry)(char *, squashfs_inode, int, int),
  5503. -   struct squashfs_fragment_entry **fragment_table,
  5504. -   squashfs_inode **inode_lookup_table);
  5505. -extern int read_sort_file(char *filename, int source, char *source_path[]);
  5506. -extern void sort_files_and_write(struct dir_info *dir);
  5507.  struct file_info *duplicate(long long file_size, long long bytes,
  5508.     unsigned int **block_list, long long *start, struct fragment **fragment,
  5509.     struct file_buffer *file_buffer, int blocks, unsigned short checksum,
  5510. -   unsigned short fragment_checksum, int checksum_flag);
  5511. +   int checksum_flag);
  5512.  struct dir_info *dir_scan1(char *, char *, struct pathnames *,
  5513.     struct dir_ent *(_readdir)(struct dir_info *), int);
  5514.  void dir_scan2(struct dir_info *dir, struct pseudo *pseudo);
  5515. -void dir_scan3(struct dir_info *root, struct dir_info *dir);
  5516. +void dir_scan3(struct dir_info *dir);
  5517.  void dir_scan4(struct dir_info *dir);
  5518.  void dir_scan5(struct dir_info *dir);
  5519. -void dir_scan6(squashfs_inode *inode, struct dir_info *dir_info);
  5520. +void dir_scan6(struct dir_info *dir);
  5521. +void dir_scan7(squashfs_inode *inode, struct dir_info *dir_info);
  5522.  struct file_info *add_non_dup(long long file_size, long long bytes,
  5523.     unsigned int *block_list, long long start, struct fragment *fragment,
  5524.     unsigned short checksum, unsigned short fragment_checksum,
  5525. -   int checksum_flag);
  5526. -extern void generate_file_priorities(struct dir_info *dir, int priority,
  5527. -   struct stat *buf);
  5528. -extern struct priority_entry *priority_list[65536];
  5529. +   int checksum_flag, int checksum_frag_flag);
  5530.  long long generic_write_table(int, void *, int, void *, int);
  5531.  void restorefs();
  5532.  struct dir_info *scan1_opendir(char *pathname, char *subpath, int depth);
  5533. -extern pthread_t *init_restore_thread(pthread_t);
  5534.  void write_filesystem_tables(struct squashfs_super_block *sBlk, int nopad);
  5535. +unsigned short get_checksum_mem(char *buff, int bytes);
  5536. +void check_usable_phys_mem(int total_mem);
  5537.  
  5538.  
  5539.  void prep_exit()
  5540. @@ -360,12 +325,14 @@
  5541.             exit(1);
  5542.         } else {
  5543.             /* signal the restore thread to restore */
  5544. -           kill(getpid(), SIGUSR1);
  5545. +           pthread_kill(*restore_thread, SIGUSR1);
  5546.             pthread_exit(NULL);
  5547.         }
  5548. -   }
  5549. -   if(delete && destination_file && !block_device)
  5550. -       unlink(destination_file);
  5551. +   } else if(delete) {
  5552. +       if(destination_file && !block_device)
  5553. +           unlink(destination_file);
  5554. +   } else if(recovery_file)
  5555. +       unlink(recovery_file);
  5556.  }
  5557.  
  5558.  
  5559. @@ -387,22 +354,20 @@
  5560.  }
  5561.  
  5562.  
  5563. +int multiply_overflowll(long long a, int multiplier)
  5564. +{
  5565. +   return (LLONG_MAX / multiplier) < a;
  5566. +}
  5567. +
  5568. +
  5569.  #define MKINODE(A) ((squashfs_inode)(((squashfs_inode) inode_bytes << 16) \
  5570.             + (((char *)A) - data_cache)))
  5571.  
  5572.  
  5573.  void restorefs()
  5574.  {
  5575. -   int i;
  5576. -
  5577.     ERROR("Exiting - restoring original filesystem!\n\n");
  5578.  
  5579. -   for(i = 0; i < 2 + processors * 2; i++)
  5580. -       pthread_cancel(thread[i]);
  5581. -   for(i = 0; i < 2 + processors * 2; i++)
  5582. -       pthread_join(thread[i], NULL);
  5583. -
  5584. -   TRACE("All threads in signal handler\n");
  5585.     bytes = sbytes;
  5586.     memcpy(data_cache, sdata_cache, cache_bytes = scache_bytes);
  5587.     memcpy(directory_data_cache, sdirectory_data_cache,
  5588. @@ -555,15 +520,11 @@
  5589.         ERROR("read_fs_bytes: Lseek on destination failed because %s, "
  5590.             "offset=0x%llx\n", strerror(errno), off);
  5591.         res = 0;
  5592. -       goto mutex_unlock;
  5593. -   }
  5594. -
  5595. -   if(read_bytes(fd, buff, bytes) < bytes) {
  5596. +   } else if(read_bytes(fd, buff, bytes) < bytes) {
  5597.         ERROR("Read on destination failed\n");
  5598.         res = 0;
  5599.     }
  5600.  
  5601. -mutex_unlock:
  5602.     pthread_cleanup_pop(1);
  5603.     return res;
  5604.  }
  5605. @@ -1076,56 +1037,28 @@
  5606.     }
  5607.     else if(type == SQUASHFS_SYMLINK_TYPE) {
  5608.         struct squashfs_symlink_inode_header *symlink = &inode_header.symlink;
  5609. -       int byte;
  5610. -       char buff[65536]; /* overflow safe */
  5611. +       int byte = strlen(dir_ent->inode->symlink);
  5612.         size_t off = offsetof(struct squashfs_symlink_inode_header, symlink);
  5613.  
  5614. -       byte = readlink(filename, buff, 65536);
  5615. -       if(byte == -1) {
  5616. -           ERROR("Failed to read symlink %s, creating empty "
  5617. -               "symlink\n", filename);
  5618. -           byte = 0;
  5619. -       }
  5620. -
  5621. -       if(byte == 65536) {
  5622. -           ERROR("Symlink %s is greater than 65536 bytes! "
  5623. -               "Creating empty symlink\n", filename);
  5624. -           byte = 0;
  5625. -       }
  5626. -
  5627.         inode = get_inode(sizeof(*symlink) + byte);
  5628.         symlink->nlink = nlink;
  5629.         symlink->symlink_size = byte;
  5630.         SQUASHFS_SWAP_SYMLINK_INODE_HEADER(symlink, inode);
  5631. -       strncpy(inode + off, buff, byte);
  5632. +       strncpy(inode + off, dir_ent->inode->symlink, byte);
  5633.         TRACE("Symbolic link inode, symlink_size %d, nlink %d\n", byte,
  5634.             nlink);
  5635.     }
  5636.     else if(type == SQUASHFS_LSYMLINK_TYPE) {
  5637.         struct squashfs_symlink_inode_header *symlink = &inode_header.symlink;
  5638. -       int byte;
  5639. -       char buff[65536]; /* overflow safe */
  5640. +       int byte = strlen(dir_ent->inode->symlink);
  5641.         size_t off = offsetof(struct squashfs_symlink_inode_header, symlink);
  5642.  
  5643. -       byte = readlink(filename, buff, 65536);
  5644. -       if(byte == -1) {
  5645. -           ERROR("Failed to read symlink %s, creating empty "
  5646. -               "symlink\n", filename);
  5647. -           byte = 0;
  5648. -       }
  5649. -
  5650. -       if(byte == 65536) {
  5651. -           ERROR("Symlink %s is greater than 65536 bytes! "
  5652. -               "Creating empty symlink\n", filename);
  5653. -           byte = 0;
  5654. -       }
  5655. -
  5656.         inode = get_inode(sizeof(*symlink) + byte +
  5657.                         sizeof(unsigned int));
  5658.         symlink->nlink = nlink;
  5659.         symlink->symlink_size = byte;
  5660.         SQUASHFS_SWAP_SYMLINK_INODE_HEADER(symlink, inode);
  5661. -       strncpy(inode + off, buff, byte);
  5662. +       strncpy(inode + off, dir_ent->inode->symlink, byte);
  5663.         SQUASHFS_SWAP_INTS(&xattr, inode + off + byte, 1);
  5664.         TRACE("Symbolic link inode, symlink_size %d, nlink %d\n", byte,
  5665.             nlink);
  5666. @@ -1363,28 +1296,82 @@
  5667.  }
  5668.  
  5669.  
  5670. -struct file_buffer *get_fragment(struct fragment *fragment)
  5671. +static struct file_buffer *get_fragment(struct fragment *fragment)
  5672.  {
  5673.     struct squashfs_fragment_entry *disk_fragment;
  5674. -   int res, size;
  5675. -   long long start_block;
  5676.     struct file_buffer *buffer, *compressed_buffer;
  5677. +   long long start_block;
  5678. +   int res, size, index = fragment->index;
  5679. +   char locked;
  5680. +
  5681. +   /*
  5682. +    * Lookup fragment block in cache.
  5683. +    * If the fragment block doesn't exist, then get the compressed version
  5684. +    * from the writer cache or off disk, and decompress it.
  5685. +    *
  5686. +    * This routine has two things which complicate the code:
  5687. +    *
  5688. +    *  1. Multiple threads can simultaneously lookup/create the
  5689. +    *     same buffer.  This means a buffer needs to be "locked"
  5690. +    *     when it is being filled in, to prevent other threads from
  5691. +    *     using it when it is not ready.  This is because we now do
  5692. +    *     fragment duplicate checking in parallel.
  5693. +    *  2. We have two caches which need to be checked for the
  5694. +    *     presence of fragment blocks: the normal fragment cache
  5695. +    *     and a "reserve" cache.  The reserve cache is used to
  5696. +    *     prevent an unnecessary pipeline stall when the fragment cache
  5697. +    *     is full of fragments waiting to be compressed.
  5698. +    */
  5699.  
  5700.     if(fragment->index == SQUASHFS_INVALID_FRAG)
  5701.         return NULL;
  5702.  
  5703. -   buffer = cache_lookup(fragment_buffer, fragment->index);
  5704. -   if(buffer)
  5705. -       return buffer;
  5706. +   pthread_cleanup_push((void *) pthread_mutex_unlock, &dup_mutex);
  5707. +   pthread_mutex_lock(&dup_mutex);
  5708.  
  5709. -   compressed_buffer = cache_lookup(writer_buffer, fragment->index +
  5710. -       FRAG_INDEX);
  5711. +again:
  5712. +   buffer = cache_lookup_nowait(fragment_buffer, index, &locked);
  5713. +   if(buffer) {
  5714. +       pthread_mutex_unlock(&dup_mutex);
  5715. +       if(locked)
  5716. +           /* got a buffer being filled in.  Wait for it */
  5717. +           cache_wait_unlock(buffer);
  5718. +       goto finished;
  5719. +   }
  5720. +
  5721. +   /* not in fragment cache, is it in the reserve cache? */
  5722. +   buffer = cache_lookup_nowait(reserve_cache, index, &locked);
  5723. +   if(buffer) {
  5724. +       pthread_mutex_unlock(&dup_mutex);
  5725. +       if(locked)
  5726. +           /* got a buffer being filled in.  Wait for it */
  5727. +           cache_wait_unlock(buffer);
  5728. +       goto finished;
  5729. +   }
  5730. +
  5731. +   /* in neither cache, try to get it from the fragment cache */
  5732. +   buffer = cache_get_nowait(fragment_buffer, index);
  5733. +   if(!buffer) {
  5734. +       /*
  5735. +        * no room, get it from the reserve cache, this is
  5736. +        * dimensioned so it will always have space (no more than
  5737. +        * processors + 1 can have an outstanding reserve buffer)
  5738. +        */
  5739. +       buffer = cache_get_nowait(reserve_cache, index);
  5740. +       if(!buffer) {
  5741. +           /* failsafe */
  5742. +           ERROR("no space in reserve cache\n");
  5743. +           goto again;
  5744. +       }
  5745. +   }
  5746. +
  5747. +   pthread_mutex_unlock(&dup_mutex);
  5748.  
  5749. -   buffer = cache_get(fragment_buffer, fragment->index, 1);
  5750. +   compressed_buffer = cache_lookup(fwriter_buffer, index);
  5751.  
  5752.     pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex);
  5753.     pthread_mutex_lock(&fragment_mutex);
  5754. -   disk_fragment = &fragment_table[fragment->index];
  5755. +   disk_fragment = &fragment_table[index];
  5756.     size = SQUASHFS_COMPRESSED_SIZE_BLOCK(disk_fragment->size);
  5757.     start_block = disk_fragment->start_block;
  5758.     pthread_cleanup_pop(1);
  5759. @@ -1395,8 +1382,14 @@
  5760.  
  5761.         if(compressed_buffer)
  5762.             data = compressed_buffer->data;
  5763. -       else
  5764. +       else {
  5765.             data = read_from_disk(start_block, size);
  5766. +           if(data == NULL) {
  5767. +               ERROR("Failed to read fragment from output"
  5768. +                   " filesystem\n");
  5769. +               BAD_ERROR("Output filesystem corrupted?\n");
  5770. +           }
  5771. +       }
  5772.  
  5773.         res = compressor_uncompress(comp, buffer->data, data, size,
  5774.             block_size, &error);
  5775. @@ -1414,58 +1407,95 @@
  5776.         }
  5777.     }
  5778.  
  5779. +   cache_unlock(buffer);
  5780.     cache_block_put(compressed_buffer);
  5781.  
  5782. +finished:
  5783. +   pthread_cleanup_pop(0);
  5784. +
  5785.     return buffer;
  5786.  }
  5787.  
  5788.  
  5789. -struct frag_locked {
  5790. -   struct file_buffer *buffer;
  5791. -   int c_byte;
  5792. -   int fragment;
  5793. -   struct frag_locked *fragment_prev;
  5794. -   struct frag_locked *fragment_next;
  5795. -};
  5796. +unsigned short get_fragment_checksum(struct file_info *file)
  5797. +{
  5798. +   struct file_buffer *frag_buffer;
  5799. +   struct append_file *append;
  5800. +   int res, index = file->fragment->index;
  5801. +   unsigned short checksum;
  5802.  
  5803. -int fragments_locked = FALSE;
  5804. -struct frag_locked *frag_locked_list = NULL;
  5805. +   if(index == SQUASHFS_INVALID_FRAG)
  5806. +       return 0;
  5807. +
  5808. +   pthread_cleanup_push((void *) pthread_mutex_unlock, &dup_mutex);
  5809. +   pthread_mutex_lock(&dup_mutex);
  5810. +   res = file->have_frag_checksum;
  5811. +   checksum = file->fragment_checksum;
  5812. +   pthread_cleanup_pop(1);
  5813. +
  5814. +   if(res)
  5815. +       return checksum;
  5816. +
  5817. +   frag_buffer = get_fragment(file->fragment);
  5818. +
  5819. +   pthread_cleanup_push((void *) pthread_mutex_unlock, &dup_mutex);
  5820. +
  5821. +   for(append = file_mapping[index]; append; append = append->next) {
  5822. +       int offset = append->file->fragment->offset;
  5823. +       int size = append->file->fragment->size;
  5824. +       unsigned short cksum =
  5825. +           get_checksum_mem(frag_buffer->data + offset, size);
  5826.  
  5827. -INSERT_LIST(fragment, struct frag_locked)
  5828. -REMOVE_LIST(fragment, struct frag_locked)
  5829. +       if(file == append->file)
  5830. +           checksum = cksum;
  5831.  
  5832. -int lock_fragments()
  5833. +       pthread_mutex_lock(&dup_mutex);
  5834. +       append->file->fragment_checksum = cksum;
  5835. +       append->file->have_frag_checksum = TRUE;
  5836. +       pthread_mutex_unlock(&dup_mutex);
  5837. +   }
  5838. +
  5839. +   cache_block_put(frag_buffer);
  5840. +   pthread_cleanup_pop(0);
  5841. +
  5842. +   return checksum;
  5843. +}
  5844. +
  5845. +
  5846. +void lock_fragments()
  5847.  {
  5848. -   int count;
  5849.     pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex);
  5850.     pthread_mutex_lock(&fragment_mutex);
  5851.     fragments_locked = TRUE;
  5852. -   count = fragments_outstanding;
  5853.     pthread_cleanup_pop(1);
  5854. -   return count;
  5855.  }
  5856.  
  5857.  
  5858.  void unlock_fragments()
  5859.  {
  5860. -   struct frag_locked *entry;
  5861. -   int compressed_size;
  5862. +   int frg, size;
  5863. +   struct file_buffer *write_buffer;
  5864.  
  5865.     pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex);
  5866.     pthread_mutex_lock(&fragment_mutex);
  5867. -   while(frag_locked_list) {
  5868. -       entry = frag_locked_list;
  5869. -       remove_fragment_list(&frag_locked_list, entry);
  5870. -       compressed_size = SQUASHFS_COMPRESSED_SIZE_BLOCK(entry->c_byte);
  5871. -       fragment_table[entry->fragment].size = entry->c_byte;
  5872. -       fragment_table[entry->fragment].start_block = bytes;
  5873. -       entry->buffer->block = bytes;
  5874. -       bytes += compressed_size;
  5875. +
  5876. +   /*
  5877. +    * Note queue_empty() is inherently racy with respect to concurrent
  5878. +    * queue get and pushes.  We avoid this because we're holding the
  5879. +    * fragment_mutex which ensures no other threads can be using the
  5880. +    * queue at this time.
  5881. +    */
  5882. +   while(!queue_empty(locked_fragment)) {
  5883. +       write_buffer = queue_get(locked_fragment);
  5884. +       frg = write_buffer->block; 
  5885. +       size = SQUASHFS_COMPRESSED_SIZE_BLOCK(fragment_table[frg].size);
  5886. +       fragment_table[frg].start_block = bytes;
  5887. +       write_buffer->block = bytes;
  5888. +       bytes += size;
  5889.         fragments_outstanding --;
  5890. -       queue_put(to_writer, entry->buffer);
  5891. +       queue_put(to_writer, write_buffer);
  5892.         TRACE("fragment_locked writing fragment %d, compressed size %d"
  5893. -           "\n", entry->fragment, compressed_size);
  5894. -       free(entry);
  5895. +           "\n", frg, size);
  5896.     }
  5897.     fragments_locked = FALSE;
  5898.     pthread_cleanup_pop(1);
  5899. @@ -1475,14 +1505,10 @@
  5900.  void add_pending_fragment(struct file_buffer *write_buffer, int c_byte,
  5901.     int fragment)
  5902.  {
  5903. -   struct frag_locked *entry = malloc(sizeof(struct frag_locked));
  5904. -   if(entry == NULL)
  5905. -       MEM_ERROR();
  5906. -   entry->buffer = write_buffer;
  5907. -   entry->c_byte = c_byte;
  5908. -   entry->fragment = fragment;
  5909. -   entry->fragment_prev = entry->fragment_next = NULL;
  5910. -   insert_fragment_list(&frag_locked_list, entry);
  5911. +   fragment_table[fragment].size = c_byte;
  5912. +   write_buffer->block = fragment;
  5913. +
  5914. +   queue_put(locked_fragment, write_buffer);
  5915.  }
  5916.  
  5917.  
  5918. @@ -1502,7 +1528,7 @@
  5919.  
  5920.  struct file_buffer *allocate_fragment()
  5921.  {
  5922. -   struct file_buffer *fragment = cache_get(fragment_buffer, fragments, 1);
  5923. +   struct file_buffer *fragment = cache_get(fragment_buffer, fragments);
  5924.  
  5925.     pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex);
  5926.     pthread_mutex_lock(&fragment_mutex);
  5927. @@ -1543,7 +1569,7 @@
  5928.     if(file_buffer == NULL || file_buffer->size == 0)
  5929.         return &empty_fragment;
  5930.  
  5931. -   fragment = eval_frag_actions(dir_ent);
  5932. +   fragment = eval_frag_actions(root_dir, dir_ent);
  5933.  
  5934.     if((*fragment) && (*fragment)->size + file_buffer->size > block_size) {
  5935.         write_fragment(*fragment);
  5936. @@ -1645,15 +1671,13 @@
  5937.  
  5938.  
  5939.  char read_from_file_buffer[SQUASHFS_FILE_MAX_SIZE];
  5940. -char *read_from_disk(long long start, unsigned int avail_bytes)
  5941. +static char *read_from_disk(long long start, unsigned int avail_bytes)
  5942.  {
  5943.     int res;
  5944.  
  5945.     res = read_fs_bytes(fd, start, avail_bytes, read_from_file_buffer);
  5946. -   if(res == 0) {
  5947. -       ERROR("Failed to read data from output filesystem\n");
  5948. -       BAD_ERROR("Output filesystem corrupted?\n");
  5949. -   }
  5950. +   if(res == 0)
  5951. +       return NULL;
  5952.  
  5953.     return read_from_file_buffer;
  5954.  }
  5955. @@ -1665,10 +1689,8 @@
  5956.     int res;
  5957.  
  5958.     res = read_fs_bytes(fd, start, avail_bytes, read_from_file_buffer2);
  5959. -   if(res == 0) {
  5960. -       ERROR("Failed to read data from output filesystem\n");
  5961. -       BAD_ERROR("Output filesystem corrupted?\n");
  5962. -   }
  5963. +   if(res == 0)
  5964. +       return NULL;
  5965.  
  5966.     return read_from_file_buffer2;
  5967.  }
  5968. @@ -1702,14 +1724,22 @@
  5969.         bytes = SQUASHFS_COMPRESSED_SIZE_BLOCK(blocks[i]);
  5970.         if(bytes == 0) /* sparse block */
  5971.             continue;
  5972. -       write_buffer = cache_lookup(writer_buffer, start);
  5973. +       write_buffer = cache_lookup(bwriter_buffer, start);
  5974.         if(write_buffer) {
  5975.             chksum = get_checksum(write_buffer->data, bytes,
  5976.                 chksum);
  5977.             cache_block_put(write_buffer);
  5978. -       } else
  5979. -           chksum = get_checksum(read_from_disk(start, bytes),
  5980. -               bytes, chksum);
  5981. +       } else {
  5982. +           void *data = read_from_disk(start, bytes);
  5983. +           if(data == NULL) { 
  5984. +               ERROR("Failed to checksum data from output"
  5985. +                   " filesystem\n");
  5986. +               BAD_ERROR("Output filesystem corrupted?\n");
  5987. +           }
  5988. +
  5989. +           chksum = get_checksum(data, bytes, chksum);
  5990. +       }
  5991. +
  5992.         l -= bytes;
  5993.         start += bytes;
  5994.     }
  5995. @@ -1741,6 +1771,8 @@
  5996.     struct fragment *frg;
  5997.     unsigned int *block_list = block_listp;
  5998.     struct file_info *dupl_ptr = dupl[DUP_HASH(file_size)];
  5999. +   struct append_file *append_file;
  6000. +   struct file_info *file;
  6001.  
  6002.     if(!duplicate_checking || file_size == 0)
  6003.         return;
  6004. @@ -1767,44 +1799,29 @@
  6005.     frg->offset = offset;
  6006.     frg->size = bytes;
  6007.  
  6008. -   add_non_dup(file_size, file_bytes, block_list, start, frg, 0, 0, FALSE);
  6009. -}
  6010. -
  6011. +   file = add_non_dup(file_size, file_bytes, block_list, start, frg, 0, 0,
  6012. +       FALSE, FALSE);
  6013.  
  6014. -int pre_duplicate(long long file_size)
  6015. -{
  6016. -   struct file_info *dupl_ptr = dupl[DUP_HASH(file_size)];
  6017. +   if(fragment == SQUASHFS_INVALID_FRAG)
  6018. +       return;
  6019.  
  6020. -   for(; dupl_ptr; dupl_ptr = dupl_ptr->next)
  6021. -       if(dupl_ptr->file_size == file_size)
  6022. -           return TRUE;
  6023. +   append_file = malloc(sizeof(struct append_file));
  6024. +   if(append_file == NULL)
  6025. +       MEM_ERROR();
  6026.  
  6027. -   return FALSE;
  6028. +   append_file->file = file;
  6029. +   append_file->next = file_mapping[fragment];
  6030. +   file_mapping[fragment] = append_file;
  6031.  }
  6032.  
  6033.  
  6034. -int pre_duplicate_frag(long long file_size, unsigned short checksum)
  6035. +int pre_duplicate(long long file_size)
  6036.  {
  6037.     struct file_info *dupl_ptr = dupl[DUP_HASH(file_size)];
  6038.  
  6039.     for(; dupl_ptr; dupl_ptr = dupl_ptr->next)
  6040. -       if(file_size == dupl_ptr->file_size && file_size ==
  6041. -               dupl_ptr->fragment->size) {
  6042. -           if(dupl_ptr->checksum_flag == FALSE) {
  6043. -               struct file_buffer *frag_buffer =
  6044. -                   get_fragment(dupl_ptr->fragment);
  6045. -               dupl_ptr->checksum =
  6046. -                   get_checksum_disk(dupl_ptr->start,
  6047. -                   dupl_ptr->bytes, dupl_ptr->block_list);
  6048. -               dupl_ptr->fragment_checksum =
  6049. -                   get_checksum_mem(frag_buffer->data +
  6050. -                   dupl_ptr->fragment->offset, file_size);
  6051. -               cache_block_put(frag_buffer);
  6052. -               dupl_ptr->checksum_flag = TRUE;
  6053. -           }
  6054. -           if(dupl_ptr->fragment_checksum == checksum)
  6055. -               return TRUE;
  6056. -       }
  6057. +       if(dupl_ptr->file_size == file_size)
  6058. +           return TRUE;
  6059.  
  6060.     return FALSE;
  6061.  }
  6062. @@ -1813,7 +1830,7 @@
  6063.  struct file_info *add_non_dup(long long file_size, long long bytes,
  6064.     unsigned int *block_list, long long start, struct fragment *fragment,
  6065.     unsigned short checksum, unsigned short fragment_checksum,
  6066. -   int checksum_flag)
  6067. +   int checksum_flag, int checksum_frag_flag)
  6068.  {
  6069.     struct file_info *dupl_ptr = malloc(sizeof(struct file_info));
  6070.  
  6071. @@ -1827,22 +1844,74 @@
  6072.     dupl_ptr->fragment = fragment;
  6073.     dupl_ptr->checksum = checksum;
  6074.     dupl_ptr->fragment_checksum = fragment_checksum;
  6075. -   dupl_ptr->checksum_flag = checksum_flag;
  6076. +   dupl_ptr->have_frag_checksum = checksum_frag_flag;
  6077. +   dupl_ptr->have_checksum = checksum_flag;
  6078. +
  6079. +   pthread_cleanup_push((void *) pthread_mutex_unlock, &dup_mutex);
  6080. +        pthread_mutex_lock(&dup_mutex);
  6081.     dupl_ptr->next = dupl[DUP_HASH(file_size)];
  6082.     dupl[DUP_HASH(file_size)] = dupl_ptr;
  6083.     dup_files ++;
  6084. +   pthread_cleanup_pop(1);
  6085.  
  6086.     return dupl_ptr;
  6087.  }
  6088.  
  6089.  
  6090. +struct fragment *frag_duplicate(struct file_buffer *file_buffer, char *dont_put)
  6091. +{
  6092. +   struct file_info *dupl_ptr;
  6093. +   struct file_buffer *buffer;
  6094. +   struct file_info *dupl_start = file_buffer->dupl_start;
  6095. +   long long file_size = file_buffer->file_size;
  6096. +   unsigned short checksum = file_buffer->checksum;
  6097. +   int res;
  6098. +
  6099. +   if(file_buffer->duplicate) {
  6100. +       TRACE("Found duplicate file, fragment %d, size %d, offset %d, "
  6101. +           "checksum 0x%x\n", dupl_start->fragment->index,
  6102. +           file_size, dupl_start->fragment->offset, checksum);
  6103. +       *dont_put = TRUE;
  6104. +       return dupl_start->fragment;
  6105. +   } else {
  6106. +       *dont_put = FALSE;
  6107. +       dupl_ptr = dupl[DUP_HASH(file_size)];
  6108. +   }
  6109. +
  6110. +   for(; dupl_ptr && dupl_ptr != dupl_start; dupl_ptr = dupl_ptr->next) {
  6111. +       if(file_size == dupl_ptr->file_size && file_size ==
  6112. +               dupl_ptr->fragment->size) {
  6113. +           if(get_fragment_checksum(dupl_ptr) == checksum) {
  6114. +               buffer = get_fragment(dupl_ptr->fragment);
  6115. +               res = memcmp(file_buffer->data, buffer->data +
  6116. +                   dupl_ptr->fragment->offset, file_size);
  6117. +               cache_block_put(buffer);
  6118. +               if(res == 0)
  6119. +                   break;
  6120. +           }
  6121. +       }
  6122. +   }
  6123. +
  6124. +   if(!dupl_ptr || dupl_ptr == dupl_start)
  6125. +       return NULL;
  6126. +
  6127. +   TRACE("Found duplicate file, fragment %d, size %d, offset %d, "
  6128. +       "checksum 0x%x\n", dupl_ptr->fragment->index, file_size,
  6129. +       dupl_ptr->fragment->offset, checksum);
  6130. +
  6131. +   return dupl_ptr->fragment;
  6132. +}
  6133. +
  6134. +
  6135.  struct file_info *duplicate(long long file_size, long long bytes,
  6136.     unsigned int **block_list, long long *start, struct fragment **fragment,
  6137.     struct file_buffer *file_buffer, int blocks, unsigned short checksum,
  6138. -   unsigned short fragment_checksum, int checksum_flag)
  6139. +   int checksum_flag)
  6140.  {
  6141.     struct file_info *dupl_ptr = dupl[DUP_HASH(file_size)];
  6142.     int frag_bytes = file_buffer ? file_buffer->size : 0;
  6143. +   unsigned short fragment_checksum = file_buffer ?
  6144. +       file_buffer->checksum : 0;
  6145.  
  6146.     for(; dupl_ptr; dupl_ptr = dupl_ptr->next)
  6147.         if(file_size == dupl_ptr->file_size && bytes == dupl_ptr->bytes
  6148. @@ -1857,27 +1926,19 @@
  6149.             if(checksum_flag == FALSE) {
  6150.                 checksum = get_checksum_disk(*start, bytes,
  6151.                     *block_list);
  6152. -               fragment_checksum =
  6153. -                   get_checksum_mem_buffer(file_buffer);
  6154.                 checksum_flag = TRUE;
  6155.             }
  6156.  
  6157. -           if(dupl_ptr->checksum_flag == FALSE) {
  6158. -               struct file_buffer *frag_buffer =
  6159. -                   get_fragment(dupl_ptr->fragment);
  6160. +           if(!dupl_ptr->have_checksum) {
  6161.                 dupl_ptr->checksum =
  6162.                     get_checksum_disk(dupl_ptr->start,
  6163.                     dupl_ptr->bytes, dupl_ptr->block_list);
  6164. -               dupl_ptr->fragment_checksum =
  6165. -                   get_checksum_mem(frag_buffer->data +
  6166. -                   dupl_ptr->fragment->offset, frag_bytes);
  6167. -               cache_block_put(frag_buffer);
  6168. -               dupl_ptr->checksum_flag = TRUE;
  6169. +               dupl_ptr->have_checksum = TRUE;
  6170.             }
  6171.  
  6172.             if(checksum != dupl_ptr->checksum ||
  6173.                     fragment_checksum !=
  6174. -                   dupl_ptr->fragment_checksum)
  6175. +                   get_fragment_checksum(dupl_ptr))
  6176.                 continue;
  6177.  
  6178.             target_start = *start;
  6179. @@ -1891,22 +1952,36 @@
  6180.  
  6181.                 if(size == 0)
  6182.                     continue;
  6183. -               target_buffer = cache_lookup(writer_buffer,
  6184. +               target_buffer = cache_lookup(bwriter_buffer,
  6185.                     target_start);
  6186.                 if(target_buffer)
  6187.                     target_data = target_buffer->data;
  6188. -               else
  6189. +               else {
  6190.                     target_data =
  6191.                         read_from_disk(target_start,
  6192.                         size);
  6193. +                   if(target_data == NULL) {
  6194. +                       ERROR("Failed to read data from"
  6195. +                           " output filesystem\n");
  6196. +                       BAD_ERROR("Output filesystem"
  6197. +                           " corrupted?\n");
  6198. +                   }
  6199. +               }
  6200.  
  6201. -               dup_buffer = cache_lookup(writer_buffer,
  6202. +               dup_buffer = cache_lookup(bwriter_buffer,
  6203.                     dup_start);
  6204.                 if(dup_buffer)
  6205.                     dup_data = dup_buffer->data;
  6206. -               else
  6207. +               else {
  6208.                     dup_data = read_from_disk2(dup_start,
  6209.                         size);
  6210. +                   if(dup_data == NULL) {
  6211. +                       ERROR("Failed to read data from"
  6212. +                           " output filesystem\n");
  6213. +                       BAD_ERROR("Output filesystem"
  6214. +                           " corrupted?\n");
  6215. +                   }
  6216. +               }
  6217.  
  6218.                 res = memcmp(target_data, dup_data, size);
  6219.                 cache_block_put(target_buffer);
  6220. @@ -1948,13 +2023,13 @@
  6221.  
  6222.  
  6223.     return add_non_dup(file_size, bytes, *block_list, *start, *fragment,
  6224. -       checksum, fragment_checksum, checksum_flag);
  6225. +       checksum, fragment_checksum, checksum_flag, TRUE);
  6226.  }
  6227.  
  6228.  
  6229.  inline int is_fragment(struct inode_info *inode)
  6230.  {
  6231. -   int file_size = inode->buf.st_size;
  6232. +   off_t file_size = inode->buf.st_size;
  6233.  
  6234.     /*
  6235.      * If this block is to be compressed differently to the
  6236. @@ -1963,33 +2038,57 @@
  6237.     if(inode->noF != noF)
  6238.         return FALSE;
  6239.  
  6240. -   return !inode->no_fragments && (file_size < block_size ||
  6241. +   return !inode->no_fragments && file_size && (file_size < block_size ||
  6242.         (inode->always_use_fragments && file_size & (block_size - 1)));
  6243.  }
  6244.  
  6245.  
  6246. +void put_file_buffer(struct file_buffer *file_buffer)
  6247. +{
  6248. +   /*
  6249. +    * Decide where to send the file buffer:
  6250. +    * - compressible non-fragment blocks go to the deflate threads,
  6251. +    * - fragments go to the process fragment threads,
  6252. +    * - all others go directly to the main thread
  6253. +    */
  6254. +   if(file_buffer->error) {
  6255. +       file_buffer->fragment = 0;
  6256. +       seq_queue_put(to_main, file_buffer);
  6257. +   } else if (file_buffer->file_size == 0)
  6258. +       seq_queue_put(to_main, file_buffer);
  6259. +   else if(file_buffer->fragment)
  6260. +       queue_put(to_process_frag, file_buffer);
  6261. +   else
  6262. +       queue_put(to_deflate, file_buffer);
  6263. +}
  6264. +
  6265. +
  6266.  static int seq = 0;
  6267.  void reader_read_process(struct dir_ent *dir_ent)
  6268.  {
  6269. +   long long bytes = 0;
  6270.     struct inode_info *inode = dir_ent->inode;
  6271.     struct file_buffer *prev_buffer = NULL, *file_buffer;
  6272. -   int status, res, byte, count = 0;
  6273. -   int file = get_pseudo_file(inode->pseudo_id)->fd;
  6274. -   int child = get_pseudo_file(inode->pseudo_id)->child;
  6275. -   long long bytes = 0;
  6276. +   int status, byte, res, child;
  6277. +   int file = pseudo_exec_file(get_pseudo_file(inode->pseudo_id), &child);
  6278. +
  6279. +   if(!file) {
  6280. +       file_buffer = cache_get_nohash(reader_buffer);
  6281. +       file_buffer->sequence = seq ++;
  6282. +       goto read_err;
  6283. +   }
  6284.  
  6285.     while(1) {
  6286. -       file_buffer = cache_get(reader_buffer, 0, 0);
  6287. +       file_buffer = cache_get_nohash(reader_buffer);
  6288.         file_buffer->sequence = seq ++;
  6289.         file_buffer->noD = inode->noD;
  6290.  
  6291.         byte = read_bytes(file, file_buffer->data, block_size);
  6292.         if(byte == -1)
  6293. -           goto read_err;
  6294. +           goto read_err2;
  6295.  
  6296.         file_buffer->size = byte;
  6297.         file_buffer->file_size = -1;
  6298. -       file_buffer->block = count ++;
  6299.         file_buffer->error = FALSE;
  6300.         file_buffer->fragment = FALSE;
  6301.         bytes += byte;
  6302. @@ -2007,7 +2106,7 @@
  6303.         progress_bar_size(1);
  6304.  
  6305.         if(prev_buffer)
  6306. -           queue_put(from_reader, prev_buffer);
  6307. +           put_file_buffer(prev_buffer);
  6308.         prev_buffer = file_buffer;
  6309.     }
  6310.  
  6311. @@ -2018,6 +2117,8 @@
  6312.     inode->buf.st_size = bytes;
  6313.  
  6314.     res = waitpid(child, &status, 0);
  6315. +   close(file);
  6316. +
  6317.     if(res == -1 || !WIFEXITED(status) || WEXITSTATUS(status) != 0)
  6318.         goto read_err;
  6319.  
  6320. @@ -2029,10 +2130,12 @@
  6321.     }
  6322.     prev_buffer->file_size = bytes;
  6323.     prev_buffer->fragment = is_fragment(inode);
  6324. -   queue_put(from_reader, prev_buffer);
  6325. +   put_file_buffer(prev_buffer);
  6326.  
  6327.     return;
  6328.  
  6329. +read_err2:
  6330. +   close(file);
  6331.  read_err:
  6332.     if(prev_buffer) {
  6333.         cache_block_put(file_buffer);
  6334. @@ -2040,7 +2143,7 @@
  6335.         file_buffer = prev_buffer;
  6336.     }
  6337.     file_buffer->error = TRUE;
  6338. -   queue_put(from_deflate, file_buffer);
  6339. +   put_file_buffer(file_buffer);
  6340.  }
  6341.  
  6342.  
  6343. @@ -2048,7 +2151,7 @@
  6344.  {
  6345.     struct stat *buf = &dir_ent->inode->buf, buf2;
  6346.     struct file_buffer *file_buffer;
  6347. -   int blocks, byte, count, expected, file, res;
  6348. +   int blocks, file, res;
  6349.     long long bytes, read_size;
  6350.     struct inode_info *inode = dir_ent->inode;
  6351.  
  6352. @@ -2058,28 +2161,22 @@
  6353.     inode->read = TRUE;
  6354.  again:
  6355.     bytes = 0;
  6356. -   count = 0;
  6357. -   file_buffer = NULL;
  6358.     read_size = buf->st_size;
  6359.     blocks = (read_size + block_size - 1) >> block_log;
  6360.  
  6361.     file = open(pathname_reader(dir_ent), O_RDONLY);
  6362.     if(file == -1) {
  6363. -       file_buffer = cache_get(reader_buffer, 0, 0);
  6364. +       file_buffer = cache_get_nohash(reader_buffer);
  6365.         file_buffer->sequence = seq ++;
  6366.         goto read_err2;
  6367.     }
  6368.  
  6369.     do {
  6370. -       expected = read_size - ((long long) count * block_size) >
  6371. -           block_size ? block_size :
  6372. -           read_size - ((long long) count * block_size);
  6373. -
  6374. -       if(file_buffer)
  6375. -           queue_put(from_reader, file_buffer);
  6376. -       file_buffer = cache_get(reader_buffer, 0, 0);
  6377. +       file_buffer = cache_get_nohash(reader_buffer);
  6378. +       file_buffer->file_size = read_size;
  6379.         file_buffer->sequence = seq ++;
  6380.         file_buffer->noD = inode->noD;
  6381. +       file_buffer->error = FALSE;
  6382.  
  6383.         /*
  6384.          * Always try to read block_size bytes from the file rather
  6385. @@ -2090,29 +2187,28 @@
  6386.          * case where the file is an exact multiple of the block_size
  6387.          * is dealt with later.
  6388.          */
  6389. -       byte = file_buffer->size = read_bytes(file, file_buffer->data,
  6390. +       file_buffer->size = read_bytes(file, file_buffer->data,
  6391.             block_size);
  6392. -
  6393. -       file_buffer->file_size = read_size;
  6394. -
  6395. -       if(byte == -1)
  6396. +       if(file_buffer->size == -1)
  6397.             goto read_err;
  6398.  
  6399. -       if(byte != expected)
  6400. -           goto restat;
  6401. +       bytes += file_buffer->size;
  6402.  
  6403. -       file_buffer->block = count;
  6404. -       file_buffer->error = FALSE;
  6405. -       file_buffer->fragment = FALSE;
  6406. +       if(blocks > 1) {
  6407. +           /* non-tail block should be exactly block_size */
  6408. +           if(file_buffer->size < block_size)
  6409. +               goto restat;
  6410.  
  6411. -       bytes += byte;
  6412. -       count ++;
  6413. -   } while(count < blocks);
  6414. +           file_buffer->fragment = FALSE;
  6415. +           put_file_buffer(file_buffer);
  6416. +       }
  6417. +   } while(-- blocks > 0);
  6418.  
  6419. +   /* Overall size including tail should match */
  6420.     if(read_size != bytes)
  6421.         goto restat;
  6422.  
  6423. -   if(expected == block_size) {
  6424. +   if(read_size && read_size % block_size == 0) {
  6425.         /*
  6426.          * Special case where we've not tried to read past the end of
  6427.          * the file.  We expect to get EOF, i.e. the file isn't larger
  6428. @@ -2130,7 +2226,7 @@
  6429.     }
  6430.  
  6431.     file_buffer->fragment = is_fragment(inode);
  6432. -   queue_put(from_reader, file_buffer);
  6433. +   put_file_buffer(file_buffer);
  6434.  
  6435.     close(file);
  6436.  
  6437. @@ -2139,7 +2235,7 @@
  6438.  restat:
  6439.     res = fstat(file, &buf2);
  6440.     if(res == -1) {
  6441. -       ERROR("Cannot stat dir/file %s because %s, ignoring\n",
  6442. +       ERROR("Cannot stat dir/file %s because %s\n",
  6443.             pathname_reader(dir_ent), strerror(errno));
  6444.         goto read_err;
  6445.     }
  6446. @@ -2148,14 +2244,14 @@
  6447.         close(file);
  6448.         memcpy(buf, &buf2, sizeof(struct stat));
  6449.         file_buffer->error = 2;
  6450. -       queue_put(from_deflate, file_buffer);
  6451. +       put_file_buffer(file_buffer);
  6452.         goto again;
  6453.     }
  6454.  read_err:
  6455.     close(file);
  6456.  read_err2:
  6457.     file_buffer->error = TRUE;
  6458. -   queue_put(from_deflate, file_buffer);
  6459. +   put_file_buffer(file_buffer);
  6460.  }
  6461.  
  6462.  
  6463. @@ -2261,6 +2357,7 @@
  6464.  
  6465.  void *deflator(void *arg)
  6466.  {
  6467. +   struct file_buffer *write_buffer = cache_get_nohash(bwriter_buffer);
  6468.     void *stream = NULL;
  6469.     int res;
  6470.  
  6471. @@ -2269,20 +2366,12 @@
  6472.         BAD_ERROR("deflator:: compressor_init failed\n");
  6473.  
  6474.     while(1) {
  6475. -       struct file_buffer *file_buffer = queue_get(from_reader);
  6476. -       struct file_buffer *write_buffer;
  6477. +       struct file_buffer *file_buffer = queue_get(to_deflate);
  6478.  
  6479. -       if(file_buffer->file_size == 0) {
  6480. +       if(sparse_files && all_zero(file_buffer)) {
  6481.             file_buffer->c_byte = 0;
  6482. -           queue_put(from_deflate, file_buffer);
  6483. -       } else if(sparse_files && all_zero(file_buffer)) {
  6484. -           file_buffer->c_byte = 0;
  6485. -           queue_put(from_deflate, file_buffer);
  6486. -       } else if(file_buffer->fragment) {
  6487. -           file_buffer->c_byte = file_buffer->size;
  6488. -           queue_put(from_deflate, file_buffer);
  6489. +           seq_queue_put(to_main, file_buffer);
  6490.         } else {
  6491. -           write_buffer = cache_get(writer_buffer, 0, 0);
  6492.             write_buffer->c_byte = mangle2(stream,
  6493.                 write_buffer->data, file_buffer->data,
  6494.                 file_buffer->size, block_size,
  6495. @@ -2295,7 +2384,8 @@
  6496.             write_buffer->fragment = FALSE;
  6497.             write_buffer->error = FALSE;
  6498.             cache_block_put(file_buffer);
  6499. -           queue_put(from_deflate, write_buffer);
  6500. +           seq_queue_put(to_main, write_buffer);
  6501. +           write_buffer = cache_get_nohash(bwriter_buffer);
  6502.         }
  6503.     }
  6504.  }
  6505. @@ -2310,18 +2400,18 @@
  6506.     if(res)
  6507.         BAD_ERROR("frag_deflator:: compressor_init failed\n");
  6508.  
  6509. +   pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex);
  6510. +
  6511.     while(1) {
  6512.         int c_byte, compressed_size;
  6513.         struct file_buffer *file_buffer = queue_get(to_frag);
  6514.         struct file_buffer *write_buffer =
  6515. -           cache_get(writer_buffer, file_buffer->block +
  6516. -           FRAG_INDEX, 1);
  6517. +           cache_get(fwriter_buffer, file_buffer->block);
  6518.  
  6519.         c_byte = mangle2(stream, write_buffer->data, file_buffer->data,
  6520.             file_buffer->size, block_size, noF, 1);
  6521.         compressed_size = SQUASHFS_COMPRESSED_SIZE_BLOCK(c_byte);
  6522.         write_buffer->size = compressed_size;
  6523. -       pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex);
  6524.         pthread_mutex_lock(&fragment_mutex);
  6525.         if(fragments_locked == FALSE) {
  6526.             fragment_table[file_buffer->block].size = c_byte;
  6527. @@ -2330,137 +2420,73 @@
  6528.             bytes += compressed_size;
  6529.             fragments_outstanding --;
  6530.             queue_put(to_writer, write_buffer);
  6531. +           pthread_mutex_unlock(&fragment_mutex);
  6532.             TRACE("Writing fragment %lld, uncompressed size %d, "
  6533.                 "compressed size %d\n", file_buffer->block,
  6534.                 file_buffer->size, compressed_size);
  6535. -       } else
  6536. +       } else {
  6537.                 add_pending_fragment(write_buffer, c_byte,
  6538.                     file_buffer->block);
  6539. -       pthread_cleanup_pop(1);
  6540. +               pthread_mutex_unlock(&fragment_mutex);
  6541. +       }
  6542.         cache_block_put(file_buffer);
  6543.     }
  6544. -}
  6545. -
  6546.  
  6547. -#define HASH_ENTRIES       256
  6548. -#define BLOCK_HASH(a)      (a % HASH_ENTRIES)
  6549. -struct file_buffer     *block_hash[HASH_ENTRIES];
  6550. -
  6551. -void push_buffer(struct file_buffer *file_buffer)
  6552. -{
  6553. -   int hash = BLOCK_HASH(file_buffer->sequence);
  6554. -
  6555. -   file_buffer->next = block_hash[hash];
  6556. -   block_hash[hash] = file_buffer;
  6557. +   pthread_cleanup_pop(0);
  6558.  }
  6559.  
  6560.  
  6561. -struct file_buffer *get_file_buffer(struct queue *queue)
  6562. +struct file_buffer *get_file_buffer()
  6563.  {
  6564. -   static unsigned int sequence = 0;
  6565. -   int hash = BLOCK_HASH(sequence);
  6566. -   struct file_buffer *file_buffer = block_hash[hash], *prev = NULL;
  6567. -
  6568. -   for(;file_buffer; prev = file_buffer, file_buffer = file_buffer->next)
  6569. -       if(file_buffer->sequence == sequence)
  6570. -           break;
  6571. -
  6572. -   if(file_buffer) {
  6573. -       if(prev)
  6574. -           prev->next = file_buffer->next;
  6575. -       else
  6576. -           block_hash[hash] = file_buffer->next;
  6577. -   } else {
  6578. -       while(1) {
  6579. -           file_buffer = queue_get(queue);
  6580. -           if(file_buffer->sequence == sequence)
  6581. -               break;
  6582. -           push_buffer(file_buffer);
  6583. -       }
  6584. -   }
  6585. -
  6586. -   sequence ++;
  6587. +   struct file_buffer *file_buffer = seq_queue_get(to_main);
  6588.  
  6589.     return file_buffer;
  6590.  }
  6591.  
  6592.  
  6593.  void write_file_empty(squashfs_inode *inode, struct dir_ent *dir_ent,
  6594. -   int *duplicate_file)
  6595. +   struct file_buffer *file_buffer, int *duplicate_file)
  6596.  {
  6597.     file_count ++;
  6598.     *duplicate_file = FALSE;
  6599. +   cache_block_put(file_buffer);
  6600.     create_inode(inode, NULL, dir_ent, SQUASHFS_FILE_TYPE, 0, 0, 0,
  6601.          NULL, &empty_fragment, NULL, 0);
  6602.  }
  6603.  
  6604.  
  6605. -void write_file_frag_dup(squashfs_inode *inode, struct dir_ent *dir_ent,
  6606. -   int size, int *duplicate_file, struct file_buffer *file_buffer,
  6607. -   unsigned short checksum)
  6608. -{
  6609. -   struct file_info *dupl_ptr;
  6610. -   struct fragment *fragment;
  6611. -   unsigned int *block_listp = NULL;
  6612. -   long long start = 0;
  6613. -
  6614. -   dupl_ptr = duplicate(size, 0, &block_listp, &start, &fragment,
  6615. -       file_buffer, 0, 0, checksum, TRUE);
  6616. -
  6617. -   if(dupl_ptr) {
  6618. -       *duplicate_file = FALSE;
  6619. -       fragment = get_and_fill_fragment(file_buffer, dir_ent);
  6620. -       dupl_ptr->fragment = fragment;
  6621. -   } else
  6622. -       *duplicate_file = TRUE;
  6623. -
  6624. -   cache_block_put(file_buffer);
  6625. -
  6626. -   total_bytes += size;
  6627. -   file_count ++;
  6628. -
  6629. -   inc_progress_bar();
  6630. -
  6631. -   create_inode(inode, NULL, dir_ent, SQUASHFS_FILE_TYPE, size, 0,
  6632. -           0, NULL, fragment, NULL, 0);
  6633. -}
  6634. -
  6635. -
  6636. -void write_file_frag(squashfs_inode *inode, struct dir_ent *dir_ent, int size,
  6637. +void write_file_frag(squashfs_inode *inode, struct dir_ent *dir_ent,
  6638.     struct file_buffer *file_buffer, int *duplicate_file)
  6639.  {
  6640. +   int size = file_buffer->file_size;
  6641.     struct fragment *fragment;
  6642. -   unsigned short checksum;
  6643. -
  6644. -   checksum = get_checksum_mem_buffer(file_buffer);
  6645. +   unsigned short checksum = file_buffer->checksum;
  6646. +   char dont_put;
  6647.  
  6648. -   if(pre_duplicate_frag(size, checksum)) {
  6649. -       write_file_frag_dup(inode, dir_ent, size, duplicate_file,
  6650. -           file_buffer, checksum);
  6651. -       return;
  6652. +   fragment = frag_duplicate(file_buffer, &dont_put);
  6653. +   *duplicate_file = !fragment;
  6654. +   if(!fragment) {
  6655. +       fragment = get_and_fill_fragment(file_buffer, dir_ent);
  6656. +       if(duplicate_checking)
  6657. +           add_non_dup(size, 0, NULL, 0, fragment, 0, checksum,
  6658. +               TRUE, TRUE);
  6659.     }
  6660. -      
  6661. -   fragment = get_and_fill_fragment(file_buffer, dir_ent);
  6662. -
  6663. -   cache_block_put(file_buffer);
  6664.  
  6665. -   if(duplicate_checking)
  6666. -       add_non_dup(size, 0, NULL, 0, fragment, 0, checksum, TRUE);
  6667. +   if(dont_put)
  6668. +       free(file_buffer);
  6669. +   else
  6670. +       cache_block_put(file_buffer);
  6671.  
  6672.     total_bytes += size;
  6673.     file_count ++;
  6674.  
  6675. -   *duplicate_file = FALSE;
  6676. -
  6677.     inc_progress_bar();
  6678.  
  6679.     create_inode(inode, NULL, dir_ent, SQUASHFS_FILE_TYPE, size, 0,
  6680.             0, NULL, fragment, NULL, 0);
  6681.  
  6682. -   if(duplicate_checking == FALSE)
  6683. +   if(!duplicate_checking)
  6684.         free_fragment(fragment);
  6685. -
  6686. -   return;
  6687.  }
  6688.  
  6689.  
  6690. @@ -2482,7 +2508,7 @@
  6691.     start = bytes;
  6692.     while (1) {
  6693.         read_size = read_buffer->file_size;
  6694. -       if(read_buffer->fragment && read_buffer->c_byte)
  6695. +       if(read_buffer->fragment)
  6696.             fragment_buffer = read_buffer;
  6697.         else {
  6698.             block_list = realloc(block_list, (block + 1) *
  6699. @@ -2493,7 +2519,7 @@
  6700.             if(read_buffer->c_byte) {
  6701.                 read_buffer->block = bytes;
  6702.                 bytes += read_buffer->size;
  6703. -               cache_rehash(read_buffer, read_buffer->block);
  6704. +               cache_hash(read_buffer, read_buffer->block);
  6705.                 file_bytes += read_buffer->size;
  6706.                 queue_put(to_writer, read_buffer);
  6707.             } else {
  6708. @@ -2506,18 +2532,19 @@
  6709.         if(read_size != -1)
  6710.             break;
  6711.  
  6712. -       read_buffer = get_file_buffer(from_deflate);
  6713. +       read_buffer = get_file_buffer();
  6714.         if(read_buffer->error)
  6715.             goto read_err;
  6716.     }
  6717.  
  6718.     unlock_fragments();
  6719.     fragment = get_and_fill_fragment(fragment_buffer, dir_ent);
  6720. -   cache_block_put(fragment_buffer);
  6721.  
  6722.     if(duplicate_checking)
  6723.         add_non_dup(read_size, file_bytes, block_list, start, fragment,
  6724. -           0, 0, FALSE);
  6725. +           0, fragment_buffer ? fragment_buffer->checksum : 0,
  6726. +           FALSE, TRUE);
  6727. +   cache_block_put(fragment_buffer);
  6728.     file_count ++;
  6729.     total_bytes += read_size;
  6730.  
  6731. @@ -2553,42 +2580,57 @@
  6732.  }
  6733.  
  6734.  
  6735. -int write_file_blocks(squashfs_inode *inode, struct dir_ent *dir_ent,
  6736. -   long long read_size, struct file_buffer *read_buffer,
  6737. -   int *duplicate_file)
  6738. +int write_file_blocks_dup(squashfs_inode *inode, struct dir_ent *dir_ent,
  6739. +   struct file_buffer *read_buffer, int *duplicate_file)
  6740.  {
  6741. -   long long file_bytes, start;
  6742. +   int block, thresh;
  6743. +   long long read_size = read_buffer->file_size;
  6744. +   long long file_bytes, dup_start, start;
  6745.     struct fragment *fragment;
  6746. -   unsigned int *block_list;
  6747. -   int block, status;
  6748. +   struct file_info *dupl_ptr;
  6749.     int blocks = (read_size + block_size - 1) >> block_log;
  6750. -   long long sparse = 0;
  6751. +   unsigned int *block_list, *block_listp;
  6752. +   struct file_buffer **buffer_list;
  6753. +   int status;
  6754. +   long long sparse = 0;
  6755.     struct file_buffer *fragment_buffer = NULL;
  6756.  
  6757. -   *duplicate_file = FALSE;
  6758. -
  6759.     block_list = malloc(blocks * sizeof(unsigned int));
  6760.     if(block_list == NULL)
  6761.         MEM_ERROR();
  6762. +   block_listp = block_list;
  6763. +
  6764. +   buffer_list = malloc(blocks * sizeof(struct file_buffer *));
  6765. +   if(buffer_list == NULL)
  6766. +       MEM_ERROR();
  6767.  
  6768.     lock_fragments();
  6769.  
  6770.     file_bytes = 0;
  6771. -   start = bytes;
  6772. +   start = dup_start = bytes;
  6773. +   thresh = blocks > bwriter_size ? blocks - bwriter_size : 0;
  6774. +
  6775.     for(block = 0; block < blocks;) {
  6776. -       if(read_buffer->fragment && read_buffer->c_byte) {
  6777. +       if(read_buffer->fragment) {
  6778.             block_list[block] = 0;
  6779. +           buffer_list[block] = NULL;
  6780.             fragment_buffer = read_buffer;
  6781.             blocks = read_size >> block_log;
  6782.         } else {
  6783.             block_list[block] = read_buffer->c_byte;
  6784. +
  6785.             if(read_buffer->c_byte) {
  6786.                 read_buffer->block = bytes;
  6787.                 bytes += read_buffer->size;
  6788. -               cache_rehash(read_buffer, read_buffer->block);
  6789.                 file_bytes += read_buffer->size;
  6790. -               queue_put(to_writer, read_buffer);
  6791. +               cache_hash(read_buffer, read_buffer->block);
  6792. +               if(block < thresh) {
  6793. +                   buffer_list[block] = NULL;
  6794. +                   queue_put(to_writer, read_buffer);
  6795. +               } else
  6796. +                   buffer_list[block] = read_buffer;
  6797.             } else {
  6798. +               buffer_list[block] = NULL;
  6799.                 sparse += read_buffer->size;
  6800.                 cache_block_put(read_buffer);
  6801.             }
  6802. @@ -2596,19 +2638,43 @@
  6803.         inc_progress_bar();
  6804.  
  6805.         if(++block < blocks) {
  6806. -           read_buffer = get_file_buffer(from_deflate);
  6807. +           read_buffer = get_file_buffer();
  6808.             if(read_buffer->error)
  6809.                 goto read_err;
  6810.         }
  6811.     }
  6812.  
  6813. +   dupl_ptr = duplicate(read_size, file_bytes, &block_listp, &dup_start,
  6814. +       &fragment, fragment_buffer, blocks, 0, FALSE);
  6815. +
  6816. +   if(dupl_ptr) {
  6817. +       *duplicate_file = FALSE;
  6818. +       for(block = thresh; block < blocks; block ++)
  6819. +           if(buffer_list[block])
  6820. +               queue_put(to_writer, buffer_list[block]);
  6821. +       fragment = get_and_fill_fragment(fragment_buffer, dir_ent);
  6822. +       dupl_ptr->fragment = fragment;
  6823. +   } else {
  6824. +       *duplicate_file = TRUE;
  6825. +       for(block = thresh; block < blocks; block ++)
  6826. +           cache_block_put(buffer_list[block]);
  6827. +       bytes = start;
  6828. +       if(thresh && !block_device) {
  6829. +           int res;
  6830. +
  6831. +           queue_put(to_writer, NULL);
  6832. +           if(queue_get(from_writer) != 0)
  6833. +               EXIT_MKSQUASHFS();
  6834. +           res = ftruncate(fd, bytes);
  6835. +           if(res != 0)
  6836. +               BAD_ERROR("Failed to truncate dest file because"
  6837. +                   "  %s\n", strerror(errno));
  6838. +       }
  6839. +   }
  6840. +
  6841.     unlock_fragments();
  6842. -   fragment = get_and_fill_fragment(fragment_buffer, dir_ent);
  6843.     cache_block_put(fragment_buffer);
  6844. -
  6845. -   if(duplicate_checking)
  6846. -       add_non_dup(read_size, file_bytes, block_list, start, fragment,
  6847. -           0, 0, FALSE);
  6848. +   free(buffer_list);
  6849.     file_count ++;
  6850.     total_bytes += read_size;
  6851.  
  6852. @@ -2623,13 +2689,11 @@
  6853.     if(sparse && (dir_ent->inode->buf.st_blocks << 9) >= read_size)
  6854.         sparse = 0;
  6855.  
  6856. -   create_inode(inode, NULL, dir_ent, SQUASHFS_FILE_TYPE, read_size, start,
  6857. -        blocks, block_list, fragment, NULL, sparse);
  6858. +   create_inode(inode, NULL, dir_ent, SQUASHFS_FILE_TYPE, read_size,
  6859. +       dup_start, blocks, block_listp, fragment, NULL, sparse);
  6860.  
  6861. -   if(duplicate_checking == FALSE) {
  6862. +   if(*duplicate_file == TRUE)
  6863.         free(block_list);
  6864. -       free_fragment(fragment);
  6865. -   }
  6866.  
  6867.     return 0;
  6868.  
  6869. @@ -2637,7 +2701,7 @@
  6870.     dec_progress_bar(block);
  6871.     status = read_buffer->error;
  6872.     bytes = start;
  6873. -   if(!block_device) {
  6874. +   if(thresh && !block_device) {
  6875.         int res;
  6876.  
  6877.         queue_put(to_writer, NULL);
  6878. @@ -2649,64 +2713,54 @@
  6879.                 strerror(errno));
  6880.     }
  6881.     unlock_fragments();
  6882. +   for(blocks = thresh; blocks < block; blocks ++)
  6883. +       cache_block_put(buffer_list[blocks]);
  6884. +   free(buffer_list);
  6885.     free(block_list);
  6886.     cache_block_put(read_buffer);
  6887.     return status;
  6888.  }
  6889.  
  6890.  
  6891. -int write_file_blocks_dup(squashfs_inode *inode, struct dir_ent *dir_ent,
  6892. -   long long read_size, struct file_buffer *read_buffer,
  6893. -   int *duplicate_file)
  6894. +int write_file_blocks(squashfs_inode *inode, struct dir_ent *dir_ent,
  6895. +   struct file_buffer *read_buffer, int *dup)
  6896.  {
  6897. -   int block, thresh;
  6898. -   long long file_bytes, dup_start, start;
  6899. +   long long read_size = read_buffer->file_size;
  6900. +   long long file_bytes, start;
  6901.     struct fragment *fragment;
  6902. -   struct file_info *dupl_ptr;
  6903. +   unsigned int *block_list;
  6904. +   int block, status;
  6905.     int blocks = (read_size + block_size - 1) >> block_log;
  6906. -   unsigned int *block_list, *block_listp;
  6907. -   struct file_buffer **buffer_list;
  6908. -   int status, num_locked_fragments;
  6909.     long long sparse = 0;
  6910.     struct file_buffer *fragment_buffer = NULL;
  6911.  
  6912. +   if(pre_duplicate(read_size))
  6913. +       return write_file_blocks_dup(inode, dir_ent, read_buffer, dup);
  6914. +
  6915. +   *dup = FALSE;
  6916. +
  6917.     block_list = malloc(blocks * sizeof(unsigned int));
  6918.     if(block_list == NULL)
  6919.         MEM_ERROR();
  6920. -   block_listp = block_list;
  6921.  
  6922. -   buffer_list = malloc(blocks * sizeof(struct file_buffer *));
  6923. -   if(buffer_list == NULL)
  6924. -       MEM_ERROR();
  6925. -
  6926. -   num_locked_fragments = lock_fragments();
  6927. +   lock_fragments();
  6928.  
  6929.     file_bytes = 0;
  6930. -   start = dup_start = bytes;
  6931. -   thresh = blocks > (writer_buffer_size - num_locked_fragments) ?
  6932. -       blocks - (writer_buffer_size - num_locked_fragments): 0;
  6933. -
  6934. +   start = bytes;
  6935.     for(block = 0; block < blocks;) {
  6936. -       if(read_buffer->fragment && read_buffer->c_byte) {
  6937. +       if(read_buffer->fragment) {
  6938.             block_list[block] = 0;
  6939. -           buffer_list[block] = NULL;
  6940.             fragment_buffer = read_buffer;
  6941.             blocks = read_size >> block_log;
  6942.         } else {
  6943.             block_list[block] = read_buffer->c_byte;
  6944. -
  6945.             if(read_buffer->c_byte) {
  6946.                 read_buffer->block = bytes;
  6947.                 bytes += read_buffer->size;
  6948. +               cache_hash(read_buffer, read_buffer->block);
  6949.                 file_bytes += read_buffer->size;
  6950. -               cache_rehash(read_buffer, read_buffer->block);
  6951. -               if(block < thresh) {
  6952. -                   buffer_list[block] = NULL;
  6953. -                   queue_put(to_writer, read_buffer);
  6954. -               } else
  6955. -                   buffer_list[block] = read_buffer;
  6956. +               queue_put(to_writer, read_buffer);
  6957.             } else {
  6958. -               buffer_list[block] = NULL;
  6959.                 sparse += read_buffer->size;
  6960.                 cache_block_put(read_buffer);
  6961.             }
  6962. @@ -2714,43 +2768,20 @@
  6963.         inc_progress_bar();
  6964.  
  6965.         if(++block < blocks) {
  6966. -           read_buffer = get_file_buffer(from_deflate);
  6967. +           read_buffer = get_file_buffer();
  6968.             if(read_buffer->error)
  6969.                 goto read_err;
  6970.         }
  6971.     }
  6972.  
  6973. -   dupl_ptr = duplicate(read_size, file_bytes, &block_listp, &dup_start,
  6974. -       &fragment, fragment_buffer, blocks, 0, 0, FALSE);
  6975. -
  6976. -   if(dupl_ptr) {
  6977. -       *duplicate_file = FALSE;
  6978. -       for(block = thresh; block < blocks; block ++)
  6979. -           if(buffer_list[block])
  6980. -               queue_put(to_writer, buffer_list[block]);
  6981. -       fragment = get_and_fill_fragment(fragment_buffer, dir_ent);
  6982. -       dupl_ptr->fragment = fragment;
  6983. -   } else {
  6984. -       *duplicate_file = TRUE;
  6985. -       for(block = thresh; block < blocks; block ++)
  6986. -           cache_block_put(buffer_list[block]);
  6987. -       bytes = start;
  6988. -       if(thresh && !block_device) {
  6989. -           int res;
  6990. -
  6991. -           queue_put(to_writer, NULL);
  6992. -           if(queue_get(from_writer) != 0)
  6993. -               EXIT_MKSQUASHFS();
  6994. -           res = ftruncate(fd, bytes);
  6995. -           if(res != 0)
  6996. -               BAD_ERROR("Failed to truncate dest file because"
  6997. -                   "  %s\n", strerror(errno));
  6998. -       }
  6999. -   }
  7000. -
  7001.     unlock_fragments();
  7002. +   fragment = get_and_fill_fragment(fragment_buffer, dir_ent);
  7003. +
  7004. +   if(duplicate_checking)
  7005. +       add_non_dup(read_size, file_bytes, block_list, start, fragment,
  7006. +           0, fragment_buffer ? fragment_buffer->checksum : 0,
  7007. +           FALSE, TRUE);
  7008.     cache_block_put(fragment_buffer);
  7009. -   free(buffer_list);
  7010.     file_count ++;
  7011.     total_bytes += read_size;
  7012.  
  7013. @@ -2765,11 +2796,13 @@
  7014.     if(sparse && (dir_ent->inode->buf.st_blocks << 9) >= read_size)
  7015.         sparse = 0;
  7016.  
  7017. -   create_inode(inode, NULL, dir_ent, SQUASHFS_FILE_TYPE, read_size,
  7018. -       dup_start, blocks, block_listp, fragment, NULL, sparse);
  7019. +   create_inode(inode, NULL, dir_ent, SQUASHFS_FILE_TYPE, read_size, start,
  7020. +        blocks, block_list, fragment, NULL, sparse);
  7021.  
  7022. -   if(*duplicate_file == TRUE)
  7023. +   if(duplicate_checking == FALSE) {
  7024.         free(block_list);
  7025. +       free_fragment(fragment);
  7026. +   }
  7027.  
  7028.     return 0;
  7029.  
  7030. @@ -2777,7 +2810,7 @@
  7031.     dec_progress_bar(block);
  7032.     status = read_buffer->error;
  7033.     bytes = start;
  7034. -   if(thresh && !block_device) {
  7035. +   if(!block_device) {
  7036.         int res;
  7037.  
  7038.         queue_put(to_writer, NULL);
  7039. @@ -2789,58 +2822,40 @@
  7040.                 strerror(errno));
  7041.     }
  7042.     unlock_fragments();
  7043. -   for(blocks = thresh; blocks < block; blocks ++)
  7044. -       cache_block_put(buffer_list[blocks]);
  7045. -   free(buffer_list);
  7046.     free(block_list);
  7047.     cache_block_put(read_buffer);
  7048.     return status;
  7049.  }
  7050.  
  7051.  
  7052. -void write_file(squashfs_inode *inode, struct dir_ent *dir_ent,
  7053. -   int *duplicate_file)
  7054. +void write_file(squashfs_inode *inode, struct dir_ent *dir, int *dup)
  7055.  {
  7056.     int status;
  7057.     struct file_buffer *read_buffer;
  7058. -   long long read_size;
  7059.  
  7060.  again:
  7061. -   read_buffer = get_file_buffer(from_deflate);
  7062. -
  7063. +   read_buffer = get_file_buffer();
  7064.     status = read_buffer->error;
  7065. -   if(status) {
  7066. -       cache_block_put(read_buffer);
  7067. -       goto file_err;
  7068. -   }
  7069. -  
  7070. -   read_size = read_buffer->file_size;
  7071.  
  7072. -   if(read_size == -1)
  7073. -       status = write_file_process(inode, dir_ent, read_buffer,
  7074. -           duplicate_file);
  7075. -   else if(read_size == 0) {
  7076. -       write_file_empty(inode, dir_ent, duplicate_file);
  7077. +   if(status)
  7078.         cache_block_put(read_buffer);
  7079. -   } else if(read_buffer->fragment && read_buffer->c_byte)
  7080. -       write_file_frag(inode, dir_ent, read_size, read_buffer,
  7081. -           duplicate_file);
  7082. -   else if(pre_duplicate(read_size))
  7083. -       status = write_file_blocks_dup(inode, dir_ent, read_size,
  7084. -           read_buffer, duplicate_file);
  7085. +   else if(read_buffer->file_size == -1)
  7086. +       status = write_file_process(inode, dir, read_buffer, dup);
  7087. +   else if(read_buffer->file_size == 0)
  7088. +       write_file_empty(inode, dir, read_buffer, dup);
  7089. +   else if(read_buffer->fragment && read_buffer->c_byte)
  7090. +       write_file_frag(inode, dir, read_buffer, dup);
  7091.     else
  7092. -       status = write_file_blocks(inode, dir_ent, read_size,
  7093. -           read_buffer, duplicate_file);
  7094. +       status = write_file_blocks(inode, dir, read_buffer, dup);
  7095.  
  7096. -file_err:
  7097.     if(status == 2) {
  7098.         ERROR("File %s changed size while reading filesystem, "
  7099. -           "attempting to re-read\n", pathname(dir_ent));
  7100. +           "attempting to re-read\n", pathname(dir));
  7101.         goto again;
  7102.     } else if(status == 1) {
  7103. -       ERROR("Failed to read file %s, creating empty file\n",
  7104. -           pathname(dir_ent));
  7105. -       write_file_empty(inode, dir_ent, duplicate_file);
  7106. +       ERROR_START("Failed to read file %s", pathname(dir));
  7107. +       ERROR_EXIT(", creating empty file\n");
  7108. +       write_file_empty(inode, dir, NULL, dup);
  7109.     }
  7110.  }
  7111.  
  7112. @@ -2921,7 +2936,8 @@
  7113.  }
  7114.  
  7115.  
  7116. -struct inode_info *lookup_inode2(struct stat *buf, int pseudo, int id)
  7117. +struct inode_info *lookup_inode3(struct stat *buf, int pseudo, int id,
  7118. +   char *symlink, int bytes)
  7119.  {
  7120.     int ino_hash = INODE_HASH(buf->st_dev, buf->st_ino);
  7121.     struct inode_info *inode;
  7122. @@ -2941,10 +2957,12 @@
  7123.         }
  7124.     }
  7125.  
  7126. -   inode = malloc(sizeof(struct inode_info));
  7127. +   inode = malloc(sizeof(struct inode_info) + bytes);
  7128.     if(inode == NULL)
  7129.         MEM_ERROR();
  7130.  
  7131. +   if(bytes)
  7132. +       memcpy(&inode->symlink, symlink, bytes);
  7133.     memcpy(&inode->buf, buf, sizeof(struct stat));
  7134.     inode->read = FALSE;
  7135.     inode->root_entry = FALSE;
  7136. @@ -2965,9 +2983,6 @@
  7137.     inode->noD = noD;
  7138.     inode->noF = noF;
  7139.  
  7140. -   if((buf->st_mode & S_IFMT) == S_IFREG)
  7141. -       progress_bar_size((buf->st_size + block_size - 1) >> block_log);
  7142. -
  7143.     inode->next = inode_info[ino_hash];
  7144.     inode_info[ino_hash] = inode;
  7145.  
  7146. @@ -2975,6 +2990,12 @@
  7147.  }
  7148.  
  7149.  
  7150. +struct inode_info *lookup_inode2(struct stat *buf, int pseudo, int id)
  7151. +{
  7152. +   return lookup_inode3(buf, pseudo, id, NULL, 0);
  7153. +}
  7154. +
  7155. +
  7156.  inline struct inode_info *lookup_inode(struct stat *buf)
  7157.  {
  7158.     return lookup_inode2(buf, 0, 0);
  7159. @@ -2983,8 +3004,12 @@
  7160.  
  7161.  inline void alloc_inode_no(struct inode_info *inode, unsigned int use_this)
  7162.  {
  7163. -   if (inode->inode_number == 0)
  7164. +   if (inode->inode_number == 0) {
  7165.         inode->inode_number = use_this ? : inode_no ++;
  7166. +       if((inode->buf.st_mode & S_IFMT) == S_IFREG)
  7167. +           progress_bar_size((inode->buf.st_size + block_size - 1)
  7168. +                                >> block_log);
  7169. +   }
  7170.  }
  7171.  
  7172.  
  7173. @@ -2999,6 +3024,7 @@
  7174.     dir_ent->source_name = source_name;
  7175.     dir_ent->nonstandard_pathname = nonstandard_pathname;
  7176.     dir_ent->our_dir = dir;
  7177. +   dir_ent->inode = NULL;
  7178.     dir_ent->next = NULL;
  7179.  
  7180.     return dir_ent;
  7181. @@ -3041,6 +3067,15 @@
  7182.     if(dir_ent->source_name)
  7183.         free(dir_ent->source_name);
  7184.  
  7185. +   if(dir_ent->nonstandard_pathname)
  7186. +       free(dir_ent->nonstandard_pathname);
  7187. +
  7188. +   /* if this entry has been associated with an inode, then we need
  7189. +    * to update the inode nlink count.  Orphaned inodes are harmless, and
  7190. +    * is easier to leave them than go to the bother of deleting them */
  7191. +   if(dir_ent->inode && !dir_ent->inode->root_entry)
  7192. +       dir_ent->inode->nlink --;
  7193. +
  7194.     free(dir_ent);
  7195.  }
  7196.  
  7197. @@ -3051,42 +3086,18 @@
  7198.  }
  7199.  
  7200.  
  7201. -
  7202.  void dir_scan(squashfs_inode *inode, char *pathname,
  7203. -   struct dir_ent *(_readdir)(struct dir_info *))
  7204. +   struct dir_ent *(_readdir)(struct dir_info *), int progress)
  7205.  {
  7206.     struct stat buf;
  7207. -   struct dir_info *dir_info = dir_scan1(pathname, "", paths, _readdir, 1);
  7208.     struct dir_ent *dir_ent;
  7209.    
  7210. -   if(dir_info == NULL)
  7211. +   root_dir = dir_scan1(pathname, "", paths, _readdir, 1);
  7212. +   if(root_dir == NULL)
  7213.         return;
  7214.  
  7215. -   /*
  7216. -    * Process most actions and any pseudo files
  7217. -    */
  7218. -   if(actions() || get_pseudo())
  7219. -       dir_scan2(dir_info, get_pseudo());
  7220. -
  7221. -   /*
  7222. -    * Process move actions
  7223. -    */
  7224. -   if(move_actions()) {
  7225. -       dir_scan3(dir_info, dir_info);
  7226. -       do_move_actions();
  7227. -   }
  7228. -
  7229. -   /*
  7230. -    * Process empty actions
  7231. -    */
  7232. -   if(empty_actions())
  7233. -       dir_scan4(dir_info);
  7234. -
  7235. -   /*
  7236. -    * Sort directories and compute the inode numbers
  7237. -    */
  7238. -   dir_scan5(dir_info);
  7239. -
  7240. +   /* Create root directory dir_ent and associated inode, and connect
  7241. +    * it to the root directory dir_info structure */
  7242.     dir_ent = create_dir_entry("", NULL, pathname,
  7243.                         scan1_opendir("", "", 0));
  7244.  
  7245. @@ -3111,21 +3122,69 @@
  7246.         dir_ent->inode = lookup_inode(&buf);
  7247.     }
  7248.  
  7249. +   dir_ent->dir = root_dir;
  7250. +   root_dir->dir_ent = dir_ent;
  7251. +
  7252. +   /*
  7253. +    * Process most actions and any pseudo files
  7254. +    */
  7255. +   if(actions() || get_pseudo())
  7256. +       dir_scan2(root_dir, get_pseudo());
  7257. +
  7258. +   /*
  7259. +    * Process move actions
  7260. +    */
  7261. +   if(move_actions()) {
  7262. +       dir_scan3(root_dir);
  7263. +       do_move_actions();
  7264. +   }
  7265. +
  7266. +   /*
  7267. +    * Process prune actions
  7268. +    */
  7269. +   if(prune_actions())
  7270. +       dir_scan4(root_dir);
  7271. +
  7272. +   /*
  7273. +    * Process empty actions
  7274. +    */
  7275. +   if(empty_actions())
  7276. +       dir_scan5(root_dir);
  7277. +
  7278. +   /*
  7279. +    * Sort directories and compute the inode numbers
  7280. +    */
  7281. +   dir_scan6(root_dir);
  7282. +
  7283.     alloc_inode_no(dir_ent->inode, root_inode_number);
  7284. -   dir_ent->dir = dir_info;
  7285. -   dir_info->dir_ent = dir_ent;
  7286.  
  7287. -   eval_actions(dir_ent);
  7288. +   eval_actions(root_dir, dir_ent);
  7289.  
  7290.     if(sorted)
  7291. -       generate_file_priorities(dir_info, 0,
  7292. -           &dir_info->dir_ent->inode->buf);
  7293. -   queue_put(to_reader, dir_info);
  7294. +       generate_file_priorities(root_dir, 0,
  7295. +           &root_dir->dir_ent->inode->buf);
  7296. +
  7297. +   if(appending) {
  7298. +       sigset_t sigmask;
  7299. +
  7300. +       restore_thread = init_restore_thread();
  7301. +       sigemptyset(&sigmask);
  7302. +       sigaddset(&sigmask, SIGINT);
  7303. +       sigaddset(&sigmask, SIGTERM);
  7304. +       sigaddset(&sigmask, SIGUSR1);
  7305. +       if(pthread_sigmask(SIG_BLOCK, &sigmask, NULL) == -1)
  7306. +           BAD_ERROR("Failed to set signal mask\n");
  7307. +       write_destination(fd, SQUASHFS_START, 4, "\0\0\0\0");
  7308. +   }
  7309. +
  7310. +   queue_put(to_reader, root_dir);
  7311. +
  7312. +   set_progressbar_state(progress);
  7313. +
  7314.     if(sorted)
  7315. -       sort_files_and_write(dir_info);
  7316. -   if(progress)
  7317. -       enable_progress_bar();
  7318. -   dir_scan6(inode, dir_info);
  7319. +       sort_files_and_write(root_dir);
  7320. +
  7321. +   dir_scan7(inode, root_dir);
  7322.     dir_ent->inode->inode = *inode;
  7323.     dir_ent->inode->type = SQUASHFS_DIR_TYPE;
  7324.  }
  7325. @@ -3187,8 +3246,9 @@
  7326.         int pass = 1, res;
  7327.  
  7328.         if(dir_name == NULL) {
  7329. -           ERROR("Bad source directory %s - skipping ...\n",
  7330. +           ERROR_START("Bad source directory %s",
  7331.                 source_path[index]);
  7332. +           ERROR_EXIT(" - skipping ...\n");
  7333.             index ++;
  7334.             continue;
  7335.         }
  7336. @@ -3213,7 +3273,7 @@
  7337.             ERROR("%s\n", dir_name);
  7338.         }
  7339.         return create_dir_entry(dir_name, basename,
  7340. -           source_path[index ++], dir);
  7341. +           strdup(source_path[index ++]), dir);
  7342.     }
  7343.     return NULL;
  7344.  }
  7345. @@ -3289,7 +3349,8 @@
  7346.     struct dir_ent *dir_ent;
  7347.  
  7348.     if(dir == NULL) {
  7349. -       ERROR("Could not open %s, skipping...\n", filename);
  7350. +       ERROR_START("Could not open %s", filename);
  7351. +       ERROR_EXIT(", skipping...\n");
  7352.         return NULL;
  7353.     }
  7354.  
  7355. @@ -3307,8 +3368,9 @@
  7356.         }
  7357.  
  7358.         if(lstat(filename, &buf) == -1) {
  7359. -           ERROR("Cannot stat dir/file %s because %s, ignoring\n",
  7360. +           ERROR_START("Cannot stat dir/file %s because %s",
  7361.                 filename, strerror(errno));
  7362. +           ERROR_EXIT(", ignoring\n");
  7363.             free_dir_entry(dir_ent);
  7364.             continue;
  7365.         }
  7366. @@ -3320,8 +3382,9 @@
  7367.                     (buf.st_mode & S_IFMT) != S_IFBLK &&
  7368.                     (buf.st_mode & S_IFMT) != S_IFIFO &&
  7369.                     (buf.st_mode & S_IFMT) != S_IFSOCK) {
  7370. -           ERROR("File %s has unrecognised filetype %d, ignoring"
  7371. -               "\n", filename, buf.st_mode & S_IFMT);
  7372. +           ERROR_START("File %s has unrecognised filetype %d",
  7373. +               filename, buf.st_mode & S_IFMT);
  7374. +           ERROR_EXIT(", ignoring\n");
  7375.             free_dir_entry(dir_ent);
  7376.             continue;
  7377.         }
  7378. @@ -3337,30 +3400,53 @@
  7379.             subpath = subpathname(dir_ent);
  7380.            
  7381.             if(eval_exclude_actions(dir_name, filename, subpath,
  7382. -                               &buf, depth)) {
  7383. +                           &buf, depth, dir_ent)) {
  7384.                 add_excluded(dir);
  7385.                 free_dir_entry(dir_ent);
  7386.                 continue;
  7387.             }
  7388.         }
  7389.  
  7390. -       if((buf.st_mode & S_IFMT) == S_IFDIR) {
  7391. +       switch(buf.st_mode & S_IFMT) {
  7392. +       case S_IFDIR:
  7393.             if(subpath == NULL)
  7394.                 subpath = subpathname(dir_ent);
  7395.  
  7396.             sub_dir = dir_scan1(filename, subpath, new,
  7397.                     scan1_readdir, depth + 1);
  7398. -           if(sub_dir == NULL) {
  7399. +           if(sub_dir) {
  7400. +               dir->directory_count ++;
  7401. +               add_dir_entry(dir_ent, sub_dir,
  7402. +                           lookup_inode(&buf));
  7403. +           } else
  7404.                 free_dir_entry(dir_ent);
  7405. -               free(new);
  7406. -               continue;
  7407. +           break;
  7408. +       case S_IFLNK: {
  7409. +           int byte;
  7410. +           static char buff[65536]; /* overflow safe */
  7411. +
  7412. +           byte = readlink(filename, buff, 65536);
  7413. +           if(byte == -1) {
  7414. +               ERROR_START("Failed to read symlink %s",
  7415. +                               filename);
  7416. +               ERROR_EXIT(", ignoring\n");
  7417. +           } else if(byte == 65536) {
  7418. +               ERROR_START("Symlink %s is greater than 65536 "
  7419. +                           "bytes!", filename);
  7420. +               ERROR_EXIT(", ignoring\n");
  7421. +           } else {
  7422. +               /* readlink doesn't 0 terminate the returned
  7423. +                * path */
  7424. +               buff[byte] = '\0';
  7425. +               add_dir_entry(dir_ent, NULL, lookup_inode3(&buf,
  7426. +                            0, 0, buff, byte + 1));
  7427.             }
  7428. +           break;
  7429. +       }
  7430. +       default:
  7431. +           add_dir_entry(dir_ent, NULL, lookup_inode(&buf));
  7432. +       }
  7433.  
  7434. -           dir->directory_count ++;
  7435. -       } else
  7436. -           sub_dir = NULL;
  7437. -
  7438. -       add_dir_entry(dir_ent, sub_dir, lookup_inode(&buf));
  7439.         free(new);
  7440.     }
  7441.  
  7442. @@ -3410,7 +3496,7 @@
  7443.         struct stat *buf = &inode_info->buf;
  7444.         char *name = dir_ent->name;
  7445.  
  7446. -       eval_actions(dir_ent);
  7447. +       eval_actions(root_dir, dir_ent);
  7448.  
  7449.         if((buf->st_mode & S_IFMT) == S_IFDIR)
  7450.             dir_scan2(dir_ent->dir, pseudo_subdir(name, pseudo));
  7451. @@ -3421,16 +3507,18 @@
  7452.         if(pseudo_ent->dev->type == 'm') {
  7453.             struct stat *buf;
  7454.             if(dir_ent == NULL) {
  7455. -               ERROR("Pseudo modify file \"%s\" does not exist "
  7456. -                   "in source filesystem.  Ignoring.\n",
  7457. +               ERROR_START("Pseudo modify file \"%s\" does "
  7458. +                   "not exist in source filesystem.",
  7459.                     pseudo_ent->pathname);
  7460. +               ERROR_EXIT("  Ignoring.\n");
  7461.                 continue;
  7462.             }
  7463.             if(dir_ent->inode->root_entry) {
  7464. -               ERROR("Pseudo modify file \"%s\" is a pre-existing"
  7465. -                   " file in the filesystem being appended"
  7466. -                   "  to.  It cannot be modified. "
  7467. -                   "Ignoring.\n", pseudo_ent->pathname);
  7468. +               ERROR_START("Pseudo modify file \"%s\" is a "
  7469. +                   "pre-existing file in the filesystem "
  7470. +                   "being appended to.  It cannot be "\
  7471. +                   "modified.", pseudo_ent->pathname);
  7472. +               ERROR_EXIT("  Ignoring.\n");
  7473.                 continue;
  7474.             }
  7475.             buf = &dir_ent->inode->buf;
  7476. @@ -3442,17 +3530,20 @@
  7477.         }
  7478.  
  7479.         if(dir_ent) {
  7480. -           if(dir_ent->inode->root_entry)
  7481. -               ERROR("Pseudo file \"%s\" is a pre-existing"
  7482. -                   " file in the filesystem being appended"
  7483. -                   "  to.  Ignoring.\n",
  7484. +           if(dir_ent->inode->root_entry) {
  7485. +               ERROR_START("Pseudo file \"%s\" is a "
  7486. +                   "pre-existing file in the filesystem "
  7487. +                   "being appended to.",
  7488.                     pseudo_ent->pathname);
  7489. -           else
  7490. -               ERROR("Pseudo file \"%s\" exists in source "
  7491. -                   "filesystem \"%s\".\nIgnoring, "
  7492. -                   "exclude it (-e/-ef) to override.\n",
  7493. +               ERROR_EXIT("  Ignoring.\n");
  7494. +           } else {
  7495. +               ERROR_START("Pseudo file \"%s\" exists in "
  7496. +                   "source filesystem \"%s\".",
  7497.                     pseudo_ent->pathname,
  7498.                     pathname(dir_ent));
  7499. +               ERROR_EXIT("\nIgnoring, exclude it (-e/-ef) to "
  7500. +                   "override.\n");
  7501. +           }
  7502.             continue;
  7503.         }
  7504.  
  7505. @@ -3473,9 +3564,9 @@
  7506.             struct dir_info *sub_dir = scan1_opendir("", subpath,
  7507.                         dir->depth + 1);
  7508.             if(sub_dir == NULL) {
  7509. -               ERROR("Could not create pseudo directory \"%s\""
  7510. -                   ", skipping...\n",
  7511. -                   pseudo_ent->pathname);
  7512. +               ERROR_START("Could not create pseudo directory "
  7513. +                   "\"%s\"", pseudo_ent->pathname);
  7514. +               ERROR_EXIT(", skipping...\n");
  7515.                 free(subpath);
  7516.                 pseudo_ino --;
  7517.                 continue;
  7518. @@ -3485,25 +3576,10 @@
  7519.             add_dir_entry(dir_ent, sub_dir,
  7520.                 lookup_inode2(&buf, PSEUDO_FILE_OTHER, 0));
  7521.         } else if(pseudo_ent->dev->type == 'f') {
  7522. -#ifdef USE_TMP_FILE
  7523. -           struct stat buf2;
  7524. -           int res = stat(pseudo_ent->dev->filename, &buf2);
  7525. -           if(res == -1) {
  7526. -               ERROR("Stat on pseudo file \"%s\" failed, "
  7527. -                   "skipping...\n", pseudo_ent->pathname);
  7528. -               pseudo_ino --;
  7529. -               continue;
  7530. -           }
  7531. -           buf.st_size = buf2.st_size;
  7532. -           add_dir_entry2(pseudo_ent->name, NULL,
  7533. -               pseudo_ent->dev->filename, NULL,
  7534. -               lookup_inode2(&buf, PSEUDO_FILE_OTHER, 0), dir);
  7535. -#else
  7536.             add_dir_entry2(pseudo_ent->name, NULL,
  7537.                 pseudo_ent->pathname, NULL,
  7538.                 lookup_inode2(&buf, PSEUDO_FILE_PROCESS,
  7539.                 pseudo_ent->dev->pseudo_id), dir);
  7540. -#endif
  7541.         } else {
  7542.             add_dir_entry2(pseudo_ent->name, NULL,
  7543.                 pseudo_ent->pathname, NULL,
  7544. @@ -3517,35 +3593,113 @@
  7545.   * dir_scan3 routines...
  7546.   * This processes the move action
  7547.   */
  7548. -void dir_scan3(struct dir_info *root, struct dir_info *dir)
  7549. +void dir_scan3(struct dir_info *dir)
  7550.  {
  7551.     struct dir_ent *dir_ent = NULL;
  7552.  
  7553.     while((dir_ent = scan2_readdir(dir, dir_ent)) != NULL) {
  7554.  
  7555. -       eval_move_actions(root, dir_ent);
  7556. +       eval_move_actions(root_dir, dir_ent);
  7557.  
  7558.         if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR)
  7559. -           dir_scan3(root, dir_ent->dir);
  7560. +           dir_scan3(dir_ent->dir);
  7561.     }
  7562.  }
  7563.  
  7564.  
  7565.  /*
  7566.   * dir_scan4 routines...
  7567. + * This processes the prune action.  This action is designed to do fine
  7568. + * grained tuning of the in-core directory structure after the exclude,
  7569. + * move and pseudo actions have been performed.  This allows complex
  7570. + * tests to be performed which are impossible at exclude time (i.e.
  7571. + * tests which rely on the in-core directory structure)
  7572. + */
  7573. +void free_dir(struct dir_info *dir)
  7574. +{
  7575. +   struct dir_ent *dir_ent = dir->list;
  7576. +
  7577. +   while(dir_ent) {
  7578. +       struct dir_ent *tmp = dir_ent;
  7579. +
  7580. +       if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR)
  7581. +           free_dir(dir_ent->dir);
  7582. +
  7583. +       dir_ent = dir_ent->next;
  7584. +       free_dir_entry(tmp);
  7585. +   }
  7586. +
  7587. +   free(dir->pathname);
  7588. +   free(dir->subpath);
  7589. +   free(dir);
  7590. +}
  7591. +  
  7592. +
  7593. +void dir_scan4(struct dir_info *dir)
  7594. +{
  7595. +   struct dir_ent *dir_ent = dir->list, *prev = NULL;
  7596. +
  7597. +   while(dir_ent) {
  7598. +       if(dir_ent->inode->root_entry) {
  7599. +           prev = dir_ent;
  7600. +           dir_ent = dir_ent->next;
  7601. +           continue;
  7602. +       }
  7603. +
  7604. +       if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR)
  7605. +           dir_scan4(dir_ent->dir);
  7606. +
  7607. +       if(eval_prune_actions(root_dir, dir_ent)) {
  7608. +           struct dir_ent *tmp = dir_ent;
  7609. +
  7610. +           if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR) {
  7611. +               free_dir(dir_ent->dir);
  7612. +               dir->directory_count --;
  7613. +           }
  7614. +
  7615. +           dir->count --;
  7616. +
  7617. +           /* remove dir_ent from list */
  7618. +           dir_ent = dir_ent->next;
  7619. +           if(prev)
  7620. +               prev->next = dir_ent;
  7621. +           else
  7622. +               dir->list = dir_ent;
  7623. +          
  7624. +           /* free it */
  7625. +           free_dir_entry(tmp);
  7626. +
  7627. +           add_excluded(dir);
  7628. +           continue;
  7629. +       }
  7630. +
  7631. +       prev = dir_ent;
  7632. +       dir_ent = dir_ent->next;
  7633. +   }
  7634. +}
  7635. +
  7636. +
  7637. +/*
  7638. + * dir_scan5 routines...
  7639.   * This processes the empty action.  This action has to be processed after
  7640.   * all other actions because the previous exclude and move actions and the
  7641.   * pseudo actions affect whether a directory is empty
  7642.   */
  7643. -void dir_scan4(struct dir_info *dir)
  7644. +void dir_scan5(struct dir_info *dir)
  7645.  {
  7646.     struct dir_ent *dir_ent = dir->list, *prev = NULL;
  7647.  
  7648.     while(dir_ent) {
  7649. +       if(dir_ent->inode->root_entry) {
  7650. +           prev = dir_ent;
  7651. +           dir_ent = dir_ent->next;
  7652. +           continue;
  7653. +       }
  7654. +
  7655.         if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR) {
  7656. -           dir_scan4(dir_ent->dir);
  7657. +           dir_scan5(dir_ent->dir);
  7658.  
  7659. -           if(eval_empty_actions(dir_ent)) {
  7660. +           if(eval_empty_actions(root_dir, dir_ent)) {
  7661.                 struct dir_ent *tmp = dir_ent;
  7662.  
  7663.                 /*
  7664. @@ -3581,7 +3735,7 @@
  7665.  
  7666.  
  7667.  /*
  7668. - * dir_scan5 routines...
  7669. + * dir_scan6 routines...
  7670.   * This sorts every directory and computes the inode numbers
  7671.   */
  7672.  
  7673. @@ -3599,7 +3753,7 @@
  7674.     struct dir_ent *cur, *l1, *l2, *next;
  7675.     int len1, len2, stride = 1;
  7676.  
  7677. -   if(dir->count < 2)
  7678. +   if(dir->list == NULL || dir->count < 2)
  7679.         return;
  7680.  
  7681.     /*
  7682. @@ -3674,7 +3828,7 @@
  7683.  }
  7684.  
  7685.  
  7686. -void dir_scan5(struct dir_info *dir)
  7687. +void dir_scan6(struct dir_info *dir)
  7688.  {
  7689.     struct dir_ent *dir_ent;
  7690.     unsigned int byte_count = 0;
  7691. @@ -3691,7 +3845,7 @@
  7692.         alloc_inode_no(dir_ent->inode, 0);
  7693.  
  7694.         if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR)
  7695. -           dir_scan5(dir_ent->dir);
  7696. +           dir_scan6(dir_ent->dir);
  7697.     }
  7698.  
  7699.     if((dir->count < 257 && byte_count < SQUASHFS_METADATA_SIZE))
  7700. @@ -3703,7 +3857,7 @@
  7701.   * dir_scan6 routines...
  7702.   * This generates the filesystem metadata and writes it out to the destination
  7703.   */
  7704. -void scan6_init_dir(struct directory *dir)
  7705. +void scan7_init_dir(struct directory *dir)
  7706.  {
  7707.     dir->buff = malloc(SQUASHFS_METADATA_SIZE);
  7708.     if(dir->buff == NULL)
  7709. @@ -3718,7 +3872,7 @@
  7710.  }
  7711.  
  7712.  
  7713. -struct dir_ent *scan6_readdir(struct directory *dir, struct dir_info *dir_info,
  7714. +struct dir_ent *scan7_readdir(struct directory *dir, struct dir_info *dir_info,
  7715.     struct dir_ent *dir_ent)
  7716.  {
  7717.     if (dir_ent == NULL)
  7718. @@ -3734,7 +3888,7 @@
  7719.  }
  7720.  
  7721.  
  7722. -void scan6_freedir(struct directory *dir)
  7723. +void scan7_freedir(struct directory *dir)
  7724.  {
  7725.     if(dir->index)
  7726.         free(dir->index);
  7727. @@ -3742,16 +3896,16 @@
  7728.  }
  7729.  
  7730.  
  7731. -void dir_scan6(squashfs_inode *inode, struct dir_info *dir_info)
  7732. +void dir_scan7(squashfs_inode *inode, struct dir_info *dir_info)
  7733.  {
  7734.     int squashfs_type;
  7735.     int duplicate_file;
  7736.     struct directory dir;
  7737.     struct dir_ent *dir_ent = NULL;
  7738.    
  7739. -   scan6_init_dir(&dir);
  7740. +   scan7_init_dir(&dir);
  7741.    
  7742. -   while((dir_ent = scan6_readdir(&dir, dir_info, dir_ent)) != NULL) {
  7743. +   while((dir_ent = scan7_readdir(&dir, dir_info, dir_ent)) != NULL) {
  7744.         struct stat *buf = &dir_ent->inode->buf;
  7745.  
  7746.         update_info(dir_ent);
  7747. @@ -3772,7 +3926,7 @@
  7748.  
  7749.                 case S_IFDIR:
  7750.                     squashfs_type = SQUASHFS_DIR_TYPE;
  7751. -                   dir_scan6(inode, dir_ent->dir);
  7752. +                   dir_scan7(inode, dir_ent->dir);
  7753.                     break;
  7754.  
  7755.                 case S_IFLNK:
  7756. @@ -3883,7 +4037,7 @@
  7757.     INFO("directory %s inode 0x%llx\n", subpathname(dir_info->dir_ent),
  7758.         *inode);
  7759.  
  7760. -   scan6_freedir(&dir);
  7761. +   scan7_freedir(&dir);
  7762.  }
  7763.  
  7764.  
  7765. @@ -3928,8 +4082,9 @@
  7766.     if(path[0] == '/' || strncmp(path, "./", 2) == 0 ||
  7767.             strncmp(path, "../", 3) == 0) {
  7768.         if(lstat(path, &buf) == -1) {
  7769. -           ERROR("Cannot stat exclude dir/file %s because %s, "
  7770. -               "ignoring\n", path, strerror(errno));
  7771. +           ERROR_START("Cannot stat exclude dir/file %s because "
  7772. +               "%s", path, strerror(errno));
  7773. +           ERROR_EXIT(", ignoring\n");
  7774.             return TRUE;
  7775.         }
  7776.         ADD_ENTRY(buf);
  7777. @@ -3941,10 +4096,11 @@
  7778.         if(res == -1)
  7779.             BAD_ERROR("asprintf failed in old_add_exclude\n");
  7780.         if(lstat(filename, &buf) == -1) {
  7781. -           if(!(errno == ENOENT || errno == ENOTDIR))
  7782. -               ERROR("Cannot stat exclude dir/file %s because "
  7783. -                   "%s, ignoring\n", filename,
  7784. -                   strerror(errno));
  7785. +           if(!(errno == ENOENT || errno == ENOTDIR)) {
  7786. +               ERROR_START("Cannot stat exclude dir/file %s "
  7787. +                   "because %s", filename, strerror(errno));
  7788. +               ERROR_EXIT(", ignoring\n");
  7789. +           }
  7790.             free(filename);
  7791.             continue;
  7792.         }
  7793. @@ -3971,39 +4127,50 @@
  7794.  }
  7795.  
  7796.  
  7797. -void initialise_threads(int readb_mbytes, int writeb_mbytes,
  7798. -   int fragmentb_mbytes)
  7799. +void initialise_threads(int readq, int fragq, int bwriteq, int fwriteq,
  7800. +   int freelst, char *destination_file)
  7801.  {
  7802.     int i;
  7803.     sigset_t sigmask, old_mask;
  7804. -   int reader_buffer_size;
  7805. -   int fragment_buffer_size;
  7806. +   int total_mem = readq;
  7807. +   int reader_size;
  7808. +   int fragment_size;
  7809. +   int fwriter_size;
  7810.     /*
  7811. -    * writer_buffer_size is global because it is needed in
  7812. +    * bwriter_size is global because it is needed in
  7813.      * write_file_blocks_dup()
  7814.      */
  7815.  
  7816.     /*
  7817. +    * Never allow the total size of the queues to be larger than
  7818. +    * physical memory
  7819. +    *
  7820. +    * When adding together the possibly user supplied values, make
  7821. +    * sure they've not been deliberately contrived to overflow an int
  7822. +    */
  7823. +   if(add_overflow(total_mem, fragq))
  7824. +       BAD_ERROR("Queue sizes rediculously too large\n");
  7825. +   total_mem += fragq;
  7826. +   if(add_overflow(total_mem, bwriteq))
  7827. +       BAD_ERROR("Queue sizes rediculously too large\n");
  7828. +   total_mem += bwriteq;
  7829. +   if(add_overflow(total_mem, fwriteq))
  7830. +       BAD_ERROR("Queue sizes rediculously too large\n");
  7831. +   total_mem += fwriteq;
  7832. +
  7833. +   check_usable_phys_mem(total_mem);
  7834. +
  7835. +   /*
  7836.      * convert from queue size in Mbytes to queue size in
  7837.      * blocks.
  7838.      *
  7839. -    * In doing so, check that the user supplied values do not
  7840. -    * overflow a signed int
  7841. +    * This isn't going to overflow an int unless there exists
  7842. +    * systems with more than 8 Petabytes of RAM!
  7843.      */
  7844. -   if(shift_overflow(readb_mbytes, 20 - block_log))
  7845. -       BAD_ERROR("Read queue is too large\n");
  7846. -   else
  7847. -       reader_buffer_size = readb_mbytes << (20 - block_log);
  7848. -  
  7849. -   if(shift_overflow(fragmentb_mbytes, 20 - block_log))
  7850. -       BAD_ERROR("Fragment queue is too large\n");
  7851. -   else
  7852. -       fragment_buffer_size = fragmentb_mbytes << (20 - block_log);
  7853. -
  7854. -   if(shift_overflow(writeb_mbytes, 20 - block_log))
  7855. -       BAD_ERROR("Write queue is too large\n");
  7856. -   else
  7857. -       writer_buffer_size = writeb_mbytes << (20 - block_log);
  7858. +   reader_size = readq << (20 - block_log);
  7859. +   fragment_size = fragq << (20 - block_log);
  7860. +   bwriter_size = bwriteq << (20 - block_log);
  7861. +   fwriter_size = fwriteq << (20 - block_log);
  7862.  
  7863.     /*
  7864.      * setup signal handlers for the main thread, these cleanup
  7865. @@ -4020,10 +4187,11 @@
  7866.     signal(SIGINT, sighandler);
  7867.     signal(SIGUSR1, sighandler);
  7868.  
  7869. -   /* block SIGQUIT this is handled by the info thread */
  7870. +   /* block SIGQUIT and SIGHUP, these are handled by the info thread */
  7871.     sigemptyset(&sigmask);
  7872.     sigaddset(&sigmask, SIGQUIT);
  7873. -   if(pthread_sigmask(SIG_BLOCK, &sigmask, &old_mask) == -1)
  7874. +   sigaddset(&sigmask, SIGHUP);
  7875. +   if(pthread_sigmask(SIG_BLOCK, &sigmask, NULL) == -1)
  7876.         BAD_ERROR("Failed to set signal mask in intialise_threads\n");
  7877.  
  7878.     /*
  7879. @@ -4050,8 +4218,9 @@
  7880.  #endif
  7881.  
  7882.         if(sysctl(mib, 2, &processors, &len, NULL, 0) == -1) {
  7883. -           ERROR("Failed to get number of available processors.  "
  7884. -               "Defaulting to 1\n");
  7885. +           ERROR_START("Failed to get number of available "
  7886. +               "processors.");
  7887. +           ERROR_EXIT("  Defaulting to 1\n");
  7888.             processors = 1;
  7889.         }
  7890.  #else
  7891. @@ -4059,44 +4228,48 @@
  7892.  #endif
  7893.     }
  7894.  
  7895. -   if(multiply_overflow(processors, 2) ||
  7896. -           add_overflow(processors * 2, 2) ||
  7897. -           multiply_overflow(processors * 2 + 2,
  7898. -                           sizeof(pthread_t)))
  7899. +   if(multiply_overflow(processors, 3) ||
  7900. +           multiply_overflow(processors * 3, sizeof(pthread_t)))
  7901.         BAD_ERROR("Processors too large\n");
  7902.  
  7903. -   thread = malloc((2 + processors * 2) * sizeof(pthread_t));
  7904. -   if(thread == NULL)
  7905. +   deflator_thread = malloc(processors * 3 * sizeof(pthread_t));
  7906. +   if(deflator_thread == NULL)
  7907.         MEM_ERROR();
  7908.  
  7909. -   deflator_thread = &thread[2];
  7910.     frag_deflator_thread = &deflator_thread[processors];
  7911. +   frag_thread = &frag_deflator_thread[processors];
  7912.  
  7913.     to_reader = queue_init(1);
  7914. -   from_reader = queue_init(reader_buffer_size);
  7915. -   to_writer = queue_init(writer_buffer_size);
  7916. +   to_deflate = queue_init(reader_size);
  7917. +   to_process_frag = queue_init(reader_size);
  7918. +   to_writer = queue_init(bwriter_size + fwriter_size);
  7919.     from_writer = queue_init(1);
  7920. -   from_deflate = queue_init(reader_buffer_size);
  7921. -   to_frag = queue_init(fragment_buffer_size);
  7922. -   reader_buffer = cache_init(block_size, reader_buffer_size);
  7923. -   writer_buffer = cache_init(block_size, writer_buffer_size);
  7924. -   fragment_buffer = cache_init(block_size, fragment_buffer_size);
  7925. -   pthread_create(&thread[0], NULL, reader, NULL);
  7926. -   pthread_create(&thread[1], NULL, writer, NULL);
  7927. +   to_frag = queue_init(fragment_size);
  7928. +   locked_fragment = queue_init(fragment_size);
  7929. +   to_main = seq_queue_init();
  7930. +   reader_buffer = cache_init(block_size, reader_size, 0, 0);
  7931. +   bwriter_buffer = cache_init(block_size, bwriter_size, 1, freelst);
  7932. +   fwriter_buffer = cache_init(block_size, fwriter_size, 1, freelst);
  7933. +   fragment_buffer = cache_init(block_size, fragment_size, 1, 0);
  7934. +   reserve_cache = cache_init(block_size, processors + 1, 1, 0);
  7935. +   pthread_create(&reader_thread, NULL, reader, NULL);
  7936. +   pthread_create(&writer_thread, NULL, writer, NULL);
  7937.     init_progress_bar();
  7938.     init_info();
  7939. -   pthread_mutex_init(&fragment_mutex, NULL);
  7940. -   pthread_cond_init(&fragment_waiting, NULL);
  7941.  
  7942.     for(i = 0; i < processors; i++) {
  7943. -       if(pthread_create(&deflator_thread[i], NULL, deflator, NULL) !=
  7944. -                0)
  7945. +       if(pthread_create(&deflator_thread[i], NULL, deflator, NULL))
  7946.             BAD_ERROR("Failed to create thread\n");
  7947.         if(pthread_create(&frag_deflator_thread[i], NULL, frag_deflator,
  7948.                 NULL) != 0)
  7949.             BAD_ERROR("Failed to create thread\n");
  7950. +       if(pthread_create(&frag_thread[i], NULL, frag_thrd,
  7951. +               (void *) destination_file) != 0)
  7952. +           BAD_ERROR("Failed to create thread\n");
  7953.     }
  7954.  
  7955. +   main_thread = pthread_self();
  7956. +
  7957.     printf("Parallel mksquashfs: Using %d processor%s\n", processors,
  7958.             processors == 1 ? "" : "s");
  7959.  
  7960. @@ -4120,12 +4293,21 @@
  7961.     inode_lookup_table = it;
  7962.  
  7963.     for(i = 0; i < INODE_HASH_SIZE; i ++) {
  7964. -       struct inode_info *inode = inode_info[i];
  7965. +       struct inode_info *inode;
  7966.  
  7967.         for(inode = inode_info[i]; inode; inode = inode->next) {
  7968.  
  7969.             inode_number = get_inode_no(inode);
  7970.  
  7971. +           /* The empty action will produce orphaned inode
  7972. +            * entries in the inode_info[] table.  These
  7973. +            * entries because they are orphaned will not be
  7974. +            * allocated an inode number in dir_scan5(), so
  7975. +            * skip any entries with the default dummy inode
  7976. +            * number of 0 */
  7977. +           if(inode_number == 0)
  7978. +               continue;
  7979. +
  7980.             SQUASHFS_SWAP_LONG_LONGS(&inode->inode,
  7981.                 &inode_lookup_table[inode_number - 1], 1);
  7982.  
  7983. @@ -4146,11 +4328,14 @@
  7984.         target ++;
  7985.  
  7986.     start = target;
  7987. -   while(*target != '/' && *target!= '\0')
  7988. +   while(*target != '/' && *target != '\0')
  7989.         target ++;
  7990.  
  7991.     *targname = strndup(start, target - start);
  7992.  
  7993. +   while(*target == '/')
  7994. +       target ++;
  7995. +
  7996.     return target;
  7997.  }
  7998.  
  7999. @@ -4550,91 +4735,23 @@
  8000.         MEM_ERROR();
  8001.  
  8002.     res = read_bytes(recoverfd, metadata, bytes);
  8003. -   if(res == -1)
  8004. -       BAD_ERROR("Failed to read recovery file, because %s\n",
  8005. -           strerror(errno));
  8006. -   if(res < bytes)
  8007. -       BAD_ERROR("Recovery file appears to be truncated\n");
  8008. -
  8009. -   write_destination(fd, 0, sizeof(struct squashfs_super_block), &sBlk);
  8010. -
  8011. -   write_destination(fd, sBlk.inode_table_start, bytes, metadata);
  8012. -
  8013. -   close(recoverfd);
  8014. -   close(fd);
  8015. -
  8016. -   printf("Successfully wrote recovery file \"%s\".  Exiting\n",
  8017. -       recovery_file);
  8018. -  
  8019. -   exit(0);
  8020. -}
  8021. -
  8022. -
  8023. -int parse_number(char *start, int *res, int size)
  8024. -{
  8025. -   char *end;
  8026. -   long number = strtol(start, &end, 10);
  8027. -
  8028. -   /*
  8029. -    * check for strtol underflow or overflow in conversion.
  8030. -    * Note: strtol can validly return LONG_MIN and LONG_MAX
  8031. -    * if the user entered these values, but, additional code
  8032. -    * to distinguish this scenario is unnecessary, because for
  8033. -    * our purposes LONG_MIN and LONG_MAX are too large anyway
  8034. -    */
  8035. -   if(number == LONG_MIN || number == LONG_MAX)
  8036. -       return 0;
  8037. -
  8038. -   /* reject negative numbers as invalid */
  8039. -   if(number < 0)
  8040. -       return 0;
  8041. -
  8042. -   /* check if long result will overflow signed int */
  8043. -   if(number > INT_MAX)
  8044. -       return 0;
  8045. -
  8046. -   if(size) {
  8047. -       /*
  8048. -        * Check for multiplier and trailing junk.
  8049. -        * But first check that a number exists before the
  8050. -        * multiplier
  8051. -        */
  8052. -       if(end == start)
  8053. -           return 0;
  8054. -
  8055. -       switch(end[0]) {
  8056. -       case 'm':
  8057. -       case 'M':
  8058. -           if(multiply_overflow((int) number, 1048576))
  8059. -               return 0;
  8060. -           number *= 1048576;
  8061. +   if(res == -1)
  8062. +       BAD_ERROR("Failed to read recovery file, because %s\n",
  8063. +           strerror(errno));
  8064. +   if(res < bytes)
  8065. +       BAD_ERROR("Recovery file appears to be truncated\n");
  8066.  
  8067. -           if(end[1] != '\0')
  8068. -               /* trailing junk after number */
  8069. -               return 0;
  8070. +   write_destination(fd, 0, sizeof(struct squashfs_super_block), &sBlk);
  8071.  
  8072. -           break;
  8073. -       case 'k':
  8074. -       case 'K':
  8075. -           if(multiply_overflow((int) number, 1024))
  8076. -               return 0;
  8077. -           number *= 1024;
  8078. +   write_destination(fd, sBlk.inode_table_start, bytes, metadata);
  8079.  
  8080. -           if(end[1] != '\0')
  8081. -               /* trailing junk after number */
  8082. -               return 0;
  8083. -       case '\0':
  8084. -           break;
  8085. -       default:
  8086. -           /* trailing junk after number */
  8087. -           return 0;
  8088. -       }
  8089. -   } else if(end[0] != '\0')
  8090. -       /* trailing junk after number */
  8091. -       return 0;
  8092. +   close(recoverfd);
  8093. +   close(fd);
  8094.  
  8095. -   *res = number;
  8096. -   return 1;
  8097. +   printf("Successfully wrote recovery file \"%s\".  Exiting\n",
  8098. +       recovery_file);
  8099. +  
  8100. +   exit(0);
  8101.  }
  8102.  
  8103.  
  8104. @@ -4674,8 +4791,6 @@
  8105.  
  8106.     close(fd);
  8107.  
  8108. -   delete_pseudo_files();
  8109. -
  8110.     if(recovery_file)
  8111.         unlink(recovery_file);
  8112.  
  8113. @@ -4750,15 +4865,233 @@
  8114.  }
  8115.  
  8116.  
  8117. +int parse_numberll(char *start, long long *res, int size)
  8118. +{
  8119. +   char *end;
  8120. +   long long number;
  8121. +
  8122. +   errno = 0; /* To distinguish success/failure after call */
  8123. +
  8124. +   number = strtoll(start, &end, 10);
  8125. +
  8126. +   /*
  8127. +    * check for strtoll underflow or overflow in conversion, and other
  8128. +    * errors.
  8129. +    */
  8130. +   if((errno == ERANGE && (number == LLONG_MIN || number == LLONG_MAX)) ||
  8131. +           (errno != 0 && number == 0))
  8132. +       return 0;
  8133. +
  8134. +   /* reject negative numbers as invalid */
  8135. +   if(number < 0)
  8136. +       return 0;
  8137. +
  8138. +   if(size) {
  8139. +       /*
  8140. +        * Check for multiplier and trailing junk.
  8141. +        * But first check that a number exists before the
  8142. +        * multiplier
  8143. +        */
  8144. +       if(end == start)
  8145. +           return 0;
  8146. +
  8147. +       switch(end[0]) {
  8148. +       case 'g':
  8149. +       case 'G':
  8150. +           if(multiply_overflowll(number, 1073741824))
  8151. +               return 0;
  8152. +           number *= 1073741824;
  8153. +
  8154. +           if(end[1] != '\0')
  8155. +               /* trailing junk after multiplier, but
  8156. +                * allow it to be "bytes" */
  8157. +               if(strcmp(end + 1, "bytes"))
  8158. +                   return 0;
  8159. +
  8160. +           break;
  8161. +       case 'm':
  8162. +       case 'M':
  8163. +           if(multiply_overflowll(number, 1048576))
  8164. +               return 0;
  8165. +           number *= 1048576;
  8166. +
  8167. +           if(end[1] != '\0')
  8168. +               /* trailing junk after multiplier, but
  8169. +                * allow it to be "bytes" */
  8170. +               if(strcmp(end + 1, "bytes"))
  8171. +                   return 0;
  8172. +
  8173. +           break;
  8174. +       case 'k':
  8175. +       case 'K':
  8176. +           if(multiply_overflowll(number, 1024))
  8177. +               return 0;
  8178. +           number *= 1024;
  8179. +
  8180. +           if(end[1] != '\0')
  8181. +               /* trailing junk after multiplier, but
  8182. +                * allow it to be "bytes" */
  8183. +               if(strcmp(end + 1, "bytes"))
  8184. +                   return 0;
  8185. +
  8186. +           break;
  8187. +       case '\0':
  8188. +           break;
  8189. +       default:
  8190. +           /* trailing junk after number */
  8191. +           return 0;
  8192. +       }
  8193. +   } else if(end[0] != '\0')
  8194. +       /* trailing junk after number */
  8195. +       return 0;
  8196. +
  8197. +   *res = number;
  8198. +   return 1;
  8199. +}
  8200. +
  8201. +
  8202. +int parse_number(char *start, int *res, int size)
  8203. +{
  8204. +   long long number;
  8205. +
  8206. +   if(!parse_numberll(start, &number, size))
  8207. +       return 0;
  8208. +  
  8209. +   /* check if long result will overflow signed int */
  8210. +   if(number > INT_MAX)
  8211. +       return 0;
  8212. +
  8213. +   *res = (int) number;
  8214. +   return 1;
  8215. +}
  8216. +
  8217. +
  8218.  int parse_num(char *arg, int *res)
  8219.  {
  8220.     return parse_number(arg, res, 0);
  8221.  }
  8222.  
  8223.  
  8224. +int get_physical_memory()
  8225. +{
  8226. +   /*
  8227. +    * Long longs are used here because with PAE, a 32-bit
  8228. +    * machine can have more than 4GB of physical memory
  8229. +    *
  8230. +    * sysconf(_SC_PHYS_PAGES) relies on /proc being mounted.
  8231. +    * If it isn't fail.
  8232. +    */
  8233. +   long long num_pages = sysconf(_SC_PHYS_PAGES);
  8234. +   long long page_size = sysconf(_SC_PAGESIZE);
  8235. +   int phys_mem = num_pages * page_size >> 20;
  8236. +
  8237. +   if(num_pages == -1 || page_size == -1)
  8238. +       return 0;
  8239. +
  8240. +   if(phys_mem < SQUASHFS_LOWMEM)
  8241. +       BAD_ERROR("Mksquashfs requires more physical memory than is "
  8242. +           "available!\n");
  8243. +
  8244. +   return phys_mem;
  8245. +}
  8246. +
  8247. +
  8248. +void check_usable_phys_mem(int total_mem)
  8249. +{
  8250. +   /*
  8251. +    * We want to allow users to use as much of their physical
  8252. +    * memory as they wish.  However, for practical reasons there are
  8253. +    * limits which need to be imposed, to protect users from themselves
  8254. +    * and to prevent people from using Mksquashfs as a DOS attack by using
  8255. +    * all physical memory.   Mksquashfs uses memory to cache data from disk
  8256. +    * to optimise performance.  It is pointless to ask it to use more
  8257. +    * than 75% of physical memory, as this causes thrashing and it is thus
  8258. +    * self-defeating.
  8259. +    */
  8260. +   int mem = get_physical_memory();
  8261. +
  8262. +   mem = (mem >> 1) + (mem >> 2); /* 75% */
  8263. +                      
  8264. +   if(total_mem > mem && mem) {
  8265. +       ERROR("Total memory requested is more than 75%% of physical "
  8266. +                       "memory.\n");
  8267. +       ERROR("Mksquashfs uses memory to cache data from disk to "
  8268. +                       "optimise performance.\n");
  8269. +       ERROR("It is pointless to ask it to use more than this amount "
  8270. +                       "of memory, as this\n");
  8271. +       ERROR("causes thrashing and it is thus self-defeating.\n");
  8272. +       BAD_ERROR("Requested memory size too large\n");
  8273. +   }
  8274. +
  8275. +   if(sizeof(void *) == 4 && total_mem > 2048) {
  8276. +       /*
  8277. +        * If we're running on a kernel with PAE or on a 64-bit kernel,
  8278. +        * then the 75% physical memory limit can still easily exceed
  8279. +        * the addressable memory by this process.
  8280. +        *
  8281. +        * Due to the typical kernel/user-space split (1GB/3GB, or
  8282. +        * 2GB/2GB), we have to conservatively assume the 32-bit
  8283. +        * processes can only address 2-3GB.  So refuse if the user
  8284. +        * tries to allocate more than 2GB.
  8285. +        */
  8286. +       ERROR("Total memory requested may exceed maximum "
  8287. +               "addressable memory by this process\n");
  8288. +       BAD_ERROR("Requested memory size too large\n");
  8289. +   }
  8290. +}
  8291. +
  8292. +
  8293. +int get_default_phys_mem()
  8294. +{
  8295. +   /*
  8296. +    * get_physical_memory() relies on /proc being mounted.
  8297. +    * If it fails, issue a warning, and use
  8298. +    * SQUASHFS_LOWMEM / SQUASHFS_TAKE as default,
  8299. +    * and allow a larger value to be set with -mem.
  8300. +    */
  8301. +   int mem = get_physical_memory();
  8302. +
  8303. +   if(mem == 0) {
  8304. +       mem = SQUASHFS_LOWMEM / SQUASHFS_TAKE;
  8305. +
  8306. +       ERROR("Warning: Cannot get size of physical memory, probably "
  8307. +               "because /proc is missing.\n");
  8308. +       ERROR("Warning: Defaulting to minimal use of %d Mbytes, use "
  8309. +               "-mem to set a better value,\n", mem);
  8310. +       ERROR("Warning: or fix /proc.\n");
  8311. +   } else
  8312. +       mem /= SQUASHFS_TAKE;
  8313. +
  8314. +   if(sizeof(void *) == 4 && mem > 640) {
  8315. +       /*
  8316. +        * If we're running on a kernel with PAE or on a 64-bit kernel,
  8317. +        * the default memory usage can exceed the addressable
  8318. +        * memory by this process.
  8319. +        * Due to the typical kernel/user-space split (1GB/3GB, or
  8320. +        * 2GB/2GB), we have to conservatively assume the 32-bit
  8321. +        * processes can only address 2-3GB.  So limit the  default
  8322. +        * usage to 640M, which gives room for other data.
  8323. +        */
  8324. +       mem = 640;
  8325. +   }
  8326. +
  8327. +   return mem;
  8328. +}
  8329. +
  8330. +
  8331. +void calculate_queue_sizes(int mem, int *readq, int *fragq, int *bwriteq,
  8332. +                           int *fwriteq)
  8333. +{
  8334. +   *readq = mem / SQUASHFS_READQ_MEM;
  8335. +   *bwriteq = mem / SQUASHFS_BWRITEQ_MEM;
  8336. +   *fwriteq = mem / SQUASHFS_FWRITEQ_MEM;
  8337. +   *fragq = mem - *readq - *bwriteq - *fwriteq;
  8338. +}
  8339. +
  8340. +
  8341.  #define VERSION() \
  8342. -   printf("mksquashfs version 4.2-git (2013/04/07)\n");\
  8343. -   printf("copyright (C) 2013 Phillip Lougher "\
  8344. +   printf("mksquashfs version 4.3-git (2014/09/12)\n");\
  8345. +   printf("copyright (C) 2014 Phillip Lougher "\
  8346.         "<phillip@squashfs.org.uk>\n\n"); \
  8347.     printf("This program is free software; you can redistribute it and/or"\
  8348.         "\n");\
  8349. @@ -4781,29 +5114,79 @@
  8350.     char *b, *root_name = NULL;
  8351.     int keep_as_directory = FALSE;
  8352.     squashfs_inode inode;
  8353. -   int readb_mbytes = READER_BUFFER_DEFAULT,
  8354. -       writeb_mbytes = WRITER_BUFFER_DEFAULT,
  8355. -       fragmentb_mbytes = FRAGMENT_BUFFER_DEFAULT;
  8356. +   int readq;
  8357. +   int fragq;
  8358. +   int bwriteq;
  8359. +   int fwriteq;
  8360. +   int total_mem = get_default_phys_mem();
  8361. +   int progress = TRUE;
  8362.     int force_progress = FALSE;
  8363.     struct file_buffer **fragment = NULL;
  8364.  
  8365. -   block_log = slog(block_size);
  8366.     if(argc > 1 && strcmp(argv[1], "-version") == 0) {
  8367.         VERSION();
  8368.         exit(0);
  8369.     }
  8370. +
  8371. +   block_log = slog(block_size);
  8372. +   calculate_queue_sizes(total_mem, &readq, &fragq, &bwriteq, &fwriteq);
  8373. +
  8374.          for(i = 1; i < argc && argv[i][0] != '-'; i++);
  8375.     if(i < 3)
  8376.         goto printOptions;
  8377.     source_path = argv + 1;
  8378.     source = i - 2;
  8379. +
  8380.     /*
  8381. -    * lookup default compressor.  Note the Makefile ensures the default
  8382. -    * compressor has been built, and so we don't need to to check
  8383. -    * for failure here
  8384. +    * Scan the command line for -comp xxx option, this is to ensure
  8385. +    * any -X compressor specific options are passed to the
  8386. +    * correct compressor
  8387.      */
  8388. -   comp = lookup_compressor(COMP_DEFAULT);
  8389.     for(; i < argc; i++) {
  8390. +       struct compressor *prev_comp = comp;
  8391. +      
  8392. +       if(strcmp(argv[i], "-comp") == 0) {
  8393. +           if(++i == argc) {
  8394. +               ERROR("%s: -comp missing compression type\n",
  8395. +                   argv[0]);
  8396. +               exit(1);
  8397. +           }
  8398. +           comp = lookup_compressor(argv[i]);
  8399. +           if(!comp->supported) {
  8400. +               ERROR("%s: Compressor \"%s\" is not supported!"
  8401. +                   "\n", argv[0], argv[i]);
  8402. +               ERROR("%s: Compressors available:\n", argv[0]);
  8403. +               display_compressors("", COMP_DEFAULT);
  8404. +               exit(1);
  8405. +           }
  8406. +           if(prev_comp != NULL && prev_comp != comp) {
  8407. +               ERROR("%s: -comp multiple conflicting -comp"
  8408. +                   " options specified on command line"
  8409. +                   ", previously %s, now %s\n", argv[0],
  8410. +                   prev_comp->name, comp->name);
  8411. +               exit(1);
  8412. +           }
  8413. +           compressor_opt_parsed = 1;
  8414. +
  8415. +       } else if(strcmp(argv[i], "-e") == 0)
  8416. +           break;
  8417. +       else if(strcmp(argv[i], "-root-becomes") == 0 ||
  8418. +               strcmp(argv[i], "-ef") == 0 ||
  8419. +               strcmp(argv[i], "-pf") == 0 ||
  8420. +               strcmp(argv[i], "-vaf") == 0 ||
  8421. +               strcmp(argv[i], "-comp") == 0)
  8422. +           i++;
  8423. +   }
  8424. +
  8425. +   /*
  8426. +    * if no -comp option specified lookup default compressor.  Note the
  8427. +    * Makefile ensures the default compressor has been built, and so we
  8428. +    * don't need to to check for failure here
  8429. +    */
  8430. +   if(comp == NULL)
  8431. +       comp = lookup_compressor(COMP_DEFAULT);
  8432. +
  8433. +   for(i = source + 2; i < argc; i++) {
  8434.         if(strcmp(argv[i], "-action") == 0 ||
  8435.                 strcmp(argv[i], "-a") ==0) {
  8436.             if(++i == argc) {
  8437. @@ -4811,54 +5194,114 @@
  8438.                     argv[0], argv[i - 1]);
  8439.                 exit(1);
  8440.             }
  8441. -           res = parse_action(argv[i]);
  8442. +           res = parse_action(argv[i], ACTION_LOG_NONE);
  8443. +           if(res == 0)
  8444. +               exit(1);
  8445. +
  8446. +       } else if(strcmp(argv[i], "-verbose-action") == 0 ||
  8447. +               strcmp(argv[i], "-va") ==0) {
  8448. +           if(++i == argc) {
  8449. +               ERROR("%s: %s missing action\n",
  8450. +                   argv[0], argv[i - 1]);
  8451. +               exit(1);
  8452. +           }
  8453. +           res = parse_action(argv[i], ACTION_LOG_VERBOSE);
  8454.             if(res == 0)
  8455.                 exit(1);
  8456.  
  8457. -       } else if(strcmp(argv[i], "-af") == 0) {
  8458. +       } else if(strcmp(argv[i], "-true-action") == 0 ||
  8459. +               strcmp(argv[i], "-ta") ==0) {
  8460.             if(++i == argc) {
  8461. -               ERROR("%s: -af missing filename\n", argv[0]);
  8462. +               ERROR("%s: %s missing action\n",
  8463. +                   argv[0], argv[i - 1]);
  8464.                 exit(1);
  8465.             }
  8466. -           if(read_action_file(argv[i]) == FALSE)
  8467. +           res = parse_action(argv[i], ACTION_LOG_TRUE);
  8468. +           if(res == 0)
  8469.                 exit(1);
  8470.  
  8471. -       } else if(strcmp(argv[i], "-comp") == 0) {
  8472. -           if(compressor_opts_parsed) {
  8473. -               ERROR("%s: -comp must appear before -X options"
  8474. -                   "\n", argv[0]);
  8475. +       } else if(strcmp(argv[i], "-false-action") == 0 ||
  8476. +               strcmp(argv[i], "-fa") ==0) {
  8477. +           if(++i == argc) {
  8478. +               ERROR("%s: %s missing action\n",
  8479. +                   argv[0], argv[i - 1]);
  8480.                 exit(1);
  8481.             }
  8482. +           res = parse_action(argv[i], ACTION_LOG_FALSE);
  8483. +           if(res == 0)
  8484. +               exit(1);
  8485. +
  8486. +       } else if(strcmp(argv[i], "-action-file") == 0 ||
  8487. +               strcmp(argv[i], "-af") ==0) {
  8488.             if(++i == argc) {
  8489. -               ERROR("%s: -comp missing compression type\n",
  8490. -                   argv[0]);
  8491. +               ERROR("%s: %s missing filename\n", argv[0],
  8492. +                           argv[i - 1]);
  8493.                 exit(1);
  8494.             }
  8495. -           comp = lookup_compressor(argv[i]);
  8496. -           if(!comp->supported) {
  8497. -               ERROR("%s: Compressor \"%s\" is not supported!"
  8498. -                   "\n", argv[0], argv[i]);
  8499. -               ERROR("%s: Compressors available:\n", argv[0]);
  8500. -               display_compressors("", COMP_DEFAULT);
  8501. +           if(read_action_file(argv[i], ACTION_LOG_NONE) == FALSE)
  8502. +               exit(1);
  8503. +
  8504. +       } else if(strcmp(argv[i], "-verbose-action-file") == 0 ||
  8505. +               strcmp(argv[i], "-vaf") ==0) {
  8506. +           if(++i == argc) {
  8507. +               ERROR("%s: %s missing filename\n", argv[0],
  8508. +                           argv[i - 1]);
  8509. +               exit(1);
  8510. +           }
  8511. +           if(read_action_file(argv[i], ACTION_LOG_VERBOSE) == FALSE)
  8512. +               exit(1);
  8513. +
  8514. +       } else if(strcmp(argv[i], "-true-action-file") == 0 ||
  8515. +               strcmp(argv[i], "-taf") ==0) {
  8516. +           if(++i == argc) {
  8517. +               ERROR("%s: %s missing filename\n", argv[0],
  8518. +                           argv[i - 1]);
  8519. +               exit(1);
  8520. +           }
  8521. +           if(read_action_file(argv[i], ACTION_LOG_TRUE) == FALSE)
  8522. +               exit(1);
  8523. +
  8524. +       } else if(strcmp(argv[i], "-false-action-file") == 0 ||
  8525. +               strcmp(argv[i], "-faf") ==0) {
  8526. +           if(++i == argc) {
  8527. +               ERROR("%s: %s missing filename\n", argv[0],
  8528. +                           argv[i - 1]);
  8529.                 exit(1);
  8530.             }
  8531. +           if(read_action_file(argv[i], ACTION_LOG_FALSE) == FALSE)
  8532. +               exit(1);
  8533. +
  8534. +       } else if(strcmp(argv[i], "-comp") == 0)
  8535. +           /* parsed previously */
  8536. +           i++;
  8537. +
  8538. +       else if(strncmp(argv[i], "-X", 2) == 0) {
  8539. +           int args;
  8540.  
  8541. -       } else if(strncmp(argv[i], "-X", 2) == 0) {
  8542. -           int args = compressor_options(comp, argv + i, argc - i);
  8543. +           if(strcmp(argv[i] + 2, "help") == 0)
  8544. +               goto print_compressor_options;
  8545. +
  8546. +           args = compressor_options(comp, argv + i, argc - i);
  8547.             if(args < 0) {
  8548.                 if(args == -1) {
  8549.                     ERROR("%s: Unrecognised compressor"
  8550.                         " option %s\n", argv[0],
  8551.                         argv[i]);
  8552. -                   ERROR("%s: Did you forget to specify"
  8553. -                       " -comp, or specify it after"
  8554. -                       " the compressor specific"
  8555. -                       " option?\n", argv[0]);
  8556. -                   }
  8557. +                   if(!compressor_opt_parsed)
  8558. +                       ERROR("%s: Did you forget to"
  8559. +                           " specify -comp?\n",
  8560. +                           argv[0]);
  8561. +print_compressor_options:
  8562. +                   ERROR("%s: selected compressor \"%s\""
  8563. +                       ".  Options supported: %s\n",
  8564. +                       argv[0], comp->name,
  8565. +                       comp->usage ? "" : "none");
  8566. +                   if(comp->usage)
  8567. +                       comp->usage();
  8568. +               }
  8569.                 exit(1);
  8570.             }
  8571.             i += args;
  8572. -           compressor_opts_parsed = 1;
  8573.  
  8574.         } else if(strcmp(argv[i], "-pf") == 0) {
  8575.             if(++i == argc) {
  8576. @@ -4910,42 +5353,68 @@
  8577.                 exit(1);
  8578.             }
  8579.         } else if(strcmp(argv[i], "-read-queue") == 0) {
  8580. -           if((++i == argc) ||
  8581. -                   !parse_num(argv[i], &readb_mbytes)) {
  8582. +           if((++i == argc) || !parse_num(argv[i], &readq)) {
  8583.                 ERROR("%s: -read-queue missing or invalid "
  8584.                     "queue size\n", argv[0]);
  8585.                 exit(1);
  8586.             }
  8587. -           if(readb_mbytes < 1) {
  8588. +           if(readq < 1) {
  8589.                 ERROR("%s: -read-queue should be 1 megabyte or "
  8590.                     "larger\n", argv[0]);
  8591.                 exit(1);
  8592.             }
  8593.         } else if(strcmp(argv[i], "-write-queue") == 0) {
  8594. -           if((++i == argc) ||
  8595. -                   !parse_num(argv[i], &writeb_mbytes)) {
  8596. +           if((++i == argc) || !parse_num(argv[i], &bwriteq)) {
  8597.                 ERROR("%s: -write-queue missing or invalid "
  8598.                     "queue size\n", argv[0]);
  8599.                 exit(1);
  8600.             }
  8601. -           if(writeb_mbytes < 1) {
  8602. -               ERROR("%s: -write-queue should be 1 megabyte "
  8603. +           if(bwriteq < 2) {
  8604. +               ERROR("%s: -write-queue should be 2 megabytes "
  8605.                     "or larger\n", argv[0]);
  8606.                 exit(1);
  8607.             }
  8608. +           fwriteq = bwriteq >> 1;
  8609. +           bwriteq -= fwriteq;
  8610.         } else if(strcmp(argv[i], "-fragment-queue") == 0) {
  8611. -           if((++i == argc) ||
  8612. -                   !parse_num(argv[i],
  8613. -                       &fragmentb_mbytes)) {
  8614. +           if((++i == argc) || !parse_num(argv[i], &fragq)) {
  8615.                 ERROR("%s: -fragment-queue missing or invalid "
  8616.                     "queue size\n", argv[0]);
  8617.                 exit(1);
  8618.             }
  8619. -           if(fragmentb_mbytes < 1) {
  8620. +           if(fragq < 1) {
  8621.                 ERROR("%s: -fragment-queue should be 1 "
  8622.                     "megabyte or larger\n", argv[0]);
  8623.                 exit(1);
  8624.             }
  8625. +       } else if(strcmp(argv[i], "-mem") == 0) {
  8626. +           long long number;
  8627. +
  8628. +           if((++i == argc) ||
  8629. +                   !parse_numberll(argv[i], &number, 1)) {
  8630. +               ERROR("%s: -mem missing or invalid mem size\n",
  8631. +                    argv[0]);
  8632. +               exit(1);
  8633. +           }
  8634. +
  8635. +           /*
  8636. +            * convert from bytes to Mbytes, ensuring the value
  8637. +            * does not overflow a signed int
  8638. +            */
  8639. +           if(number >= (1LL << 51)) {
  8640. +               ERROR("%s: -mem invalid mem size\n", argv[0]);
  8641. +               exit(1);
  8642. +           }
  8643. +
  8644. +           total_mem = number / 1048576;
  8645. +           if(total_mem < (SQUASHFS_LOWMEM / SQUASHFS_TAKE)) {
  8646. +               ERROR("%s: -mem should be %d Mbytes or "
  8647. +                   "larger\n", argv[0],
  8648. +                   SQUASHFS_LOWMEM / SQUASHFS_TAKE);
  8649. +               exit(1);
  8650. +           }
  8651. +           calculate_queue_sizes(total_mem, &readq, &fragq,
  8652. +               &bwriteq, &fwriteq);
  8653.         } else if(strcmp(argv[i], "-b") == 0) {
  8654.             if(++i == argc) {
  8655.                 ERROR("%s: -b missing block size\n", argv[0]);
  8656. @@ -5067,6 +5536,9 @@
  8657.         else if(strcmp(argv[i], "-keep-as-directory") == 0)
  8658.             keep_as_directory = TRUE;
  8659.  
  8660. +       else if(strcmp(argv[i], "-exit-on-error") == 0)
  8661. +           exit_on_error = TRUE;
  8662. +
  8663.         else if(strcmp(argv[i], "-root-becomes") == 0) {
  8664.             if(++i == argc) {
  8665.                 ERROR("%s: -root-becomes: missing name\n",
  8666. @@ -5086,8 +5558,10 @@
  8667.             ERROR("\t\t\tCompressors available:\n");
  8668.             display_compressors("\t\t\t", COMP_DEFAULT);
  8669.             ERROR("-b <block_size>\t\tset data block to "
  8670. -               "<block_size>.  Default %d bytes\n",
  8671. -               SQUASHFS_FILE_SIZE);
  8672. +               "<block_size>.  Default 128 Kbytes\n");
  8673. +           ERROR("\t\t\tOptionally a suffix of K or M can be"
  8674. +               " given to specify\n\t\t\tKbytes or Mbytes"
  8675. +               " respectively\n");
  8676.             ERROR("-no-exports\t\tdon't make the filesystem "
  8677.                 "exportable via NFS\n");
  8678.             ERROR("-no-sparse\t\tdon't detect sparse files\n");
  8679. @@ -5145,6 +5619,8 @@
  8680.             ERROR("\nMksquashfs runtime options:\n");
  8681.             ERROR("-version\t\tprint version, licence and "
  8682.                 "copyright message\n");
  8683. +           ERROR("-exit-on-error\t\ttreat normally ignored errors "
  8684. +               "as fatal\n");
  8685.             ERROR("-recover <name>\t\trecover filesystem data "
  8686.                 "using recovery file <name>\n");
  8687.             ERROR("-no-recovery\t\tdon't generate a recovery "
  8688. @@ -5157,15 +5633,11 @@
  8689.             ERROR("-processors <number>\tUse <number> processors."
  8690.                 "  By default will use number of\n");
  8691.             ERROR("\t\t\tprocessors available\n");
  8692. -           ERROR("-read-queue <size>\tSet input queue to <size> "
  8693. -               "Mbytes.  Default %d Mbytes\n",
  8694. -               READER_BUFFER_DEFAULT);
  8695. -           ERROR("-write-queue <size>\tSet output queue to <size> "
  8696. -               "Mbytes.  Default %d Mbytes\n",
  8697. -               WRITER_BUFFER_DEFAULT);
  8698. -           ERROR("-fragment-queue <size>\tSet fragment queue to "
  8699. -               "<size> Mbytes.  Default %d Mbytes\n",
  8700. -               FRAGMENT_BUFFER_DEFAULT);
  8701. +           ERROR("-mem <size>\t\tUse <size> physical memory.  "
  8702. +               "Currently set to %dM\n", total_mem);
  8703. +           ERROR("\t\t\tOptionally a suffix of K, M or G can be"
  8704. +               " given to specify\n\t\t\tKbytes, Mbytes or"
  8705. +               " Gbytes respectively\n");
  8706.             ERROR("\nMiscellaneous options:\n");
  8707.             ERROR("-root-owned\t\talternative name for -all-root"
  8708.                 "\n");
  8709. @@ -5177,6 +5649,8 @@
  8710.                 "-noF\n");
  8711.             ERROR("-noXattrCompression\talternative name for "
  8712.                 "-noX\n");
  8713. +           ERROR("\n-Xhelp\t\t\tprint compressor options for"
  8714. +               " selected compressor\n");
  8715.             ERROR("\nCompressors available and compressor specific "
  8716.                 "options:\n");
  8717.             display_compressor_usage(COMP_DEFAULT);
  8718. @@ -5201,6 +5675,11 @@
  8719.         progress = force_progress;
  8720.        
  8721.  #ifdef SQUASHFS_TRACE
  8722. +   /*
  8723. +    * Disable progress bar if full debug tracing is enabled.
  8724. +    * The progress bar in this case just gets in the way of the
  8725. +    * debug trace output
  8726. +    */
  8727.     progress = FALSE;
  8728.  #endif
  8729.  
  8730. @@ -5269,6 +5748,7 @@
  8731.                 strcmp(argv[i], "-sort") == 0 ||
  8732.                 strcmp(argv[i], "-pf") == 0 ||
  8733.                 strcmp(argv[i], "-af") == 0 ||
  8734. +               strcmp(argv[i], "-vaf") == 0 ||
  8735.                 strcmp(argv[i], "-comp") == 0)
  8736.             i++;
  8737.  
  8738. @@ -5298,6 +5778,7 @@
  8739.                 strcmp(argv[i], "-ef") == 0 ||
  8740.                 strcmp(argv[i], "-pf") == 0 ||
  8741.                 strcmp(argv[i], "-af") == 0 ||
  8742. +               strcmp(argv[i], "-vaf") == 0 ||
  8743.                 strcmp(argv[i], "-comp") == 0)
  8744.             i++;
  8745.  
  8746. @@ -5324,7 +5805,8 @@
  8747.         comp_opts = SQUASHFS_COMP_OPTS(sBlk.flags);
  8748.     }
  8749.  
  8750. -   initialise_threads(readb_mbytes, writeb_mbytes, fragmentb_mbytes);
  8751. +   initialise_threads(readq, fragq, bwriteq, fwriteq, delete,
  8752. +       destination_file);
  8753.  
  8754.     res = compressor_init(comp, &stream, SQUASHFS_METADATA_SIZE, 0);
  8755.     if(res)
  8756. @@ -5366,7 +5848,6 @@
  8757.             SQUASHFS_INODE_BLK(sBlk.root_inode),
  8758.             root_inode_offset =
  8759.             SQUASHFS_INODE_OFFSET(sBlk.root_inode);
  8760. -       sigset_t sigmask, old_mask;
  8761.  
  8762.         if((bytes = read_filesystem(root_name, fd, &sBlk, &inode_table,
  8763.                 &data_cache, &directory_table,
  8764. @@ -5385,7 +5866,7 @@
  8765.                 "device or file use -noappend\n");
  8766.             EXIT_MKSQUASHFS();
  8767.         }
  8768. -       if((fragments = sBlk.fragments)) {
  8769. +       if((append_fragments = fragments = sBlk.fragments)) {
  8770.             fragment_table = realloc((char *) fragment_table,
  8771.                 ((fragments + FRAG_SIZE - 1) & ~(FRAG_SIZE - 1))
  8772.                  * sizeof(struct squashfs_fragment_entry));
  8773. @@ -5437,14 +5918,7 @@
  8774.         sid_count = id_count;
  8775.         write_recovery_data(&sBlk);
  8776.         save_xattrs();
  8777. -       restore_thread = init_restore_thread(pthread_self());
  8778. -       sigemptyset(&sigmask);
  8779. -       sigaddset(&sigmask, SIGINT);
  8780. -       sigaddset(&sigmask, SIGTERM);
  8781. -       sigaddset(&sigmask, SIGUSR1);
  8782. -       if(pthread_sigmask(SIG_BLOCK, &sigmask, &old_mask) == -1)
  8783. -           BAD_ERROR("Failed to set signal mask\n");
  8784. -       write_destination(fd, SQUASHFS_START, 4, "\0\0\0\0");
  8785. +       appending = TRUE;
  8786.  
  8787.         /*
  8788.          * set the filesystem state up to be able to append to the
  8789. @@ -5493,33 +5967,22 @@
  8790.  
  8791.         inode_count = file_count + dir_count + sym_count + dev_count +
  8792.             fifo_count + sock_count;
  8793. -
  8794. -       /*
  8795. -        * The default use freelist before growing cache policy behaves
  8796. -        * poorly with appending - with many deplicates the caches
  8797. -        * do not grow due to the fact that large queues of outstanding
  8798. -        * fragments/writer blocks do not occur, leading to small caches
  8799. -        * and un-uncessary performance loss to frequent cache
  8800. -        * replacement in the small caches.  Therefore with appending
  8801. -        * change the policy to grow the caches before reusing blocks
  8802. -        * from the freelist
  8803. -        */
  8804. -       first_freelist = FALSE;
  8805.     }
  8806.  
  8807.     if(path)
  8808.         paths = add_subdir(paths, path);
  8809.  
  8810.     dump_actions();
  8811. +   dump_pseudos();
  8812.  
  8813.     if(delete && !keep_as_directory && source == 1 &&
  8814.             S_ISDIR(source_buf.st_mode))
  8815. -       dir_scan(&inode, source_path[0], scan1_readdir);
  8816. +       dir_scan(&inode, source_path[0], scan1_readdir, progress);
  8817.     else if(!keep_as_directory && source == 1 &&
  8818.             S_ISDIR(source_buf.st_mode))
  8819. -       dir_scan(&inode, source_path[0], scan1_single_readdir);
  8820. +       dir_scan(&inode, source_path[0], scan1_single_readdir, progress);
  8821.     else
  8822. -       dir_scan(&inode, "", scan1_encomp_readdir);
  8823. +       dir_scan(&inode, "", scan1_encomp_readdir, progress);
  8824.     sBlk.root_inode = inode;
  8825.     sBlk.inodes = inode_count;
  8826.     sBlk.s_magic = SQUASHFS_MAGIC;
  8827. @@ -5532,7 +5995,6 @@
  8828.         no_xattrs, comp_opts);
  8829.     sBlk.mkfs_time = time(NULL);
  8830.  
  8831. -   disable_progress_bar();
  8832.     disable_info();
  8833.  
  8834.     while((fragment = get_frag_action(fragment)))
  8835. @@ -5551,6 +6013,7 @@
  8836.     if(queue_get(from_writer) != 0)
  8837.         EXIT_MKSQUASHFS();
  8838.  
  8839. +   set_progressbar_state(FALSE);
  8840.     write_filesystem_tables(&sBlk, nopad);
  8841.  
  8842.     return 0;
  8843. diff -Nru squashfs-tools-4.2+20130409/mksquashfs.h squashfs-tools-4.3+20140919/mksquashfs.h
  8844. --- squashfs-tools-4.2+20130409/mksquashfs.h    2013-05-09 10:39:11.000000000 +0200
  8845. +++ squashfs-tools-4.3+20140919/mksquashfs.h    2015-07-20 21:03:05.000000000 +0200
  8846. @@ -4,8 +4,8 @@
  8847.   * Squashfs
  8848.   *
  8849.   * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011
  8850. - * 2012
  8851. - * Phillip Lougher <phillip@lougher.demon.co.uK>
  8852. + * 2012, 2013, 2014
  8853. + * Phillip Lougher <phillip@squashfs.org.uk>
  8854.   *
  8855.   * This program is free software; you can redistribute it and/or
  8856.   * modify it under the terms of the GNU General Public License
  8857. @@ -25,22 +25,6 @@
  8858.   *
  8859.   */
  8860.  
  8861. -#if __BYTE_ORDER == __BIG_ENDIAN
  8862. -#define SQUASHFS_SWAP_SHORTS(s, d, n) swap_le16_num(s, d, n)
  8863. -#define SQUASHFS_SWAP_INTS(s, d, n) swap_le32_num(s, d, n)
  8864. -#define SQUASHFS_SWAP_LONG_LONGS(s, d, n) swap_le64_num(s, d, n)
  8865. -
  8866. -#define SWAP_LE16(s, d)        swap_le16(s, d)
  8867. -#define SWAP_LE32(s, d)        swap_le32(s, d)
  8868. -#define SWAP_LE64(s, d)        swap_le64(s, d)
  8869. -#else
  8870. -#define SQUASHFS_MEMCPY(s, d, n)   memcpy(d, s, n)
  8871. -#define SQUASHFS_SWAP_SHORTS(s, d, n)  memcpy(d, s, n * sizeof(short))
  8872. -#define SQUASHFS_SWAP_INTS(s, d, n)    memcpy(d, s, n * sizeof(int))
  8873. -#define SQUASHFS_SWAP_LONG_LONGS(s, d, n) \
  8874. -                   memcpy(d, s, n * sizeof(long long))
  8875. -#endif
  8876. -
  8877.  struct dir_info {
  8878.     char            *pathname;
  8879.     char            *subpath;
  8880. @@ -79,6 +63,47 @@
  8881.     char            always_use_fragments;
  8882.     char            noD;
  8883.     char            noF;
  8884. +   char            symlink[0];
  8885. +};
  8886. +
  8887. +/* in memory file info */
  8888. +struct file_info {
  8889. +   long long       file_size;
  8890. +   long long       bytes;
  8891. +   long long       start;
  8892. +   unsigned int        *block_list;
  8893. +   struct file_info    *next;
  8894. +   struct fragment     *fragment;
  8895. +   unsigned short      checksum;
  8896. +   unsigned short      fragment_checksum;
  8897. +   char            have_frag_checksum;
  8898. +   char            have_checksum;
  8899. +};
  8900. +
  8901. +/* fragment block data structures */
  8902. +struct fragment {
  8903. +   unsigned int        index;
  8904. +   int         offset;
  8905. +   int         size;
  8906. +};
  8907. +
  8908. +/* in memory uid tables */
  8909. +#define ID_ENTRIES 256
  8910. +#define ID_HASH(id) (id & (ID_ENTRIES - 1))
  8911. +#define ISA_UID 1
  8912. +#define ISA_GID 2
  8913. +
  8914. +struct id {
  8915. +   unsigned int id;
  8916. +   int index;
  8917. +   char    flags;
  8918. +   struct id *next;
  8919. +};
  8920. +
  8921. +/* fragment to file mapping used when appending */
  8922. +struct append_file {
  8923. +   struct file_info *file;
  8924. +   struct append_file *next;
  8925.  };
  8926.  
  8927.  #define PSEUDO_FILE_OTHER  1
  8928. @@ -88,7 +113,42 @@
  8929.  #define IS_PSEUDO_PROCESS(a)   ((a)->pseudo_file & PSEUDO_FILE_PROCESS)
  8930.  #define IS_PSEUDO_OTHER(a) ((a)->pseudo_file & PSEUDO_FILE_OTHER)
  8931.  
  8932. +/*
  8933. + * Amount of physical memory to use by default, and the default queue
  8934. + * ratios
  8935. + */
  8936. +#define SQUASHFS_TAKE 4
  8937. +#define SQUASHFS_READQ_MEM 4
  8938. +#define SQUASHFS_BWRITEQ_MEM 4
  8939. +#define SQUASHFS_FWRITEQ_MEM 4
  8940. +
  8941. +/*
  8942. + * Lowest amount of physical memory considered viable for Mksquashfs
  8943. + * to run in Mbytes
  8944. + */
  8945. +#define SQUASHFS_LOWMEM 64
  8946. +
  8947.  /* offset of data in compressed metadata blocks (allowing room for
  8948.   * compressed size */
  8949.  #define BLOCK_OFFSET 2
  8950. +
  8951. +extern struct cache *reader_buffer, *fragment_buffer, *reserve_cache;
  8952. +struct cache *bwriter_buffer, *fwriter_buffer;
  8953. +extern struct queue *to_reader, *to_deflate, *to_writer, *from_writer,
  8954. +   *to_frag, *locked_fragment, *to_process_frag;
  8955. +extern struct append_file **file_mapping;
  8956. +extern struct seq_queue *to_main;
  8957. +extern pthread_mutex_t fragment_mutex, dup_mutex;
  8958. +extern struct squashfs_fragment_entry *fragment_table;
  8959. +extern struct compressor *comp;
  8960. +extern int block_size;
  8961. +extern struct file_info *dupl[];
  8962. +extern int read_fs_bytes(int, long long, int, void *);
  8963. +extern void add_file(long long, long long, long long, unsigned int *, int,
  8964. +   unsigned int, int, int);
  8965. +extern struct id *create_id(unsigned int);
  8966. +extern unsigned int get_uid(unsigned int);
  8967. +extern unsigned int get_guid(unsigned int);
  8968. +extern int read_bytes(int, void *, int);
  8969. +extern unsigned short get_checksum_mem(char *, int);
  8970.  #endif
  8971. diff -Nru squashfs-tools-4.2+20130409/par_mksquashfs/README squashfs-tools-4.3+20140919/par_mksquashfs/README
  8972. --- squashfs-tools-4.2+20130409/par_mksquashfs/README   2013-05-09 10:39:11.000000000 +0200
  8973. +++ squashfs-tools-4.3+20140919/par_mksquashfs/README   1970-01-01 01:00:00.000000000 +0100
  8974. @@ -1,2 +0,0 @@
  8975. -par_mksquashfs is now the standard mksquashfs, and so this directory is now empty.
  8976. -
  8977. diff -Nru squashfs-tools-4.2+20130409/process_fragments.c squashfs-tools-4.3+20140919/process_fragments.c
  8978. --- squashfs-tools-4.2+20130409/process_fragments.c 1970-01-01 01:00:00.000000000 +0100
  8979. +++ squashfs-tools-4.3+20140919/process_fragments.c 2015-07-20 21:03:05.000000000 +0200
  8980. @@ -0,0 +1,370 @@
  8981. +/*
  8982. + * Create a squashfs filesystem.  This is a highly compressed read only
  8983. + * filesystem.
  8984. + *
  8985. + * Copyright (c) 2014
  8986. + * Phillip Lougher <phillip@squashfs.org.uk>
  8987. + *
  8988. + * This program is free software; you can redistribute it and/or
  8989. + * modify it under the terms of the GNU General Public License
  8990. + * as published by the Free Software Foundation; either version 2,
  8991. + * or (at your option) any later version.
  8992. + *
  8993. + * This program is distributed in the hope that it will be useful,
  8994. + * but WITHOUT ANY WARRANTY; without even the implied warranty of
  8995. + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  8996. + * GNU General Public License for more details.
  8997. + *
  8998. + * You should have received a copy of the GNU General Public License
  8999. + * along with this program; if not, write to the Free Software
  9000. + * Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  9001. + *
  9002. + * process_fragments.c
  9003. + */
  9004. +
  9005. +#include <pthread.h>
  9006. +#include <sys/ioctl.h>
  9007. +#include <unistd.h>
  9008. +#include <signal.h>
  9009. +#include <sys/time.h>
  9010. +#include <string.h>
  9011. +#include <stdio.h>
  9012. +#include <math.h>
  9013. +#include <stdarg.h>
  9014. +#include <errno.h>
  9015. +#include <stdlib.h>
  9016. +#include <dirent.h>
  9017. +#include <sys/types.h>
  9018. +#include <sys/stat.h>
  9019. +#include <fcntl.h>
  9020. +
  9021. +#include "caches-queues-lists.h"
  9022. +#include "squashfs_fs.h"
  9023. +#include "mksquashfs.h"
  9024. +#include "error.h"
  9025. +#include "progressbar.h"
  9026. +#include "info.h"
  9027. +#include "compressor.h"
  9028. +#include "process_fragments.h"
  9029. +
  9030. +#define FALSE 0
  9031. +#define TRUE 1
  9032. +
  9033. +extern struct queue *to_process_frag;
  9034. +extern struct seq_queue *to_main;
  9035. +extern int sparse_files;
  9036. +
  9037. +/*
  9038. + * Compute 16 bit BSD checksum over the data, and check for sparseness
  9039. + */
  9040. +static int checksum_sparse(struct file_buffer *file_buffer)
  9041. +{
  9042. +   unsigned char *b = (unsigned char *) file_buffer->data;
  9043. +   unsigned short chksum = 0;
  9044. +   int bytes = file_buffer->size, sparse = TRUE, value;
  9045. +
  9046. +   while(bytes --) {
  9047. +       chksum = (chksum & 1) ? (chksum >> 1) | 0x8000 : chksum >> 1;
  9048. +       value = *b++;
  9049. +       if(value) {
  9050. +           sparse = FALSE;
  9051. +           chksum += value;
  9052. +       }
  9053. +   }
  9054. +
  9055. +   file_buffer->checksum = chksum;
  9056. +   return sparse;
  9057. +}
  9058. +
  9059. +
  9060. +static int read_filesystem(int fd, long long byte, int bytes, void *buff)
  9061. +{
  9062. +   off_t off = byte;
  9063. +
  9064. +   TRACE("read_filesystem: reading from position 0x%llx, bytes %d\n",
  9065. +       byte, bytes);
  9066. +
  9067. +   if(lseek(fd, off, SEEK_SET) == -1) {
  9068. +       ERROR("read_filesystem: Lseek on destination failed because %s, "
  9069. +           "offset=0x%llx\n", strerror(errno), off);
  9070. +       return 0;
  9071. +   } else if(read_bytes(fd, buff, bytes) < bytes) {
  9072. +       ERROR("Read on destination failed\n");
  9073. +       return 0;
  9074. +   }
  9075. +
  9076. +   return 1;
  9077. +}
  9078. +
  9079. +
  9080. +static struct file_buffer *get_fragment(struct fragment *fragment,
  9081. +   char *data_buffer, int fd)
  9082. +{
  9083. +   struct squashfs_fragment_entry *disk_fragment;
  9084. +   struct file_buffer *buffer, *compressed_buffer;
  9085. +   long long start_block;
  9086. +   int res, size, index = fragment->index;
  9087. +   char locked;
  9088. +
  9089. +   /*
  9090. +    * Lookup fragment block in cache.
  9091. +    * If the fragment block doesn't exist, then get the compressed version
  9092. +    * from the writer cache or off disk, and decompress it.
  9093. +    *
  9094. +    * This routine has two things which complicate the code:
  9095. +    *
  9096. +    *  1. Multiple threads can simultaneously lookup/create the
  9097. +    *     same buffer.  This means a buffer needs to be "locked"
  9098. +    *     when it is being filled in, to prevent other threads from
  9099. +    *     using it when it is not ready.  This is because we now do
  9100. +    *     fragment duplicate checking in parallel.
  9101. +    *  2. We have two caches which need to be checked for the
  9102. +    *     presence of fragment blocks: the normal fragment cache
  9103. +    *     and a "reserve" cache.  The reserve cache is used to
  9104. +    *     prevent an unnecessary pipeline stall when the fragment cache
  9105. +    *     is full of fragments waiting to be compressed.
  9106. +    */
  9107. +   pthread_cleanup_push((void *) pthread_mutex_unlock, &dup_mutex);
  9108. +   pthread_mutex_lock(&dup_mutex);
  9109. +
  9110. +again:
  9111. +   buffer = cache_lookup_nowait(fragment_buffer, index, &locked);
  9112. +   if(buffer) {
  9113. +       pthread_mutex_unlock(&dup_mutex);
  9114. +       if(locked)
  9115. +           /* got a buffer being filled in.  Wait for it */
  9116. +           cache_wait_unlock(buffer);
  9117. +       goto finished;
  9118. +   }
  9119. +
  9120. +   /* not in fragment cache, is it in the reserve cache? */
  9121. +   buffer = cache_lookup_nowait(reserve_cache, index, &locked);
  9122. +   if(buffer) {
  9123. +       pthread_mutex_unlock(&dup_mutex);
  9124. +       if(locked)
  9125. +           /* got a buffer being filled in.  Wait for it */
  9126. +           cache_wait_unlock(buffer);
  9127. +       goto finished;
  9128. +   }
  9129. +
  9130. +   /* in neither cache, try to get it from the fragment cache */
  9131. +   buffer = cache_get_nowait(fragment_buffer, index);
  9132. +   if(!buffer) {
  9133. +       /*
  9134. +        * no room, get it from the reserve cache, this is
  9135. +        * dimensioned so it will always have space (no more than
  9136. +        * processors + 1 can have an outstanding reserve buffer)
  9137. +        */
  9138. +       buffer = cache_get_nowait(reserve_cache, index);
  9139. +       if(!buffer) {
  9140. +           /* failsafe */
  9141. +           ERROR("no space in reserve cache\n");
  9142. +           goto again;
  9143. +       }
  9144. +   }
  9145. +
  9146. +   pthread_mutex_unlock(&dup_mutex);
  9147. +
  9148. +   compressed_buffer = cache_lookup(fwriter_buffer, index);
  9149. +
  9150. +   pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex);
  9151. +   pthread_mutex_lock(&fragment_mutex);
  9152. +   disk_fragment = &fragment_table[index];
  9153. +   size = SQUASHFS_COMPRESSED_SIZE_BLOCK(disk_fragment->size);
  9154. +   start_block = disk_fragment->start_block;
  9155. +   pthread_cleanup_pop(1);
  9156. +
  9157. +   if(SQUASHFS_COMPRESSED_BLOCK(disk_fragment->size)) {
  9158. +       int error;
  9159. +       char *data;
  9160. +
  9161. +       if(compressed_buffer)
  9162. +           data = compressed_buffer->data;
  9163. +       else {
  9164. +           res = read_filesystem(fd, start_block, size, data_buffer);
  9165. +           if(res == 0) {
  9166. +               ERROR("Failed to read fragment from output"
  9167. +                   " filesystem\n");
  9168. +               BAD_ERROR("Output filesystem corrupted?\n");
  9169. +           }
  9170. +           data = data_buffer;
  9171. +       }
  9172. +
  9173. +       res = compressor_uncompress(comp, buffer->data, data, size,
  9174. +           block_size, &error);
  9175. +       if(res == -1)
  9176. +           BAD_ERROR("%s uncompress failed with error code %d\n",
  9177. +               comp->name, error);
  9178. +   } else if(compressed_buffer)
  9179. +       memcpy(buffer->data, compressed_buffer->data, size);
  9180. +   else {
  9181. +       res = read_filesystem(fd, start_block, size, buffer->data);
  9182. +       if(res == 0) {
  9183. +           ERROR("Failed to read fragment from output "
  9184. +               "filesystem\n");
  9185. +           BAD_ERROR("Output filesystem corrupted?\n");
  9186. +       }
  9187. +   }
  9188. +
  9189. +   cache_unlock(buffer);
  9190. +   cache_block_put(compressed_buffer);
  9191. +
  9192. +finished:
  9193. +   pthread_cleanup_pop(0);
  9194. +
  9195. +   return buffer;
  9196. +}
  9197. +
  9198. +
  9199. +struct file_buffer *get_fragment_cksum(struct file_info *file,
  9200. +   char *data_buffer, int fd, unsigned short *checksum)
  9201. +{
  9202. +   struct file_buffer *frag_buffer;
  9203. +   struct append_file *append;
  9204. +   int index = file->fragment->index;
  9205. +
  9206. +   frag_buffer = get_fragment(file->fragment, data_buffer, fd);
  9207. +
  9208. +   pthread_cleanup_push((void *) pthread_mutex_unlock, &dup_mutex);
  9209. +
  9210. +   for(append = file_mapping[index]; append; append = append->next) {
  9211. +       int offset = append->file->fragment->offset;
  9212. +       int size = append->file->fragment->size;
  9213. +       char *data = frag_buffer->data + offset;
  9214. +       unsigned short cksum = get_checksum_mem(data, size);
  9215. +
  9216. +       if(file == append->file)
  9217. +           *checksum = cksum;
  9218. +
  9219. +       pthread_mutex_lock(&dup_mutex);
  9220. +       append->file->fragment_checksum = cksum;
  9221. +       append->file->have_frag_checksum = TRUE;
  9222. +       pthread_mutex_unlock(&dup_mutex);
  9223. +   }
  9224. +
  9225. +   pthread_cleanup_pop(0);
  9226. +
  9227. +   return frag_buffer;
  9228. +}
  9229. +
  9230. +
  9231. +void *frag_thrd(void *destination_file)
  9232. +{
  9233. +   sigset_t sigmask, old_mask;
  9234. +   char *data_buffer;
  9235. +   int fd;
  9236. +
  9237. +   sigemptyset(&sigmask);
  9238. +   sigaddset(&sigmask, SIGINT);
  9239. +   sigaddset(&sigmask, SIGTERM);
  9240. +   sigaddset(&sigmask, SIGUSR1);
  9241. +   pthread_sigmask(SIG_BLOCK, &sigmask, &old_mask);
  9242. +
  9243. +   fd = open(destination_file, O_RDONLY);
  9244. +   if(fd == -1)
  9245. +       BAD_ERROR("frag_thrd: can't open destination for reading\n");
  9246. +
  9247. +   data_buffer = malloc(SQUASHFS_FILE_MAX_SIZE);
  9248. +   if(data_buffer == NULL)
  9249. +       MEM_ERROR();
  9250. +
  9251. +   pthread_cleanup_push((void *) pthread_mutex_unlock, &dup_mutex);
  9252. +
  9253. +   while(1) {
  9254. +       struct file_buffer *file_buffer = queue_get(to_process_frag);
  9255. +       struct file_buffer *buffer;
  9256. +       int sparse = checksum_sparse(file_buffer);
  9257. +       struct file_info *dupl_ptr;
  9258. +       long long file_size;
  9259. +       unsigned short checksum;
  9260. +       char flag;
  9261. +       int res;
  9262. +
  9263. +       if(sparse_files && sparse) {
  9264. +           file_buffer->c_byte = 0;
  9265. +           file_buffer->fragment = FALSE;
  9266. +       } else
  9267. +           file_buffer->c_byte = file_buffer->size;
  9268. +
  9269. +       /*
  9270. +        * Specutively pull into the fragment cache any fragment blocks
  9271. +        * which contain fragments which *this* fragment may be
  9272. +        * be a duplicate.
  9273. +        *
  9274. +        * By ensuring the fragment block is in cache ahead of time
  9275. +        * should eliminate the parallelisation stall when the
  9276. +        * main thread needs to read the fragment block to do a
  9277. +        * duplicate check on it.
  9278. +        *
  9279. +        * If this is a fragment belonging to a larger file
  9280. +        * (with additional blocks) then ignore it.  Here we're
  9281. +        * interested in the "low hanging fruit" of files which
  9282. +        * consist of only a fragment
  9283. +        */
  9284. +       if(file_buffer->file_size != file_buffer->size) {
  9285. +           seq_queue_put(to_main, file_buffer);
  9286. +           continue;
  9287. +       }
  9288. +
  9289. +       file_size = file_buffer->file_size;
  9290. +
  9291. +       pthread_mutex_lock(&dup_mutex);
  9292. +       dupl_ptr = dupl[DUP_HASH(file_size)];
  9293. +       pthread_mutex_unlock(&dup_mutex);
  9294. +
  9295. +       file_buffer->dupl_start = dupl_ptr;
  9296. +       file_buffer->duplicate = FALSE;
  9297. +
  9298. +       for(; dupl_ptr; dupl_ptr = dupl_ptr->next) {
  9299. +           if(file_size != dupl_ptr->file_size ||
  9300. +                   file_size != dupl_ptr->fragment->size)
  9301. +               continue;
  9302. +
  9303. +           pthread_mutex_lock(&dup_mutex);
  9304. +           flag = dupl_ptr->have_frag_checksum;
  9305. +           checksum = dupl_ptr->fragment_checksum;
  9306. +           pthread_mutex_unlock(&dup_mutex);
  9307. +
  9308. +           /*
  9309. +            * If we have the checksum and it matches then
  9310. +            * read in the fragment block.
  9311. +            *
  9312. +            * If we *don't* have the checksum, then we are
  9313. +            * appending, and the fragment block is on the
  9314. +            * "old" filesystem.  Read it in and checksum
  9315. +            * the entire fragment buffer
  9316. +            */
  9317. +           if(!flag) {
  9318. +               buffer = get_fragment_cksum(dupl_ptr,
  9319. +                   data_buffer, fd, &checksum);
  9320. +               if(checksum != file_buffer->checksum) {
  9321. +                   cache_block_put(buffer);
  9322. +                   continue;
  9323. +               }
  9324. +           } else if(checksum == file_buffer->checksum)
  9325. +               buffer = get_fragment(dupl_ptr->fragment,
  9326. +                   data_buffer, fd);
  9327. +           else
  9328. +               continue;
  9329. +
  9330. +           res = memcmp(file_buffer->data, buffer->data +
  9331. +               dupl_ptr->fragment->offset, file_size);
  9332. +           cache_block_put(buffer);
  9333. +           if(res == 0) {
  9334. +               struct file_buffer *dup = malloc(sizeof(*dup));
  9335. +               if(dup == NULL)
  9336. +                   MEM_ERROR();
  9337. +               memcpy(dup, file_buffer, sizeof(*dup));
  9338. +               cache_block_put(file_buffer);
  9339. +               dup->dupl_start = dupl_ptr;
  9340. +               dup->duplicate = TRUE;
  9341. +               file_buffer = dup;
  9342. +               break;
  9343. +           }
  9344. +       }
  9345. +
  9346. +       seq_queue_put(to_main, file_buffer);
  9347. +   }
  9348. +
  9349. +   pthread_cleanup_pop(0);
  9350. +}
  9351. diff -Nru squashfs-tools-4.2+20130409/process_fragments.h squashfs-tools-4.3+20140919/process_fragments.h
  9352. --- squashfs-tools-4.2+20130409/process_fragments.h 1970-01-01 01:00:00.000000000 +0100
  9353. +++ squashfs-tools-4.3+20140919/process_fragments.h 2015-07-20 21:03:05.000000000 +0200
  9354. @@ -0,0 +1,30 @@
  9355. +#ifndef PROCESS_FRAGMENTS_H
  9356. +#define PROCESS_FRAGMENTS_H
  9357. +/*
  9358. + * Create a squashfs filesystem.  This is a highly compressed read only
  9359. + * filesystem.
  9360. + *
  9361. + * Copyright (c) 2014
  9362. + * Phillip Lougher <phillip@squashfs.org.uk>
  9363. + *
  9364. + * This program is free software; you can redistribute it and/or
  9365. + * modify it under the terms of the GNU General Public License
  9366. + * as published by the Free Software Foundation; either version 2,
  9367. + * or (at your option) any later version.
  9368. + *
  9369. + * This program is distributed in the hope that it will be useful,
  9370. + * but WITHOUT ANY WARRANTY; without even the implied warranty of
  9371. + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  9372. + * GNU General Public License for more details.
  9373. + *
  9374. + * You should have received a copy of the GNU General Public License
  9375. + * along with this program; if not, write to the Free Software
  9376. + * Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  9377. + *
  9378. + * process_fragments.h
  9379. + */
  9380. +
  9381. +#define DUP_HASH(a) (a & 0xffff)
  9382. +
  9383. +extern void *frag_thrd(void *);
  9384. +#endif
  9385. diff -Nru squashfs-tools-4.2+20130409/progressbar.c squashfs-tools-4.3+20140919/progressbar.c
  9386. --- squashfs-tools-4.2+20130409/progressbar.c   2013-05-09 10:39:11.000000000 +0200
  9387. +++ squashfs-tools-4.3+20140919/progressbar.c   2015-07-20 21:03:05.000000000 +0200
  9388. @@ -2,7 +2,7 @@
  9389.   * Create a squashfs filesystem.  This is a highly compressed read only
  9390.   * filesystem.
  9391.   *
  9392. - * Copyright (c) 2012, 2013
  9393. + * Copyright (c) 2012, 2013, 2014
  9394.   * Phillip Lougher <phillip@squashfs.org.uk>
  9395.   *
  9396.   * This program is free software; you can redistribute it and/or
  9397. @@ -35,7 +35,15 @@
  9398.  
  9399.  #include "error.h"
  9400.  
  9401. -int progress_enabled = 0;
  9402. +#define FALSE 0
  9403. +#define TRUE 1
  9404. +
  9405. +/* flag whether progressbar display is enabled or not */
  9406. +int display_progress_bar = FALSE;
  9407. +
  9408. +/* flag whether the progress bar is temporarily disbled */
  9409. +int temp_disabled = FALSE;
  9410. +
  9411.  int rotate = 0;
  9412.  int cur_uncompressed = 0, estimated_uncompressed = 0;
  9413.  int columns;
  9414. @@ -50,7 +58,7 @@
  9415.  
  9416.     if(ioctl(1, TIOCGWINSZ, &winsize) == -1) {
  9417.         if(isatty(STDOUT_FILENO))
  9418. -           printf("TIOCGWINSZ ioctl failed, defaulting to 80 "
  9419. +           ERROR("TIOCGWINSZ ioctl failed, defaulting to 80 "
  9420.                 "columns\n");
  9421.         columns = 80;
  9422.     } else
  9423. @@ -132,21 +140,39 @@
  9424.  
  9425.  void enable_progress_bar()
  9426.  {
  9427. +   pthread_cleanup_push((void *) pthread_mutex_unlock, &progress_mutex);
  9428.     pthread_mutex_lock(&progress_mutex);
  9429. -   progress_enabled = 1;
  9430. -   pthread_mutex_unlock(&progress_mutex);
  9431. +   if(display_progress_bar)
  9432. +       progress_bar(cur_uncompressed, estimated_uncompressed, columns);
  9433. +   temp_disabled = FALSE;
  9434. +   pthread_cleanup_pop(1);
  9435.  }
  9436.  
  9437.  
  9438.  void disable_progress_bar()
  9439.  {
  9440. +   pthread_cleanup_push((void *) pthread_mutex_unlock, &progress_mutex);
  9441.     pthread_mutex_lock(&progress_mutex);
  9442. -   if(progress_enabled)  {
  9443. -       progress_bar(cur_uncompressed, estimated_uncompressed, columns);
  9444. +   if(display_progress_bar)
  9445.         printf("\n");
  9446. +   temp_disabled = TRUE;
  9447. +   pthread_cleanup_pop(1);
  9448. +}
  9449. +
  9450. +
  9451. +void set_progressbar_state(int state)
  9452. +{
  9453. +   pthread_cleanup_push((void *) pthread_mutex_unlock, &progress_mutex);
  9454. +   pthread_mutex_lock(&progress_mutex);
  9455. +   if(display_progress_bar != state) {
  9456. +       if(display_progress_bar && !temp_disabled) {
  9457. +           progress_bar(cur_uncompressed, estimated_uncompressed,
  9458. +               columns);
  9459. +           printf("\n");
  9460. +       }
  9461. +       display_progress_bar = state;
  9462.     }
  9463. -   progress_enabled = 0;
  9464. -   pthread_mutex_unlock(&progress_mutex);
  9465. +   pthread_cleanup_pop(1);
  9466.  }
  9467.  
  9468.  
  9469. @@ -158,7 +184,7 @@
  9470.  
  9471.     if(ioctl(1, TIOCGWINSZ, &winsize) == -1) {
  9472.         if(isatty(STDOUT_FILENO))
  9473. -           printf("TIOCGWINSZ ioctl failed, defaulting to 80 "
  9474. +           ERROR("TIOCGWINSZ ioctl failed, defaulting to 80 "
  9475.                 "columns\n");
  9476.         columns = 80;
  9477.     } else
  9478. @@ -181,12 +207,11 @@
  9479.         if(res == -1 && errno != EINTR)
  9480.             BAD_ERROR("nanosleep failed in progress thread\n");
  9481.  
  9482. -       if(progress_enabled) {
  9483. -           pthread_mutex_lock(&progress_mutex);
  9484. +       pthread_mutex_lock(&progress_mutex);
  9485. +       if(display_progress_bar && !temp_disabled)
  9486.             progress_bar(cur_uncompressed, estimated_uncompressed,
  9487.                 columns);
  9488. -           pthread_mutex_unlock(&progress_mutex);
  9489. -       }
  9490. +       pthread_mutex_unlock(&progress_mutex);
  9491.     }
  9492.  }
  9493.  
  9494. @@ -201,16 +226,17 @@
  9495.  {
  9496.     va_list ap;
  9497.  
  9498. +   pthread_cleanup_push((void *) pthread_mutex_unlock, &progress_mutex);
  9499.     pthread_mutex_lock(&progress_mutex);
  9500.  
  9501. -   if(progress_enabled)
  9502. +   if(display_progress_bar && !temp_disabled)
  9503.         fprintf(stderr, "\n");
  9504.  
  9505.     va_start(ap, fmt);
  9506.     vfprintf(stderr, fmt, ap);
  9507.     va_end(ap);
  9508.  
  9509. -   pthread_mutex_unlock(&progress_mutex);
  9510. +   pthread_cleanup_pop(1);
  9511.  }
  9512.  
  9513.  
  9514. @@ -218,15 +244,16 @@
  9515.  {
  9516.     va_list ap;
  9517.  
  9518. +   pthread_cleanup_push((void *) pthread_mutex_unlock, &progress_mutex);
  9519.     pthread_mutex_lock(&progress_mutex);
  9520.  
  9521. -   if(progress_enabled)
  9522. +   if(display_progress_bar && !temp_disabled)
  9523.         printf("\n");
  9524.  
  9525.     va_start(ap, fmt);
  9526.     vprintf(fmt, ap);
  9527.     va_end(ap);
  9528.  
  9529. -   pthread_mutex_unlock(&progress_mutex);
  9530. +   pthread_cleanup_pop(1);
  9531.  }
  9532.  
  9533. diff -Nru squashfs-tools-4.2+20130409/progressbar.h squashfs-tools-4.3+20140919/progressbar.h
  9534. --- squashfs-tools-4.2+20130409/progressbar.h   2013-05-09 10:39:11.000000000 +0200
  9535. +++ squashfs-tools-4.3+20140919/progressbar.h   2015-07-20 21:03:05.000000000 +0200
  9536. @@ -1,8 +1,10 @@
  9537. +#ifndef PROGRESSBAR_H
  9538. +#define PROGRESSBAR_H
  9539.  /*
  9540.   * Create a squashfs filesystem.  This is a highly compressed read only
  9541.   * filesystem.
  9542.   *
  9543. - * Copyright (c) 2012, 2013
  9544. + * Copyright (c) 2012, 2013, 2014
  9545.   * Phillip Lougher <phillip@squashfs.org.uk>
  9546.   *
  9547.   * This program is free software; you can redistribute it and/or
  9548. @@ -28,3 +30,5 @@
  9549.  extern void enable_progress_bar();
  9550.  extern void disable_progress_bar();
  9551.  extern void init_progress_bar();
  9552. +extern void set_progressbar_state(int);
  9553. +#endif
  9554. diff -Nru squashfs-tools-4.2+20130409/pseudo.c squashfs-tools-4.3+20140919/pseudo.c
  9555. --- squashfs-tools-4.2+20130409/pseudo.c    2013-05-09 10:39:11.000000000 +0200
  9556. +++ squashfs-tools-4.3+20140919/pseudo.c    2015-07-20 21:03:05.000000000 +0200
  9557. @@ -2,7 +2,7 @@
  9558.   * Create a squashfs filesystem.  This is a highly compressed read only
  9559.   * filesystem.
  9560.   *
  9561. - * Copyright (c) 2009, 2010, 2012
  9562. + * Copyright (c) 2009, 2010, 2012, 2014
  9563.   * Phillip Lougher <phillip@squashfs.org.uk>
  9564.   *
  9565.   * This program is free software; you can redistribute it and/or
  9566. @@ -36,6 +36,7 @@
  9567.  
  9568.  #include "pseudo.h"
  9569.  #include "error.h"
  9570. +#include "progressbar.h"
  9571.  
  9572.  #define TRUE 1
  9573.  #define FALSE 0
  9574. @@ -46,32 +47,6 @@
  9575.  struct pseudo *pseudo = NULL;
  9576.  int pseudo_count = 0;
  9577.  
  9578. -static void dump_pseudo(struct pseudo *pseudo, char *string)
  9579. -{
  9580. -   int i, res;
  9581. -   char *path;
  9582. -
  9583. -   for(i = 0; i < pseudo->names; i++) {
  9584. -       struct pseudo_entry *entry = &pseudo->name[i];
  9585. -       if(string) {
  9586. -           res = asprintf(&path, "%s/%s", string, entry->name);
  9587. -           if(res == -1)
  9588. -               BAD_ERROR("asprintf failed in dump_pseudo\n");
  9589. -       } else
  9590. -           path = entry->name;
  9591. -       if(entry->pseudo == NULL)
  9592. -           ERROR("%s %c %o %d %d %d %d\n", path, entry->dev->type,
  9593. -               entry->dev->mode, entry->dev->uid,
  9594. -               entry->dev->gid, entry->dev->major,
  9595. -               entry->dev->minor);
  9596. -       else
  9597. -           dump_pseudo(entry->pseudo, path);
  9598. -       if(string)
  9599. -           free(path);
  9600. -   }
  9601. -}
  9602. -
  9603. -
  9604.  static char *get_component(char *target, char **targname)
  9605.  {
  9606.     char *start;
  9607. @@ -80,11 +55,14 @@
  9608.         target ++;
  9609.  
  9610.     start = target;
  9611. -   while(*target != '/' && *target!= '\0')
  9612. +   while(*target != '/' && *target != '\0')
  9613.         target ++;
  9614.  
  9615.     *targname = strndup(start, target - start);
  9616.  
  9617. +   while(*target == '/')
  9618. +       target ++;
  9619. +
  9620.     return target;
  9621.  }
  9622.  
  9623. @@ -154,16 +132,23 @@
  9624.                     pseudo->name[i].pseudo =
  9625.                         add_pseudo(NULL, pseudo_dev,
  9626.                         target, alltarget);
  9627. -               else
  9628. -                   ERROR("%s already exists as a non "
  9629. -                       "directory.  Ignoring %s!\n",
  9630. -                        targname, alltarget);
  9631. +               else {
  9632. +                   ERROR_START("%s already exists as a "
  9633. +                       "non directory.",
  9634. +                       pseudo->name[i].name);
  9635. +                   ERROR_EXIT(".  Ignoring %s!\n",
  9636. +                       alltarget);
  9637. +               }
  9638.             } else if(memcmp(pseudo_dev, pseudo->name[i].dev,
  9639. -                   sizeof(struct pseudo_dev)) != 0)
  9640. -               ERROR("%s already exists as a different pseudo "
  9641. -                   "definition.  Ignoring!\n", alltarget);
  9642. -           else ERROR("%s already exists as an identical "
  9643. -                   "pseudo definition!\n", alltarget);
  9644. +                   sizeof(struct pseudo_dev)) != 0) {
  9645. +               ERROR_START("%s already exists as a different "
  9646. +                   "pseudo definition.", alltarget);
  9647. +               ERROR_EXIT("  Ignoring!\n");
  9648. +           } else {
  9649. +               ERROR_START("%s already exists as an identical "
  9650. +                   "pseudo definition!", alltarget);
  9651. +               ERROR_EXIT("  Ignoring!\n");
  9652. +           }
  9653.         } else {
  9654.             if(target[0] == '\0') {
  9655.                 /*
  9656. @@ -176,10 +161,13 @@
  9657.                     pseudo->name[i].pathname =
  9658.                         strdup(alltarget);
  9659.                     pseudo->name[i].dev = pseudo_dev;
  9660. -               } else
  9661. -                   ERROR("%s already exists as a different"
  9662. -                       " pseudo definition.  Ignoring"
  9663. -                       " %s!\n", targname, alltarget);
  9664. +               } else {
  9665. +                   ERROR_START("%s already exists as a "
  9666. +                       "different pseudo definition.",
  9667. +                       pseudo->name[i].name);
  9668. +                   ERROR_EXIT("  Ignoring %s!\n",
  9669. +                       alltarget);
  9670. +               }
  9671.             } else
  9672.                 /* recurse adding child components */
  9673.                 add_pseudo(pseudo->name[i].pseudo, pseudo_dev,
  9674. @@ -227,75 +215,40 @@
  9675.  }
  9676.  
  9677.  
  9678. -int exec_file(char *command, struct pseudo_dev *dev)
  9679. +int pseudo_exec_file(struct pseudo_dev *dev, int *child)
  9680.  {
  9681. -   int child, res;
  9682. -   static pid_t pid = -1;
  9683. -   int pipefd[2];
  9684. -#ifdef USE_TMP_FILE
  9685. -   char *filename;
  9686. -   int status;
  9687. -   static int number = 0;
  9688. -#endif
  9689. -
  9690. -   if(pid == -1)
  9691. -       pid = getpid();
  9692. +   int res, pipefd[2];
  9693.  
  9694. -#ifdef USE_TMP_FILE
  9695. -   res = asprintf(&filename, "/tmp/squashfs_pseudo_%d_%d", pid, number ++);
  9696. -   if(res == -1) {
  9697. -       ERROR("asprint failed in exec_file()\n");
  9698. -       return -1;
  9699. -   }
  9700. -   pipefd[1] = open(filename, O_CREAT | O_TRUNC | O_RDWR, S_IRWXU);
  9701. -   if(pipefd[1] == -1) {
  9702. -       ERROR("Executing dynamic pseudo file, open failed\n");
  9703. -       free(filename);
  9704. -       return -1;
  9705. -   }
  9706. -#else
  9707.     res = pipe(pipefd);
  9708.     if(res == -1) {
  9709.         ERROR("Executing dynamic pseudo file, pipe failed\n");
  9710. -       return -1;
  9711. +       return 0;
  9712.     }
  9713. -#endif
  9714.  
  9715. -   child = fork();
  9716. -   if(child == -1) {
  9717. +   *child = fork();
  9718. +   if(*child == -1) {
  9719.         ERROR("Executing dynamic pseudo file, fork failed\n");
  9720.         goto failed;
  9721.     }
  9722.  
  9723. -   if(child == 0) {
  9724. +   if(*child == 0) {
  9725. +       close(pipefd[0]);
  9726.         close(STDOUT_FILENO);
  9727.         res = dup(pipefd[1]);
  9728.         if(res == -1)
  9729.             exit(EXIT_FAILURE);
  9730.  
  9731. -       execl("/bin/sh", "sh", "-c", command, (char *) NULL);
  9732. +       execl("/bin/sh", "sh", "-c", dev->command, (char *) NULL);
  9733.         exit(EXIT_FAILURE);
  9734.     }
  9735.  
  9736. -#ifdef USE_TMP_FILE
  9737. -   res = waitpid(child, &status, 0);
  9738.     close(pipefd[1]);
  9739. -   if(res != -1 && WIFEXITED(status) && WEXITSTATUS(status) == 0) {
  9740. -       dev->filename = filename;
  9741. -       return 0;
  9742. -   }
  9743. +   return pipefd[0];
  9744. +
  9745.  failed:
  9746. -   unlink(filename);
  9747. -   free(filename);
  9748. -   return -1;
  9749. -#else
  9750. +   close(pipefd[0]);
  9751.     close(pipefd[1]);
  9752. -   dev->fd = pipefd[0];
  9753. -   dev->child = child;
  9754.     return 0;
  9755. -failed:
  9756. -   return -1;
  9757. -#endif
  9758.  }
  9759.  
  9760.  
  9761. @@ -311,17 +264,6 @@
  9762.  }
  9763.  
  9764.  
  9765. -void delete_pseudo_files()
  9766. -{
  9767. -#ifdef USE_TMP_FILE
  9768. -   int i;
  9769. -
  9770. -   for(i = 0; i < pseudo_count; i++)
  9771. -       unlink(pseudo_file[i]->filename);
  9772. -#endif
  9773. -}
  9774. -
  9775. -
  9776.  struct pseudo_dev *get_pseudo_file(int pseudo_id)
  9777.  {
  9778.     return pseudo_file[pseudo_id];
  9779. @@ -513,20 +455,8 @@
  9780.     dev->gid = gid;
  9781.     dev->major = major;
  9782.     dev->minor = minor;
  9783. -
  9784.     if(type == 'f') {
  9785. -       int res;
  9786. -
  9787. -       printf("Executing dynamic pseudo file\n");
  9788. -       printf("\t\"%s\"\n", orig_def);
  9789. -       res = exec_file(def, dev);
  9790. -       if(res == -1) {
  9791. -           ERROR("Failed to execute dynamic pseudo file definition"
  9792. -               " \"%s\"\n", orig_def);
  9793. -           free(filename);
  9794. -           free(dev);
  9795. -           return FALSE;
  9796. -       }
  9797. +       dev->command = strdup(def);
  9798.         add_pseudo_file(dev);
  9799.     }
  9800.  
  9801. @@ -557,3 +487,41 @@
  9802.  {
  9803.     return pseudo;
  9804.  }
  9805. +
  9806. +
  9807. +#ifdef SQUASHFS_TRACE
  9808. +static void dump_pseudo(struct pseudo *pseudo, char *string)
  9809. +{
  9810. +   int i, res;
  9811. +   char *path;
  9812. +
  9813. +   for(i = 0; i < pseudo->names; i++) {
  9814. +       struct pseudo_entry *entry = &pseudo->name[i];
  9815. +       if(string) {
  9816. +           res = asprintf(&path, "%s/%s", string, entry->name);
  9817. +           if(res == -1)
  9818. +               BAD_ERROR("asprintf failed in dump_pseudo\n");
  9819. +       } else
  9820. +           path = entry->name;
  9821. +       if(entry->dev)
  9822. +           ERROR("%s %c 0%o %d %d %d %d\n", path, entry->dev->type,
  9823. +               entry->dev->mode & ~S_IFMT, entry->dev->uid,
  9824. +               entry->dev->gid, entry->dev->major,
  9825. +               entry->dev->minor);
  9826. +       if(entry->pseudo)
  9827. +           dump_pseudo(entry->pseudo, path);
  9828. +       if(string)
  9829. +           free(path);
  9830. +   }
  9831. +}
  9832. +
  9833. +
  9834. +void dump_pseudos()
  9835. +{
  9836. +   dump_pseudo(pseudo, NULL);
  9837. +}
  9838. +#else
  9839. +void dump_pseudos()
  9840. +{
  9841. +}
  9842. +#endif
  9843. diff -Nru squashfs-tools-4.2+20130409/pseudo.h squashfs-tools-4.3+20140919/pseudo.h
  9844. --- squashfs-tools-4.2+20130409/pseudo.h    2013-05-09 10:39:11.000000000 +0200
  9845. +++ squashfs-tools-4.3+20140919/pseudo.h    2015-07-20 21:03:05.000000000 +0200
  9846. @@ -1,8 +1,10 @@
  9847. +#ifndef PSEUDO_H
  9848. +#define PSEUDO_H
  9849.  /*
  9850.   * Create a squashfs filesystem.  This is a highly compressed read only
  9851.   * filesystem.
  9852.   *
  9853. - * Copyright (c) 2009, 2010
  9854. + * Copyright (c) 2009, 2010, 2014
  9855.   * Phillip Lougher <phillip@squashfs.org.uk>
  9856.   *
  9857.   * This program is free software; you can redistribute it and/or
  9858. @@ -29,11 +31,7 @@
  9859.     unsigned int    major;
  9860.     unsigned int    minor;
  9861.     int     pseudo_id;
  9862. -   int     fd;
  9863. -   int     child;
  9864. -#ifdef USE_TMP_FILE
  9865. -   char        *filename;
  9866. -#endif
  9867. +   char        *command;
  9868.  };
  9869.  
  9870.  struct pseudo_entry {
  9871. @@ -54,5 +52,7 @@
  9872.  extern struct pseudo *pseudo_subdir(char *, struct pseudo *);
  9873.  extern struct pseudo_entry *pseudo_readdir(struct pseudo *);
  9874.  extern struct pseudo_dev *get_pseudo_file(int);
  9875. -extern void delete_pseudo_files();
  9876. +extern int pseudo_exec_file(struct pseudo_dev *, int *);
  9877.  extern struct pseudo *get_pseudo();
  9878. +extern void dump_pseudos();
  9879. +#endif
  9880. diff -Nru squashfs-tools-4.2+20130409/read_fs.c squashfs-tools-4.3+20140919/read_fs.c
  9881. --- squashfs-tools-4.2+20130409/read_fs.c   2013-05-09 10:39:11.000000000 +0200
  9882. +++ squashfs-tools-4.3+20140919/read_fs.c   2015-07-20 21:03:05.000000000 +0200
  9883. @@ -3,7 +3,7 @@
  9884.   * filesystem.
  9885.   *
  9886.   * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
  9887. - * 2012, 2013
  9888. + * 2012, 2013, 2014
  9889.   * Phillip Lougher <phillip@squashfs.org.uk>
  9890.   *
  9891.   * This program is free software; you can redistribute it and/or
  9892. @@ -33,6 +33,7 @@
  9893.  #include <string.h>
  9894.  #include <sys/mman.h>
  9895.  #include <limits.h>
  9896. +#include <dirent.h>
  9897.  
  9898.  #ifndef linux
  9899.  #define __BYTE_ORDER BYTE_ORDER
  9900. @@ -46,19 +47,10 @@
  9901.  
  9902.  #include "squashfs_fs.h"
  9903.  #include "squashfs_swap.h"
  9904. -#include "read_fs.h"
  9905.  #include "compressor.h"
  9906.  #include "xattr.h"
  9907.  #include "error.h"
  9908. -
  9909. -extern int read_fs_bytes(int, long long, int, void *);
  9910. -extern int add_file(long long, long long, long long, unsigned int *, int,
  9911. -   unsigned int, int, int);
  9912. -extern void *create_id(unsigned int);
  9913. -extern unsigned int get_uid(unsigned int);
  9914. -extern unsigned int get_guid(unsigned int);
  9915. -
  9916. -static struct compressor *comp;
  9917. +#include "mksquashfs.h"
  9918.  
  9919.  int read_block(int fd, long long start, long long *next, int expected,
  9920.                                 void *block)
  9921. @@ -207,7 +199,7 @@
  9922.      */
  9923.     *root_inode_size = bytes - (*root_inode_block + root_inode_offset);
  9924.     bytes = *root_inode_block + root_inode_offset;
  9925. -   SQUASHFS_SWAP_DIR_INODE_HEADER(&dir_inode->dir, *inode_table + bytes);
  9926. +   SQUASHFS_SWAP_DIR_INODE_HEADER(*inode_table + bytes, &dir_inode->dir);
  9927.    
  9928.     if(dir_inode->base.inode_type == SQUASHFS_DIR_TYPE)
  9929.         directory_start_block = dir_inode->dir.start_block;
  9930. @@ -215,8 +207,8 @@
  9931.         if(*root_inode_size < sizeof(struct squashfs_ldir_inode_header))
  9932.             /* corrupted filesystem */
  9933.             goto corrupted;
  9934. -       SQUASHFS_SWAP_LDIR_INODE_HEADER(&dir_inode->ldir,
  9935. -           *inode_table + bytes);
  9936. +       SQUASHFS_SWAP_LDIR_INODE_HEADER(*inode_table + bytes,
  9937. +           &dir_inode->ldir);
  9938.         directory_start_block = dir_inode->ldir.start_block;
  9939.     } else
  9940.         /* bad type, corrupted filesystem */
  9941. @@ -225,12 +217,17 @@
  9942.     get_uid(id_table[dir_inode->base.uid]);
  9943.     get_guid(id_table[dir_inode->base.guid]);
  9944.  
  9945. +   /* allocate fragment to file mapping table */
  9946. +   file_mapping = calloc(sBlk->fragments, sizeof(struct append_file *));
  9947. +   if(file_mapping == NULL)
  9948. +       MEM_ERROR();
  9949. +
  9950.     for(cur_ptr = *inode_table; cur_ptr < *inode_table + bytes; files ++) {
  9951.         if(NO_INODE_BYTES(squashfs_base_inode_header))
  9952.             /* corrupted filesystem */
  9953.             goto corrupted;
  9954.  
  9955. -       SQUASHFS_SWAP_BASE_INODE_HEADER(&base, cur_ptr);
  9956. +       SQUASHFS_SWAP_BASE_INODE_HEADER(cur_ptr, &base);
  9957.  
  9958.         TRACE("scan_inode_table: processing inode @ byte position "
  9959.             "0x%x, type 0x%x\n",
  9960. @@ -251,7 +248,7 @@
  9961.                 /* corrupted filesystem */
  9962.                 goto corrupted;
  9963.  
  9964. -           SQUASHFS_SWAP_REG_INODE_HEADER(&inode, cur_ptr);
  9965. +           SQUASHFS_SWAP_REG_INODE_HEADER(cur_ptr, &inode);
  9966.  
  9967.             frag_bytes = inode.fragment == SQUASHFS_INVALID_FRAG ?
  9968.                 0 : inode.file_size % sBlk->block_size;
  9969. @@ -273,7 +270,7 @@
  9970.                 MEM_ERROR();
  9971.  
  9972.             cur_ptr += sizeof(inode);
  9973. -           SQUASHFS_SWAP_INTS(block_list, cur_ptr, blocks);
  9974. +           SQUASHFS_SWAP_INTS(cur_ptr, block_list, blocks);
  9975.  
  9976.             *uncompressed_file += inode.file_size;
  9977.             (*file_count) ++;
  9978. @@ -283,6 +280,12 @@
  9979.                     SQUASHFS_COMPRESSED_SIZE_BLOCK
  9980.                                 (block_list[i]);
  9981.  
  9982. +           if(inode.fragment != SQUASHFS_INVALID_FRAG &&
  9983. +                   inode.fragment >= sBlk->fragments) {
  9984. +               free(block_list);
  9985. +               goto corrupted;
  9986. +           }
  9987. +
  9988.             add_file(start, inode.file_size, file_bytes,
  9989.                 block_list, blocks, inode.fragment,
  9990.                 inode.offset, frag_bytes);
  9991. @@ -300,7 +303,7 @@
  9992.                 /* corrupted filesystem */
  9993.                 goto corrupted;
  9994.  
  9995. -           SQUASHFS_SWAP_LREG_INODE_HEADER(&inode, cur_ptr);
  9996. +           SQUASHFS_SWAP_LREG_INODE_HEADER(cur_ptr, &inode);
  9997.  
  9998.             frag_bytes = inode.fragment == SQUASHFS_INVALID_FRAG ?
  9999.                 0 : inode.file_size % sBlk->block_size;
  10000. @@ -323,7 +326,7 @@
  10001.                 MEM_ERROR();
  10002.  
  10003.             cur_ptr += sizeof(inode);
  10004. -           SQUASHFS_SWAP_INTS(block_list, cur_ptr, blocks);
  10005. +           SQUASHFS_SWAP_INTS(cur_ptr, block_list, blocks);
  10006.  
  10007.             *uncompressed_file += inode.file_size;
  10008.             (*file_count) ++;
  10009. @@ -333,6 +336,12 @@
  10010.                     SQUASHFS_COMPRESSED_SIZE_BLOCK
  10011.                                 (block_list[i]);
  10012.  
  10013. +           if(inode.fragment != SQUASHFS_INVALID_FRAG &&
  10014. +                   inode.fragment >= sBlk->fragments) {
  10015. +               free(block_list);
  10016. +               goto corrupted;
  10017. +           }
  10018. +
  10019.             add_file(start, inode.file_size, file_bytes,
  10020.                 block_list, blocks, inode.fragment,
  10021.                 inode.offset, frag_bytes);
  10022. @@ -348,7 +357,7 @@
  10023.                 /* corrupted filesystem */
  10024.                 goto corrupted;
  10025.  
  10026. -           SQUASHFS_SWAP_SYMLINK_INODE_HEADER(&inode, cur_ptr);
  10027. +           SQUASHFS_SWAP_SYMLINK_INODE_HEADER(cur_ptr, &inode);
  10028.  
  10029.             (*sym_count) ++;
  10030.  
  10031. @@ -374,7 +383,7 @@
  10032.                 /* corrupted filesystem */
  10033.                 goto corrupted;
  10034.                
  10035. -           SQUASHFS_SWAP_DIR_INODE_HEADER(&dir_inode, cur_ptr);
  10036. +           SQUASHFS_SWAP_DIR_INODE_HEADER(cur_ptr, &dir_inode);
  10037.  
  10038.             if(dir_inode.start_block < directory_start_block)
  10039.                 *uncompressed_directory += dir_inode.file_size;
  10040. @@ -391,7 +400,7 @@
  10041.                 /* corrupted filesystem */
  10042.                 goto corrupted;
  10043.  
  10044. -           SQUASHFS_SWAP_LDIR_INODE_HEADER(&dir_inode, cur_ptr);
  10045. +           SQUASHFS_SWAP_LDIR_INODE_HEADER(cur_ptr, &dir_inode);
  10046.  
  10047.             if(dir_inode.start_block < directory_start_block)
  10048.                 *uncompressed_directory += dir_inode.file_size;
  10049. @@ -406,7 +415,7 @@
  10050.                     /* corrupted filesystem */
  10051.                     goto corrupted;
  10052.            
  10053. -               SQUASHFS_SWAP_DIR_INDEX(&index, cur_ptr);
  10054. +               SQUASHFS_SWAP_DIR_INDEX(cur_ptr, &index);
  10055.  
  10056.                 if(NO_BYTES(index.size + 1))
  10057.                     /* corrupted filesystem */
  10058. @@ -531,7 +540,7 @@
  10059.     /* Check the compression type */
  10060.     comp = lookup_compressor_id(sBlk->compression);
  10061.     if(!comp->supported) {
  10062. -       ERROR("Filesystem on %s uses %s compression, this is"
  10063. +       ERROR("Filesystem on %s uses %s compression, this is "
  10064.             "unsupported by this version\n", source, comp->name);
  10065.         ERROR("Compressors available:\n");
  10066.         display_compressors("", "");
  10067. @@ -548,6 +557,10 @@
  10068.      * is still called to set the default options (the defaults may have
  10069.      * been changed by the user specifying options on the command
  10070.      * line which need to be over-ridden).
  10071. +    *
  10072. +    * Compressor_extract_options is also used to ensure that
  10073. +    * we know how decompress a filesystem compressed with these
  10074. +    * compression options.
  10075.      */
  10076.     if(SQUASHFS_COMP_OPTS(sBlk->flags)) {
  10077.         bytes = read_block(fd, sizeof(*sBlk), NULL, 0, buffer);
  10078. @@ -579,7 +592,7 @@
  10079.         SQUASHFS_UNCOMPRESSED_XATTRS(sBlk->flags) ? "un" : "");
  10080.     printf("\tFragments are %spresent in the filesystem\n",
  10081.         SQUASHFS_NO_FRAGMENTS(sBlk->flags) ? "not " : "");
  10082. -   printf("\tAlways_use_fragments option is %sspecified\n",
  10083. +   printf("\tAlways-use-fragments option is %sspecified\n",
  10084.         SQUASHFS_ALWAYS_FRAGMENTS(sBlk->flags) ? "" : "not ");
  10085.     printf("\tDuplicates are %sremoved\n",
  10086.         SQUASHFS_DUPLICATES(sBlk->flags) ? "" : "not ");
  10087. @@ -651,7 +664,7 @@
  10088.  
  10089.     bytes = offset;
  10090.     while(bytes < size) {          
  10091. -       SQUASHFS_SWAP_DIR_HEADER(&dirh, directory_table + bytes);
  10092. +       SQUASHFS_SWAP_DIR_HEADER(directory_table + bytes, &dirh);
  10093.  
  10094.         dir_count = dirh.count + 1;
  10095.         TRACE("squashfs_readdir: Read directory header @ byte position "
  10096. @@ -659,7 +672,7 @@
  10097.         bytes += sizeof(dirh);
  10098.  
  10099.         while(dir_count--) {
  10100. -           SQUASHFS_SWAP_DIR_ENTRY(dire, directory_table + bytes);
  10101. +           SQUASHFS_SWAP_DIR_ENTRY(directory_table + bytes, dire);
  10102.             bytes += sizeof(*dire);
  10103.  
  10104.             memcpy(dire->name, directory_table + bytes,
  10105. diff -Nru squashfs-tools-4.2+20130409/read_fs.h squashfs-tools-4.3+20140919/read_fs.h
  10106. --- squashfs-tools-4.2+20130409/read_fs.h   2013-05-09 10:39:11.000000000 +0200
  10107. +++ squashfs-tools-4.3+20140919/read_fs.h   2015-07-20 21:03:05.000000000 +0200
  10108. @@ -3,7 +3,7 @@
  10109.  /*
  10110.   * Squashfs
  10111.   *
  10112. - * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010
  10113. + * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2013
  10114.   * Phillip Lougher <phillip@squashfs.org.uk>
  10115.   *
  10116.   * This program is free software; you can redistribute it and/or
  10117. @@ -23,20 +23,12 @@
  10118.   * read_fs.h
  10119.   *
  10120.   */
  10121. -
  10122. -#if __BYTE_ORDER == __BIG_ENDIAN
  10123. -#define SQUASHFS_SWAP_SHORTS(d, s, n) swap_le16_num(s, d, n)
  10124. -#define SQUASHFS_SWAP_INTS(d, s, n) swap_le32_num(s, d, n)
  10125. -#define SQUASHFS_SWAP_LONG_LONGS(d, s, n) swap_le64_num(s, d, n)
  10126. -
  10127. -#define SWAP_LE16(d, s)        swap_le16(s, d)
  10128. -#define SWAP_LE32(d, s)        swap_le32(s, d)
  10129. -#define SWAP_LE64(d, s)        swap_le64(s, d)
  10130. -#else
  10131. -#define SQUASHFS_MEMCPY(d, s, n)   memcpy(d, s, n)
  10132. -#define SQUASHFS_SWAP_SHORTS(d, s, n)  memcpy(d, s, n * sizeof(short))
  10133. -#define SQUASHFS_SWAP_INTS(d, s, n)    memcpy(d, s, n * sizeof(int))
  10134. -#define SQUASHFS_SWAP_LONG_LONGS(d, s, n) \
  10135. -                   memcpy(d, s, n * sizeof(long long))
  10136. -#endif
  10137. +extern struct compressor *read_super(int, struct squashfs_super_block *,
  10138. +   char *);
  10139. +extern long long read_filesystem(char *, int, struct squashfs_super_block *,
  10140. +char **, char **, char **, char **, unsigned int *, unsigned int *,
  10141. +unsigned int *, unsigned int *, unsigned int *, int *, int *, int *, int *,
  10142. +int *, int *, long long *, unsigned int *, unsigned int *, unsigned int *,
  10143. +unsigned int *, void (push_directory_entry)(char *, squashfs_inode, int, int),
  10144. +struct squashfs_fragment_entry **, squashfs_inode **);
  10145.  #endif
  10146. diff -Nru squashfs-tools-4.2+20130409/read_xattrs.c squashfs-tools-4.3+20140919/read_xattrs.c
  10147. --- squashfs-tools-4.2+20130409/read_xattrs.c   2013-05-09 10:39:11.000000000 +0200
  10148. +++ squashfs-tools-4.3+20140919/read_xattrs.c   2015-07-20 21:03:05.000000000 +0200
  10149. @@ -41,7 +41,6 @@
  10150.  
  10151.  #include "squashfs_fs.h"
  10152.  #include "squashfs_swap.h"
  10153. -#include "read_fs.h"
  10154.  #include "xattr.h"
  10155.  #include "error.h"
  10156.  
  10157. @@ -338,7 +337,7 @@
  10158.                 MEM_ERROR();
  10159.         }
  10160.            
  10161. -       SQUASHFS_SWAP_XATTR_ENTRY(&entry, xptr);
  10162. +       SQUASHFS_SWAP_XATTR_ENTRY(xptr, &entry);
  10163.         xptr += sizeof(entry);
  10164.  
  10165.         res = read_xattr_entry(&xattr_list[j], &entry, xptr);
  10166. @@ -361,15 +360,15 @@
  10167.             void *ool_xptr;
  10168.  
  10169.             xptr += sizeof(val);
  10170. -           SQUASHFS_SWAP_LONG_LONGS(&xattr, xptr, 1);
  10171. +           SQUASHFS_SWAP_LONG_LONGS(xptr, &xattr, 1);
  10172.             xptr += sizeof(xattr); 
  10173.             start = SQUASHFS_XATTR_BLK(xattr) + xattr_table_start;
  10174.             offset = SQUASHFS_XATTR_OFFSET(xattr);
  10175.             ool_xptr = xattrs + get_xattr_block(start) + offset;
  10176. -           SQUASHFS_SWAP_XATTR_VAL(&val, ool_xptr);
  10177. +           SQUASHFS_SWAP_XATTR_VAL(ool_xptr, &val);
  10178.             xattr_list[j].value = ool_xptr + sizeof(val);
  10179.         } else {
  10180. -           SQUASHFS_SWAP_XATTR_VAL(&val, xptr);
  10181. +           SQUASHFS_SWAP_XATTR_VAL(xptr, &val);
  10182.             xattr_list[j].value = xptr + sizeof(val);
  10183.             xptr += sizeof(val) + val.vsize;
  10184.         }
  10185. diff -Nru squashfs-tools-4.2+20130409/restore.c squashfs-tools-4.3+20140919/restore.c
  10186. --- squashfs-tools-4.2+20130409/restore.c   2013-05-09 10:39:11.000000000 +0200
  10187. +++ squashfs-tools-4.3+20140919/restore.c   2015-07-20 21:03:05.000000000 +0200
  10188. @@ -2,7 +2,7 @@
  10189.   * Create a squashfs filesystem.  This is a highly compressed read only
  10190.   * filesystem.
  10191.   *
  10192. - * Copyright (c) 2013
  10193. + * Copyright (c) 2013, 2014
  10194.   * Phillip Lougher <phillip@squashfs.org.uk>
  10195.   *
  10196.   * This program is free software; you can redistribute it and/or
  10197. @@ -36,22 +36,30 @@
  10198.  #include <sys/types.h>
  10199.  #include <sys/stat.h>
  10200.  
  10201. +#include "caches-queues-lists.h"
  10202.  #include "squashfs_fs.h"
  10203.  #include "mksquashfs.h"
  10204.  #include "error.h"
  10205.  #include "progressbar.h"
  10206.  #include "info.h"
  10207.  
  10208. -pthread_t restore_thread, main_thread;
  10209. -int interrupted = 0;
  10210. +#define FALSE 0
  10211. +#define TRUE 1
  10212.  
  10213. +extern pthread_t reader_thread, writer_thread, main_thread;
  10214. +extern pthread_t *deflator_thread, *frag_deflator_thread, *frag_thread;
  10215. +extern struct queue *to_deflate, *to_writer, *to_frag, *to_process_frag;
  10216. +extern struct seq_queue *to_main;
  10217.  extern void restorefs();
  10218. +extern int processors;
  10219.  
  10220. +static int interrupted = 0;
  10221. +static pthread_t restore_thread;
  10222.  
  10223.  void *restore_thrd(void *arg)
  10224.  {
  10225.     sigset_t sigmask, old_mask;
  10226. -   int sig;
  10227. +   int i, sig;
  10228.  
  10229.     sigemptyset(&sigmask);
  10230.     sigaddset(&sigmask, SIGINT);
  10231. @@ -62,31 +70,86 @@
  10232.     while(1) {
  10233.         sigwait(&sigmask, &sig);
  10234.  
  10235. -       if(sig == SIGINT || sig == SIGTERM) {
  10236. -           interrupted ++;
  10237. -
  10238. -           if(interrupted == 1) {
  10239. -               ERROR("Interrupting will restore original "
  10240. -                   "filesystem!\n");
  10241. -                       ERROR("Interrupt again to quit\n");
  10242. -           }
  10243. +       if((sig == SIGINT || sig == SIGTERM) && !interrupted) {
  10244. +           ERROR("Interrupting will restore original "
  10245. +               "filesystem!\n");
  10246. +                   ERROR("Interrupt again to quit\n");
  10247. +           interrupted = TRUE;
  10248. +           continue;
  10249.         }
  10250.  
  10251. -       if(interrupted == 2 || sig == SIGUSR1) {
  10252. -           disable_progress_bar();
  10253. -           disable_info();
  10254. -           pthread_cancel(main_thread);
  10255. -           pthread_join(main_thread, NULL);
  10256. +       /* kill main thread/worker threads and restore */
  10257. +       set_progressbar_state(FALSE);
  10258. +       disable_info();
  10259. +
  10260. +       /* first kill the reader thread */
  10261. +       pthread_cancel(reader_thread);
  10262. +       pthread_join(reader_thread, NULL);
  10263. +
  10264. +       /*
  10265. +        * then flush the reader to deflator thread(s) output queue.
  10266. +        * The deflator thread(s) will idle
  10267. +        */
  10268. +       queue_flush(to_deflate);
  10269. +
  10270. +       /* now kill the deflator thread(s) */
  10271. +       for(i = 0; i < processors; i++)
  10272. +           pthread_cancel(deflator_thread[i]);
  10273. +       for(i = 0; i < processors; i++)
  10274. +           pthread_join(deflator_thread[i], NULL);
  10275. +
  10276. +       /*
  10277. +        * then flush the reader to process fragment thread(s) output
  10278. +        * queue.  The process fragment thread(s) will idle
  10279. +        */
  10280. +       queue_flush(to_process_frag);
  10281. +
  10282. +       /* now kill the process fragment thread(s) */
  10283. +       for(i = 0; i < processors; i++)
  10284. +           pthread_cancel(frag_thread[i]);
  10285. +       for(i = 0; i < processors; i++)
  10286. +           pthread_join(frag_thread[i], NULL);
  10287. +
  10288. +       /*
  10289. +        * then flush the reader/deflator/process fragment to main
  10290. +        * thread output queue.  The main thread will idle
  10291. +        */
  10292. +       seq_queue_flush(to_main);
  10293. +
  10294. +       /* now kill the main thread */
  10295. +       pthread_cancel(main_thread);
  10296. +       pthread_join(main_thread, NULL);
  10297. +
  10298. +       /* then flush the main thread to fragment deflator thread(s)
  10299. +        * queue.  The fragment deflator thread(s) will idle
  10300. +        */
  10301. +       queue_flush(to_frag);
  10302. +
  10303. +       /* now kill the fragment deflator thread(s) */
  10304. +       for(i = 0; i < processors; i++)
  10305. +           pthread_cancel(frag_deflator_thread[i]);
  10306. +       for(i = 0; i < processors; i++)
  10307. +           pthread_join(frag_deflator_thread[i], NULL);
  10308. +
  10309. +       /*
  10310. +        * then flush the main thread/fragment deflator thread(s)
  10311. +        * to writer thread queue.  The writer thread will idle
  10312. +        */
  10313. +       queue_flush(to_writer);
  10314. +
  10315. +       /* now kill the writer thread */
  10316. +       pthread_cancel(writer_thread);
  10317. +       pthread_join(writer_thread, NULL);
  10318.  
  10319. -           restorefs();
  10320. -       }
  10321. +       TRACE("All threads cancelled\n");
  10322. +
  10323. +       restorefs();
  10324.     }
  10325.  }
  10326.  
  10327.  
  10328. -pthread_t *init_restore_thread(pthread_t thread)
  10329. +pthread_t *init_restore_thread()
  10330.  {
  10331. -   main_thread = thread;
  10332.     pthread_create(&restore_thread, NULL, restore_thrd, NULL);
  10333.     return &restore_thread;
  10334.  }
  10335. diff -Nru squashfs-tools-4.2+20130409/restore.h squashfs-tools-4.3+20140919/restore.h
  10336. --- squashfs-tools-4.2+20130409/restore.h   1970-01-01 01:00:00.000000000 +0100
  10337. +++ squashfs-tools-4.3+20140919/restore.h   2015-07-20 21:03:05.000000000 +0200
  10338. @@ -0,0 +1,28 @@
  10339. +#ifndef RESTORE_H
  10340. +#define RESTORE_H
  10341. +/*
  10342. + * Create a squashfs filesystem.  This is a highly compressed read only
  10343. + * filesystem.
  10344. + *
  10345. + * Copyright (c) 2013, 2014
  10346. + * Phillip Lougher <phillip@squashfs.org.uk>
  10347. + *
  10348. + * This program is free software; you can redistribute it and/or
  10349. + * modify it under the terms of the GNU General Public License
  10350. + * as published by the Free Software Foundation; either version 2,
  10351. + * or (at your option) any later version.
  10352. + *
  10353. + * This program is distributed in the hope that it will be useful,
  10354. + * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10355. + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10356. + * GNU General Public License for more details.
  10357. + *
  10358. + * You should have received a copy of the GNU General Public License
  10359. + * along with this program; if not, write to the Free Software
  10360. + * Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  10361. + *
  10362. + * restore.h
  10363. + */
  10364. +
  10365. +extern pthread_t *init_restore_thread();
  10366. +#endif
  10367. diff -Nru squashfs-tools-4.2+20130409/sort.c squashfs-tools-4.3+20140919/sort.c
  10368. --- squashfs-tools-4.2+20130409/sort.c  2013-05-09 10:39:11.000000000 +0200
  10369. +++ squashfs-tools-4.3+20140919/sort.c  2015-07-20 21:03:05.000000000 +0200
  10370. @@ -3,7 +3,7 @@
  10371.   * filesystem.
  10372.   *
  10373.   * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2012,
  10374. - * 2013
  10375. + * 2013, 2014
  10376.   * Phillip Lougher <phillip@squashfs.org.uk>
  10377.   *
  10378.   * This program is free software; you can redistribute it and/or
  10379. @@ -42,6 +42,7 @@
  10380.  #include "mksquashfs.h"
  10381.  #include "sort.h"
  10382.  #include "error.h"
  10383. +#include "progressbar.h"
  10384.  
  10385.  int mkisofs_style = -1;
  10386.  
  10387. @@ -170,9 +171,10 @@
  10388.     }
  10389.  
  10390.  error:
  10391. -        ERROR("Cannot stat sortlist entry \"%s\"\n", path);
  10392. +        ERROR_START("Cannot stat sortlist entry \"%s\"\n", path);
  10393.          ERROR("This is probably because you're using the wrong file\n");
  10394. -        ERROR("path relative to the source directories\n");
  10395. +        ERROR("path relative to the source directories.");
  10396. +   ERROR_EXIT("  Ignoring");
  10397.     /*
  10398.      * Historical note
  10399.      * Failure to stat a sortlist entry is deliberately ignored, even
  10400. diff -Nru squashfs-tools-4.2+20130409/sort.h squashfs-tools-4.3+20140919/sort.h
  10401. --- squashfs-tools-4.2+20130409/sort.h  2013-05-09 10:39:11.000000000 +0200
  10402. +++ squashfs-tools-4.3+20140919/sort.h  2015-07-20 21:03:05.000000000 +0200
  10403. @@ -4,7 +4,7 @@
  10404.  /*
  10405.   * Squashfs
  10406.   *
  10407. - * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010
  10408. + * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2013
  10409.   * Phillip Lougher <phillip@squashfs.org.uk>
  10410.   *
  10411.   * This program is free software; you can redistribute it and/or
  10412. @@ -28,4 +28,10 @@
  10413.     struct dir_ent *dir;
  10414.     struct priority_entry *next;
  10415.  };
  10416. +
  10417. +extern int read_sort_file(char *, int, char *[]);
  10418. +extern void sort_files_and_write(struct dir_info *);
  10419. +extern void generate_file_priorities(struct dir_info *, int priority,
  10420. +   struct stat *);
  10421. +extern struct  priority_entry *priority_list[65536];
  10422.  #endif
  10423. diff -Nru squashfs-tools-4.2+20130409/squashfs_compat.h squashfs-tools-4.3+20140919/squashfs_compat.h
  10424. --- squashfs-tools-4.2+20130409/squashfs_compat.h   2013-05-09 10:39:11.000000000 +0200
  10425. +++ squashfs-tools-4.3+20140919/squashfs_compat.h   2015-07-20 21:03:05.000000000 +0200
  10426. @@ -3,7 +3,7 @@
  10427.  /*
  10428.   * Squashfs
  10429.   *
  10430. - * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010
  10431. + * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2014
  10432.   * Phillip Lougher <phillip@squashfs.org.uk>
  10433.   *
  10434.   * This program is free software; you can redistribute it and/or
  10435. @@ -50,7 +50,7 @@
  10436.     unsigned int        flags:8;
  10437.     unsigned int        no_uids:8;
  10438.     unsigned int        no_guids:8;
  10439. -   unsigned int        mkfs_time /* time of filesystem creation */;
  10440. +   int         mkfs_time /* time of filesystem creation */;
  10441.     squashfs_inode      root_inode;
  10442.     unsigned int        block_size;
  10443.     unsigned int        fragments;
  10444. @@ -71,38 +71,55 @@
  10445.     unsigned char       name[0];
  10446.  } __attribute__ ((packed));
  10447.  
  10448. -#define SQUASHFS_BASE_INODE_HEADER_3       \
  10449. -   unsigned int        inode_type:4;   \
  10450. -   unsigned int        mode:12;    \
  10451. -   unsigned int        uid:8;      \
  10452. -   unsigned int        guid:8;     \
  10453. -   unsigned int        mtime;      \
  10454. -   unsigned int        inode_number;
  10455. -
  10456.  struct squashfs_base_inode_header_3 {
  10457. -   SQUASHFS_BASE_INODE_HEADER_3;
  10458. +   unsigned int        inode_type:4;
  10459. +   unsigned int        mode:12;
  10460. +   unsigned int        uid:8;
  10461. +   unsigned int        guid:8;
  10462. +   int         mtime;
  10463. +   unsigned int        inode_number;
  10464.  } __attribute__ ((packed));
  10465.  
  10466.  struct squashfs_ipc_inode_header_3 {
  10467. -   SQUASHFS_BASE_INODE_HEADER_3;
  10468. +   unsigned int        inode_type:4;
  10469. +   unsigned int        mode:12;
  10470. +   unsigned int        uid:8;
  10471. +   unsigned int        guid:8;
  10472. +   int         mtime;
  10473. +   unsigned int        inode_number;
  10474.     unsigned int        nlink;
  10475.  } __attribute__ ((packed));
  10476.  
  10477.  struct squashfs_dev_inode_header_3 {
  10478. -   SQUASHFS_BASE_INODE_HEADER_3;
  10479. +   unsigned int        inode_type:4;
  10480. +   unsigned int        mode:12;
  10481. +   unsigned int        uid:8;
  10482. +   unsigned int        guid:8;
  10483. +   int         mtime;
  10484. +   unsigned int        inode_number;
  10485.     unsigned int        nlink;
  10486.     unsigned short      rdev;
  10487.  } __attribute__ ((packed));
  10488.    
  10489.  struct squashfs_symlink_inode_header_3 {
  10490. -   SQUASHFS_BASE_INODE_HEADER_3;
  10491. +   unsigned int        inode_type:4;
  10492. +   unsigned int        mode:12;
  10493. +   unsigned int        uid:8;
  10494. +   unsigned int        guid:8;
  10495. +   int         mtime;
  10496. +   unsigned int        inode_number;
  10497.     unsigned int        nlink;
  10498.     unsigned short      symlink_size;
  10499.     char            symlink[0];
  10500.  } __attribute__ ((packed));
  10501.  
  10502.  struct squashfs_reg_inode_header_3 {
  10503. -   SQUASHFS_BASE_INODE_HEADER_3;
  10504. +   unsigned int        inode_type:4;
  10505. +   unsigned int        mode:12;
  10506. +   unsigned int        uid:8;
  10507. +   unsigned int        guid:8;
  10508. +   int         mtime;
  10509. +   unsigned int        inode_number;
  10510.     squashfs_block      start_block;
  10511.     unsigned int        fragment;
  10512.     unsigned int        offset;
  10513. @@ -111,7 +128,12 @@
  10514.  } __attribute__ ((packed));
  10515.  
  10516.  struct squashfs_lreg_inode_header_3 {
  10517. -   SQUASHFS_BASE_INODE_HEADER_3;
  10518. +   unsigned int        inode_type:4;
  10519. +   unsigned int        mode:12;
  10520. +   unsigned int        uid:8;
  10521. +   unsigned int        guid:8;
  10522. +   int         mtime;
  10523. +   unsigned int        inode_number;
  10524.     unsigned int        nlink;
  10525.     squashfs_block      start_block;
  10526.     unsigned int        fragment;
  10527. @@ -121,7 +143,12 @@
  10528.  } __attribute__ ((packed));
  10529.  
  10530.  struct squashfs_dir_inode_header_3 {
  10531. -   SQUASHFS_BASE_INODE_HEADER_3;
  10532. +   unsigned int        inode_type:4;
  10533. +   unsigned int        mode:12;
  10534. +   unsigned int        uid:8;
  10535. +   unsigned int        guid:8;
  10536. +   int         mtime;
  10537. +   unsigned int        inode_number;
  10538.     unsigned int        nlink;
  10539.     unsigned int        file_size:19;
  10540.     unsigned int        offset:13;
  10541. @@ -130,7 +157,12 @@
  10542.  } __attribute__  ((packed));
  10543.  
  10544.  struct squashfs_ldir_inode_header_3 {
  10545. -   SQUASHFS_BASE_INODE_HEADER_3;
  10546. +   unsigned int        inode_type:4;
  10547. +   unsigned int        mode:12;
  10548. +   unsigned int        uid:8;
  10549. +   unsigned int        guid:8;
  10550. +   int         mtime;
  10551. +   unsigned int        inode_number;
  10552.     unsigned int        nlink;
  10553.     unsigned int        file_size:27;
  10554.     unsigned int        offset:13;
  10555. @@ -449,7 +481,7 @@
  10556.     unsigned int        mode:12; /* protection */
  10557.     unsigned int        uid:4; /* index into uid table */
  10558.     unsigned int        guid:4; /* index into guid table */
  10559. -   unsigned int        mtime;
  10560. +   int         mtime;
  10561.     unsigned int        start_block;
  10562.     unsigned int        file_size:32;
  10563.     unsigned short      block_list[0];
  10564. @@ -462,7 +494,7 @@
  10565.     unsigned int        guid:4; /* index into guid table */
  10566.     unsigned int        file_size:19;
  10567.     unsigned int        offset:13;
  10568. -   unsigned int        mtime;
  10569. +   int         mtime;
  10570.     unsigned int        start_block:24;
  10571.  } __attribute__  ((packed));
  10572.  
  10573. @@ -582,7 +614,7 @@
  10574.     unsigned int        mode:12; /* protection */
  10575.     unsigned int        uid:8; /* index into uid table */
  10576.     unsigned int        guid:8; /* index into guid table */
  10577. -   unsigned int        mtime;
  10578. +   int         mtime;
  10579.     unsigned int        start_block;
  10580.     unsigned int        fragment;
  10581.     unsigned int        offset;
  10582. @@ -597,7 +629,7 @@
  10583.     unsigned int        guid:8; /* index into guid table */
  10584.     unsigned int        file_size:19;
  10585.     unsigned int        offset:13;
  10586. -   unsigned int        mtime;
  10587. +   int         mtime;
  10588.     unsigned int        start_block:24;
  10589.  } __attribute__  ((packed));
  10590.  
  10591. @@ -608,7 +640,7 @@
  10592.     unsigned int        guid:8; /* index into guid table */
  10593.     unsigned int        file_size:27;
  10594.     unsigned int        offset:13;
  10595. -   unsigned int        mtime;
  10596. +   int         mtime;
  10597.     unsigned int        start_block:24;
  10598.     unsigned int        i_count:16;
  10599.     struct squashfs_dir_index_2 index[0];
  10600. diff -Nru squashfs-tools-4.2+20130409/squashfs_fs.h squashfs-tools-4.3+20140919/squashfs_fs.h
  10601. --- squashfs-tools-4.2+20130409/squashfs_fs.h   2013-05-09 10:39:11.000000000 +0200
  10602. +++ squashfs-tools-4.3+20140919/squashfs_fs.h   2015-07-20 21:03:05.000000000 +0200
  10603. @@ -4,7 +4,7 @@
  10604.   * Squashfs
  10605.   *
  10606.   * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2012,
  10607. - * 2013
  10608. + * 2013, 2014
  10609.   * Phillip Lougher <phillip@squashfs.org.uk>
  10610.   *
  10611.   * This program is free software; you can redistribute it and/or
  10612. @@ -276,11 +276,12 @@
  10613.  #define LZMA_COMPRESSION   2
  10614.  #define LZO_COMPRESSION        3
  10615.  #define XZ_COMPRESSION     4
  10616. +#define LZ4_COMPRESSION        5
  10617.  
  10618.  struct squashfs_super_block {
  10619.     unsigned int        s_magic;
  10620.     unsigned int        inodes;
  10621. -   unsigned int        mkfs_time /* time of filesystem creation */;
  10622. +   int         mkfs_time /* time of filesystem creation */;
  10623.     unsigned int        block_size;
  10624.     unsigned int        fragments;
  10625.     unsigned short      compression;
  10626. @@ -311,7 +312,7 @@
  10627.     unsigned short      mode;
  10628.     unsigned short      uid;
  10629.     unsigned short      guid;
  10630. -   unsigned int        mtime;
  10631. +   int         mtime;
  10632.     unsigned int        inode_number;
  10633.  };
  10634.  
  10635. @@ -320,7 +321,7 @@
  10636.     unsigned short      mode;
  10637.     unsigned short      uid;
  10638.     unsigned short      guid;
  10639. -   unsigned int        mtime;
  10640. +   int         mtime;
  10641.     unsigned int        inode_number;
  10642.     unsigned int        nlink;
  10643.  };
  10644. @@ -330,7 +331,7 @@
  10645.     unsigned short      mode;
  10646.     unsigned short      uid;
  10647.     unsigned short      guid;
  10648. -   unsigned int        mtime;
  10649. +   int         mtime;
  10650.     unsigned int        inode_number;
  10651.     unsigned int        nlink;
  10652.     unsigned int        xattr;
  10653. @@ -341,7 +342,7 @@
  10654.     unsigned short      mode;
  10655.     unsigned short      uid;
  10656.     unsigned short      guid;
  10657. -   unsigned int        mtime;
  10658. +   int         mtime;
  10659.     unsigned int        inode_number;
  10660.     unsigned int        nlink;
  10661.     unsigned int        rdev;
  10662. @@ -352,7 +353,7 @@
  10663.     unsigned short      mode;
  10664.     unsigned short      uid;
  10665.     unsigned short      guid;
  10666. -   unsigned int        mtime;
  10667. +   int         mtime;
  10668.     unsigned int        inode_number;
  10669.     unsigned int        nlink;
  10670.     unsigned int        rdev;
  10671. @@ -364,7 +365,7 @@
  10672.     unsigned short      mode;
  10673.     unsigned short      uid;
  10674.     unsigned short      guid;
  10675. -   unsigned int        mtime;
  10676. +   int         mtime;
  10677.     unsigned int        inode_number;
  10678.     unsigned int        nlink;
  10679.     unsigned int        symlink_size;
  10680. @@ -376,7 +377,7 @@
  10681.     unsigned short      mode;
  10682.     unsigned short      uid;
  10683.     unsigned short      guid;
  10684. -   unsigned int        mtime;
  10685. +   int         mtime;
  10686.     unsigned int        inode_number;
  10687.     unsigned int        start_block;
  10688.     unsigned int        fragment;
  10689. @@ -390,7 +391,7 @@
  10690.     unsigned short      mode;
  10691.     unsigned short      uid;
  10692.     unsigned short      guid;
  10693. -   unsigned int        mtime;
  10694. +   int         mtime;
  10695.     unsigned int        inode_number;
  10696.     squashfs_block      start_block;
  10697.     long long       file_size;
  10698. @@ -407,7 +408,7 @@
  10699.     unsigned short      mode;
  10700.     unsigned short      uid;
  10701.     unsigned short      guid;
  10702. -   unsigned int        mtime;
  10703. +   int         mtime;
  10704.     unsigned int        inode_number;
  10705.     unsigned int        start_block;
  10706.     unsigned int        nlink;
  10707. @@ -421,7 +422,7 @@
  10708.     unsigned short      mode;
  10709.     unsigned short      uid;
  10710.     unsigned short      guid;
  10711. -   unsigned int        mtime;
  10712. +   int         mtime;
  10713.     unsigned int        inode_number;
  10714.     unsigned int        nlink;
  10715.     unsigned int        file_size;
  10716. diff -Nru squashfs-tools-4.2+20130409/squashfs_swap.h squashfs-tools-4.3+20140919/squashfs_swap.h
  10717. --- squashfs-tools-4.2+20130409/squashfs_swap.h 2013-05-09 10:39:11.000000000 +0200
  10718. +++ squashfs-tools-4.3+20140919/squashfs_swap.h 2015-07-20 21:03:05.000000000 +0200
  10719. @@ -3,7 +3,7 @@
  10720.  /*
  10721.   * Squashfs
  10722.   *
  10723. - * Copyright (c) 2008, 2009, 2010
  10724. + * Copyright (c) 2008, 2009, 2010, 2013, 2014
  10725.   * Phillip Lougher <phillip@squashfs.org.uk>
  10726.   *
  10727.   * This program is free software; you can redistribute it and/or
  10728. @@ -45,7 +45,7 @@
  10729.  #define _SQUASHFS_SWAP_SUPER_BLOCK(s, d, SWAP_FUNC) {\
  10730.     SWAP_FUNC(32, s, d, s_magic, struct squashfs_super_block);\
  10731.     SWAP_FUNC(32, s, d, inodes, struct squashfs_super_block);\
  10732. -   SWAP_FUNC(32, s, d, mkfs_time, struct squashfs_super_block);\
  10733. +   SWAP_FUNC##S(32, s, d, mkfs_time, struct squashfs_super_block);\
  10734.     SWAP_FUNC(32, s, d, block_size, struct squashfs_super_block);\
  10735.     SWAP_FUNC(32, s, d, fragments, struct squashfs_super_block);\
  10736.     SWAP_FUNC(16, s, d, compression, struct squashfs_super_block);\
  10737. @@ -75,7 +75,7 @@
  10738.     SWAP_FUNC(16, s, d, mode, struct squashfs_base_inode_header);\
  10739.     SWAP_FUNC(16, s, d, uid, struct squashfs_base_inode_header);\
  10740.     SWAP_FUNC(16, s, d, guid, struct squashfs_base_inode_header);\
  10741. -   SWAP_FUNC(32, s, d, mtime, struct squashfs_base_inode_header);\
  10742. +   SWAP_FUNC##S(32, s, d, mtime, struct squashfs_base_inode_header);\
  10743.     SWAP_FUNC(32, s, d, inode_number, struct squashfs_base_inode_header);\
  10744.  }
  10745.  
  10746. @@ -84,7 +84,7 @@
  10747.     SWAP_FUNC(16, s, d, mode, struct squashfs_ipc_inode_header);\
  10748.     SWAP_FUNC(16, s, d, uid, struct squashfs_ipc_inode_header);\
  10749.     SWAP_FUNC(16, s, d, guid, struct squashfs_ipc_inode_header);\
  10750. -   SWAP_FUNC(32, s, d, mtime, struct squashfs_ipc_inode_header);\
  10751. +   SWAP_FUNC##S(32, s, d, mtime, struct squashfs_ipc_inode_header);\
  10752.     SWAP_FUNC(32, s, d, inode_number, struct squashfs_ipc_inode_header);\
  10753.     SWAP_FUNC(32, s, d, nlink, struct squashfs_ipc_inode_header);\
  10754.  }
  10755. @@ -94,7 +94,7 @@
  10756.     SWAP_FUNC(16, s, d, mode, struct squashfs_lipc_inode_header);\
  10757.     SWAP_FUNC(16, s, d, uid, struct squashfs_lipc_inode_header);\
  10758.     SWAP_FUNC(16, s, d, guid, struct squashfs_lipc_inode_header);\
  10759. -   SWAP_FUNC(32, s, d, mtime, struct squashfs_lipc_inode_header);\
  10760. +   SWAP_FUNC##S(32, s, d, mtime, struct squashfs_lipc_inode_header);\
  10761.     SWAP_FUNC(32, s, d, inode_number, struct squashfs_lipc_inode_header);\
  10762.     SWAP_FUNC(32, s, d, nlink, struct squashfs_lipc_inode_header);\
  10763.     SWAP_FUNC(32, s, d, xattr, struct squashfs_lipc_inode_header);\
  10764. @@ -105,7 +105,7 @@
  10765.     SWAP_FUNC(16, s, d, mode, struct squashfs_dev_inode_header);\
  10766.     SWAP_FUNC(16, s, d, uid, struct squashfs_dev_inode_header);\
  10767.     SWAP_FUNC(16, s, d, guid, struct squashfs_dev_inode_header);\
  10768. -   SWAP_FUNC(32, s, d, mtime, struct squashfs_dev_inode_header);\
  10769. +   SWAP_FUNC##S(32, s, d, mtime, struct squashfs_dev_inode_header);\
  10770.     SWAP_FUNC(32, s, d, inode_number, struct squashfs_dev_inode_header);\
  10771.     SWAP_FUNC(32, s, d, nlink, struct squashfs_dev_inode_header);\
  10772.     SWAP_FUNC(32, s, d, rdev, struct squashfs_dev_inode_header);\
  10773. @@ -116,7 +116,7 @@
  10774.     SWAP_FUNC(16, s, d, mode, struct squashfs_ldev_inode_header);\
  10775.     SWAP_FUNC(16, s, d, uid, struct squashfs_ldev_inode_header);\
  10776.     SWAP_FUNC(16, s, d, guid, struct squashfs_ldev_inode_header);\
  10777. -   SWAP_FUNC(32, s, d, mtime, struct squashfs_ldev_inode_header);\
  10778. +   SWAP_FUNC##S(32, s, d, mtime, struct squashfs_ldev_inode_header);\
  10779.     SWAP_FUNC(32, s, d, inode_number, struct squashfs_ldev_inode_header);\
  10780.     SWAP_FUNC(32, s, d, nlink, struct squashfs_ldev_inode_header);\
  10781.     SWAP_FUNC(32, s, d, rdev, struct squashfs_ldev_inode_header);\
  10782. @@ -128,7 +128,7 @@
  10783.     SWAP_FUNC(16, s, d, mode, struct squashfs_symlink_inode_header);\
  10784.     SWAP_FUNC(16, s, d, uid, struct squashfs_symlink_inode_header);\
  10785.     SWAP_FUNC(16, s, d, guid, struct squashfs_symlink_inode_header);\
  10786. -   SWAP_FUNC(32, s, d, mtime, struct squashfs_symlink_inode_header);\
  10787. +   SWAP_FUNC##S(32, s, d, mtime, struct squashfs_symlink_inode_header);\
  10788.     SWAP_FUNC(32, s, d, inode_number, struct squashfs_symlink_inode_header);\
  10789.     SWAP_FUNC(32, s, d, nlink, struct squashfs_symlink_inode_header);\
  10790.     SWAP_FUNC(32, s, d, symlink_size, struct squashfs_symlink_inode_header);\
  10791. @@ -139,7 +139,7 @@
  10792.     SWAP_FUNC(16, s, d, mode, struct squashfs_reg_inode_header);\
  10793.     SWAP_FUNC(16, s, d, uid, struct squashfs_reg_inode_header);\
  10794.     SWAP_FUNC(16, s, d, guid, struct squashfs_reg_inode_header);\
  10795. -   SWAP_FUNC(32, s, d, mtime, struct squashfs_reg_inode_header);\
  10796. +   SWAP_FUNC##S(32, s, d, mtime, struct squashfs_reg_inode_header);\
  10797.     SWAP_FUNC(32, s, d, inode_number, struct squashfs_reg_inode_header);\
  10798.     SWAP_FUNC(32, s, d, start_block, struct squashfs_reg_inode_header);\
  10799.     SWAP_FUNC(32, s, d, fragment, struct squashfs_reg_inode_header);\
  10800. @@ -152,7 +152,7 @@
  10801.     SWAP_FUNC(16, s, d, mode, struct squashfs_lreg_inode_header);\
  10802.     SWAP_FUNC(16, s, d, uid, struct squashfs_lreg_inode_header);\
  10803.     SWAP_FUNC(16, s, d, guid, struct squashfs_lreg_inode_header);\
  10804. -   SWAP_FUNC(32, s, d, mtime, struct squashfs_lreg_inode_header);\
  10805. +   SWAP_FUNC##S(32, s, d, mtime, struct squashfs_lreg_inode_header);\
  10806.     SWAP_FUNC(32, s, d, inode_number, struct squashfs_lreg_inode_header);\
  10807.     SWAP_FUNC(64, s, d, start_block, struct squashfs_lreg_inode_header);\
  10808.     SWAP_FUNC(64, s, d, file_size, struct squashfs_lreg_inode_header);\
  10809. @@ -168,7 +168,7 @@
  10810.     SWAP_FUNC(16, s, d, mode, struct squashfs_dir_inode_header);\
  10811.     SWAP_FUNC(16, s, d, uid, struct squashfs_dir_inode_header);\
  10812.     SWAP_FUNC(16, s, d, guid, struct squashfs_dir_inode_header);\
  10813. -   SWAP_FUNC(32, s, d, mtime, struct squashfs_dir_inode_header);\
  10814. +   SWAP_FUNC##S(32, s, d, mtime, struct squashfs_dir_inode_header);\
  10815.     SWAP_FUNC(32, s, d, inode_number, struct squashfs_dir_inode_header);\
  10816.     SWAP_FUNC(32, s, d, start_block, struct squashfs_dir_inode_header);\
  10817.     SWAP_FUNC(32, s, d, nlink, struct squashfs_dir_inode_header);\
  10818. @@ -182,7 +182,7 @@
  10819.     SWAP_FUNC(16, s, d, mode, struct squashfs_ldir_inode_header);\
  10820.     SWAP_FUNC(16, s, d, uid, struct squashfs_ldir_inode_header);\
  10821.     SWAP_FUNC(16, s, d, guid, struct squashfs_ldir_inode_header);\
  10822. -   SWAP_FUNC(32, s, d, mtime, struct squashfs_ldir_inode_header);\
  10823. +   SWAP_FUNC##S(32, s, d, mtime, struct squashfs_ldir_inode_header);\
  10824.     SWAP_FUNC(32, s, d, inode_number, struct squashfs_ldir_inode_header);\
  10825.     SWAP_FUNC(32, s, d, nlink, struct squashfs_ldir_inode_header);\
  10826.     SWAP_FUNC(32, s, d, file_size, struct squashfs_ldir_inode_header);\
  10827. @@ -281,6 +281,14 @@
  10828.  #define SQUASHFS_SWAP_LOOKUP_BLOCKS(s, d, n) SQUASHFS_SWAP_LONG_LONGS(s, d, n)
  10829.  #define SQUASHFS_SWAP_ID_BLOCKS(s, d, n) SQUASHFS_SWAP_LONG_LONGS(s, d, n)
  10830.  
  10831. +#define SQUASHFS_SWAP_SHORTS(s, d, n) swap_le16_num(s, d, n)
  10832. +#define SQUASHFS_SWAP_INTS(s, d, n) swap_le32_num(s, d, n)
  10833. +#define SQUASHFS_SWAP_LONG_LONGS(s, d, n) swap_le64_num(s, d, n)
  10834. +
  10835. +#define SWAP_LE16(s, d)        swap_le16(s, d)
  10836. +#define SWAP_LE32(s, d)        swap_le32(s, d)
  10837. +#define SWAP_LE64(s, d)        swap_le64(s, d)
  10838. +
  10839.  /* big endian architecture swap in-place macros */
  10840.  #define SQUASHFS_INSWAP_SUPER_BLOCK(s) \
  10841.             _SQUASHFS_SWAP_SUPER_BLOCK(s, s, INSWAP_LE)
  10842. @@ -323,8 +331,9 @@
  10843.  #define INSWAP_LE(bits, s, d, field, type) \
  10844.             (s)->field = inswap_le##bits((s)->field)
  10845.  #define INSWAP_LES(bits, s, d, field, type) \
  10846. -           (s)->field = (short) inswap_le##bits((unsigned short) \
  10847. -               (s)->field)
  10848. +           (s)->field = INSWAP_LES##bits((s)->field)
  10849. +#define INSWAP_LES16(num) (short) inswap_le16((unsigned short) (num))
  10850. +#define INSWAP_LES32(num) (int) inswap_le32((unsigned int) (num))
  10851.  #define SQUASHFS_INSWAP_INODE_T(s) s = inswap_le64(s)
  10852.  #define SQUASHFS_INSWAP_FRAGMENT_INDEXES(s, n) inswap_le64_num(s, n)
  10853.  #define SQUASHFS_INSWAP_LOOKUP_BLOCKS(s, n) inswap_le64_num(s, n)
  10854. @@ -335,49 +344,55 @@
  10855.  #else
  10856.  /* little endian architecture, just copy */
  10857.  #define SQUASHFS_SWAP_SUPER_BLOCK(s, d)    \
  10858. -       SQUASHFS_MEMCPY(s, d, sizeof(*(s)))
  10859. +       SQUASHFS_MEMCPY(s, d, sizeof(struct squashfs_super_block))
  10860.  #define SQUASHFS_SWAP_DIR_INDEX(s, d) \
  10861. -       SQUASHFS_MEMCPY(s, d, sizeof(*(s)))
  10862. +       SQUASHFS_MEMCPY(s, d, sizeof(struct squashfs_dir_index))
  10863.  #define SQUASHFS_SWAP_BASE_INODE_HEADER(s, d) \
  10864. -       SQUASHFS_MEMCPY(s, d, sizeof(*(s)))
  10865. +       SQUASHFS_MEMCPY(s, d, sizeof(struct squashfs_base_inode_header))
  10866.  #define SQUASHFS_SWAP_IPC_INODE_HEADER(s, d) \
  10867. -       SQUASHFS_MEMCPY(s, d, sizeof(*(s)))
  10868. +       SQUASHFS_MEMCPY(s, d, sizeof(struct squashfs_ipc_inode_header))
  10869.  #define SQUASHFS_SWAP_LIPC_INODE_HEADER(s, d) \
  10870. -       SQUASHFS_MEMCPY(s, d, sizeof(*(s)))
  10871. +       SQUASHFS_MEMCPY(s, d, sizeof(struct squashfs_lipc_inode_header))
  10872.  #define SQUASHFS_SWAP_DEV_INODE_HEADER(s, d) \
  10873. -       SQUASHFS_MEMCPY(s, d, sizeof(*(s)))
  10874. +       SQUASHFS_MEMCPY(s, d, sizeof(struct squashfs_dev_inode_header))
  10875.  #define SQUASHFS_SWAP_LDEV_INODE_HEADER(s, d) \
  10876. -       SQUASHFS_MEMCPY(s, d, sizeof(*(s)))
  10877. +       SQUASHFS_MEMCPY(s, d, sizeof(struct squashfs_ldev_inode_header))
  10878.  #define SQUASHFS_SWAP_SYMLINK_INODE_HEADER(s, d) \
  10879. -       SQUASHFS_MEMCPY(s, d, sizeof(*(s)))
  10880. +       SQUASHFS_MEMCPY(s, d, sizeof(struct squashfs_symlink_inode_header))
  10881.  #define SQUASHFS_SWAP_REG_INODE_HEADER(s, d) \
  10882. -       SQUASHFS_MEMCPY(s, d, sizeof(*(s)))
  10883. +       SQUASHFS_MEMCPY(s, d, sizeof(struct squashfs_reg_inode_header))
  10884.  #define SQUASHFS_SWAP_LREG_INODE_HEADER(s, d) \
  10885. -       SQUASHFS_MEMCPY(s, d, sizeof(*(s)))
  10886. +       SQUASHFS_MEMCPY(s, d, sizeof(struct squashfs_lreg_inode_header))
  10887.  #define SQUASHFS_SWAP_DIR_INODE_HEADER(s, d) \
  10888. -       SQUASHFS_MEMCPY(s, d, sizeof(*(s)))
  10889. +       SQUASHFS_MEMCPY(s, d, sizeof(struct squashfs_dir_inode_header))
  10890.  #define SQUASHFS_SWAP_LDIR_INODE_HEADER(s, d) \
  10891. -       SQUASHFS_MEMCPY(s, d, sizeof(*(s)))
  10892. +       SQUASHFS_MEMCPY(s, d, sizeof(struct squashfs_ldir_inode_header))
  10893.  #define SQUASHFS_SWAP_DIR_ENTRY(s, d) \
  10894. -       SQUASHFS_MEMCPY(s, d, sizeof(*(s)))
  10895. +       SQUASHFS_MEMCPY(s, d, sizeof(struct squashfs_dir_entry))
  10896.  #define SQUASHFS_SWAP_DIR_HEADER(s, d) \
  10897. -       SQUASHFS_MEMCPY(s, d, sizeof(*(s)))
  10898. +       SQUASHFS_MEMCPY(s, d, sizeof(struct squashfs_dir_header))
  10899.  #define SQUASHFS_SWAP_FRAGMENT_ENTRY(s, d) \
  10900. -       SQUASHFS_MEMCPY(s, d, sizeof(*(s)))
  10901. +       SQUASHFS_MEMCPY(s, d, sizeof(struct squashfs_fragment_entry))
  10902.  #define SQUASHFS_SWAP_XATTR_ENTRY(s, d) \
  10903. -       SQUASHFS_MEMCPY(s, d, sizeof(*(s)))
  10904. +       SQUASHFS_MEMCPY(s, d, sizeof(struct squashfs_xattr_entry))
  10905.  #define SQUASHFS_SWAP_XATTR_VAL(s, d) \
  10906. -       SQUASHFS_MEMCPY(s, d, sizeof(*(s)))
  10907. +       SQUASHFS_MEMCPY(s, d, sizeof(struct squashfs_xattr_val))
  10908.  #define SQUASHFS_SWAP_XATTR_ID(s, d) \
  10909. -       SQUASHFS_MEMCPY(s, d, sizeof(*(s)))
  10910. +       SQUASHFS_MEMCPY(s, d, sizeof(struct squashfs_xattr_id))
  10911.  #define SQUASHFS_SWAP_XATTR_TABLE(s, d) \
  10912. -       SQUASHFS_MEMCPY(s, d, sizeof(*(s)))
  10913. +       SQUASHFS_MEMCPY(s, d, sizeof(struct squashfs_xattr_table))
  10914.  #define SQUASHFS_SWAP_INODE_T(s, d) SQUASHFS_SWAP_LONG_LONGS(s, d, 1)
  10915.  #define SQUASHFS_SWAP_FRAGMENT_INDEXES(s, d, n) \
  10916.             SQUASHFS_SWAP_LONG_LONGS(s, d, n)
  10917.  #define SQUASHFS_SWAP_LOOKUP_BLOCKS(s, d, n) SQUASHFS_SWAP_LONG_LONGS(s, d, n)
  10918.  #define SQUASHFS_SWAP_ID_BLOCKS(s, d, n) SQUASHFS_SWAP_LONG_LONGS(s, d, n)
  10919.  
  10920. +#define SQUASHFS_MEMCPY(s, d, n)   memcpy(d, s, n)
  10921. +#define SQUASHFS_SWAP_SHORTS(s, d, n)  memcpy(d, s, n * sizeof(short))
  10922. +#define SQUASHFS_SWAP_INTS(s, d, n)    memcpy(d, s, n * sizeof(int))
  10923. +#define SQUASHFS_SWAP_LONG_LONGS(s, d, n) \
  10924. +                   memcpy(d, s, n * sizeof(long long))
  10925. +
  10926.  /* little endian architecture, data already in place so do nothing */
  10927.  #define SQUASHFS_INSWAP_SUPER_BLOCK(s)
  10928.  #define SQUASHFS_INSWAP_DIR_INDEX(s)
  10929. diff -Nru squashfs-tools-4.2+20130409/unsquash-4.c squashfs-tools-4.3+20140919/unsquash-4.c
  10930. --- squashfs-tools-4.2+20130409/unsquash-4.c    2013-05-09 10:39:11.000000000 +0200
  10931. +++ squashfs-tools-4.3+20140919/unsquash-4.c    2015-07-20 21:03:05.000000000 +0200
  10932. @@ -24,7 +24,6 @@
  10933.  
  10934.  #include "unsquashfs.h"
  10935.  #include "squashfs_swap.h"
  10936. -#include "read_fs.h"
  10937.  
  10938.  static struct squashfs_fragment_entry *fragment_table;
  10939.  static unsigned int *id_table;
  10940. @@ -109,7 +108,7 @@
  10941.         EXIT_UNSQUASH("read_inode: inode table block %lld not found\n",
  10942.             start);        
  10943.  
  10944. -   SQUASHFS_SWAP_BASE_INODE_HEADER(&header.base, block_ptr);
  10945. +   SQUASHFS_SWAP_BASE_INODE_HEADER(block_ptr, &header.base);
  10946.  
  10947.     i.uid = (uid_t) id_table[header.base.uid];
  10948.     i.gid = (uid_t) id_table[header.base.guid];
  10949. @@ -122,7 +121,7 @@
  10950.         case SQUASHFS_DIR_TYPE: {
  10951.             struct squashfs_dir_inode_header *inode = &header.dir;
  10952.  
  10953. -           SQUASHFS_SWAP_DIR_INODE_HEADER(inode, block_ptr);
  10954. +           SQUASHFS_SWAP_DIR_INODE_HEADER(block_ptr, inode);
  10955.  
  10956.             i.data = inode->file_size;
  10957.             i.offset = inode->offset;
  10958. @@ -133,7 +132,7 @@
  10959.         case SQUASHFS_LDIR_TYPE: {
  10960.             struct squashfs_ldir_inode_header *inode = &header.ldir;
  10961.  
  10962. -           SQUASHFS_SWAP_LDIR_INODE_HEADER(inode, block_ptr);
  10963. +           SQUASHFS_SWAP_LDIR_INODE_HEADER(block_ptr, inode);
  10964.  
  10965.             i.data = inode->file_size;
  10966.             i.offset = inode->offset;
  10967. @@ -144,7 +143,7 @@
  10968.         case SQUASHFS_FILE_TYPE: {
  10969.             struct squashfs_reg_inode_header *inode = &header.reg;
  10970.  
  10971. -           SQUASHFS_SWAP_REG_INODE_HEADER(inode, block_ptr);
  10972. +           SQUASHFS_SWAP_REG_INODE_HEADER(block_ptr, inode);
  10973.  
  10974.             i.data = inode->file_size;
  10975.             i.frag_bytes = inode->fragment == SQUASHFS_INVALID_FRAG
  10976. @@ -164,7 +163,7 @@
  10977.         case SQUASHFS_LREG_TYPE: {
  10978.             struct squashfs_lreg_inode_header *inode = &header.lreg;
  10979.  
  10980. -           SQUASHFS_SWAP_LREG_INODE_HEADER(inode, block_ptr);
  10981. +           SQUASHFS_SWAP_LREG_INODE_HEADER(block_ptr, inode);
  10982.  
  10983.             i.data = inode->file_size;
  10984.             i.frag_bytes = inode->fragment == SQUASHFS_INVALID_FRAG
  10985. @@ -185,7 +184,7 @@
  10986.         case SQUASHFS_LSYMLINK_TYPE: {
  10987.             struct squashfs_symlink_inode_header *inode = &header.symlink;
  10988.  
  10989. -           SQUASHFS_SWAP_SYMLINK_INODE_HEADER(inode, block_ptr);
  10990. +           SQUASHFS_SWAP_SYMLINK_INODE_HEADER(block_ptr, inode);
  10991.  
  10992.             i.symlink = malloc(inode->symlink_size + 1);
  10993.             if(i.symlink == NULL)
  10994. @@ -198,9 +197,9 @@
  10995.             i.data = inode->symlink_size;
  10996.  
  10997.             if(header.base.inode_type == SQUASHFS_LSYMLINK_TYPE)
  10998. -               SQUASHFS_SWAP_INTS(&i.xattr, block_ptr +
  10999. +               SQUASHFS_SWAP_INTS(block_ptr +
  11000.                     sizeof(struct squashfs_symlink_inode_header) +
  11001. -                   inode->symlink_size, 1);
  11002. +                   inode->symlink_size, &i.xattr, 1);
  11003.             else
  11004.                 i.xattr = SQUASHFS_INVALID_XATTR;
  11005.             break;
  11006. @@ -209,7 +208,7 @@
  11007.         case SQUASHFS_CHRDEV_TYPE: {
  11008.             struct squashfs_dev_inode_header *inode = &header.dev;
  11009.  
  11010. -           SQUASHFS_SWAP_DEV_INODE_HEADER(inode, block_ptr);
  11011. +           SQUASHFS_SWAP_DEV_INODE_HEADER(block_ptr, inode);
  11012.  
  11013.             i.data = inode->rdev;
  11014.             i.xattr = SQUASHFS_INVALID_XATTR;
  11015. @@ -219,7 +218,7 @@
  11016.         case SQUASHFS_LCHRDEV_TYPE: {
  11017.             struct squashfs_ldev_inode_header *inode = &header.ldev;
  11018.  
  11019. -           SQUASHFS_SWAP_LDEV_INODE_HEADER(inode, block_ptr);
  11020. +           SQUASHFS_SWAP_LDEV_INODE_HEADER(block_ptr, inode);
  11021.  
  11022.             i.data = inode->rdev;
  11023.             i.xattr = inode->xattr;
  11024. @@ -234,7 +233,7 @@
  11025.         case SQUASHFS_LSOCKET_TYPE: {
  11026.             struct squashfs_lipc_inode_header *inode = &header.lipc;
  11027.  
  11028. -           SQUASHFS_SWAP_LIPC_INODE_HEADER(inode, block_ptr);
  11029. +           SQUASHFS_SWAP_LIPC_INODE_HEADER(block_ptr, inode);
  11030.  
  11031.             i.data = 0;
  11032.             i.xattr = inode->xattr;
  11033. @@ -299,7 +298,7 @@
  11034.     size = (*i)->data + bytes - 3;
  11035.  
  11036.     while(bytes < size) {          
  11037. -       SQUASHFS_SWAP_DIR_HEADER(&dirh, directory_table + bytes);
  11038. +       SQUASHFS_SWAP_DIR_HEADER(directory_table + bytes, &dirh);
  11039.    
  11040.         dir_count = dirh.count + 1;
  11041.         TRACE("squashfs_opendir: Read directory header @ byte position "
  11042. @@ -311,7 +310,7 @@
  11043.             goto corrupted;
  11044.  
  11045.         while(dir_count--) {
  11046. -           SQUASHFS_SWAP_DIR_ENTRY(dire, directory_table + bytes);
  11047. +           SQUASHFS_SWAP_DIR_ENTRY(directory_table + bytes, dire);
  11048.  
  11049.             bytes += sizeof(*dire);
  11050.  
  11051. diff -Nru squashfs-tools-4.2+20130409/unsquashfs.c squashfs-tools-4.3+20140919/unsquashfs.c
  11052. --- squashfs-tools-4.2+20130409/unsquashfs.c    2013-05-09 10:39:11.000000000 +0200
  11053. +++ squashfs-tools-4.3+20140919/unsquashfs.c    2015-07-20 21:03:05.000000000 +0200
  11054. @@ -3,7 +3,7 @@
  11055.   * filesystem.
  11056.   *
  11057.   * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011,
  11058. - * 2012, 2013
  11059. + * 2012, 2013, 2014
  11060.   * Phillip Lougher <phillip@squashfs.org.uk>
  11061.   *
  11062.   * This program is free software; you can redistribute it and/or
  11063. @@ -26,9 +26,9 @@
  11064.  #include "unsquashfs.h"
  11065.  #include "squashfs_swap.h"
  11066.  #include "squashfs_compat.h"
  11067. -#include "read_fs.h"
  11068.  #include "compressor.h"
  11069.  #include "xattr.h"
  11070. +#include "unsquashfs_info.h"
  11071.  #include "stdarg.h"
  11072.  
  11073.  #include <sys/sysinfo.h>
  11074. @@ -39,8 +39,8 @@
  11075.  #include <ctype.h>
  11076.  
  11077.  struct cache *fragment_cache, *data_cache;
  11078. -struct queue *to_reader, *to_deflate, *to_writer, *from_writer;
  11079. -pthread_t *thread, *deflator_thread;
  11080. +struct queue *to_reader, *to_inflate, *to_writer, *from_writer;
  11081. +pthread_t *thread, *inflator_thread;
  11082.  pthread_mutex_t    fragment_mutex;
  11083.  
  11084.  /* user options that control parallelisation */
  11085. @@ -69,7 +69,6 @@
  11086.  int columns;
  11087.  int rotate = 0;
  11088.  pthread_mutex_t    screen_mutex;
  11089. -pthread_cond_t progress_wait;
  11090.  int progress = TRUE, progress_enabled = FALSE;
  11091.  unsigned int total_blocks = 0, total_files = 0, total_inodes = 0;
  11092.  unsigned int cur_blocks = 0;
  11093. @@ -121,7 +120,6 @@
  11094.  };
  11095.  
  11096.  void progress_bar(long long current, long long max, int columns);
  11097. -void update_progress_bar();
  11098.  
  11099.  #define MAX_LINE 16384
  11100.  
  11101. @@ -226,6 +224,21 @@
  11102.  }
  11103.  
  11104.  
  11105. +void dump_queue(struct queue *queue)
  11106. +{
  11107. +   pthread_mutex_lock(&queue->mutex);
  11108. +
  11109. +   printf("Max size %d, size %d%s\n", queue->size - 1,  
  11110. +       queue->readp <= queue->writep ? queue->writep - queue->readp :
  11111. +           queue->size - queue->readp + queue->writep,
  11112. +       queue->readp == queue->writep ? " (EMPTY)" :
  11113. +           ((queue->writep + 1) % queue->size) == queue->readp ?
  11114. +           " (FULL)" : "");
  11115. +
  11116. +   pthread_mutex_unlock(&queue->mutex);
  11117. +}
  11118. +
  11119. +
  11120.  /* Called with the cache mutex held */
  11121.  void insert_hash_table(struct cache *cache, struct cache_entry *entry)
  11122.  {
  11123. @@ -272,7 +285,7 @@
  11124.  /* Called with the cache mutex held */
  11125.  void remove_free_list(struct cache *cache, struct cache_entry *entry)
  11126.  {
  11127. -   if(entry->free_prev == NULL && entry->free_next == NULL)
  11128. +   if(entry->free_prev == NULL || entry->free_next == NULL)
  11129.         /* not in free list */
  11130.         return;
  11131.     else if(entry->free_prev == entry && entry->free_next == entry) {
  11132. @@ -300,6 +313,7 @@
  11133.     cache->max_buffers = max_buffers;
  11134.     cache->buffer_size = buffer_size;
  11135.     cache->count = 0;
  11136. +   cache->used = 0;
  11137.     cache->free_list = NULL;
  11138.     memset(cache->hash_table, 0, sizeof(struct cache_entry *) * 65536);
  11139.     cache->wait_free = FALSE;
  11140. @@ -316,7 +330,7 @@
  11141.  {
  11142.     /*
  11143.      * Get a block out of the cache.  If the block isn't in the cache
  11144. -    * it is added and queued to the reader() and deflate() threads for
  11145. +    * it is added and queued to the reader() and inflate() threads for
  11146.      * reading off disk and decompression.  The cache grows until max_blocks
  11147.      * is reached, once this occurs existing discarded blocks on the free
  11148.      * list are reused
  11149. @@ -332,11 +346,14 @@
  11150.  
  11151.     if(entry) {
  11152.         /*
  11153. -        * found the block in the cache, increment used count and
  11154. -        * if necessary remove from free list so it won't disappear
  11155. +        * found the block in the cache.  If the block is currently unused
  11156. +        * remove it from the free list and increment cache used count.
  11157.          */
  11158. +       if(entry->used == 0) {
  11159. +           cache->used ++;
  11160. +           remove_free_list(cache, entry);
  11161. +       }
  11162.         entry->used ++;
  11163. -       remove_free_list(cache, entry);
  11164.         pthread_mutex_unlock(&cache->mutex);
  11165.     } else {
  11166.         /*
  11167. @@ -369,7 +386,11 @@
  11168.         }
  11169.  
  11170.         /*
  11171. -        * initialise block and insert into the hash table
  11172. +        * Initialise block and insert into the hash table.
  11173. +        * Increment used which tracks how many buffers in the
  11174. +        * cache are actively in use (the other blocks, count - used,
  11175. +        * are in the cache and available for lookup, but can also be
  11176. +        * re-used).
  11177.          */
  11178.         entry->block = block;
  11179.         entry->size = size;
  11180. @@ -377,6 +398,7 @@
  11181.         entry->error = FALSE;
  11182.         entry->pending = TRUE;
  11183.         insert_hash_table(cache, entry);
  11184. +       cache->used ++;
  11185.  
  11186.         /*
  11187.          * queue to read thread to read and ultimately (via the
  11188. @@ -446,6 +468,7 @@
  11189.     entry->used --;
  11190.     if(entry->used == 0) {
  11191.         insert_free_list(entry->cache, entry);
  11192. +       entry->cache->used --;
  11193.  
  11194.         /*
  11195.          * if the wait_free flag is set, one or more threads may be
  11196. @@ -461,6 +484,18 @@
  11197.  }
  11198.  
  11199.  
  11200. +void dump_cache(struct cache *cache)
  11201. +{
  11202. +   pthread_mutex_lock(&cache->mutex);
  11203. +
  11204. +   printf("Max buffers %d, Current size %d, Used %d,  %s\n",
  11205. +       cache->max_buffers, cache->count, cache->used,
  11206. +       cache->free_list ?  "Free buffers" : "No free buffers");
  11207. +
  11208. +   pthread_mutex_unlock(&cache->mutex);
  11209. +}
  11210. +
  11211. +
  11212.  char *modestr(char *str, int mode)
  11213.  {
  11214.     int i;
  11215. @@ -1223,11 +1258,14 @@
  11216.         target ++;
  11217.  
  11218.     start = target;
  11219. -   while(*target != '/' && *target!= '\0')
  11220. +   while(*target != '/' && *target != '\0')
  11221.         target ++;
  11222.  
  11223.     *targname = strndup(start, target - start);
  11224.  
  11225. +   while(*target == '/')
  11226. +       target ++;
  11227. +
  11228.     return target;
  11229.  }
  11230.  
  11231. @@ -1554,26 +1592,27 @@
  11232.         if(res == -1)
  11233.             EXIT_UNSQUASH("asprintf failed in dir_scan\n");
  11234.  
  11235. -       if(type == SQUASHFS_DIR_TYPE)
  11236. +       if(type == SQUASHFS_DIR_TYPE) {
  11237.             dir_scan(pathname, start_block, offset, new);
  11238. -       else if(new == NULL) {
  11239. +           free(pathname);
  11240. +       } else if(new == NULL) {
  11241. +           update_info(pathname);
  11242. +
  11243.             i = s_ops.read_inode(start_block, offset);
  11244.  
  11245.             if(lsonly || info)
  11246.                 print_filename(pathname, i);
  11247.  
  11248. -           if(!lsonly) {
  11249. +           if(!lsonly)
  11250.                 create_inode(pathname, i);
  11251. -               update_progress_bar();
  11252. -               }
  11253.  
  11254.             if(i->type == SQUASHFS_SYMLINK_TYPE ||
  11255.                     i->type == SQUASHFS_LSYMLINK_TYPE)
  11256.                 free(i->symlink);
  11257. -       }
  11258. +       } else
  11259. +           free(pathname);
  11260.  
  11261.         free_subdir(new);
  11262. -       free(pathname);
  11263.     }
  11264.  
  11265.     if(!lsonly)
  11266. @@ -1609,7 +1648,7 @@
  11267.         printf("Compression %s\n", comp->name);
  11268.  
  11269.         if(SQUASHFS_COMP_OPTS(sBlk.s.flags)) {
  11270. -           char buffer[SQUASHFS_METADATA_SIZE];
  11271. +           char buffer[SQUASHFS_METADATA_SIZE] __attribute__ ((aligned));
  11272.             int bytes;
  11273.  
  11274.             bytes = read_block(fd, sizeof(sBlk.s), NULL, 0, buffer);
  11275. @@ -1637,7 +1676,7 @@
  11276.             printf("Fragments are %scompressed\n",
  11277.                 SQUASHFS_UNCOMPRESSED_FRAGMENTS(sBlk.s.flags) ?
  11278.                 "un" : "");
  11279. -           printf("Always_use_fragments option is %sspecified\n",
  11280. +           printf("Always-use-fragments option is %sspecified\n",
  11281.                 SQUASHFS_ALWAYS_FRAGMENTS(sBlk.s.flags) ? "" :
  11282.                 "not ");
  11283.         }
  11284. @@ -1698,6 +1737,42 @@
  11285.  }
  11286.  
  11287.  
  11288. +int check_compression(struct compressor *comp)
  11289. +{
  11290. +   int res, bytes = 0;
  11291. +   char buffer[SQUASHFS_METADATA_SIZE] __attribute__ ((aligned));
  11292. +
  11293. +   if(!comp->supported) {
  11294. +       ERROR("Filesystem uses %s compression, this is "
  11295. +           "unsupported by this version\n", comp->name);
  11296. +       ERROR("Decompressors available:\n");
  11297. +       display_compressors("", "");
  11298. +       return 0;
  11299. +   }
  11300. +
  11301. +   /*
  11302. +    * Read compression options from disk if present, and pass to
  11303. +    * the compressor to ensure we know how to decompress a filesystem
  11304. +    * compressed with these compression options.
  11305. +    *
  11306. +    * Note, even if there is no compression options we still call the
  11307. +    * compressor because some compression options may be mandatory
  11308. +    * for some compressors.
  11309. +    */
  11310. +   if(SQUASHFS_COMP_OPTS(sBlk.s.flags)) {
  11311. +       bytes = read_block(fd, sizeof(sBlk.s), NULL, 0, buffer);
  11312. +       if(bytes == 0) {
  11313. +           ERROR("Failed to read compressor options\n");
  11314. +           return 0;
  11315. +       }
  11316. +   }
  11317. +
  11318. +   res = compressor_check_options(comp, sBlk.s.block_size, buffer, bytes);
  11319. +
  11320. +   return res != -1;
  11321. +}
  11322. +
  11323. +
  11324.  int read_super(char *source)
  11325.  {
  11326.     squashfs_super_block_3 sBlk_3;
  11327. @@ -1897,10 +1972,10 @@
  11328.  
  11329.         if(res && SQUASHFS_COMPRESSED_BLOCK(entry->size))
  11330.             /*
  11331. -            * queue successfully read block to the deflate
  11332. +            * queue successfully read block to the inflate
  11333.              * thread(s) for further processing
  11334.              */
  11335. -           queue_put(to_deflate, entry);
  11336. +           queue_put(to_inflate, entry);
  11337.         else
  11338.             /*
  11339.              * block has either been successfully read and is
  11340. @@ -2019,12 +2094,12 @@
  11341.  /*
  11342.   * decompress thread.  This decompresses buffers queued by the read thread
  11343.   */
  11344. -void *deflator(void *arg)
  11345. +void *inflator(void *arg)
  11346.  {
  11347.     char tmp[block_size];
  11348.  
  11349.     while(1) {
  11350. -       struct cache_entry *entry = queue_get(to_deflate);
  11351. +       struct cache_entry *entry = queue_get(to_inflate);
  11352.         int error, res;
  11353.  
  11354.         res = compressor_uncompress(comp, tmp, entry->data,
  11355. @@ -2049,8 +2124,7 @@
  11356.  
  11357.  void *progress_thread(void *arg)
  11358.  {
  11359. -   struct timeval timeval;
  11360. -   struct timespec timespec;
  11361. +   struct timespec requested_time, remaining;
  11362.     struct itimerval itimerval;
  11363.     struct winsize winsize;
  11364.  
  11365. @@ -2070,22 +2144,22 @@
  11366.     itimerval.it_interval.tv_usec = 250000;
  11367.     setitimer(ITIMER_REAL, &itimerval, NULL);
  11368.  
  11369. -   pthread_cond_init(&progress_wait, NULL);
  11370. +   requested_time.tv_sec = 0;
  11371. +   requested_time.tv_nsec = 250000000;
  11372.  
  11373. -   pthread_mutex_lock(&screen_mutex);
  11374.     while(1) {
  11375. -       gettimeofday(&timeval, NULL);
  11376. -       timespec.tv_sec = timeval.tv_sec;
  11377. -       if(timeval.tv_usec + 250000 > 999999)
  11378. -           timespec.tv_sec++;
  11379. -       timespec.tv_nsec = ((timeval.tv_usec + 250000) % 1000000) *
  11380. -           1000;
  11381. -       pthread_cond_timedwait(&progress_wait, &screen_mutex,
  11382. -           &timespec);
  11383. -       if(progress_enabled)
  11384. +       int res = nanosleep(&requested_time, &remaining);
  11385. +
  11386. +       if(res == -1 && errno != EINTR)
  11387. +           EXIT_UNSQUASH("nanosleep failed in progress thread\n");
  11388. +
  11389. +       if(progress_enabled) {
  11390. +           pthread_mutex_lock(&screen_mutex);
  11391.             progress_bar(sym_count + dev_count +
  11392.                 fifo_count + cur_blocks, total_inodes -
  11393.                 total_files + total_blocks, columns);
  11394. +           pthread_mutex_unlock(&screen_mutex);
  11395. +       }
  11396.     }
  11397.  }
  11398.  
  11399. @@ -2096,11 +2170,23 @@
  11400.     int i, max_files, res;
  11401.     sigset_t sigmask, old_mask;
  11402.  
  11403. +   /* block SIGQUIT and SIGHUP, these are handled by the info thread */
  11404.     sigemptyset(&sigmask);
  11405. -   sigaddset(&sigmask, SIGINT);
  11406.     sigaddset(&sigmask, SIGQUIT);
  11407. -   if(sigprocmask(SIG_BLOCK, &sigmask, &old_mask) == -1)
  11408. -       EXIT_UNSQUASH("Failed to set signal mask in intialise_threads"
  11409. +   sigaddset(&sigmask, SIGHUP);
  11410. +   if(pthread_sigmask(SIG_BLOCK, &sigmask, NULL) == -1)
  11411. +       EXIT_UNSQUASH("Failed to set signal mask in initialise_threads"
  11412. +           "\n");
  11413. +
  11414. +   /*
  11415. +    * temporarily block these signals so the created sub-threads will
  11416. +    * ignore them, ensuring the main thread handles them
  11417. +    */
  11418. +   sigemptyset(&sigmask);
  11419. +   sigaddset(&sigmask, SIGINT);
  11420. +   sigaddset(&sigmask, SIGTERM);
  11421. +   if(pthread_sigmask(SIG_BLOCK, &sigmask, &old_mask) == -1)
  11422. +       EXIT_UNSQUASH("Failed to set signal mask in initialise_threads"
  11423.             "\n");
  11424.  
  11425.     if(processors == -1) {
  11426. @@ -2132,14 +2218,14 @@
  11427.     thread = malloc((3 + processors) * sizeof(pthread_t));
  11428.     if(thread == NULL)
  11429.         EXIT_UNSQUASH("Out of memory allocating thread descriptors\n");
  11430. -   deflator_thread = &thread[3];
  11431. +   inflator_thread = &thread[3];
  11432.  
  11433.     /*
  11434. -    * dimensioning the to_reader and to_deflate queues.  The size of
  11435. +    * dimensioning the to_reader and to_inflate queues.  The size of
  11436.      * these queues is directly related to the amount of block
  11437.      * read-ahead possible.  To_reader queues block read requests to
  11438. -    * the reader thread and to_deflate queues block decompression
  11439. -    * requests to the deflate thread(s) (once the block has been read by
  11440. +    * the reader thread and to_inflate queues block decompression
  11441. +    * requests to the inflate thread(s) (once the block has been read by
  11442.      * the reader thread).  The amount of read-ahead is determined by
  11443.      * the combined size of the data_block and fragment caches which
  11444.      * determine the total number of blocks which can be "in flight"
  11445. @@ -2162,7 +2248,7 @@
  11446.      *
  11447.      * dimensioning the to_writer queue.  The size of this queue is
  11448.      * directly related to the amount of block read-ahead possible.
  11449. -    * However, unlike the to_reader and to_deflate queues, this is
  11450. +    * However, unlike the to_reader and to_inflate queues, this is
  11451.      * complicated by the fact the to_writer queue not only contains
  11452.      * entries for fragments and data_blocks but it also contains
  11453.      * file entries, one per open file in the read-ahead.
  11454. @@ -2196,7 +2282,7 @@
  11455.     open_init(max_files);
  11456.  
  11457.     /*
  11458. -    * allocate to_reader, to_deflate and to_writer queues.  Set based on
  11459. +    * allocate to_reader, to_inflate and to_writer queues.  Set based on
  11460.      * open file limit and cache size, unless open file limit is unlimited,
  11461.      * in which case set purely based on cache limits
  11462.      *
  11463. @@ -2209,7 +2295,7 @@
  11464.             EXIT_UNSQUASH("Data queue size is too large\n");
  11465.  
  11466.         to_reader = queue_init(max_files + data_buffer_size);
  11467. -       to_deflate = queue_init(max_files + data_buffer_size);
  11468. +       to_inflate = queue_init(max_files + data_buffer_size);
  11469.         to_writer = queue_init(max_files * 2 + data_buffer_size);
  11470.     } else {
  11471.         int all_buffers_size;
  11472. @@ -2225,7 +2311,7 @@
  11473.                             " too large\n");
  11474.  
  11475.         to_reader = queue_init(all_buffers_size);
  11476. -       to_deflate = queue_init(all_buffers_size);
  11477. +       to_inflate = queue_init(all_buffers_size);
  11478.         to_writer = queue_init(all_buffers_size * 2);
  11479.     }
  11480.  
  11481. @@ -2236,10 +2322,11 @@
  11482.     pthread_create(&thread[0], NULL, reader, NULL);
  11483.     pthread_create(&thread[1], NULL, writer, NULL);
  11484.     pthread_create(&thread[2], NULL, progress_thread, NULL);
  11485. +   init_info();
  11486.     pthread_mutex_init(&fragment_mutex, NULL);
  11487.  
  11488.     for(i = 0; i < processors; i++) {
  11489. -       if(pthread_create(&deflator_thread[i], NULL, deflator, NULL) !=
  11490. +       if(pthread_create(&inflator_thread[i], NULL, inflator, NULL) !=
  11491.                  0)
  11492.             EXIT_UNSQUASH("Failed to create thread\n");
  11493.     }
  11494. @@ -2247,8 +2334,8 @@
  11495.     printf("Parallel unsquashfs: Using %d processor%s\n", processors,
  11496.             processors == 1 ? "" : "s");
  11497.  
  11498. -   if(sigprocmask(SIG_SETMASK, &old_mask, NULL) == -1)
  11499. -       EXIT_UNSQUASH("Failed to set signal mask in intialise_threads"
  11500. +   if(pthread_sigmask(SIG_SETMASK, &old_mask, NULL) == -1)
  11501. +       EXIT_UNSQUASH("Failed to set signal mask in initialise_threads"
  11502.             "\n");
  11503.  }
  11504.  
  11505. @@ -2256,7 +2343,7 @@
  11506.  void enable_progress_bar()
  11507.  {
  11508.     pthread_mutex_lock(&screen_mutex);
  11509. -   progress_enabled = TRUE;
  11510. +   progress_enabled = progress;
  11511.     pthread_mutex_unlock(&screen_mutex);
  11512.  }
  11513.  
  11514. @@ -2264,19 +2351,16 @@
  11515.  void disable_progress_bar()
  11516.  {
  11517.     pthread_mutex_lock(&screen_mutex);
  11518. +   if(progress_enabled) {
  11519. +       progress_bar(sym_count + dev_count + fifo_count + cur_blocks,
  11520. +           total_inodes - total_files + total_blocks, columns);
  11521. +       printf("\n");
  11522. +   }
  11523.     progress_enabled = FALSE;
  11524.     pthread_mutex_unlock(&screen_mutex);
  11525.  }
  11526.  
  11527.  
  11528. -void update_progress_bar()
  11529. -{
  11530. -   pthread_mutex_lock(&screen_mutex);
  11531. -   pthread_cond_signal(&progress_wait);
  11532. -   pthread_mutex_unlock(&screen_mutex);
  11533. -}
  11534. -
  11535. -
  11536.  void progressbar_error(char *fmt, ...)
  11537.  {
  11538.     va_list ap;
  11539. @@ -2393,8 +2477,8 @@
  11540.  
  11541.  
  11542.  #define VERSION() \
  11543. -   printf("unsquashfs version 4.2-git (2013/03/13)\n");\
  11544. -   printf("copyright (C) 2013 Phillip Lougher "\
  11545. +   printf("unsquashfs version 4.3 (2014/05/12)\n");\
  11546. +   printf("copyright (C) 2014 Phillip Lougher "\
  11547.         "<phillip@squashfs.org.uk>\n\n");\
  11548.         printf("This program is free software; you can redistribute it and/or"\
  11549.         "\n");\
  11550. @@ -2535,6 +2619,11 @@
  11551.         progress = FALSE;
  11552.  
  11553.  #ifdef SQUASHFS_TRACE
  11554. +   /*
  11555. +    * Disable progress bar if full debug tracing is enabled.
  11556. +    * The progress bar in this case just gets in the way of the
  11557. +    * debug trace output
  11558. +    */
  11559.     progress = FALSE;
  11560.  #endif
  11561.  
  11562. @@ -2608,13 +2697,8 @@
  11563.         exit(0);
  11564.     }
  11565.  
  11566. -   if(!comp->supported) {
  11567. -       ERROR("Filesystem uses %s compression, this is "
  11568. -           "unsupported by this version\n", comp->name);
  11569. -       ERROR("Decompressors available:\n");
  11570. -       display_compressors("", "");
  11571. +   if(!check_compression(comp))
  11572.         exit(1);
  11573. -   }
  11574.  
  11575.     block_size = sBlk.s.block_size;
  11576.     block_log = sBlk.s.block_log;
  11577. @@ -2707,8 +2791,7 @@
  11578.     printf("%d inodes (%d blocks) to write\n\n", total_inodes,
  11579.         total_inodes - total_files + total_blocks);
  11580.  
  11581. -   if(progress)
  11582. -       enable_progress_bar();
  11583. +   enable_progress_bar();
  11584.  
  11585.     dir_scan(dest, SQUASHFS_INODE_BLK(sBlk.s.root_inode),
  11586.         SQUASHFS_INODE_OFFSET(sBlk.s.root_inode), paths);
  11587. @@ -2716,11 +2799,7 @@
  11588.     queue_put(to_writer, NULL);
  11589.     queue_get(from_writer);
  11590.  
  11591. -   if(progress) {
  11592. -       disable_progress_bar();
  11593. -       progress_bar(sym_count + dev_count + fifo_count + cur_blocks,
  11594. -           total_inodes - total_files + total_blocks, columns);
  11595. -   }
  11596. +   disable_progress_bar();
  11597.  
  11598.     if(!lsonly) {
  11599.         printf("\n");
  11600. diff -Nru squashfs-tools-4.2+20130409/unsquashfs.h squashfs-tools-4.3+20140919/unsquashfs.h
  11601. --- squashfs-tools-4.2+20130409/unsquashfs.h    2013-05-09 10:39:11.000000000 +0200
  11602. +++ squashfs-tools-4.3+20140919/unsquashfs.h    2015-07-20 21:03:05.000000000 +0200
  11603. @@ -1,8 +1,10 @@
  11604. +#ifndef UNSQUASHFS_H
  11605. +#define UNSQUASHFS_H
  11606.  /*
  11607.   * Unsquash a squashfs filesystem.  This is a highly compressed read only
  11608.   * filesystem.
  11609.   *
  11610. - * Copyright (c) 2009, 2010, 2013
  11611. + * Copyright (c) 2009, 2010, 2013, 2014
  11612.   * Phillip Lougher <phillip@squashfs.org.uk>
  11613.   *
  11614.   * This program is free software; you can redistribute it and/or
  11615. @@ -121,6 +123,7 @@
  11616.  struct cache {
  11617.     int max_buffers;
  11618.     int count;
  11619. +   int used;
  11620.     int buffer_size;
  11621.     int wait_free;
  11622.     int wait_pending;
  11623. @@ -232,11 +235,17 @@
  11624.  extern int inode_number;
  11625.  extern int lookup_type[];
  11626.  extern int fd;
  11627. +extern struct queue *to_reader, *to_inflate, *to_writer;
  11628. +extern struct cache *fragment_cache, *data_cache;
  11629.  
  11630.  /* unsquashfs.c */
  11631.  extern int lookup_entry(struct hash_table_entry **, long long);
  11632.  extern int read_fs_bytes(int fd, long long, int, void *);
  11633.  extern int read_block(int, long long, long long *, int, void *);
  11634. +extern void enable_progress_bar();
  11635. +extern void disable_progress_bar();
  11636. +extern void dump_queue(struct queue *);
  11637. +extern void dump_cache(struct cache *);
  11638.  
  11639.  /* unsquash-1.c */
  11640.  extern void read_block_list_1(unsigned int *, char *, int);
  11641. @@ -266,3 +275,4 @@
  11642.  extern struct dir *squashfs_opendir_4(unsigned int, unsigned int,
  11643.     struct inode **);
  11644.  extern int read_uids_guids_4();
  11645. +#endif
  11646. diff -Nru squashfs-tools-4.2+20130409/unsquashfs_info.c squashfs-tools-4.3+20140919/unsquashfs_info.c
  11647. --- squashfs-tools-4.2+20130409/unsquashfs_info.c   1970-01-01 01:00:00.000000000 +0100
  11648. +++ squashfs-tools-4.3+20140919/unsquashfs_info.c   2015-07-20 21:03:05.000000000 +0200
  11649. @@ -0,0 +1,145 @@
  11650. +/*
  11651. + * Create a squashfs filesystem.  This is a highly compressed read only
  11652. + * filesystem.
  11653. + *
  11654. + * Copyright (c) 2013
  11655. + * Phillip Lougher <phillip@squashfs.org.uk>
  11656. + *
  11657. + * This program is free software; you can redistribute it and/or
  11658. + * modify it under the terms of the GNU General Public License
  11659. + * as published by the Free Software Foundation; either version 2,
  11660. + * or (at your option) any later version.
  11661. + *
  11662. + * This program is distributed in the hope that it will be useful,
  11663. + * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11664. + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11665. + * GNU General Public License for more details.
  11666. + *
  11667. + * You should have received a copy of the GNU General Public License
  11668. + * along with this program; if not, write to the Free Software
  11669. + * Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  11670. + *
  11671. + * unsquashfs_info.c
  11672. + */
  11673. +
  11674. +#include <pthread.h>
  11675. +#include <sys/ioctl.h>
  11676. +#include <unistd.h>
  11677. +#include <signal.h>
  11678. +#include <sys/time.h>
  11679. +#include <stdio.h>
  11680. +#include <math.h>
  11681. +#include <stdarg.h>
  11682. +#include <errno.h>
  11683. +#include <stdlib.h>
  11684. +#include <dirent.h>
  11685. +#include <sys/types.h>
  11686. +#include <sys/stat.h>
  11687. +#include <string.h>
  11688. +
  11689. +#include "squashfs_fs.h"
  11690. +#include "unsquashfs.h"
  11691. +#include "error.h"
  11692. +
  11693. +static int silent = 0;
  11694. +char *pathname = NULL;
  11695. +
  11696. +pthread_t info_thread;
  11697. +
  11698. +
  11699. +void disable_info()
  11700. +{
  11701. +   if(pathname)
  11702. +       free(pathname);
  11703. +
  11704. +   pathname = NULL;
  11705. +}
  11706. +
  11707. +
  11708. +void update_info(char *name)
  11709. +{
  11710. +   if(pathname)
  11711. +       free(pathname);
  11712. +
  11713. +   pathname = name;
  11714. +}
  11715. +
  11716. +
  11717. +void dump_state()
  11718. +{
  11719. +   disable_progress_bar();
  11720. +
  11721. +   printf("Queue and cache status dump\n");
  11722. +   printf("===========================\n");
  11723. +
  11724. +   printf("file buffer read queue (main thread -> reader thread)\n");
  11725. +   dump_queue(to_reader);
  11726. +
  11727. +   printf("file buffer decompress queue (reader thread -> inflate"
  11728. +                           " thread(s))\n");
  11729. +   dump_queue(to_inflate);
  11730. +
  11731. +   printf("file buffer write queue (main thread -> writer thread)\n");
  11732. +   dump_queue(to_writer);
  11733. +
  11734. +   printf("\nbuffer cache (uncompressed blocks and compressed blocks "
  11735. +                           "'in flight')\n");
  11736. +   dump_cache(data_cache);
  11737. +
  11738. +   printf("fragment buffer cache (uncompressed frags and compressed"
  11739. +                       " frags 'in flight')\n");
  11740. +   dump_cache(fragment_cache);
  11741. +
  11742. +   enable_progress_bar();
  11743. +}
  11744. +
  11745. +
  11746. +void *info_thrd(void *arg)
  11747. +{
  11748. +   sigset_t sigmask;
  11749. +   struct timespec timespec = { .tv_sec = 1, .tv_nsec = 0 };
  11750. +   int sig, waiting = 0;
  11751. +
  11752. +   sigemptyset(&sigmask);
  11753. +   sigaddset(&sigmask, SIGQUIT);
  11754. +   sigaddset(&sigmask, SIGHUP);
  11755. +
  11756. +   while(1) {
  11757. +       if(waiting)
  11758. +           sig = sigtimedwait(&sigmask, NULL, &timespec);
  11759. +       else
  11760. +           sig = sigwaitinfo(&sigmask, NULL);
  11761. +
  11762. +       if(sig == -1) {
  11763. +           switch(errno) {
  11764. +           case EAGAIN:
  11765. +               /* interval timed out */
  11766. +               waiting = 0;
  11767. +               /* FALLTHROUGH */
  11768. +           case EINTR:
  11769. +               /* if waiting, the wait will be longer, but
  11770. +                  that's OK */
  11771. +               continue;
  11772. +           default:
  11773. +               BAD_ERROR("sigtimedwait/sigwaitinfo failed "
  11774. +                   "because %s\n", strerror(errno));
  11775. +           }
  11776. +       }
  11777. +
  11778. +       if(sig == SIGQUIT && !waiting) {
  11779. +           if(pathname)
  11780. +               INFO("%s\n", pathname);
  11781. +
  11782. +           /* set one second interval period, if ^\ received
  11783. +              within then, dump queue and cache status */
  11784. +           waiting = 1;
  11785. +       } else
  11786. +           dump_state();
  11787. +   }
  11788. +}
  11789. +
  11790. +
  11791. +void init_info()
  11792. +{
  11793. +   pthread_create(&info_thread, NULL, info_thrd, NULL);
  11794. +}
  11795. diff -Nru squashfs-tools-4.2+20130409/unsquashfs_info.h squashfs-tools-4.3+20140919/unsquashfs_info.h
  11796. --- squashfs-tools-4.2+20130409/unsquashfs_info.h   1970-01-01 01:00:00.000000000 +0100
  11797. +++ squashfs-tools-4.3+20140919/unsquashfs_info.h   2015-07-20 21:03:05.000000000 +0200
  11798. @@ -0,0 +1,30 @@
  11799. +#ifndef UNSQUASHFS_INFO_H
  11800. +#define UNSQUASHFS_INFO_H
  11801. +/*
  11802. + * Create a squashfs filesystem.  This is a highly compressed read only
  11803. + * filesystem.
  11804. + *
  11805. + * Copyright (c) 2013, 2014
  11806. + * Phillip Lougher <phillip@squashfs.org.uk>
  11807. + *
  11808. + * This program is free software; you can redistribute it and/or
  11809. + * modify it under the terms of the GNU General Public License
  11810. + * as published by the Free Software Foundation; either version 2,
  11811. + * or (at your option) any later version.
  11812. + *
  11813. + * This program is distributed in the hope that it will be useful,
  11814. + * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11815. + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11816. + * GNU General Public License for more details.
  11817. + *
  11818. + * You should have received a copy of the GNU General Public License
  11819. + * along with this program; if not, write to the Free Software
  11820. + * Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  11821. + *
  11822. + * unsquashfs_info.h
  11823. + */
  11824. +
  11825. +extern void disable_info();
  11826. +extern void update_info(char *);
  11827. +extern void init_info();
  11828. +#endif
  11829. diff -Nru squashfs-tools-4.2+20130409/xattr.c squashfs-tools-4.3+20140919/xattr.c
  11830. --- squashfs-tools-4.2+20130409/xattr.c 2013-05-09 10:39:11.000000000 +0200
  11831. +++ squashfs-tools-4.3+20140919/xattr.c 2015-07-20 21:03:05.000000000 +0200
  11832. @@ -2,7 +2,7 @@
  11833.   * Create a squashfs filesystem.  This is a highly compressed read only
  11834.   * filesystem.
  11835.   *
  11836. - * Copyright (c) 2008, 2009, 2010, 2012
  11837. + * Copyright (c) 2008, 2009, 2010, 2012, 2014
  11838.   * Phillip Lougher <phillip@squashfs.org.uk>
  11839.   *
  11840.   * This program is free software; you can redistribute it and/or
  11841. @@ -41,6 +41,7 @@
  11842.  #include "mksquashfs.h"
  11843.  #include "xattr.h"
  11844.  #include "error.h"
  11845. +#include "progressbar.h"
  11846.  
  11847.  /* compressed xattr table */
  11848.  static char *xattr_table = NULL;
  11849. @@ -65,7 +66,7 @@
  11850.  static int sxattr_ids = 0;
  11851.  
  11852.  /* xattr hash table for value duplicate detection */
  11853. -static struct xattr_list *dupl[65536];
  11854. +static struct xattr_list *dupl_value[65536];
  11855.  
  11856.  /* xattr hash table for id duplicate detection */
  11857.  static struct dupl_id *dupl_id[65536];
  11858. @@ -120,10 +121,12 @@
  11859.     while(1) {
  11860.         size = llistxattr(filename, NULL, 0);
  11861.         if(size <= 0) {
  11862. -           if(size < 0 && errno != ENOTSUP)
  11863. -               ERROR("llistxattr for %s failed in read_attrs,"
  11864. -                   " because %s\n", filename,
  11865. +           if(size < 0 && errno != ENOTSUP) {
  11866. +               ERROR_START("llistxattr for %s failed in "
  11867. +                   "read_attrs, because %s", filename,
  11868.                     strerror(errno));
  11869. +               ERROR_EXIT(".  Ignoring");
  11870. +           }
  11871.             return 0;
  11872.         }
  11873.  
  11874. @@ -138,9 +141,10 @@
  11875.                 /* xattr list grew?  Try again */
  11876.                 continue;
  11877.             else {
  11878. -               ERROR("llistxattr for %s failed in read_attrs,"
  11879. -                   " because %s\n", filename,
  11880. +               ERROR_START("llistxattr for %s failed in "
  11881. +                   "read_attrs, because %s", filename,
  11882.                     strerror(errno));
  11883. +               ERROR_EXIT(".  Ignoring");
  11884.                 return 0;
  11885.             }
  11886.         }
  11887. @@ -169,9 +173,10 @@
  11888.             vsize = lgetxattr(filename, xattr_list[i].full_name,
  11889.                                 NULL, 0);
  11890.             if(vsize < 0) {
  11891. -               ERROR("lgetxattr failed for %s in read_attrs,"
  11892. -                   " because %s\n", filename,
  11893. +               ERROR_START("lgetxattr failed for %s in "
  11894. +                   "read_attrs, because %s", filename,
  11895.                     strerror(errno));
  11896. +               ERROR_EXIT(".  Ignoring");
  11897.                 free(xattr_list[i].full_name);
  11898.                 goto failed;
  11899.             }
  11900. @@ -188,9 +193,10 @@
  11901.                     /* xattr grew?  Try again */
  11902.                     continue;
  11903.                 else {
  11904. -                   ERROR("lgetxattr failed for %s in "
  11905. -                       "read_attrs, because %s\n",
  11906. +                   ERROR_START("lgetxattr failed for %s "
  11907. +                       "in read_attrs, because %s",
  11908.                         filename, strerror(errno));
  11909. +                   ERROR_EXIT(".  Ignoring");
  11910.                     free(xattr_list[i].full_name);
  11911.                     goto failed;
  11912.                 }
  11913. @@ -342,7 +348,7 @@
  11914.  
  11915.     /* Check if this is a duplicate of an existing value */
  11916.     xattr->vchecksum = get_checksum(xattr->value, xattr->vsize, 0);
  11917. -   for(entry = dupl[xattr->vchecksum]; entry; entry = entry->vnext) {
  11918. +   for(entry = dupl_value[xattr->vchecksum]; entry; entry = entry->vnext) {
  11919.         if(entry->vsize != xattr->vsize)
  11920.             continue;
  11921.        
  11922. @@ -355,8 +361,8 @@
  11923.          * No duplicate exists, add to hash table, and mark as
  11924.          * requiring writing
  11925.          */
  11926. -       xattr->vnext = dupl[xattr->vchecksum];
  11927. -       dupl[xattr->vchecksum] = xattr;
  11928. +       xattr->vnext = dupl_value[xattr->vchecksum];
  11929. +       dupl_value[xattr->vchecksum] = xattr;
  11930.         xattr->ool_value = SQUASHFS_INVALID_BLK;
  11931.     } else {
  11932.         /*
  11933. diff -Nru squashfs-tools-4.2+20130409/xattr.h squashfs-tools-4.3+20140919/xattr.h
  11934. --- squashfs-tools-4.2+20130409/xattr.h 2013-05-09 10:39:11.000000000 +0200
  11935. +++ squashfs-tools-4.3+20140919/xattr.h 2015-07-20 21:03:05.000000000 +0200
  11936. @@ -1,8 +1,10 @@
  11937. +#ifndef XATTR_H
  11938. +#define XATTR_H
  11939.  /*
  11940.   * Create a squashfs filesystem.  This is a highly compressed read only
  11941.   * filesystem.
  11942.   *
  11943. - * Copyright (c) 2010, 2012, 2013
  11944. + * Copyright (c) 2010, 2012, 2013, 2014
  11945.   * Phillip Lougher <phillip@squashfs.org.uk>
  11946.   *
  11947.   * This program is free software; you can redistribute it and/or
  11948. @@ -124,7 +126,7 @@
  11949.  }
  11950.  
  11951.  
  11952. -static inline struct xattr_list *get_xattr(int i, unsigned int *count, int)
  11953. +static inline struct xattr_list *get_xattr(int i, unsigned int *count, int j)
  11954.  {
  11955.     return NULL;
  11956.  }
  11957. @@ -145,4 +147,4 @@
  11958.  #define XOPT_STR " (unsupported)"
  11959.  #define XATTR_DEF 1
  11960.  #endif
  11961. -
  11962. +#endif
  11963. diff -Nru squashfs-tools-4.2+20130409/xz_wrapper.c squashfs-tools-4.3+20140919/xz_wrapper.c
  11964. --- squashfs-tools-4.2+20130409/xz_wrapper.c    2013-05-09 10:39:11.000000000 +0200
  11965. +++ squashfs-tools-4.3+20140919/xz_wrapper.c    2015-07-20 21:03:05.000000000 +0200
  11966. @@ -41,8 +41,6 @@
  11967.     { NULL, LZMA_VLI_UNKNOWN, 0 }
  11968.  };
  11969.  
  11970. -static struct comp_opts comp_opts;
  11971. -
  11972.  static int filter_count = 1;
  11973.  static int dictionary_size = 0;
  11974.  static float dictionary_percent = 0;
  11975. @@ -56,10 +54,11 @@
  11976.   * -Xbcj
  11977.   * -Xdict-size
  11978.   *
  11979. - * This function returns 1 on successful parsing of an option
  11980. - *         -1 if the option was unrecognised, or
  11981. - *         -2 if the option was recognised, but otherwise bad in
  11982. - *            some way (e.g. invalid parameter)
  11983. + * This function returns:
  11984. + * >=0 (number of additional args parsed) on success
  11985. + * -1 if the option was unrecognised, or
  11986. + * -2 if the option was recognised, but otherwise bad in
  11987. + *    some way (e.g. invalid parameter)
  11988.   *
  11989.   * Note: this function sets internal compressor state, but does not
  11990.   * pass back the results of the parsing other than success/failure.
  11991. @@ -226,6 +225,7 @@
  11992.   */
  11993.  static void *xz_dump_options(int block_size, int *size)
  11994.  {
  11995. +   static struct comp_opts comp_opts;
  11996.     int flags = 0, i;
  11997.  
  11998.     /*
  11999. @@ -329,7 +329,7 @@
  12000.     int i, n;
  12001.  
  12002.     /* check passed comp opts struct is of the correct length */
  12003. -   if(size != sizeof(comp_opts))
  12004. +   if(size != sizeof(struct comp_opts))
  12005.         goto failed;
  12006.  
  12007.     SQUASHFS_INSWAP_COMP_OPTS(comp_opts);
  12008. @@ -494,8 +494,12 @@
  12009.     lzma_ret res = lzma_stream_buffer_decode(&memlimit, 0, NULL,
  12010.             src, &src_pos, size, dest, &dest_pos, outsize);
  12011.  
  12012. -   *error = res;
  12013. -   return res == LZMA_OK && size == (int) src_pos ? (int) dest_pos : -1;
  12014. +   if(res == LZMA_OK && size == (int) src_pos)
  12015. +       return (int) dest_pos;
  12016. +   else {
  12017. +       *error = res;
  12018. +       return -1;
  12019. +   }
  12020.  }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!