This week only. Pastebin PRO Accounts Christmas Special! Don't miss out!Want more features on Pastebin? Sign Up, it's FREE!
Guest

GFVGFVGFVGFV

By: a guest on Aug 8th, 2012  |  syntax: None  |  size: 3.79 KB  |  views: 33  |  expires: Never
download  |  raw  |  embed  |  report abuse  |  print
Text below is selected. Please press Ctrl+C to copy to your clipboard. (⌘+C on Mac)
  1. define GFV GetFieldValue('transforms.conf');
  2.  
  3. -- Variant A --
  4. students = load 'roster' using PigStorage();
  5.  
  6. passing = filter students by GFV(*, 'threadId') != 'F'
  7.                          and GFV(*, 'threadId') != 'E'
  8.                          and GFV(*, 'threadId') != 'G'
  9.                          and GFV(*, 'threadId') != 'H'
  10.                          and GFV(*, 'threadId') != 'I'
  11.                          and GFV(*, 'threadId') != 'J'
  12.                          and GFV(*, 'threadId') != 'K'
  13.                          and GFV(*, 'threadId') != 'L'
  14.                          and GFV(*, 'threadId') != 'M'
  15.                          and GFV(*, 'threadId') != 'N'
  16.                          and GFV(*, 'threadId') != 'O'
  17.                          and GFV(*, 'threadId') != 'P'
  18.                          and GFV(*, 'threadId') != 'Q'
  19.                          and GFV(*, 'threadId') != 'R'
  20.                          and GFV(*, 'threadId') != 'S'
  21.                          and GFV(*, 'threadId') != 'T'
  22.                          and GFV(*, 'threadId') != 'U'
  23.                          and GFV(*, 'threadId') != 'V'
  24.                          and GFV(*, 'threadId') != 'W'
  25.                          and GFV(*, 'threadId') != 'X'
  26.                          and GFV(*, 'threadId') != 'Y'
  27.                          and GFV(*, 'threadId') != 'Z';
  28.  
  29. pruneFields = foreach passing generate
  30.                     (int) GFV(*, 'logRecordType') as age,
  31.                               GFV(*, 'timestamp') as name,
  32.                               GFV(*, 'requestId') as grade,
  33.                                GFV(*, 'threadId') as hometown;
  34. -- -- -- -- --
  35.  
  36. -- Variant B --
  37. -- students = load 'roster' using PigStorage()
  38. --                 as (age,
  39. --                     name:chararray,
  40. --                     grade:chararray,
  41. --                     hometown:chararray);
  42. --
  43. -- passing = filter students by grade != 'F'
  44. --                          and grade != 'E'
  45. --                          and grade != 'G'
  46. --                          and grade != 'H'
  47. --                          and grade != 'I'
  48. --                          and grade != 'J'
  49. --                          and grade != 'K'
  50. --                          and grade != 'L'
  51. --                          and grade != 'M'
  52. --                          and grade != 'N'
  53. --                          and grade != 'O'
  54. --                          and grade != 'P'
  55. --                          and grade != 'Q'
  56. --                          and grade != 'R'
  57. --                          and grade != 'S'
  58. --                          and grade != 'T'
  59. --                          and grade != 'U'
  60. --                          and grade != 'V'
  61. --                          and grade != 'W'
  62. --                          and grade != 'X'
  63. --                          and grade != 'Y'
  64. --                          and grade != 'Z';
  65. --
  66. -- pruneFields = foreach passing generate (int) age, name, grade, hometown;
  67. -- -- -- -- --
  68.  
  69. split pruneFields into aStudents if grade == 'A',
  70.                        bStudents if grade == 'B',
  71.                        cStudents if grade == 'C',
  72.                        dStudents if grade == 'D';
  73.  
  74. aStudents = foreach aStudents generate *, 'Superb' as description;
  75. bStudents = foreach bStudents generate *, 'Good' as description;
  76. cStudents = foreach cStudents generate *, 'Average' as description;
  77. dStudents = foreach dStudents generate *, 'Okay' as description;
  78.  
  79. mergeAll = union aStudents, bStudents, cStudents, dStudents;
  80.  
  81. agg = group mergeAll by (hometown, description) parallel 1;
  82.  
  83. report = foreach agg {
  84.     names = mergeAll.name;
  85.     uniqNames = distinct names;
  86.     generate group, COUNT_STAR(mergeAll) as numStudents,
  87.                         COUNT(uniqNames) as numNames,
  88.                        AVG(mergeAll.age) as avgAge;
  89. }
  90.  
  91. store report into 'roster-out';
clone this paste RAW Paste Data