Advertisement
Guest User

Untitled

a guest
Jul 26th, 2011
275
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.55 KB | None | 0 0
  1. #include <algorithm>
  2. #include <limits>
  3. #include <Pipes.hh>
  4. #include <string>
  5. #include <StringUtils.hh>
  6. #include <TemplateFactory.hh>
  7.  
  8. using namespace std;
  9.  
  10. class WordCountMapper : public HadoopPipes::Mapper {
  11. public:
  12. // constructor: does nothing
  13. WordCountMapper( HadoopPipes::TaskContext& context ) {
  14. }
  15.  
  16. // map function: receives a line, outputs (word,"1")
  17. // to reducer.
  18. void map( HadoopPipes::MapContext& context ) {
  19. //--- get line of text ---
  20. string line = context.getInputValue();
  21.  
  22. //--- split it into words ---
  23. vector< string > words =
  24. HadoopUtils::splitString( line, " " );
  25.  
  26. //--- emit each word tuple (word, "1" ) ---
  27. for ( unsigned int i=0; i < words.size(); i++ ) {
  28. context.emit( words[i], HadoopUtils::toString( 1 ) );
  29. }
  30. }
  31. };
  32.  
  33. class WordCountReducer : public HadoopPipes::Reducer {
  34. public:
  35. // constructor: does nothing
  36. WordCountReducer(HadoopPipes::TaskContext& context) {
  37. }
  38.  
  39. // reduce function
  40. void reduce(HadoopPipes::ReduceContext& context) {
  41. int count = 0;
  42.  
  43. //--- get all tuples with the same key, and count their numbers ---
  44. while ( context.nextValue() ) {
  45. count += HadoopUtils::toInt( context.getInputValue() );
  46. }
  47.  
  48. //--- emit (word, count) ---
  49. context.emit(context.getInputKey(), HadoopUtils::toString( count ));
  50. }
  51. };
  52.  
  53. int main(int argc, char *argv[])
  54. {
  55. HadoopPipes::TemplateFactory2<WordCountMapper,
  56. WordCountReducer> factory;
  57.  
  58. return HadoopPipes::runTask(factory);
  59. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement