API tools faq
paste
View difference between Paste ID: uPUAs9T2 and fiKFUTru
SHOW: | | - or go back to the newest paste.
1
grunt> ec = LOAD 'cassandra://abc/ec' USING CassandraStorage();
2
grunt> DESCRIBE ec;
3
ec: {key: chararray,columns: {(name: (),value: long)}}
4
5
grunt> DUMP ec;

6
2013-07-19 12:22:25,951 [main] INFO  org.apache.pig.tools.pigstats.ScriptState - Pig features used in the script: UNKNOWN

7
2013-07-19 12:22:25,954 [main] INFO  org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler - File concatenation threshold: 100 optimistic? false

8
2013-07-19 12:22:25,956 [main] INFO  org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MultiQueryOptimizer - MR plan size before optimization: 1

9
2013-07-19 12:22:25,956 [main] INFO  org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MultiQueryOptimizer - MR plan size after optimization: 1

10
2013-07-19 12:22:25,959 [main] INFO  org.apache.pig.tools.pigstats.ScriptState - Pig script settings are added to the job

11
2013-07-19 12:22:25,959 [main] INFO  org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler - mapred.job.reduce.markreset.buffer.percent is not set, set to default 0.3

12
2013-07-19 12:22:25,970 [main] INFO  org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler - Setting up single store job

13
2013-07-19 12:22:25,976 [main] INFO  org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher - 1 map-reduce job(s) waiting for submission.

14
2013-07-19 12:22:25,982 [Thread-15] WARN  org.apache.hadoop.mapred.JobClient - No job jar file set.  User classes may not be found. See JobConf(Class) or JobConf#setJar(String).

15
2013-07-19 12:22:26,001 [Thread-15] INFO  org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil - Total input paths (combined) to process : 1

16
2013-07-19 12:22:26,057 [Thread-15] WARN  org.apache.hadoop.conf.Configuration - file:/tmp/hadoop-konrad/mapred/local/localRunner/job_local823738536_0003.xml:a attempt to override final parameter: mapred.system.dir;  Ignoring.

17
2013-07-19 12:22:26,057 [Thread-15] WARN  org.apache.hadoop.conf.Configuration - file:/tmp/hadoop-konrad/mapred/local/localRunner/job_local823738536_0003.xml:a attempt to override final parameter: mapred.local.dir;  Ignoring.

18
2013-07-19 12:22:26,057 [Thread-15] WARN  org.apache.hadoop.conf.Configuration - file:/tmp/hadoop-konrad/mapred/local/localRunner/job_local823738536_0003.xml:a attempt to override final parameter: mapred.job.tracker.http.address;  Ignoring.

19
2013-07-19 12:22:26,059 [Thread-15] WARN  org.apache.hadoop.conf.Configuration - file:/tmp/hadoop-konrad/mapred/local/localRunner/job_local823738536_0003.xml:a attempt to override final parameter: fs.default.name;  Ignoring.

20
2013-07-19 12:22:26,059 [Thread-15] WARN  org.apache.hadoop.conf.Configuration - file:/tmp/hadoop-konrad/mapred/local/localRunner/job_local823738536_0003.xml:a attempt to override final parameter: mapred.jobtracker.maxtasks.per.job;  Ignoring.

21
2013-07-19 12:22:26,059 [Thread-15] WARN  org.apache.hadoop.conf.Configuration - file:/tmp/hadoop-konrad/mapred/local/localRunner/job_local823738536_0003.xml:a attempt to override final parameter: mapred.job.tracker;  Ignoring.

22
2013-07-19 12:22:26,061 [Thread-16] INFO  org.apache.hadoop.mapred.LocalJobRunner - Waiting for map tasks

23
2013-07-19 12:22:26,061 [pool-6-thread-1] INFO  org.apache.hadoop.mapred.LocalJobRunner - Starting task: attempt_local823738536_0003_m_000000_0

24
2013-07-19 12:22:26,078 [pool-6-thread-1] INFO  org.apache.hadoop.mapred.Task -  Using ResourceCalculatorPlugin : org.apache.hadoop.util.LinuxResourceCalculatorPlugin@2b52b6f5

25
2013-07-19 12:22:26,079 [pool-6-thread-1] INFO  org.apache.hadoop.mapred.MapTask - Processing split: Number of splits :2

26
Total Length = 0

27
Input split[0]:

28
   Length = 0

29
  Locations:

30
    localhost

31
32
-----------------------

33
Input split[1]:

34
   Length = 0

35
  Locations:

36
    localhost

37
38
-----------------------

39
40
2013-07-19 12:22:26,081 [pool-6-thread-1] INFO  org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigRecordReader - Current split being processed ColumnFamilySplit((-2169904116074813774, '-1] @[localhost])

41
2013-07-19 12:22:26,105 [Thread-16] INFO  org.apache.hadoop.mapred.LocalJobRunner - Map task executor complete.

42
2013-07-19 12:22:26,106 [Thread-16] WARN  org.apache.hadoop.mapred.FileOutputCommitter - Output path is null in cleanup

43
2013-07-19 12:22:26,108 [Thread-16] WARN  org.apache.hadoop.mapred.LocalJobRunner - job_local823738536_0003

44
java.lang.Exception: java.lang.IndexOutOfBoundsException

45
	at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:354)

46
Caused by: java.lang.IndexOutOfBoundsException

47
	at java.nio.Buffer.checkIndex(Buffer.java:518)

48
	at java.nio.HeapByteBuffer.getLong(HeapByteBuffer.java:391)

49
	at org.apache.cassandra.utils.ByteBufferUtil.toLong(ByteBufferUtil.java:421)

50
	at org.apache.cassandra.cql.jdbc.JdbcLong.compose(JdbcLong.java:94)

51
	at org.apache.cassandra.db.marshal.LongType.compose(LongType.java:34)

52
	at org.apache.cassandra.db.marshal.LongType.compose(LongType.java:26)

53
	at org.apache.cassandra.hadoop.pig.AbstractCassandraStorage.columnToTuple(AbstractCassandraStorage.java:129)

54
	at org.apache.cassandra.hadoop.pig.CassandraStorage.getNext(CassandraStorage.java:243)

55
	at org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigRecordReader.nextKeyValue(PigRecordReader.java:194)

56
	at org.apache.hadoop.mapred.MapTask$NewTrackingRecordReader.nextKeyValue(MapTask.java:531)

57
	at org.apache.hadoop.mapreduce.MapContext.nextKeyValue(MapContext.java:67)

58
	at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:144)

59
	at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:764)

60
	at org.apache.hadoop.mapred.MapTask.run(MapTask.java:364)

61
	at org.apache.hadoop.mapred.LocalJobRunner$Job$MapTaskRunnable.run(LocalJobRunner.java:223)

62
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:439)

63
	at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)

64
	at java.util.concurrent.FutureTask.run(FutureTask.java:138)

65
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:895)

66
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:918)

67
	at java.lang.Thread.run(Thread.java:662)

68
2013-07-19 12:22:26,477 [main] INFO  org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher - HadoopJobId: job_local823738536_0003

69
2013-07-19 12:22:26,477 [main] INFO  org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher - 0% complete

70
2013-07-19 12:22:31,485 [main] INFO  org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher - job job_local823738536_0003 has failed! Stop running all dependent jobs

71
2013-07-19 12:22:31,485 [main] INFO  org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher - 100% complete

72
2013-07-19 12:22:31,486 [main] ERROR org.apache.pig.tools.pigstats.PigStatsUtil - 1 map reduce job(s) failed!

73
2013-07-19 12:22:31,486 [main] INFO  org.apache.pig.tools.pigstats.SimplePigStats - Detected Local mode. Stats reported below may be incomplete

74
2013-07-19 12:22:31,486 [main] INFO  org.apache.pig.tools.pigstats.SimplePigStats - Script Statistics: 

75
76
HadoopVersion	PigVersion	UserId	StartedAt	FinishedAt	Features

77
1.2.0	0.10.0	konrad	2013-07-19 12:22:25	2013-07-19 12:22:31	UNKNOWN

78
79
Failed!

80
81
Failed Jobs:

82
JobId	Alias	Feature	Message	Outputs

83
job_local823738536_0003	ec	MAP_ONLY	Message: Job failed! Error - NA	file:/tmp/temp1312341768/tmp-1393597214,

84
85
Input(s):

86-
Failed to read data from "cassandra://woowanalytics/events_logins_occurs"

86+
Failed to read data from "cassandra://abc/ec"

87
88
Output(s):

89
Failed to produce result in "file:/tmp/temp1312341768/tmp-1393597214"

90
91
Job DAG:

92
job_local823738536_0003

93
94
95
2013-07-19 12:22:31,487 [main] INFO  org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher - Failed!

96
2013-07-19 12:22:31,489 [main] ERROR org.apache.pig.tools.grunt.Grunt - ERROR 1066: Unable to open iterator for alias ec

97
Details at logfile: /home/konrad/pig_1374228132069.log

98
grunt> 2013-07-19 12:22:32,066 [communication thread] INFO  org.apache.hadoop.mapred.LocalJobRunner - 

99
100
101
from cqlsh:

102
103-
CREATE TABLE events_logins_occurs (

103+
CREATE TABLE ec (

104
  item_id text PRIMARY KEY,

105
  gr_1 counter,

106
  gr_10 counter,

107
  gr_11 counter,

108
  gr_2 counter,

109
  gr_3 counter,

110
  gr_4 counter,

111
  gr_5 counter,

112
  gr_6 counter,

113
  gr_7 counter,

114
  gr_8 counter,

115
  gr_9 counter

116
) WITH

117
  bloom_filter_fp_chance=0.010000 AND

118
  caching='KEYS_ONLY' AND

119
  comment='' AND

120
  dclocal_read_repair_chance=0.000000 AND

121
  gc_grace_seconds=864000 AND

122
  read_repair_chance=0.100000 AND

123
  replicate_on_write='true' AND

124
  populate_io_cache_on_flush='false' AND

125
  compaction={'class': 'SizeTieredCompactionStrategy'} AND

126
  compression={'sstable_compression': 'SnappyCompressor'};

127
128-
CREATE INDEX events_logins_occurs_gr_10_idx ON events_logins_occurs (gr_10);

128+
CREATE INDEX ec_gr_10_idx ON ec (gr_10);

129
130-
CREATE INDEX events_logins_occurs_gr_11_idx ON events_logins_occurs (gr_11);

130+
CREATE INDEX ec_gr_11_idx ON ec (gr_11);

131
132-
CREATE INDEX events_logins_occurs_gr_2_idx ON events_logins_occurs (gr_2);

132+
CREATE INDEX ec_gr_2_idx ON ec (gr_2);

133
134-
CREATE INDEX events_logins_occurs_gr_3_idx ON events_logins_occurs (gr_3);

134+
CREATE INDEX ec_gr_3_idx ON ec (gr_3);

135
136-
CREATE INDEX events_logins_occurs_gr_4_idx ON events_logins_occurs (gr_4);

136+
CREATE INDEX ec_gr_4_idx ON ec (gr_4);

137
138-
CREATE INDEX events_logins_occurs_gr_5_idx ON events_logins_occurs (gr_5);

138+
CREATE INDEX ec_gr_5_idx ON ec (gr_5);

139
140-
CREATE INDEX events_logins_occurs_gr_6_idx ON events_logins_occurs (gr_6);

140+
CREATE INDEX ec_gr_6_idx ON ec (gr_6);

141
142-
CREATE INDEX events_logins_occurs_gr_7_idx ON events_logins_occurs (gr_7);

142+
CREATE INDEX ec_gr_7_idx ON ec (gr_7);

143
144-
CREATE INDEX events_logins_occurs_gr_8_idx ON events_logins_occurs (gr_8);

144+
CREATE INDEX ec_gr_8_idx ON ec (gr_8);

145
146-
CREATE INDEX events_logins_occurs_gr_9_idx ON events_logins_occurs (gr_9);

146+
CREATE INDEX ec_gr_9_idx ON ec (gr_9);

147
148
149
 item_id | gr_1 | gr_10 | gr_11 | gr_2 | gr_3 | gr_4 | gr_5 | gr_6 | gr_7 | gr_8 | gr_9

150
---------+------+-------+-------+------+------+------+------+------+------+------+------

151
       3 |    2 |  null |  null |    1 |    1 |    1 |    2 |    1 |    1 |    1 | null

152
       2 |    3 |     2 |  null |    1 |    3 |    1 |    1 |    2 |    2 |    3 |    3

153
       1 |    1 |     2 |  null |    2 | null |    2 |    1 |    1 |    1 | null |    1
        

    


            


                                


        

            



                
    

        Public Pastes
    

    
            

    

                    

        

    





    


    
    
    
    
    
    
    
    




    


    



                

            We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.             OK, I Understand
        

    
                

            

                
                    
                
            

            

                Not a member of Pastebin yet?

                Sign Up, it unlocks many cool features!