profile_5nodes

Query (id=754caf76397b19d3:1be6cb02414de49a)
  - AsyncTotalTime: 0
  - TotalTime: 0
  Summary
    Session ID: b345703f00825ab0:60309011dc1cd7a3
    Session Type: BEESWAX
    Start Time: 2014-06-18 10:22:52.733483000
    End Time: 2014-06-18 10:23:00.363519000
    Query Type: DML
    Query State: FINISHED
    Query Status: OK
    Impala Version: impalad version 1.3.1-cdh4 RELEASE (build 907481bf45b248a7bb3bb077d54831a71f484e5f)
    User: hadoop
    Network Address: ::ffff:x.y.z..a:port
    Default Db: test_parquet_part
    Sql Statement: insert INTO transactions PARTITION(year=2000) SELECT id, customer_id, book_id, quantity, transaction_date from test_external_part.transactions WHERE year=2000
    Plan:
----------------
Estimated Per-Host Requirements: Memory=343.32MB VCores=1

F00:PLAN FRAGMENT [PARTITION=RANDOM]
  WRITE TO HDFS [test_parquet_part.transactions, OVERWRITE=false, PARTITION-KEYS=(2000)]
  |  partitions=1
  |  hosts=5 per-host-mem=167.32MB
  |
  00:SCAN HDFS [test_external_part.transactions, PARTITION=RANDOM]
     partitions=1/15 size=851.69MB
     table stats: 263673113 rows total
     column stats: all
     hosts=5 per-host-mem=176.00MB
     tuple-ids=0 row-size=48B cardinality=18275489
----------------
    Estimated Per-Host Mem: 359994032
    Estimated Per-Host VCores: 1
    Request Pool: default-pool
    - AsyncTotalTime: 0
    - TotalTime: 0
  ImpalaServer
    - AsyncTotalTime: 0
    - ClientFetchWaitTimer: 14145809
    - MetastoreUpdateTimer: 2053720636
    - RowMaterializationTimer: 0
    - TotalTime: 0
  Execution Profile 754caf76397b19d3:1be6cb02414de49a
    Insert Stats:
Partition: year=2000/
  BytesWritten: 421.17 MB
  Per Column Sizes:
    book_id: 94.33 MB
    customer_id: 93.25 MB
    id: 79.62 MB
    quantity: 13.12 MB
    transaction_date: 140.84 MB
    Per Node Peak Memory Usage: DataNode1.domain.name:port(175.74 MB) DataNode2.domain.name:port(183.80 MB) DataNode5.domain.name:port(167.97 MB) DataNode3.domain.name:port(100.88 MB) DataNode4.domain.name:port(100.88 MB)
    - AsyncTotalTime: 0
    - FinalizationTimer: 153692181
      - FileDeletionTimer: 58978091
      - FileMoveTimer: 58116648
      - Overwrite/PartitionCreationTimer: 26622048
    - TotalTime: 5483750544
    Averaged Fragment 0
      split sizes:  min: 128.00 MB, max: 256.00 MB, avg: 170.34 MB, stddev: 46.77 MB
      completion times: min:2s313ms  max:5s237ms  mean: 3s900ms  stddev:981.932ms
      execution rates: min:33.59 MB/sec  max:55.32 MB/sec  mean:44.45 MB/sec  stddev:7.79 MB/sec
      num instances: 5
      - AverageThreadTokens: 1.8065079365079364
      - PeakMemoryUsage: 152941712
      - PrepareTime: 50083
      - RowsProduced: 3655097
      - TotalCpuTime: 4983494295
      - TotalNetworkReceiveTime: 0
      - TotalNetworkSendTime: 0
      - TotalStorageWaitTime: 2502582780
      - TotalTime: 3894164210
      CodeGen
        - CodegenTime: 181
        - CompileTime: 44464303
        - LoadTime: 11291071
        - ModuleFileSize: 108040
        - TotalTime: 56296341
      HdfsTableSink
        - BytesWritten: 88325167
        - EncodeTimer: 845860806
        - FinalizePartitionFileTimer: 1530076070
        - HdfsWriteTimer: 1409855885
        - PeakMemoryUsage: 127548150
        - RowsInserted: 3655097
        - TmpFileCreateTimer: 27963774
        - TotalTime: 2405319399
      HDFS_SCAN_NODE (id=0)
        - AverageHdfsReadThreadConcurrency: 1.0
        - AverageScannerThreadConcurrency: 1.4333333333333333
        - BytesRead: 178614480
        - BytesReadDataNodeCache: 0
        - BytesReadLocal: 178613865
        - BytesReadShortCircuit: 178613865
        - NumDisksAccessed: 1
        - NumScannerThreadsStarted: 1
        - PeakMemoryUsage: 30605312
        - PerReadThreadRawHdfsThroughput: 82249536
        - RowsRead: 3655097
        - RowsReturned: 3655097
        - RowsReturnedRate: 2712426
        - ScanRangesComplete: 1
        - ScannerThreadsInvoluntaryContextSwitches: 124
        - ScannerThreadsTotalWallClockTime: 3532027956
          - DelimiterParseTime: 162742257
          - MaterializeTupleTime(*): 800519521
          - ScannerThreadsSysTime: 4799000
          - ScannerThreadsUserTime: 965652400
        - ScannerThreadsVoluntaryContextSwitches: 90
        - TotalRawHdfsReadTime(*): 2258346009
        - TotalReadThroughput: 64831773
        - TotalTime: 1486192361
    Fragment 0
      - AsyncTotalTime: 0
      - TotalTime: 0
      Instance 754caf76397b19d3:1be6cb02414de49c (host=DataNode2.domain.name:port)
        Hdfs split stats (<volume id>:<# splits>/<split lengths>): 0:2/256.00 MB
        - AsyncTotalTime: 0
        - AverageThreadTokens: 2.2
        - PeakMemoryUsage: 192730256
        - PrepareTime: 46235
        - RowsProduced: 5498758
        - TotalCpuTime: 6813475047
        - TotalNetworkReceiveTime: 0
        - TotalNetworkSendTime: 0
        - TotalStorageWaitTime: 4599419050
        - TotalTime: 5220078172
        CodeGen
          - AsyncTotalTime: 0
          - CodegenTime: 159
          - CompileTime: 43891505
          - LoadTime: 7183744
          - ModuleFileSize: 108040
          - TotalTime: 51524321
        HdfsTableSink
          - AsyncTotalTime: 0
          - BytesWritten: 132607405
          - EncodeTimer: 1261734977
          - FinalizePartitionFileTimer: 2110027568
          - HdfsWriteTimer: 2024262885
          - PeakMemoryUsage: 158946448
          - RowsInserted: 5498758
          - TmpFileCreateTimer: 34911840
          - TotalTime: 3408901942
        HDFS_SCAN_NODE (id=0)
          Hdfs split stats (<volume id>:<# splits>/<split lengths>): 0:2/256.00 MB
          Hdfs Read Thread Concurrency Bucket: 0:0% 1:100% 2:0%
          File Formats: TEXT/NONE:2
          ExecOption: Codegen enabled: 0 out of 2
          - AsyncTotalTime: 0
          - AverageHdfsReadThreadConcurrency: 1.0
          - AverageScannerThreadConcurrency: 2.0
          - BytesRead: 268437504
          - BytesReadDataNodeCache: 0
          - BytesReadLocal: 268436480
          - BytesReadShortCircuit: 268436480
          - NumDisksAccessed: 1
          - NumScannerThreadsStarted: 2
          - PeakMemoryUsage: 33775616
          - PerReadThreadRawHdfsThroughput: 88101698
          - RowsRead: 5498758
          - RowsReturned: 5498758
          - RowsReturnedRate: 3042828
          - ScanRangesComplete: 2
          - ScannerThreadsInvoluntaryContextSwitches: 183
          - ScannerThreadsTotalWallClockTime: 6136132122
            - DelimiterParseTime: 248314944
            - MaterializeTupleTime(*): 1207349698
            - ScannerThreadsSysTime: 19996000
            - ScannerThreadsUserTime: 1446779000
          - ScannerThreadsVoluntaryContextSwitches: 102
          - TotalRawHdfsReadTime(*): 3046904990
          - TotalReadThroughput: 78293674
          - TotalTime: 1807120313
      Instance 754caf76397b19d3:1be6cb02414de49b (host=DataNode1.domain.name:port)
        Hdfs split stats (<volume id>:<# splits>/<split lengths>): 0:2/172.23 MB
        - AsyncTotalTime: 0
        - AverageThreadTokens: 2.111111111111111
        - PeakMemoryUsage: 184280208
        - PrepareTime: 43879
        - RowsProduced: 3690077
        - TotalCpuTime: 5659865638
        - TotalNetworkReceiveTime: 0
        - TotalNetworkSendTime: 0
        - TotalStorageWaitTime: 3633646344
        - TotalTime: 4541521457
        CodeGen
          - AsyncTotalTime: 0
          - CodegenTime: 154
          - CompileTime: 43973489
          - LoadTime: 26040206
          - ModuleFileSize: 108040
          - TotalTime: 70467145
        HdfsTableSink
          - AsyncTotalTime: 0
          - BytesWritten: 89164406
          - EncodeTimer: 861715092
          - FinalizePartitionFileTimer: 1629742080
          - HdfsWriteTimer: 1504999606
          - PeakMemoryUsage: 158946448
          - RowsInserted: 3690077
          - TmpFileCreateTimer: 17410834
          - TotalTime: 2510351544
        HDFS_SCAN_NODE (id=0)
          Hdfs split stats (<volume id>:<# splits>/<split lengths>): 0:2/172.23 MB
          Hdfs Read Thread Concurrency Bucket: 0:0% 1:100% 2:0%
          File Formats: TEXT/NONE:2
          ExecOption: Codegen enabled: 0 out of 2
          - AsyncTotalTime: 0
          - AverageHdfsReadThreadConcurrency: 1.0
          - AverageScannerThreadConcurrency: 1.6666666666666667
          - BytesRead: 180598464
          - BytesReadDataNodeCache: 0
          - BytesReadLocal: 180598464
          - BytesReadShortCircuit: 180598464
          - NumDisksAccessed: 1
          - NumScannerThreadsStarted: 2
          - PeakMemoryUsage: 33775616
          - PerReadThreadRawHdfsThroughput: 63868992
          - RowsRead: 3690077
          - RowsReturned: 3690077
          - RowsReturnedRate: 1819183
          - ScanRangesComplete: 2
          - ScannerThreadsInvoluntaryContextSwitches: 142
          - ScannerThreadsTotalWallClockTime: 4677818661
            - DelimiterParseTime: 167520671
            - MaterializeTupleTime(*): 821017082
            - ScannerThreadsSysTime: 3999000
            - ScannerThreadsUserTime: 991848000
          - ScannerThreadsVoluntaryContextSwitches: 92
          - TotalRawHdfsReadTime(*): 2827639154
          - TotalReadThroughput: 57402944
          - TotalTime: 2028424558
      Instance 754caf76397b19d3:1be6cb02414de49e (host=DataNode3.domain.name:port)
        Hdfs split stats (<volume id>:<# splits>/<split lengths>): 0:1/128.00 MB
        - AsyncTotalTime: 0
        - AverageThreadTokens: 1.5714285714285714
        - PeakMemoryUsage: 105784464
        - PrepareTime: 41560
        - RowsProduced: 2749191
        - TotalCpuTime: 4639203067
        - TotalNetworkReceiveTime: 0
        - TotalNetworkSendTime: 0
        - TotalStorageWaitTime: 1658910414
        - TotalTime: 3806581977
        CodeGen
          - AsyncTotalTime: 0
          - CodegenTime: 154
          - CompileTime: 43651796
          - LoadTime: 7805914
          - ModuleFileSize: 108040
          - TotalTime: 51982862
        HdfsTableSink
          - AsyncTotalTime: 0
          - BytesWritten: 66567863
          - EncodeTimer: 658828231
          - FinalizePartitionFileTimer: 1371725596
          - HdfsWriteTimer: 1264036070
          - PeakMemoryUsage: 80450704
          - RowsInserted: 2749191
          - TmpFileCreateTimer: 18521554
          - TotalTime: 2050169869
        HDFS_SCAN_NODE (id=0)
          Hdfs split stats (<volume id>:<# splits>/<split lengths>): 0:1/128.00 MB
          Hdfs Read Thread Concurrency Bucket: 0:0% 1:100% 2:0%
          File Formats: TEXT/NONE:1
          ExecOption: Codegen enabled: 0 out of 1
          - AsyncTotalTime: 0
          - AverageHdfsReadThreadConcurrency: 1.0
          - AverageScannerThreadConcurrency: 1.0
          - BytesRead: 134218752
          - BytesReadDataNodeCache: 0
          - BytesReadLocal: 134217728
          - BytesReadShortCircuit: 134217728
          - NumDisksAccessed: 1
          - NumScannerThreadsStarted: 1
          - PeakMemoryUsage: 25325568
          - PerReadThreadRawHdfsThroughput: 58113575
          - RowsRead: 2749191
          - RowsReturned: 2749191
          - RowsReturnedRate: 1566945
          - ScanRangesComplete: 1
          - ScannerThreadsInvoluntaryContextSwitches: 94
          - ScannerThreadsTotalWallClockTime: 2436686085
            - DelimiterParseTime: 119908593
            - MaterializeTupleTime(*): 605411989
            - ScannerThreadsSysTime: 0
            - ScannerThreadsUserTime: 729889000
          - ScannerThreadsVoluntaryContextSwitches: 84
          - TotalRawHdfsReadTime(*): 2309593755
          - TotalReadThroughput: 54525952
          - TotalTime: 1754490726
      Instance 754caf76397b19d3:1be6cb02414de49d (host=DataNode5.domain.name:port)
        Hdfs split stats (<volume id>:<# splits>/<split lengths>): 0:2/167.46 MB
        - AsyncTotalTime: 0
        - AverageThreadTokens: 1.75
        - PeakMemoryUsage: 176129168
        - PrepareTime: 45069
        - RowsProduced: 3588681
        - TotalCpuTime: 4648706631
        - TotalNetworkReceiveTime: 0
        - TotalNetworkSendTime: 0
        - TotalStorageWaitTime: 2065003295
        - TotalTime: 3591876918
        CodeGen
          - AsyncTotalTime: 0
          - CodegenTime: 138
          - CompileTime: 44129340
          - LoadTime: 7507659
          - ModuleFileSize: 108040
          - TotalTime: 52069184
        HdfsTableSink
          - AsyncTotalTime: 0
          - BytesWritten: 86731573
          - EncodeTimer: 820217644
          - FinalizePartitionFileTimer: 1569556560
          - HdfsWriteTimer: 1396340681
          - PeakMemoryUsage: 158946448
          - RowsInserted: 3588681
          - TmpFileCreateTimer: 37370751
          - TotalTime: 2428401127
        HDFS_SCAN_NODE (id=0)
          Hdfs split stats (<volume id>:<# splits>/<split lengths>): 0:2/167.46 MB
          Hdfs Read Thread Concurrency Bucket: 0:0% 1:100% 2:0%
          File Formats: TEXT/NONE:2
          ExecOption: Codegen enabled: 0 out of 2
          - AsyncTotalTime: 0
          - AverageHdfsReadThreadConcurrency: 1.0
          - AverageScannerThreadConcurrency: 1.5
          - BytesRead: 175598929
          - BytesReadDataNodeCache: 0
          - BytesReadLocal: 175598929
          - BytesReadShortCircuit: 175598929
          - NumDisksAccessed: 1
          - NumScannerThreadsStarted: 2
          - PeakMemoryUsage: 33775616
          - PerReadThreadRawHdfsThroughput: 95693578
          - RowsRead: 3588681
          - RowsReturned: 3588681
          - RowsReturnedRate: 3091524
          - ScanRangesComplete: 2
          - ScannerThreadsInvoluntaryContextSwitches: 111
          - ScannerThreadsTotalWallClockTime: 3066334040
            - DelimiterParseTime: 159015457
            - MaterializeTupleTime(*): 769671311
            - ScannerThreadsSysTime: 0
            - ScannerThreadsUserTime: 935857000
          - ScannerThreadsVoluntaryContextSwitches: 91
          - TotalRawHdfsReadTime(*): 1835012674
          - TotalReadThroughput: 66827432
          - TotalTime: 1160812557
      Instance 754caf76397b19d3:1be6cb02414de49f (host=DataNode4.domain.name:port)
        Hdfs split stats (<volume id>:<# splits>/<split lengths>): 0:1/128.00 MB
        - AsyncTotalTime: 0
        - AverageThreadTokens: 1.4
        - PeakMemoryUsage: 105784464
        - PrepareTime: 73674
        - RowsProduced: 2748782
        - TotalCpuTime: 3156221095
        - TotalNetworkReceiveTime: 0
        - TotalNetworkSendTime: 0
        - TotalStorageWaitTime: 555934798
        - TotalTime: 2310762526
        CodeGen
          - AsyncTotalTime: 0
          - CodegenTime: 300
          - CompileTime: 46675385
          - LoadTime: 7917832
          - ModuleFileSize: 108040
          - TotalTime: 55438197
        HdfsTableSink
          - AsyncTotalTime: 0
          - BytesWritten: 66554592
          - EncodeTimer: 626808088
          - FinalizePartitionFileTimer: 969328547
          - HdfsWriteTimer: 859640183
          - PeakMemoryUsage: 80450704
          - RowsInserted: 2748782
          - TmpFileCreateTimer: 31603894
          - TotalTime: 1628772516
        HDFS_SCAN_NODE (id=0)
          Hdfs split stats (<volume id>:<# splits>/<split lengths>): 0:1/128.00 MB
          Hdfs Read Thread Concurrency Bucket: 0:0% 1:100% 2:0%
          File Formats: TEXT/NONE:1
          ExecOption: Codegen enabled: 0 out of 1
          - AsyncTotalTime: 0
          - AverageHdfsReadThreadConcurrency: 1.0
          - AverageScannerThreadConcurrency: 1.0
          - BytesRead: 134218752
          - BytesReadDataNodeCache: 0
          - BytesReadLocal: 134217728
          - BytesReadShortCircuit: 134217728
          - NumDisksAccessed: 1
          - NumScannerThreadsStarted: 1
          - PeakMemoryUsage: 26374144
          - PerReadThreadRawHdfsThroughput: 105469838
          - RowsRead: 2748782
          - RowsReturned: 2748782
          - RowsReturnedRate: 4041650
          - ScanRangesComplete: 1
          - ScannerThreadsInvoluntaryContextSwitches: 92
          - ScannerThreadsTotalWallClockTime: 1343168874
            - DelimiterParseTime: 118951624
            - MaterializeTupleTime(*): 599147529
            - ScannerThreadsSysTime: 0
            - ScannerThreadsUserTime: 723889000
          - ScannerThreadsVoluntaryContextSwitches: 81
          - TotalRawHdfsReadTime(*): 1272579474
          - TotalReadThroughput: 67108864
          - TotalTime: 680113652