Guest User

Untitled

a guest
Jun 19th, 2018
85
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 3.44 KB | None | 0 0
  1. diff -urB sqoop-1.1.0+8/src/java/com/cloudera/sqoop/hive/HiveImport.java sqoopfixed/src/java/com/cloudera/sqoop/hive/HiveImport.java
  2. --- sqoop-1.1.0+8/src/java/com/cloudera/sqoop/hive/HiveImport.java 2010-10-10 18:43:50.000000000 -0400
  3. +++ sqoopfixed/src/java/com/cloudera/sqoop/hive/HiveImport.java 2010-10-21 10:46:12.000000000 -0400
  4. @@ -99,12 +99,7 @@
  5. private void removeTempLogs(String tableName) throws IOException {
  6. FileSystem fs = FileSystem.get(configuration);
  7. String warehouseDir = options.getWarehouseDir();
  8. - Path tablePath;
  9. - if (warehouseDir != null) {
  10. - tablePath = new Path(new Path(warehouseDir), tableName);
  11. - } else {
  12. - tablePath = new Path(tableName);
  13. - }
  14. + Path tablePath = new Path(tableName);
  15.  
  16. Path logsPath = new Path(tablePath, "_logs");
  17. if (fs.exists(logsPath)) {
  18. diff -urB sqoop-1.1.0+8/src/java/com/cloudera/sqoop/hive/TableDefWriter.java sqoopfixed/src/java/com/cloudera/sqoop/hive/TableDefWriter.java
  19. --- sqoop-1.1.0+8/src/java/com/cloudera/sqoop/hive/TableDefWriter.java 2010-10-10 18:43:50.000000000 -0400
  20. +++ sqoopfixed/src/java/com/cloudera/sqoop/hive/TableDefWriter.java 2010-10-21 10:36:53.000000000 -0400
  21. @@ -177,14 +177,7 @@
  22. * @return the LOAD DATA statement to import the data in HDFS into hive.
  23. */
  24. public String getLoadDataStmt() throws IOException {
  25. - String warehouseDir = options.getWarehouseDir();
  26. - if (null == warehouseDir) {
  27. - warehouseDir = "";
  28. - } else if (!warehouseDir.endsWith(File.separator)) {
  29. - warehouseDir = warehouseDir + File.separator;
  30. - }
  31. -
  32. - String tablePath = warehouseDir + inputTableName;
  33. + String tablePath = inputTableName;
  34. FileSystem fs = FileSystem.get(configuration);
  35. Path finalPath = new Path(tablePath).makeQualified(fs);
  36. String finalPathStr = finalPath.toString();
  37. diff -urB sqoop-1.1.0+8/src/java/com/cloudera/sqoop/util/AppendUtils.java sqoopfixed/src/java/com/cloudera/sqoop/util/AppendUtils.java
  38. --- sqoop-1.1.0+8/src/java/com/cloudera/sqoop/util/AppendUtils.java 2010-10-10 18:43:50.000000000 -0400
  39. +++ sqoopfixed/src/java/com/cloudera/sqoop/util/AppendUtils.java 2010-10-21 11:03:04.000000000 -0400
  40. @@ -68,9 +68,6 @@
  41. Path userDestDir = null;
  42. if (options.getTargetDir() != null) {
  43. userDestDir = new Path(options.getTargetDir());
  44. - } else if (options.getWarehouseDir() != null) {
  45. - userDestDir = new Path(options.getWarehouseDir(),
  46. - context.getTableName());
  47. } else {
  48. userDestDir = new Path(context.getTableName());
  49. }
  50. @@ -94,7 +91,20 @@
  51. } else {
  52. LOG.info("Appending to directory " + userDestDir.getName());
  53. // Get the right next partition for the imported files
  54. - nextPartition = getNextPartition(fs, userDestDir);
  55. +
  56. + // if the warehousedir is specified and a hive import is going to occur,
  57. + // then we need the next partition based on the files in the hive DB
  58. + // folder, not based on the ones in userDestDir (because that's not the
  59. + // final resting place for the files)
  60. + Path finalDestDir = userDestDir;
  61. + String hiveTable = options.getHiveTableName();
  62. + if (hiveTable == null)
  63. + hiveTable = context.getTableName();
  64. + if (options.getWarehouseDir() != null && options.doHiveImport())
  65. + finalDestDir = new Path(options.getWarehouseDir(), hiveTable);
  66. +
  67. + LOG.debug("Checking " + finalDestDir.toString() + " for next partition");
  68. + nextPartition = getNextPartition(fs, finalDestDir);
  69. }
  70.  
  71. // move files
Add Comment
Please, Sign In to add comment