Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- diff -urB sqoop-1.1.0+8/src/java/com/cloudera/sqoop/hive/HiveImport.java sqoopfixed/src/java/com/cloudera/sqoop/hive/HiveImport.java
- --- sqoop-1.1.0+8/src/java/com/cloudera/sqoop/hive/HiveImport.java 2010-10-10 18:43:50.000000000 -0400
- +++ sqoopfixed/src/java/com/cloudera/sqoop/hive/HiveImport.java 2010-10-21 10:46:12.000000000 -0400
- @@ -99,12 +99,7 @@
- private void removeTempLogs(String tableName) throws IOException {
- FileSystem fs = FileSystem.get(configuration);
- String warehouseDir = options.getWarehouseDir();
- - Path tablePath;
- - if (warehouseDir != null) {
- - tablePath = new Path(new Path(warehouseDir), tableName);
- - } else {
- - tablePath = new Path(tableName);
- - }
- + Path tablePath = new Path(tableName);
- Path logsPath = new Path(tablePath, "_logs");
- if (fs.exists(logsPath)) {
- diff -urB sqoop-1.1.0+8/src/java/com/cloudera/sqoop/hive/TableDefWriter.java sqoopfixed/src/java/com/cloudera/sqoop/hive/TableDefWriter.java
- --- sqoop-1.1.0+8/src/java/com/cloudera/sqoop/hive/TableDefWriter.java 2010-10-10 18:43:50.000000000 -0400
- +++ sqoopfixed/src/java/com/cloudera/sqoop/hive/TableDefWriter.java 2010-10-21 10:36:53.000000000 -0400
- @@ -177,14 +177,7 @@
- * @return the LOAD DATA statement to import the data in HDFS into hive.
- */
- public String getLoadDataStmt() throws IOException {
- - String warehouseDir = options.getWarehouseDir();
- - if (null == warehouseDir) {
- - warehouseDir = "";
- - } else if (!warehouseDir.endsWith(File.separator)) {
- - warehouseDir = warehouseDir + File.separator;
- - }
- -
- - String tablePath = warehouseDir + inputTableName;
- + String tablePath = inputTableName;
- FileSystem fs = FileSystem.get(configuration);
- Path finalPath = new Path(tablePath).makeQualified(fs);
- String finalPathStr = finalPath.toString();
- diff -urB sqoop-1.1.0+8/src/java/com/cloudera/sqoop/util/AppendUtils.java sqoopfixed/src/java/com/cloudera/sqoop/util/AppendUtils.java
- --- sqoop-1.1.0+8/src/java/com/cloudera/sqoop/util/AppendUtils.java 2010-10-10 18:43:50.000000000 -0400
- +++ sqoopfixed/src/java/com/cloudera/sqoop/util/AppendUtils.java 2010-10-21 11:03:04.000000000 -0400
- @@ -68,9 +68,6 @@
- Path userDestDir = null;
- if (options.getTargetDir() != null) {
- userDestDir = new Path(options.getTargetDir());
- - } else if (options.getWarehouseDir() != null) {
- - userDestDir = new Path(options.getWarehouseDir(),
- - context.getTableName());
- } else {
- userDestDir = new Path(context.getTableName());
- }
- @@ -94,7 +91,20 @@
- } else {
- LOG.info("Appending to directory " + userDestDir.getName());
- // Get the right next partition for the imported files
- - nextPartition = getNextPartition(fs, userDestDir);
- +
- + // if the warehousedir is specified and a hive import is going to occur,
- + // then we need the next partition based on the files in the hive DB
- + // folder, not based on the ones in userDestDir (because that's not the
- + // final resting place for the files)
- + Path finalDestDir = userDestDir;
- + String hiveTable = options.getHiveTableName();
- + if (hiveTable == null)
- + hiveTable = context.getTableName();
- + if (options.getWarehouseDir() != null && options.doHiveImport())
- + finalDestDir = new Path(options.getWarehouseDir(), hiveTable);
- +
- + LOG.debug("Checking " + finalDestDir.toString() + " for next partition");
- + nextPartition = getNextPartition(fs, finalDestDir);
- }
- // move files
Add Comment
Please, Sign In to add comment