Advertisement
Guest User

Binary_Attachment_BulkAPI

a guest
May 9th, 2014
330
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Java 7.62 KB | None | 0 0
  1.     /**
  2.      * Create and BulkLoad batches using a CSV file. The file into the appropriate
  3.      * size batch files.
  4.      */
  5.     private static List<BatchInfo> createBatchesFromCSVFile(BulkConnection connection,
  6.             JobInfo jobInfo, String csvFileName
  7.     ) throws IOException,
  8.     AsyncApiException {
  9.         List<BatchInfo> batchInfos = new ArrayList<BatchInfo>();
  10.         BufferedReader rdr = new BufferedReader(new InputStreamReader(
  11.                 new FileInputStream(csvFileName)));
  12.         // read the CSV header row
  13.         String hdr = rdr.readLine();
  14.         byte[] headerBytes = (hdr + "\n").getBytes("US-ASCII");
  15.         int headerBytesLength = headerBytes.length;
  16.         File tmpFile = File.createTempFile("bulkAPIInsert", ".csv");
  17.  
  18.         // Split the CSV file into multiple batches
  19.         try {
  20.             FileOutputStream tmpOut = new FileOutputStream(tmpFile);
  21.             int maxBytesPerBatch = 10000000; // 10 million bytes per batch
  22.             int maxRowsPerBatch = 10000; // 10 thousand rows per batch 90001
  23.             int currentBytes = 0;
  24.             int currentLines = 0;
  25.             String nextLine;
  26.  
  27.             while ((nextLine = rdr.readLine()) != null) {
  28.                 byte[] bytes = (nextLine + "\n").getBytes("US-ASCII"); //TODO
  29.                 if (currentBytes + bytes.length > maxBytesPerBatch
  30.                         || currentLines > maxRowsPerBatch) {
  31.                     createBatch(new FileOutputStream(tmpFile), tmpFile, batchInfos, connection, jobInfo);
  32.                     currentBytes = 0;
  33.                     currentLines = 0;
  34.                 }
  35.                 if (currentBytes == 0) { // This is read first time or everytime i think
  36.                     tmpOut = new FileOutputStream(tmpFile);
  37.                     tmpOut.write(headerBytes);
  38.                     currentBytes = headerBytesLength;
  39.                     currentLines = 1;
  40.                 }
  41.                 tmpOut.write(bytes);
  42.                 currentBytes += bytes.length;
  43.                 currentLines++;
  44.             }
  45.  
  46.             if (currentLines > 1) {
  47.                 createBatch(new FileOutputStream(tmpFile), tmpFile, batchInfos, connection, jobInfo);
  48.             }
  49.         } finally {
  50.             if(!tmpFile.delete())
  51.                 tmpFile.deleteOnExit();
  52.             rdr.close();
  53.         }
  54.         //System.out.println("BatchInfos created is: "+batchInfos);
  55.         return batchInfos;
  56.     } // END private List<BatchInfo> createBatchesFromCSVFile()
  57.  
  58.     /**
  59.      * Create a batch by BulkLoading the contents of the file. This closes the
  60.      * output stream.
  61.      */
  62.     private static void createBatch(FileOutputStream tmpOut, File tmpFile,
  63.             List<BatchInfo> batchInfos, BulkConnection connection, JobInfo jobInfo)
  64.     throws IOException, AsyncApiException {
  65.         tmpOut.flush();
  66.         tmpOut.close();
  67.         FileInputStream tmpInputStream = new FileInputStream(tmpFile);
  68.         try {
  69.             Map<String, InputStream> attachments = new HashMap<String, InputStream>();
  70.             attachments.put("MyFile1.txt", new FileInputStream("C:\\Users\\user\\Desktop\\upload\\2014-04-18_231314.png"));
  71.            
  72.             BatchInfo batchInfo = connection.createBatchWithInputStreamAttachments(jobInfo, tmpInputStream, attachments);
  73.                     //connection.createBatchFromStream(jobInfo, tmpInputStream);
  74.             System.out.println(batchInfo!=null ? "BatchInfo is not null :)" : " BatchInfo is Null :(");
  75.            
  76.             batchInfos.add(batchInfo);
  77.             //          System.out.println("batchInfo: "+batchInfo);
  78.         } finally {
  79.             tmpInputStream.close();
  80.         }
  81.     } // END private void createBatch()
  82.  
  83.  
  84.     private static JobInfo createJob(String fileName, BulkConnection bc)
  85.             throws AsyncApiException {
  86.         JobInfo job = new JobInfo();
  87.         // operation = OperationEnum.Insert
  88.         job.setOperation(OperationEnum.insert);
  89.         // object = Attachment
  90.         job.setObject("Attachment");
  91.         // contentType = ZIP_CSV
  92.         job.setContentType(ContentType.ZIP_CSV);
  93.         job = bc.createJob(job);
  94.         return job;
  95.     } // END public static void createJob(String fileName, BulkConnection bc)
  96.  
  97.     /**
  98.      * Wait for a job to complete by polling the Bulk API.
  99.      */
  100.     private static void awaitCompletion(BulkConnection connection, JobInfo job,
  101.             List<BatchInfo> batchInfoList) throws AsyncApiException {
  102.         long sleepTime = 0L;
  103.         Set<String> incomplete = new HashSet<String>();
  104.         for (BatchInfo bi : batchInfoList) {
  105.             incomplete.add(bi.getId());
  106.         }
  107.         while (!incomplete.isEmpty()) {
  108.             try {
  109.                 Thread.sleep(sleepTime);
  110.             } catch (InterruptedException e) {}
  111.             sleepTime = 10000L;
  112.             BatchInfo[] statusList = connection.getBatchInfoList(job.getId())
  113.             .getBatchInfo();
  114.             for (BatchInfo b : statusList) {
  115.                 if (b.getState() == BatchStateEnum.Completed) {
  116.                     if (incomplete.remove(b.getId())) {}
  117.                 }
  118.                 else if(b.getState() == BatchStateEnum.Failed){ //TODO TODO TODO its to be notified every time
  119.                     System.out.println("\n  Reason of Failure: "+b.getStateMessage()+".\n  " +
  120.                             "Number of Records Processed: "+b.getNumberRecordsProcessed());
  121.                 }
  122.             } // END for (BatchInfo b : statusList)
  123.         } // END while (!incomplete.isEmpty())
  124.        
  125.     } // END private static void awaitCompletion()
  126.    
  127.     private static void checkResults(BulkConnection connection, JobInfo job,
  128.             List<BatchInfo> batchInfoList, String fileName) throws AsyncApiException, IOException {
  129.         for (BatchInfo b : batchInfoList) {
  130.             CSVReader rdr = new CSVReader(
  131.                     new InputStreamReader(connection.getBatchResultStream(
  132.                             job.getId(), b.getId())));
  133.             List<String> resultHeader = new LinkedList<String>(
  134.                     Arrays.asList(rdr.readNext()));
  135.             int resultCols = resultHeader.size();
  136.             List<String> row;
  137.             String[] rows;
  138.             List<String> prevRow;
  139.  
  140.             while ((rows = rdr.readNext()) != null) {
  141.                 row = new LinkedList<String>(Arrays.asList(rows));
  142.                 Map<String, String> resultInfo = new HashMap<String, String>();
  143.                 for (int i = 0; i < resultCols; i++) {
  144.                     resultInfo.put(resultHeader.get(i), row.get(i));
  145.                 }
  146.  
  147.                 boolean success = Boolean
  148.                         .valueOf(resultInfo.get("Success"));
  149.                 boolean created = Boolean
  150.                         .valueOf(resultInfo.get("Created"));
  151.                 String id = resultInfo.get("Id");
  152.                 String error = resultInfo.get("Error");
  153.                 if (success && created) {
  154.                     System.out.println("The record id: "+id+" is successfully created.");
  155.                 } else if (!success) {
  156.                     System.out.println("Error Occurred. Error: "+error);
  157.                 }
  158.                 if (success && !created) {
  159.                     System.out.println("The record is updated with id: "+id);
  160.                 }
  161.                 row.clear();
  162.                 resultInfo.clear();
  163.             } // END while
  164.             rdr.close();
  165.             resultHeader.clear();
  166.  
  167.         } // END for
  168.  
  169.     } // END private static void checkResults()
  170.  
  171.     public static void uploadBinaryAttachmentUsingBulkAPI(BulkConnection bConnection)
  172.             throws AsyncApiException, IOException {
  173.         // Create a Job for Batches with Binary Attachments
  174.         String fileName = "C:\\Users\\user\\Desktop\\upload\\request1.CSV";
  175.         // Call the createJob()
  176.         JobInfo job = createJob(fileName, bConnection);
  177.         // Create a Batch with Binary Attachments
  178.         if (job != null) {
  179.             System.out.println("Job is created.");
  180.             List<BatchInfo> batchInfoList = new ArrayList<BatchInfo>();
  181.             Map<String, InputStream> attachments = new HashMap<String, InputStream>();
  182.             attachments.put("MyFile1.txt", new FileInputStream("C:\\Users\\user\\Desktop\\upload\\2014-04-18_231314.png"));
  183.             try {
  184.                 batchInfoList = createBatchesFromCSVFile(bConnection, job,
  185.                         fileName);
  186.                 System.out.println(batchInfoList.size()>0 ? "batchInfoList have some elements :) " : "No elements there in batchInfoList :<");
  187.             } catch (AsyncApiException ex){}
  188.             if (batchInfoList.size() > 0) {
  189.                 System.out.println("closing the Job.");
  190.                 closeJob(bConnection, job.getId());
  191.                 System.out.println("waiting for completion of Job ...");
  192.                 awaitCompletion(bConnection, job, batchInfoList);
  193.                 System.out.println("Checking for the results ...");
  194.                 checkResults(bConnection, job, batchInfoList, fileName);
  195.             }
  196.         } // END if (job != null)
  197.         // You send data in batches in separate HTTP POST requests.
  198.         // Make the upload using BulkAPI
  199.     } // END public static void createRequestTextFile(BulkConnection bConnection)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement