SHARE
TWEET

Untitled

a guest May 19th, 2019 71 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. package org.datavec.transform.basic;
  2.  
  3. import org.datavec.api.records.reader.RecordReader;
  4. import org.datavec.api.records.reader.impl.csv.CSVRecordReader;
  5. import org.datavec.api.split.FileSplit;
  6. import org.datavec.api.transform.TransformProcess;
  7. import org.datavec.api.transform.schema.Schema;
  8. import org.datavec.api.transform.transform.sequence.SequenceOffsetTransform;
  9. import org.datavec.api.writable.Writable;
  10. import org.datavec.local.transforms.LocalTransformExecutor;
  11. import org.joda.time.DateTimeZone;
  12. import org.nd4j.linalg.io.ClassPathResource;
  13.  
  14. import java.io.File;
  15. import java.util.ArrayList;
  16. import java.util.Arrays;
  17. import java.util.List;
  18.  
  19. public class myExample {
  20.  
  21.     public static void main(String[] args) throws  Exception {
  22.  
  23.         Schema inputDataSchema = new Schema.Builder()
  24.             .addColumnString("DateTimeString")
  25.             .addColumnsString("CustomerID", "MerchantID")
  26.             .addColumnInteger("NumItemsInTransaction")
  27.             .addColumnCategorical("MerchantCountryCode", Arrays.asList("USA","CAN","FR","MX"))
  28.             .addColumnDouble("TransactionAmountUSD",0.0,null,false,false)   //$0.0 or more, no maximum limit, no NaN and no Infinite values
  29.             .addColumnCategorical("FraudLabel", Arrays.asList("Fraud","Legit"))
  30.             .build();
  31.  
  32.         TransformProcess tp = new TransformProcess.Builder(inputDataSchema)
  33.             .removeAllColumnsExceptFor("DateTimeString","TransactionAmountUSD")
  34.             .stringToTimeTransform("DateTimeString","YYYY-MM-DD HH:mm:ss.SSS", DateTimeZone.UTC)
  35.             .offsetSequence(Arrays.asList("TransactionAmountUSD"),1, SequenceOffsetTransform.OperationType.NewColumn)
  36.             .build();
  37.  
  38.         File inputFile = new ClassPathResource("BasicDataVecExample/exampledata.csv").getFile();
  39.  
  40.         //Define input reader and output writer:
  41.         RecordReader rr = new CSVRecordReader(0, ',');
  42.         //RecordReader rr = new CSVVariableSlidingWindowRecordReader(5, 2);
  43.         rr.initialize(new FileSplit(inputFile));
  44.  
  45.         //Process the data:
  46.         List<List<Writable>> originalData = new ArrayList<>();
  47.         while(rr.hasNext()){
  48.             originalData.add(rr.next());
  49.         }
  50.  
  51.         List<List<Writable>> processedData = LocalTransformExecutor.execute(originalData, tp);
  52.  
  53.         int numRows = 5;
  54.  
  55.         System.out.println("=== BEFORE ===");
  56.  
  57.         for (int i=0;i<=numRows;i++) {
  58.             System.out.println(originalData.get(i));
  59.         }
  60.  
  61.         System.out.println("=== AFTER ===");
  62.         for (int i=0;i<=numRows;i++) {
  63.             System.out.println(processedData.get(i));
  64.         }
  65.  
  66.     }
  67.  
  68. }
  69.  
  70. /* ERROR I GET:
  71. Exception in thread "main" java.lang.UnsupportedOperationException: SequenceOffsetTransform cannot be applied to non-sequence data
  72.     at org.datavec.api.transform.transform.sequence.SequenceOffsetTransform.map(SequenceOffsetTransform.java:159)
  73.     at org.datavec.local.transforms.transform.LocalTransformFunction.apply(LocalTransformFunction.java:48)
  74.     at org.datavec.local.transforms.transform.LocalTransformFunction.apply(LocalTransformFunction.java:32)
  75.     at org.datavec.local.transforms.LocalTransformExecutor.lambda$execute$3(LocalTransformExecutor.java:340)
  76.     at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
  77.     at java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1382)
  78.     at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:481)
  79.     at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471)
  80.     at java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:708)
  81.     at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
  82.     at java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:499)
  83.     at org.datavec.local.transforms.LocalTransformExecutor.execute(LocalTransformExecutor.java:340)
  84.     at org.datavec.local.transforms.LocalTransformExecutor.execute(LocalTransformExecutor.java:93)
  85.     at org.datavec.transform.basic.myExample.main(myExample.java:51)
  86. */
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!
 
Top