require(reticulate)
source_python("clean_wf.py")
drv <- dbDriver("PostgreSQL")
con <- dbConnect(drv, dbname = "xx",
                 host = "xxx", port = 5432,
                 user = "xx", password = 'xxxx')
# fn = "~/Dropbox/Notebooks/tmp/wf_abba_global2018012.v65.txt"
# fn = "/Users/jxiong1/Dropbox/Notebooks/tmp/wf_abba_global2018064.v65.txt"
wf_reader = function(fn) {
df = read.csv(text = clean_wfdata(fn), header=F, sep=',')
names(df) = unlist(strsplit('lon,lat,temp4,temp11,size,temp,ecosys,flag,sat,yd,time', ','))
df$datetime = strptime(sprintf("%s-%04d", df$yd, df$time), format = "%Y%j-%H%M")
drops <- c("yd","time")
df[ , !(names(df) %in% drops)]
print(c(basename(fn), dbWriteTable(con, 'wf_abba', head(df), row.names=FALSE, append=TRUE)))
}

flist = list.files(path = "/Users/jxiong1/Dropbox/Notebooks/tmp", pattern = 'txt', full.names=T)
lapply(flist, wf_reader)

dbDisconnect(con)