learning_stage

import csv
import datetime
import numpy
import os
import yaml
import math
import numpy as np
from itertools import islice
from nupic.algorithms.sdr_classifier_factory import SDRClassifierFactory
from nupic.algorithms.spatial_pooler import SpatialPooler
from nupic.algorithms.temporal_memory import TemporalMemory

from nupic.encoders.date import DateEncoder
from nupic.encoders.random_distributed_scalar import \
  RandomDistributedScalarEncoder
from nupic.encoders.multi import  MultiEncoder
from  nupic.encoders.scalar import ScalarEncoder
from nupic.algorithms.sdr_classifier import  SDRClassifier
from time import  time
import json
import urllib2
import capnp
from nupic.proto.SpatialPoolerProto_capnp import  SpatialPoolerProto
from nupic.proto.TemporalMemoryProto_capnp import TemporalMemoryProto
from nupic.algorithms.spatial_pooler import SpatialPooler

from pytz import timezone

_INPUT_FILE_PATH =  ("test1.csv")
PARAMS_PATH =  ("model_iot.yaml")
result_testing =  [0]
result_testing1 = [0]
result_testing2 = [0]
result_testing3 = [0]
result_testing4 = [0]
result_testing5 = [0]
result_testing7 = [0]

sum_event  = []
sum_event2 = []
sum_event3 = []
sum_event4 = []
sum_event5 = []
sum_event6 = []
sum_event7 = []
res2 = []
evnt2 = []
res5 = []
res7 = []
evnt7 = []
evnt5 =[]
res3 = []
evnt3  = []
flow_x = []
bezline = [] # row 10
encoding_csv = [] # row 9
sum_flow = []
# parameter file
PARAMS_PATH =  ("model_iot.yaml")
#base dataset
INPUT = ("real_dataset.csv")
#Work dataset
PREPARE = ("prepare_model.csv")
TEST =("test.csv")
pressure = []
flow_rate =[]
event = []
# Massive metrics
metric_massive = []
#Massive with results of learning
result_learining = [0]
#Massive with results of testing

flow = []
pres = []
ev = []
sred = []
p=[]
tr =[]
l =[]
pl=[]
dr=[]
fl = []
fl_r = []
tnt =[]
v1 = []
v =[]
switch_event = [] # switch event
encod_value = 0
val = 0
ev1 = 0
ev2 = 0
ev3 = 0
ev4 = 0
ev5 = 0
ev6 = 0
ev7 = 0

event1 = []
event2 = []
event3 = []
event4 = []
event5 = []
event6 = []
event7 = []
event10 = []

event11 = []
event12 = []
event13 = []
event14 = []
event15 = []
event16 = []
event17 = []
event18 = []

even = 0
pr = []
dr = []
dx = 0
kr = 0
event18 = []

with open("real_dataset3.csv") as fin:

    reader = csv.reader(fin, delimiter=',')
    headers = reader.next()
    reader.next()
    reader.next()
    for row in reader:
         pressure.append(float(row[1]))
         flow_rate.append(float(row[2]))
         event.append(row[3])
    for tt in range(0,29900):
        event10.append(event[tt])


for cur in xrange(len(event10)): # cur - current index in massive event
     kr = map(int,event[cur].split(",")) #  Parse string

     for tir in xrange(len(kr)):

        if kr[tir] == 0:
            ev1 = 0
            ev2 = 0
            ev3 = 0
            ev4 = 0
            ev5 = 0
            ev6 = 0
            ev7 = 0
        if kr[tir] == 5:
            ev1 = 0
            ev2 = 0
            ev3 = 0
            ev4 = 0
            ev5 = 1
            ev6 = 0
            ev7 = 0
        if kr[tir] == 7:
            ev1 = 0
            ev2 = 0
            ev3 = 0
            ev4 = 0
            ev5 = 0
            ev6 = 0
            ev7 = 1
        if kr[tir] == 2 :
            ev1 = 0
            ev2 = 1
            ev3 = 0
            ev4 = 0
            ev5 = 0
            ev6 = 0
            ev7 = 0
        if kr[tir] == 3:
            ev1 = 0
            ev2 = 0
            ev3 = 1
            ev4 = 0
            ev5 = 0
            ev6 = 0
            ev7 = 0
        if kr[tir] == 1:
            ev1 = 1
            ev2 = 0
            ev3 = 0
            ev4 = 0
            ev5 = 0
            ev6 = 0
            ev7 = 0
        if kr[tir] == 4:
            ev1 = 0
            ev2 = 0
            ev3 = 0
            ev4 = 1
            ev5 = 0
            ev6 = 0
            ev7 = 0
        if kr[tir] == 6:
            ev1 = 0
            ev2 = 0
            ev3 = 0
            ev4 = 0
            ev5 = 0
            ev6 = 1
            ev7 = 0
        event1.append(ev1)

        event2.append(ev2)

        event3.append(ev3)

        event4.append(ev4)

        event5.append(ev5)

        event6.append(ev6)

        event7.append(ev7)

        if tir == 0:
         event11.append(ev1)
         event12.append(ev2)
         event13.append(ev3)
         event14.append(ev4)
         event15.append(ev5)
         event16.append(ev6)
         event17.append(ev7)

        if len(kr)>1 and tir != 0:
                event11[cur] += event1[tir]
                event12[cur] += event2[tir]
                event13[cur] += event3[tir]
                event14[cur] += event4[tir]
                event15[cur] += event5[tir]
                event16[cur] += event6[tir]
                event17[cur] += event7[tir]

     event1 =[]
     event2 =[]
     event3 =[]
     event4 =[]
     event5 =[]
     event6 =[]
     event7 =[]


with open("real_dataset3.csv") as tes1:
  # global max_square_10
  reader = csv.reader(tes1, delimiter=',')
  headers = reader.next()
  reader.next()
  reader.next()
  encoding = []
  enc = []
vr = []
pr = []
dr =[]
bezline = []
sum_el = 0

val_massive =[]
val_massive_permanent = []
def diff(left_r):
    global val_massive,sum_sred,index
    val_massive.append(pressure[left_r])
    if(len(val_massive)>1):

        #val_massive.append(srw[xs+1])
        for i in range(2,len(pressure)):
            sum_sred = math.ceil(sum(val_massive)/len(val_massive))
            enc_value = (sum_sred - val_massive[-1])
            sq = math.sqrt(enc_value**2)

            if ((flow_rate[left_r + i-1] - flow_rate[left_r + i - 2])<0.2 and (flow_rate[left_r + i-2] - flow_rate[left_r + i - 1])<0.2):

                    print "Flow_rate-1", flow_rate[left_r + i - 1] - flow_rate[left_r + i - 2]
            else:
                            del val_massive[-1]
                            # print "Lenny:",len(val_massive)
                            try:
                                sum_sred_1 = sum(val_massive) / len(val_massive)
                            except ZeroDivisionError:
                                pass
                            for vts in range(0, len(val_massive)):
                                positive_value = sum_sred_1 - val_massive[vts]

                                encoding.append(round(positive_value, 2))
                                bezline.append(round(sum_sred_1, 2))
                            break

    else:
        val_massive.append(pressure[left_r+1])

        for i in xrange(2,len(pressure)):
            sum_sred = math.ceil(sum(val_massive) / len(val_massive))
            enc_value = sum_sred - val_massive[-1]

            sq = math.sqrt(enc_value**2)

            if ((flow_rate[left_r + i - 1] - flow_rate[left_r + i - 2]) < 0.2 and (flow_rate[left_r + i - 2] - flow_rate[left_r + i - 1]) < 0.2):

                    val_massive.append(pressure[left_r+i])

            else:
                            del val_massive[-1]
                            # print "Lenny:",len(val_massive)
                            try:
                                sum_sred_1 = sum(val_massive) / len(val_massive)
                            except ZeroDivisionError:
                                pass

                            for vts in range(0, len(val_massive)):
                                positive_value = sum_sred_1 - val_massive[vts]
                                encoding.append(round(positive_value, 2))
                                bezline.append(round(sum_sred_1, 2))
                            break
    index = i +left_r

    val_massive = []

ty = 1
diff(ty-1)

while (index <29900):
     diff(index-1)
vrs = []

print "Min flow:",min(flow_rate)
print "Max flow:",max(flow_rate)
print "Baseline",len(bezline)
print "Bezline min:",min(bezline)
print "Bezline max:" ,max(bezline)

headers = ("flow","event","event1","event2","event3","event4","event5","event6","event7","encoding","bezline","pressure")
type = ("float","int","int","int","int","int","int","int","int","float","float","float")
meta = (" "," "," "," "," ")
with open('test3.csv', 'w') as csv_file:
   csv_writer = csv.writer(csv_file)
   csv_writer.writerow(headers)
   csv_writer.writerow(type)
   csv_writer.writerow(meta)
   for pot in range(0, 29899):
       csv_writer.writerow([flow_rate[pot],event[pot],event11[pot],event12[pot],event13[pot],event14[pot],event15[pot],event16[pot],event17[pot],encoding[pot],bezline[pot],pressure[pot]])

with open("test3.csv", "r") as fin:
    reader = csv.reader(fin)
    headers = reader.next()
    reader.next()
    reader.next()
    for row in reader:

         sum_event.append(int(row[2]))
         sum_event2.append(int(row[3]))
         sum_event3.append(int(row[4]))
         sum_event4.append(int(row[5]))
         sum_event5.append(int(row[6]))
         sum_event6.append(int(row[7]))
         sum_event7.append(int(row[8]))
         encoding_csv.append(float(row[9]))
         bezline.append(float(row[10]))

         sum_flow.append(float(row[0]))


with open(PARAMS_PATH, "r") as f:
    modelParams = yaml.safe_load(f)["modelParams"]
    # enParams = modelParams["sensorParams"]["encoders"]
    spParams = modelParams["spParams"]
    tmParams = modelParams["tmParams"]
    clParams = modelParams["clParams"]
eventEncoder = ScalarEncoder(name="event", w=7, n=14, minval=0, maxval=1,forced=True)
eventEncoder1 = ScalarEncoder(name="event1", w=7, n=14, minval=0, maxval=1,forced=True)
eventEncoder7 = ScalarEncoder(name="event7", w=7, n=14, minval=0, maxval=1,forced=True)
eventEncoder2 = ScalarEncoder(name="event2", w=7, n=14, minval=0, maxval=1,forced=True)
#eventEncoder2 = ScalarEncoder(name="event2", w=9, n=18, minval=0, maxval=1,forced=True)
baselineEncoder = ScalarEncoder(name = "baseline",w = 21, n=1365,minval= 51,maxval=63,forced= True)
flowEncoder = ScalarEncoder(name="flow", w=15, n = 900, minval=0, maxval=5,forced = True)
encodingWidth = (eventEncoder.getWidth()+eventEncoder1.getWidth()+eventEncoder2.getWidth()+flowEncoder.getWidth()+baselineEncoder.getWidth())

encodingWidth1 =(eventEncoder1.getWidth()
                 + flowEncoder.getWidth())

sp = SpatialPooler(
    inputDimensions=(encodingWidth,),
    columnDimensions=(spParams["columnCount"],),
    potentialPct=spParams["potentialPct"],
    potentialRadius=encodingWidth,
    globalInhibition=spParams["globalInhibition"],
    localAreaDensity=spParams["localAreaDensity"],
    numActiveColumnsPerInhArea=spParams["numActiveColumnsPerInhArea"],
    synPermInactiveDec=spParams["synPermInactiveDec"],
    synPermActiveInc=spParams["synPermActiveInc"],
    synPermConnected=spParams["synPermConnected"],
    boostStrength=spParams["boostStrength"],
    seed=spParams["seed"],
    wrapAround=True
)



tm = TemporalMemory(
    columnDimensions=(tmParams["columnCount"],),
    cellsPerColumn=tmParams["cellsPerColumn"],
    activationThreshold=tmParams["activationThreshold"],
    initialPermanence=tmParams["initialPerm"],
    connectedPermanence=spParams["synPermConnected"],
    minThreshold=tmParams["minThreshold"],
    maxNewSynapseCount=tmParams["newSynapseCount"],
    permanenceIncrement=tmParams["permanenceInc"],
    permanenceDecrement=tmParams["permanenceDec"],
    predictedSegmentDecrement=tmParams["predictedSegmentDecrement"],
    maxSegmentsPerCell=tmParams["maxSegmentsPerCell"],
    maxSynapsesPerSegment=tmParams["maxSynapsesPerSegment"],
    seed=tmParams["seed"]
)

classifier = SDRClassifier(
    steps  = [1],alpha=clParams["alpha"], verbosity= clParams["verbosity"]
)
classifier1 = SDRClassifier(
    steps  = [1],alpha=clParams["alpha"], verbosity= clParams["verbosity"]
)
classifier2 = SDRClassifier(
    steps  = [1],alpha=clParams["alpha"], verbosity= clParams["verbosity"]
)

with open("real_dataset3.csv", "r") as fin:
    reader = csv.reader(fin)
    headers = reader.next()
    reader.next()
    reader.next()
    for row in reader:
        pressure.append(float(row[1]))

print "Columcount",spParams["columnCount"]
def runLearning(numRecords):
 # global  activeCells,encoding
  learning_time = time()
  with open("test3.csv", "r") as fin:
    reader = csv.reader(fin)
    headers = reader.next()
    reader.next()
    reader.next()

    for count, record in enumerate(reader):
      print "Count",count
      if count >= numRecords: break

      # Convert data string into Python date object.
      #dateString = datetime.datetime.strptime(record[0], "%m/%d/%y %H:%M")
      # Convert data value string into float.
      event_value = float(record[2]) # device 1
      event_value_3 = float(record[4]) # device 3
      event_value_2 = float(record[3]) #device 2
      # event_value_7 = float(record[8]) # device 7
      bezline_all = float(record[10])
      pres_data    = float(record[11])
      flow_value  = float(record[0])
      # To encode, we need to provide zero-filled numpy arrays for the encoders
      # to populate.
      eventBits = numpy.zeros(eventEncoder.getWidth())
      eventBits_2 = numpy.zeros(eventEncoder2.getWidth())
      eventBits_3 = numpy.zeros(eventEncoder1.getWidth())


      baseline_Bits = numpy.zeros(baselineEncoder.getWidth())
      flowBits = numpy.zeros(flowEncoder.getWidth())


      # Now we call the encoders to create bit representations for each value.
      eventEncoder.encodeIntoArray(event_value, eventBits)
      eventEncoder1.encodeIntoArray(event_value_3,eventBits_3)
      eventEncoder2.encodeIntoArray(event_value_2,eventBits_2)


      baselineEncoder.encodeIntoArray(bezline_all,baseline_Bits)
      flowEncoder.encodeIntoArray(flow_value, flowBits)


      # Concatenate all these encodings into one large encoding for Spatial
      # Pooling.
      encoding = numpy.concatenate(
        [eventBits,eventBits_2,eventBits_3,flowBits,baseline_Bits,]
      )

      # enc2 = numpy.concatenate([eventBits,eventBits_2,eventBits_3])
      # enc2 = np.pad(enc2, (0, encodingWidth - len(enc2)), 'constant')
      # Create an array to represent active columns, all initially zero. This
      # will be populated by the compute method below. It must have the same
      # dimensions as the Spatial Pooler.

      activeColumns = numpy.zeros(spParams["columnCount"])
      # activeColumns1 = numpy.zeros(spParams["columnCount"])


      # Execute Spatial Pooling algorithm over input space.

      sp.compute(encoding,True,activeColumns)

      # sp.compute(enc2,True,activeColumns)
     # sp.compute(encoding1, True, activeColumns)

      activeColumnIndices = numpy.nonzero(activeColumns)[0]

      # Execute Temporal Memory algorithm over active mini-columns.
      tm.compute(activeColumnIndices, learn=True)

      activeCells = tm.getActiveCells()

      # Get the bucket info for this input value for classification.
    #   bucketIdx = eventEncoder.getBucketIndices(event_value)[0]
      bucketIdx  =  eventEncoder.getBucketIndices(event_value)[0]
      bucketIdx_2 = eventEncoder2.getBucketIndices(event_value_2)[0]
      bucketIdx_3 = eventEncoder1.getBucketIndices(event_value_3)[0]
      # bucketIdx_7 = eventEncoder7.getBucketIndices(event_value_7)[0]r.getWidth()+eventEncoder1.getWidth()+ eventEncoder2.getWidth()


      print "BucketIdx",bucketIdx
      print "BucketIdx_2",bucketIdx_2
      print "BucketIdx_3",bucketIdx_3
      # Run classifier to translate active cells back to scalar value.
      classifierResult = classifier.compute(
        recordNum=count,
        patternNZ=activeCells,
        classification={
          "bucketIdx": bucketIdx,
          "actValue": event_value
        },
        learn=True,
        infer=False
      )
      classifierResult1 = classifier1.compute(
        recordNum=count,
        patternNZ=activeCells,
        classification={
          "bucketIdx": bucketIdx_3,
          "actValue": event_value_3
        },
        learn=True,
        infer=False
      )

      classifierResult2 = classifier2.compute(
        recordNum=count,
        patternNZ=activeCells,
        classification={
          "bucketIdx": bucketIdx_2,
          "actValue": event_value_2
        },
        learn=True,
        infer=False
      )
      learning_time_end = time()
      print "Time",(learning_time - learning_time_end)
      if (count%100) == 0:
        with open("out_sp.tmp", "w") as f1:
            sp.writeToFile(f1)
        with open("out_tm.tmp", "w") as f2:
            tm.writeToFile(f2)
        with open("out_classifier.tmp", "w") as f3:
            classifier.writeToFile(f3)
        with open("out_classifier1.tmp", "w") as f4:
            classifier1.writeToFile(f4)
        with open("out_classifier2.tmp", "w") as f5:
            classifier2.writeToFile(f5)
  # builder = SpatialPoolerProto.new_message()
  # sp.write(builder)
  # serializedMessage = builder.to_bytes_packed()
  # builder1 = TemporalMemoryProto.new_message()
  # tm.write(builder1)
  # serializedMessage = builder1.to_bytes_packed()
if __name__ == "__main__":
  runLearning(20000)