Project

General

Profile

GUImonitor.py

GUI data logger 310818 - Jonathan Ang, 08/31/2018 07:26 PM

 
1
# -*- coding: utf-8 -*-
2
"""
3
Created on Thu Aug  9 15:33:00 2018
4

5
@author: Jonathan
6
"""
7
import numpy as np
8
import json
9
import requests
10
import random
11
import time, threading
12
import tkinter as tk
13
from tkinter import messagebox  #does not automatically get imported
14
import sklearn
15
from sklearn.ensemble import GradientBoostingClassifier
16
from sklearn.naive_bayes import GaussianNB
17
from sklearn.model_selection import train_test_split
18
from sklearn.gaussian_process import GaussianProcessClassifier
19
from sklearn.neural_network import MLPClassifier
20
from sklearn.gaussian_process.kernels import RBF
21
from sklearn import svm
22
from sklearn.covariance import EllipticEnvelope
23
from sklearn.ensemble import IsolationForest
24
from sklearn.neighbors import LocalOutlierFactor
25
import pickle
26
import multiprocessing
27
from sklearn.externals import joblib
28
np.random.seed(1)
29
random.seed(2)
30

    
31
global data_pool     #This is where all approved data is stored
32
global data_list
33
global system_state
34
global clf
35
global readout_list
36

    
37
data_pool = []
38
data_list = []
39
system_state = -1
40
count = 0
41

    
42
global Finish 
43
Finish = False
44

    
45
global setlist, readoutlist, losslist 
46

    
47
#%%############################################################################
48
##### Helper Functions
49
###############################################################################
50
#%%
51
def getparams():
52
    """
53
    Gets the list of ACNET parameters from the file different files and loads it into
54
    a paramslist for fetching
55
    
56
    Parameters
57
    ----------
58
    none
59
          
60
    Returns
61
    -------
62
    none
63
    
64
    """
65
    global paramslist, superparamslist, bpms_list, rfgradients_list, toroids_list,  trims_list, intensities_list
66
    
67
    paramslist = []        
68
    
69
    rfgradients_list = list(np.transpose(np.genfromtxt('rfgradients.txt',dtype='str'))) #switcher inside here N:D122
70
    toroids_list = list(np.transpose(np.genfromtxt('toroids.txt',dtype='str')))
71
    trims_list = list(np.transpose(np.genfromtxt('trims.txt',dtype='str')))
72
    intensities_list = list(np.transpose(np.genfromtxt('Intensities.txt',dtype='str')))
73

    
74
    
75
    paramslist = list(np.transpose(np.genfromtxt('LE_Essentials.txt',dtype='str')))
76
    #paramslist += bpms_list + rfgradients_list + toroids_list + trims_list
77
    #superparamslist = [bpms_list, rfgradients_list, toroids_list,  trims_list, intensities_list] #a list of lists
78
    
79
#%%
80
def intensityprinter():
81
    """
82
    Grabs the label at a specific point of time 
83
    """
84
    intensity_url = format_url(intensities_list)
85

    
86
    try:
87
        intensityvals = requests.get(intensity_url) 
88
        rawdata = intensityvals.text
89
        rawlist2 = [item.split('=')for item in rawdata.split('\n')][:-1]
90
        rawlist3 = [float(item[1][:-4]) for item in rawlist2]
91
        try:
92
            Losses = float((rawlist3[0] - rawlist3[-1])/rawlist3[0])
93
            transmittance = 'Cumulative losses from N:B441I to N:B603I at ' + str(Losses) + '%'
94
        except: 
95
            if rawlist3[0] == 0:
96
                transmittance = 'Zero reading on N:B101BI'
97
            else:
98
                transmittance = 'Bad Reading on Intensity Monitors'
99
    except:
100
        transmittance = 'Request Error'
101
    
102
    textbox.insert(tk.END, transmittance + '\n')
103
    print(transmittance)
104

    
105

    
106
#%%
107
def format_url(readout_list):
108
    """
109
    Formats requested readout list into a parsable URL for extraction. PH PV PI
110
            
111
    Parameters
112
    ----------
113
        readout_list: List of strings of device names eg ['N:C1CGRD', 'N:C1ESPA',......].
114
          
115
    Returns
116
    -------
117
        base_url : String. URL the parameter dataset is to be fetched from. 
118
    
119
    """
120
    device_string = ','.join(str(readout) for readout in readout_list)
121
    base_url = """http://www-ad.fnal.gov/cgi-bin/acl.pl?acl=device_list/create+devs+devices=\'{}\';read_list+device_list=devs""".format(device_string)
122
    
123
    return base_url
124

    
125
#%%
126
def pgrab(readout_list):
127
    """
128
    Grabs a single data point D_t = (x_1, x_2, x_3,.....x_m,y_1,y_2,.....,y_n) and 
129
    at a given time where x_i represent the m parameter inputs and y_n represent
130
    the n readout values.
131
            
132
    Parameters
133
    ----------
134
        readout_list: List of strings of device names eg ['N:C1CGRD', 'N:C1ESPA',......].
135
          
136
    Returns
137
    -------
138
        np.array(datalist) : Numpy array. Numpy array of values  
139
    
140
    """
141
    request_url = format_url(readout_list)
142
    datalist = []
143

    
144
    try:
145
        readout_val = requests.get(request_url) 
146
        rawdata = readout_val.text
147
        rawlist2 = [item.split('=')for item in rawdata.split('\n')][:-1]
148
        rawlist3 = [item[1][:-4] for item in rawlist2]
149
        for item in rawlist3:
150
            try:
151
                datalist.append(float(item))
152
            except: 
153
                #datalist.append('valueERROR')
154
                datalist.append(222222222222)
155
    except:
156
        #datalist.append('FetchERROR')
157
        datalist.append(111111111111)
158
    return np.array(datalist) 
159

    
160
#%%
161
getparams()
162
#%%############################################################################
163
##### Main Functions
164
###############################################################################
165
#%%
166
def Labeller(readout_list, system_state):
167
    """
168
    Grabs incoming data every 5 seconds and labels it as good or bad depending 
169
    on the label input 
170

171
    Extracts information about a device.
172
            
173
    Parameters
174
    ----------
175
        device_name: String. Name of device eg 'N:C1CGRD'
176
   
177
    Returns
178
    -------
179
        none
180
    
181
    """
182
    global running
183
    
184
    def des(system_state):
185
        if system_state == 1:
186
            return 'good'
187
        if system_state == -1:
188
            return 'bad'   
189
        
190
    data = pgrab(readout_list)
191
    try:
192
        if data[0] == 222222222222 or data[0] == 111111111111: #ignore incomplete sets
193
            print(des(system_state) + " data NOT saved at " + str(time.ctime()))
194
            textbox.insert(tk.END, str(time.ctime()) + ' ' + des(system_state) + " data NOT saved"+ "\n")    
195
        else:
196
            data = np.append(data,system_state)        # adds label to end of parameter set
197
            data_list.append(list(data))        # adds data to data_list
198
            print(des(system_state) + " data saved at " + str(time.ctime()))
199
            textbox.insert(tk.END,str(time.ctime()) + ' ' +  des(system_state) + " data saved"+ "\n")
200

    
201
    except:
202
            print(des(system_state) + " data NOT saved at " + str(time.ctime()))
203
            textbox.insert(tk.END, str(time.ctime()) + ' ' + des(system_state) + " data NOT saved"+ "\n")    
204
      
205
        
206
def goodlabel():
207
    """
208
    Continuously grabs data from ACNET and saves it in a temporary list, labelling
209
    the data as good. 
210
            
211
    Parameters
212
    ----------
213
        device_name: String. Name of device eg 'N:C1CGRD'
214
   
215
    Returns
216
    -------
217
        none
218
    
219
    """
220
    Labeller(paramslist, 1)
221
    global running
222
    goodbutton.config(disabledforeground="green",state=tk.DISABLED,text='Logging good data')
223
    badbutton.config(state=tk.DISABLED)
224
    autobutton.config(state=tk.DISABLED)
225
    savebutton.config(state=tk.DISABLED)
226
    clearbutton.config(state=tk.DISABLED)
227
    clearallbutton.config(state=tk.DISABLED)
228
    running = GUI.after(1000, goodlabel)  # reschedule event in 2 seconds
229

    
230

    
231
def badlabel():
232
    """
233
    Continuously grabs data from ACNET and saves it in a temporary list, labelling
234
    the data as BAD. 
235
            
236
    Parameters
237
    ----------
238
        device_name: String. Name of device eg 'N:C1CGRD'
239
   
240
    Returns
241
    -------
242
        none
243
    
244
    """
245
    Labeller(paramslist, -1)
246
    global running
247
    goodbutton.config(state=tk.DISABLED)
248
    badbutton.config(disabledforeground="red",state=tk.DISABLED,text='Logging bad data')
249
    autobutton.config(state=tk.DISABLED)
250
    savebutton.config(state=tk.DISABLED)
251
    clearbutton.config(state=tk.DISABLED)
252
    clearallbutton.config(state=tk.DISABLED)
253
    running = GUI.after(1000, badlabel)  # reschedule event in 2 seconds
254
 
255
    
256
def autolabel():
257
    """
258
    Automatically label points based on an inbuilt rule based system
259
    
260
    Parameters
261
    ----------
262
    none
263
          
264
    Returns
265
    -------
266
    none
267
    
268
    """
269
    data = pgrab(paramslist + intensities_list) 
270
    def des(system_state):
271
        if system_state == 1:
272
            return 'good'
273
        if system_state == -1:
274
            return 'bad'   
275
    try:
276
        if data[0] == 222222222222 or data[0] == 111111111111: #ignore incomplete sets
277
            print(des(system_state) + " data NOT saved at " + str(time.ctime()))
278
            textbox.insert(tk.END, str(time.ctime()) + ' ' + des(system_state) + " data NOT saved"+ "\n")    
279
        else:
280
            Trasmit =  (data[-2] - data[-1]) / data[-2] * 100
281
            #data = np.append(data,str(time.ctime()))        # adds label to end of parameter set
282
            data_list.append(list(data) + [str(time.ctime())])        # adds data to data_list
283
            print(" data saved at " + str(time.ctime()))
284
            
285
            if Trasmit > 80:
286
                textbox.insert(tk.END,str(time.ctime()) + " good data saved"+ "\n")
287
                
288
            else:   
289
                textbox.insert(tk.END,str(time.ctime()) + " data saved"+ "\n")
290
            textbox.insert(tk.END,'Cumulative losses from N:B441I to N:B603I at ' + str(Trasmit)[:6] + '%' + "\n")
291

    
292
    except:
293
            print(" data NOT saved at " + str(time.ctime()))
294
            textbox.insert(tk.END, str(time.ctime()) + " unacceptable data NOT saved"+ "\n")    
295
          
296
    global running
297
    goodbutton.config(state=tk.DISABLED)
298
    badbutton.config(state=tk.DISABLED)
299
    autobutton.config(disabledforeground="blue",state=tk.DISABLED,text='automatic logging')
300
    savebutton.config(state=tk.DISABLED)
301
    clearbutton.config(state=tk.DISABLED)
302
    clearallbutton.config(state=tk.DISABLED)
303
    running = GUI.after(1000, autolabel)  # reschedule event in 2 seconds    
304
    
305

    
306
def save():
307
    """
308
    Saves data 
309
    
310
    Parameters
311
    ----------
312
    none
313
          
314
    Returns
315
    -------
316
    none
317
    
318
    """
319
    global data_list
320
    print("saving data at "  + str(time.ctime()))
321
    filename = saveloc.get()
322
    
323
    if filename == '':
324
        print("No file name entered")
325
        messagebox.showerror("Error", "No file name entered")
326
        
327
    if len(data_list) == 0:
328
        print("No saved data")
329
        messagebox.showerror("Error", "No data in cache")
330
        
331
    else:
332
        savefilename = filename + ".txt"
333
        data_pool.extend(data_list)
334
        with open(savefilename, 'w') as outfile:
335
            json.dump(data_pool, outfile)  
336
        print("data saved in " + savefilename + " at "  + str(time.ctime()))
337
        textbox.insert(tk.END,str(time.ctime()) + ' ' +  "data saved in " + savefilename + "\n")
338
    goodbutton.config(state=tk.NORMAL)
339
    badbutton.config(state=tk.NORMAL)
340

    
341

    
342
def clear():
343
    """
344
    Clears local data_list
345
    
346
    Parameters
347
    ----------
348
    none
349
          
350
    Returns
351
    -------
352
    none
353
    
354
    """
355
    global data_list 
356
    data_list = []
357
    print("Latest cache cleared at " + str(time.ctime()))
358
    textbox.insert(tk.END,str(time.ctime()) + ' ' +  "Latest cache cleared" + "\n")
359

    
360
def clearall():
361
    """
362
    Clears all recorded but unsaved files.
363
    
364
    Parameters
365
    ----------
366
    none
367
          
368
    Returns
369
    -------
370
    none
371
    
372
    """
373
    global data_list, data_pool
374
    data_list = []
375
    data_pool = []
376
    print("Cache cleared at " + str(time.ctime()))
377
    textbox.insert(tk.END,str(time.ctime()) + ' ' +  "Cache cleared" + "\n")
378

    
379
def merge():
380
    """
381
    Merges different files
382
    
383
    Parameters
384
    ----------
385
    none
386
          
387
    Returns
388
    -------
389
    none
390
    
391
    """
392
    mergefile1 = mloc1.get()
393
    mergefile2 = mloc2.get()
394
    mergeout = moutloc.get()
395
    
396
    if mergeout == '':
397
        print("No file name entered")
398
        messagebox.showerror("Error", "No file name entered")
399
        
400
    mergefile1name = mergefile1 + ".txt"
401
    mergefile2name = mergefile2 + ".txt"
402
    
403
    with open(mergefile1name) as json_data:
404
        merge1data = json.load(json_data) 
405
    
406
    with open(mergefile2name) as json_data:
407
        merge2data = json.load(json_data) 
408
    
409
    if len(merge1data) == 0 or len(merge2data) == 0:
410
        print("No saved data in file")
411
        messagebox.showerror("Error", "No data to merge")
412
        
413
    moutdata = merge1data + merge2data
414
    
415
    moutfilename =  mergeout + ".txt"
416
    with open(moutfilename, 'w') as outfile:
417
        json.dump(moutdata, outfile)  
418
    print("data saved in " + moutfilename + " at "  + str(time.ctime()))    
419
    textbox.insert(tk.END, str(time.ctime()) + "data saved in " + moutfilename + "\n")
420
    
421
def key_pressed():
422
    """
423
    Stop recording
424
    
425
    Parameters
426
    ----------
427
    none
428
          
429
    Returns
430
    -------
431
    none
432
    
433
    """
434
    print("Recording stopped at " + str(time.ctime()))
435
    textbox.insert(tk.END, str(time.ctime()) + " Recording stopped" + "\n")
436
    GUI.after_cancel(running)
437
    goodbutton.config(disabledforeground="grey",text='Log good data', state=tk.NORMAL)
438
    badbutton.config(disabledforeground="grey",text='Log bad data', state=tk.NORMAL)
439
    autobutton.config(disabledforeground="grey",text='Auto Log', state=tk.NORMAL)
440
    savebutton.config(state=tk.NORMAL)
441
    clearbutton.config(state=tk.NORMAL)
442
    clearallbutton.config(state=tk.NORMAL)
443
    
444
#%%    
445
def choosealgo():
446
    """
447
    Choose the algorithm I want to run, labelled by integers
448
    
449
    Parameters
450
    ----------
451
    none
452
          
453
    Returns
454
    -------
455
    none
456
    
457
    """
458
    choice = Algoselection.get()
459
    
460
    if choice == 1:
461
        NaiveBayes()
462
    elif choice == 2:
463
        kNN()
464
    elif choice == 3: 
465
        BDT()
466
    elif choice == 4:  
467
        SVC()
468
    elif choice == 5:  
469
        GaussPross()
470
    elif choice == 6:  
471
        NN()
472
    else:
473
        print("Bad choices detected")
474
        messagebox.showerror("Error", "No algorithm selected")  
475
    
476
#%%  
477

    
478
def savepredictor():
479
    """
480
    Saves the predictor in a *.txt file with the name specified in savefilename
481
    
482
    Parameters
483
    ----------
484
    none
485
          
486
    Returns
487
    -------
488
    none
489
    
490
    """
491
    global clf
492
    filename = savepredictorloc.get()
493
    
494
    if filename == '':
495
        print("No file name entered")
496
        messagebox.showerror("Error", "No file name entered")
497
    else:
498
        savefilename = filename + '.pkl'
499
        joblib.dump(clf, savefilename) 
500
        print("predictor saved in " + savefilename + " at "  + str(time.ctime()))
501
        textbox.insert(tk.END, str(time.ctime()) + "predictor saved in " + savefilename + "\n")
502
    
503
def loadpredictor():
504
    """
505
    load the predictor in a file to reduce training times in the future 
506
    
507
    Parameters
508
    ----------
509
    none
510
          
511
    Returns
512
    -------
513
    none
514
    
515
    """
516
    global clf
517
    filename = loadpredictorloc.get()
518
    
519
    if filename == '':
520
        print("No file name entered")
521
        messagebox.showerror("Error", "No file name entered")
522
    else:
523
        loadfilename = filename + '.pkl'
524
        clf = joblib.load(loadfilename) 
525
        print("predictor loaded from " + loadfilename + " at "  + str(time.ctime()))
526
        textbox.insert(tk.END, str(time.ctime()) + " predictor loaded from " + loadfilename + "\n")
527
    
528
    
529
#%%  
530
def runpredictor():
531
    """
532
    Runs the predictor once to predict if there's something wrong with the system
533
    
534
    Parameters
535
    ----------
536
    none
537
          
538
    Returns
539
    -------
540
    none
541
    
542
    """
543
    global clf, running
544
    
545
    def des(system_state):
546
        if system_state == 1:
547
            return 'Good'
548
        if system_state == -1:
549
            return 'Bad'
550
    while Finish == True:
551
        data = pgrab(paramslist)
552
        data = data.reshape(1,-1)
553
        try:
554
            prediction = clf.predict(data)   
555
            print(des(prediction) + " values seen at " + str(time.ctime()))
556
            textbox.insert(tk.END, str(time.ctime()) + ' ' + des(prediction) + " values expected"+ "\n")
557
            intensityprinter()
558
            time.sleep(2)
559
        except: #ignore incomplete sets
560
            print("Prediction not possible at " + str(time.ctime()))
561
            textbox.insert(tk.END, str(time.ctime()) + " Prediction not possible at this time " + "\n")
562
    
563
            time.sleep(2)
564
            
565
    
566
#%%  
567
def runningpredictor():
568
    global running, Finish
569
    Finish = True
570
    Process = threading.Thread(target=runpredictor)        
571
    Process.start()
572
    goodbutton.config(state=tk.DISABLED)
573
    badbutton.config(state=tk.DISABLED)
574
    autobutton.config(state=tk.DISABLED)
575
    netbutton.config(state=tk.DISABLED)
576
    savebutton.config(state=tk.DISABLED)
577
    clearbutton.config(state=tk.DISABLED)
578
    clearallbutton.config(state=tk.DISABLED)
579
    clearallbutton.config(state=tk.DISABLED)
580
    loadpredictornetbutton.config(state=tk.DISABLED)
581
    savepredictornetbutton.config(state=tk.DISABLED)
582
    runpredictornetbutton.config(state=tk.DISABLED)
583

    
584
    
585
#%%  
586
def stoppredictor():
587
    global running, Finish
588
    Finish = False
589
    goodbutton.config(disabledforeground="grey",text='Log good data', state=tk.NORMAL)
590
    badbutton.config(disabledforeground="grey",text='Log bad data', state=tk.NORMAL)
591
    badbutton.config(disabledforeground="grey",text='AUTO LOG', state=tk.NORMAL)
592
    netbutton.config(state=tk.NORMAL)
593
    savebutton.config(state=tk.NORMAL)
594
    clearbutton.config(state=tk.NORMAL)
595
    clearallbutton.config(state=tk.NORMAL)
596
    
597
    loadpredictornetbutton.config(state=tk.NORMAL)
598
    savepredictornetbutton.config(state=tk.NORMAL)
599
    runpredictornetbutton.config(state=tk.NORMAL)
600
    print("Prediction stopped at " + str(time.ctime()))
601
    textbox.insert(tk.END, str(time.ctime()) + " Prediction stopped " + "\n")
602
 
603
#%%
604
        
605
def uchoosealgo():
606
    """
607
    Choose the algorithm I want to run
608
    """
609
    choice = Algoselection.get()
610
    
611
    if choice == 1:
612
        oneclasssvm()
613
    elif choice == 2:
614
        EENV()
615
    elif choice == 3: 
616
        iForest()
617
    elif choice == 4:  
618
        LOF()
619
    else:
620
        print("Bad choices detected")
621
        messagebox.showerror("Error", "No algorithm selected")  
622

    
623

    
624
def usavepredictor():
625
    """
626
    Saves the predictor in a file to reduce training times in the future 
627
    """
628
    global clf
629
    filename = usavepredictorloc.get()
630
    
631
    if filename == '':
632
        print("No file name entered")
633
        messagebox.showerror("Error", "No file name entered")
634
    else:
635
        savefilename = filename + '.pkl'
636
        joblib.dump(clf, savefilename) 
637
        print("predictor saved in " + savefilename + " at "  + str(time.ctime()))
638
        textbox.insert(tk.END, str(time.ctime()) + "predictor saved in " + savefilename + "\n")
639
    
640
def uloadpredictor():
641
    """
642
    load the predictor in a file to reduce training times in the future 
643
    """
644
    global clf
645
    filename = uloadpredictorloc.get()
646
    
647
    if filename == '':
648
        print("No file name entered")
649
        messagebox.showerror("Error", "No file name entered")
650
    else:
651
        loadfilename = filename + '.pkl'
652
        clf = joblib.load(loadfilename) 
653
        print("predictor loaded from " + loadfilename + " at "  + str(time.ctime()))
654
        textbox.insert(tk.END, str(time.ctime()) + " predictor loaded from " + loadfilename + "\n")
655
    
656

    
657
def urunpredictor():
658
    """
659
    Runs the predictor once to predict if there's something wrong with the system
660
    """
661
    global clf, running
662
    
663
    def des(system_state):
664
        if system_state[0] == 1:
665
            return 'Good'
666
        if system_state[0] == -1:
667
            return 'Bad'
668
        else:
669
            return "DESERROR"
670
    while Finish == True:
671
        data = pgrab(paramslist)
672
        data = data.reshape(1,-1)
673
        #prediction = clf.predict(data)  
674
        #print(des(prediction) + " values seen at " + str(time.ctime()))
675
        try:
676
            global prediction
677
            prediction = clf.predict(data)  ###need to fine tune as only LOF here 
678
            print(des(prediction) + " values seen at " + str(time.ctime()))
679
            textbox.insert(tk.END, str(time.ctime()) + ' ' + des(prediction) + " values expected"+ "\n")
680
            intensityprinter()
681
            time.sleep(2)
682
        except: #ignore incomplete sets
683
            print("Prediction not possible at " + str(time.ctime()))
684
            textbox.insert(tk.END, str(time.ctime()) + " Prediction not possible at this time " + "\n")
685
    
686
            time.sleep(2)
687
            
688

    
689
def urunningpredictor():
690
    global running, Finish
691
    Finish = True
692
    Process = threading.Thread(target=urunpredictor)        
693
    Process.start()
694
    goodbutton.config(state=tk.DISABLED)
695
    badbutton.config(state=tk.DISABLED)
696
    autobutton.config(state=tk.DISABLED)
697
    netbutton.config(state=tk.DISABLED)
698
    savebutton.config(state=tk.DISABLED)
699
    clearbutton.config(state=tk.DISABLED)
700
    clearallbutton.config(state=tk.DISABLED)
701
    clearallbutton.config(state=tk.DISABLED)
702
    loadpredictornetbutton.config(state=tk.DISABLED)
703
    savepredictornetbutton.config(state=tk.DISABLED)
704
    runpredictornetbutton.config(state=tk.DISABLED)
705

    
706
def ustoppredictor():
707
    global running, Finish
708
    Finish = False
709
    goodbutton.config(disabledforeground="grey",text='Log good data', state=tk.NORMAL)
710
    badbutton.config(disabledforeground="grey",text='Log bad data', state=tk.NORMAL)
711
    autobutton.config(disabledforeground="grey",text='AUTO LOG', state=tk.NORMAL)
712
    netbutton.config(state=tk.NORMAL)
713
    savebutton.config(state=tk.NORMAL)
714
    clearbutton.config(state=tk.NORMAL)
715
    clearallbutton.config(state=tk.NORMAL)
716
    
717
    loadpredictornetbutton.config(state=tk.NORMAL)
718
    savepredictornetbutton.config(state=tk.NORMAL)
719
    runpredictornetbutton.config(state=tk.NORMAL)
720
    print("Prediction stopped at " + str(time.ctime()))
721
    textbox.insert(tk.END, str(time.ctime()) + " Prediction stopped " + "\n")
722
    
723
#%%############################################################################
724
### Prediction Algorithms   
725
#%%############################################################################
726
    
727
def NaiveBayes():
728
    """
729
    Implements Naive Bayes
730
    """  
731
    print("Loading json file data")   
732
    
733
    global data,X,Y  
734
    filename = loadloc.get()
735
    
736
    if filename == '':
737
        print("No file name entered")
738
        messagebox.showerror("Error", "No file name entered")
739
    else:
740
        loadfilename = filename + ".txt"
741
        with open(loadfilename) as json_data:
742
            data = json.load(json_data) 
743
        print("Naive Bayes Training started at " + str(time.ctime()))
744
        
745
    X = np.array([i[:-1]for i in data])
746
    Y = np.array([i[-1]for i in data])  
747
    X_train, X_test, Y_train, Y_test = train_test_split(X, Y, random_state=0, test_size=0.20)   
748

    
749
    ## no validation
750
    global clf
751
    clf = GaussianNB()
752
    Y_pred = clf.fit(X_train, Y_train).predict(X_test)
753
    
754
    print("Naive Bayes Training ended at " + str(time.ctime()))
755
    print("Number of mislabeled points out of a total %d points : %d" % (X_test.shape[0],(Y_test != Y_pred).sum()))
756
    print(clf.score(X_test, Y_test))
757

    
758

    
759
def kNN():
760
    """
761
    Implements Nearest Neighbours classifier
762
    """   
763
    print("Loading json file data")   
764
    
765
    global data,X,Y  
766
    filename = loadloc.get()
767
    
768
    if filename == '':
769
        print("No file name entered")
770
        messagebox.showerror("Error", "No file name entered")
771
    else:
772
        loadfilename = filename + ".txt"
773
        with open(loadfilename) as json_data:
774
            data = json.load(json_data) 
775
        print("kNN Training started at " + str(time.ctime()))
776
        
777
    X = np.array([i[:-1]for i in data])
778
    Y = np.array([i[-1]for i in data])  
779
    X_train, X_test, Y_train, Y_test = train_test_split(X, Y, random_state=0, test_size=0.20)   
780
    
781
    global clf
782
    clf = sklearn.neighbors.KNeighborsClassifier(n_neighbors=6)
783
    Y_pred = clf.fit(X_train, Y_train).predict(X_test)
784
    
785
    print("kNN Training ended at " + str(time.ctime()))
786
    print("Number of mislabeled points out of a total %d points : %d" % (X_test.shape[0],(Y_test != Y_pred).sum()))
787
    print(clf.score(X_test, Y_test))
788

    
789

    
790
def BDT():
791
    """
792
    Implements Linear Boosted Tree
793
    """   
794
    print("Loading json file data")   
795
    
796
    global data,X,Y  
797
    filename = loadloc.get()
798
    
799
    if filename == '':
800
        print("No file name entered")
801
        messagebox.showerror("Error", "No file name entered")
802
    else:
803
        loadfilename = filename + ".txt"
804
        with open(loadfilename) as json_data:
805
            data = json.load(json_data) 
806
        print("BDT Training started at " + str(time.ctime()))
807
        
808
    X = np.array([i[:-1]for i in data])
809
    Y = np.array([i[-1]for i in data])  
810
    X_train, X_test, Y_train, Y_test = train_test_split(X, Y, random_state=0, test_size=0.20)   
811

    
812
    global clf
813
    clf = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0, max_depth=3, random_state=0).fit(X_train, Y_train)
814
    Y_pred = clf.fit(X_train, Y_train).predict(X_test)
815
    
816
    print("BDT Training ended at " + str(time.ctime()))
817
    print("Number of mislabeled points out of a total %d points : %d" % (X_test.shape[0],(Y_test != Y_pred).sum()))
818
    print(clf.score(X_test, Y_test))
819

    
820
def SVC():
821
    """
822
    Implements Support Vector Machine Classifier
823
    """  
824
    print("Loading json file data")   
825
    
826
    global data,X,Y  
827
    filename = loadloc.get()
828
    
829
    if filename == '':
830
        print("No file name entered")
831
        messagebox.showerror("Error", "No file name entered")
832
    else:
833
        loadfilename = filename + ".txt"
834
        with open(loadfilename) as json_data:
835
            data = json.load(json_data) 
836
        print("SVC Training started at " + str(time.ctime()))
837
        
838
    X = np.array([i[:-1]for i in data])
839
    Y = np.array([i[-1]for i in data])  
840
    X_train, X_test, Y_train, Y_test = train_test_split(X, Y, random_state=0, test_size=0.20)   
841
    
842
    global clf
843
    clf = svm.SVC(gamma=2, C=1)
844
    
845
    Y_pred = clf.fit(X_train, Y_train).predict(X_test)
846
    print("SVC Training ended at " + str(time.ctime()))
847
    print("Number of mislabeled points out of a total %d points : %d" % (X_test.shape[0],(Y_test != Y_pred).sum()))
848
    print(clf.score(X_test, Y_test))
849

    
850

    
851
def GaussPross():
852
    """
853
    Implements Gaussian Process Classifier
854
    """  
855
    print("Loading json file data")   
856
    
857
    global data,X,Y  
858
    filename = loadloc.get()
859
    
860
    if filename == '':
861
        print("No file name entered")
862
        messagebox.showerror("Error", "No file name entered")
863
    else:
864
        loadfilename = filename + ".txt"
865
        with open(loadfilename) as json_data:
866
            data = json.load(json_data) 
867
        print("Gaussian process Training started at " + str(time.ctime()))
868
        
869
    X = np.array([i[:-1]for i in data])
870
    Y = np.array([i[-1]for i in data])  
871
    X_train, X_test, Y_train, Y_test = train_test_split(X, Y, random_state=0, test_size=0.20)   
872
    
873
    global clf
874
    clf = GaussianProcessClassifier(1.0 * RBF(1.0))
875
    
876
    Y_pred = clf.fit(X_train, Y_train).predict(X_test)
877
    print("Gaussian process Training ended at " + str(time.ctime()))
878
    print("Number of mislabeled points out of a total %d points : %d" % (X_test.shape[0],(Y_test != Y_pred).sum()))
879
    print(clf.score(X_test, Y_test))
880

    
881

    
882
def NN():
883
    """
884
    Implements Neural Net Classifier
885
    """  
886
    print("Loading json file data")   
887
    
888
    global data,X,Y  
889
    filename = loadloc.get()
890
    
891
    if filename == '':
892
        print("No file name entered")
893
        messagebox.showerror("Error", "No file name entered")
894
    else:
895
        loadfilename = filename + ".txt"
896
        with open(loadfilename) as json_data:
897
            data = json.load(json_data) 
898
        print("Neural Network Training started at " + str(time.ctime()))
899
        
900
    X = np.array([i[:-1]for i in data])
901
    Y = np.array([i[-1]for i in data])  
902
    X_train, X_test, Y_train, Y_test = train_test_split(X, Y, random_state=0, test_size=0.20)   
903
    
904
    global clf
905
    clf = MLPClassifier(alpha=1)
906
    
907
    Y_pred = clf.fit(X_train, Y_train).predict(X_test)
908
    print("Neural Network Training ended at " + str(time.ctime()))
909
    print("Number of mislabeled points out of a total %d points : %d" % (X_test.shape[0],(Y_test != Y_pred).sum()))
910
    print(clf.score(X_test, Y_test))
911

    
912
#%%
913
def oneclasssvm():
914
    """
915
    Implements One-Class SVM anomaly detection
916
    """
917
    global data,X
918
    outliers_fraction = 0.05
919
    filename = uloadloc.get()   
920
    
921
    if filename == '':
922
        print("No file name entered")
923
        messagebox.showerror("Error", "No file name entered")
924
    else:
925
        loadfilename = filename + ".txt"
926
        with open(loadfilename) as json_data:
927
            data = json.load(json_data) 
928
        print("One Class SVM Training started at " + str(time.ctime()))
929
     
930
    global clf
931
    X = np.array([i[:-1]for i in data])
932
    clf = svm.OneClassSVM(nu=0.95 * outliers_fraction + 0.05,kernel="rbf", gamma=0.1)
933
    clf.fit(X)
934
   
935
    print("One Class SVM Training ended at " + str(time.ctime()))
936

    
937
def EENV():
938
    """
939
    Implements Ellpitic Envelope
940
    """
941
    global data,X
942
    filename = uloadloc.get()   
943
    
944
    if filename == '':
945
        print("No file name entered")
946
        messagebox.showerror("Error", "No file name entered")
947
    else:
948
        loadfilename = filename + ".txt"
949
        with open(loadfilename) as json_data:
950
            data = json.load(json_data) 
951
        print("Ellpitic Envelope Training started at " + str(time.ctime()))
952
     
953
    global clf
954
    X = np.array([i[:-1]for i in data])
955
    clf = EllipticEnvelope(contamination=0.05)
956
    clf.fit(X)
957
   
958
    print("Ellpitic Envelope Training ended at " + str(time.ctime()))
959

    
960

    
961
def iForest():
962
    """
963
    Implements One-Class SVM anomaly detection
964
    """
965
    global data,X
966
    filename = uloadloc.get()   
967
    
968
    if filename == '':
969
        print("No file name entered")
970
        messagebox.showerror("Error", "No file name entered")
971
    else:
972
        loadfilename = filename + ".txt"
973
        with open(loadfilename) as json_data:
974
            data = json.load(json_data) 
975
        print("Isolation Forest Training started at " + str(time.ctime()))
976
     
977
    global clf
978
    X = np.array([i[:-1]for i in data])
979
    clf = IsolationForest(max_samples=200 ,contamination=0.05,random_state=1)
980
    clf.fit(X)
981
   
982
    print("Isolation Forest Training ended at " + str(time.ctime()))
983

    
984
def LOF():
985
    """
986
    Local Outlier Factor anomaly detection
987
    """
988
    global data,X
989
    filename = uloadloc.get()   
990
    
991
    if filename == '':
992
        print("No file name entered")
993
        messagebox.showerror("Error", "No file name entered")
994
    else:
995
        loadfilename = filename + ".txt"
996
        with open(loadfilename) as json_data:
997
            data = json.load(json_data) 
998
        print("Local Outlier Factor Training started at " + str(time.ctime()))
999
     
1000
    global clf
1001
    X = np.array([i[:-1]for i in data])
1002
    clf = LocalOutlierFactor(n_neighbors=35, contamination=0.05)
1003
    clf.fit(X)
1004
   
1005
    print("Local Outlier Factor Training ended at " + str(time.ctime()))    
1006
    
1007
#%%############################################################################
1008
##### Graphical User Interface (GUI)
1009
###############################################################################
1010
#%%  
1011
GUI = tk.Tk()
1012
GUI.title('Operator Utilities')
1013
GUI.geometry('1020x600')
1014
#GUI.after(2000, task) 
1015

    
1016
saveloc = tk.StringVar()
1017
loadloc = tk.StringVar()
1018
mloc1 = tk.StringVar()
1019
mloc2 = tk.StringVar()
1020
moutloc = tk.StringVar()
1021
savepredictorloc = tk.StringVar()
1022
loadpredictorloc = tk.StringVar()
1023

    
1024
usaveloc = tk.StringVar()
1025
uloadloc = tk.StringVar()
1026
umloc1 = tk.StringVar()
1027
umloc2 = tk.StringVar()
1028
umoutloc = tk.StringVar()
1029
usavepredictorloc = tk.StringVar()
1030
uloadpredictorloc = tk.StringVar()
1031

    
1032
###############################################################################
1033
Saveutils = tk.LabelFrame(GUI, text="Save Utilities", padx=5, pady=5)
1034
Saveutils.grid(row=0, columnspan=7, sticky='WE', \
1035
                 padx=5, pady=5, ipadx=5, ipady=5)
1036

    
1037
loadfilelabel = tk.Label(Saveutils, text="Select the label to begin recording")
1038
loadfilelabel.grid(row=0, column=0, sticky='E', padx=5, pady=2)
1039

    
1040
goodbutton = tk.Button(Saveutils, text='Log good data',command=goodlabel)
1041
goodbutton.config(height = 3, width = 12 )
1042
goodbutton.grid(row=1, column=0, sticky='E', padx=5, pady=2)
1043

    
1044
badbutton = tk.Button(Saveutils, text='Log bad data',command=badlabel)
1045
badbutton.config(height = 3, width = 12)
1046
badbutton.grid(row=1, column=1, sticky='E', padx=5, pady=2)
1047

    
1048
autobutton = tk.Button(Saveutils, text='Auto Log',command=autolabel)
1049
#autobutton.config(height = 3, width = 12)
1050
autobutton.grid(row=6, column=0, sticky='E', padx=5, pady=2)
1051

    
1052
stopbutton = tk.Button(Saveutils, text='Stop logging',command=key_pressed)
1053
stopbutton.config(height = 3, width = 12)
1054
stopbutton.grid(row=1, column=2, sticky='E', padx=5, pady=2)
1055

    
1056
savefilelabel = tk.Label(Saveutils, text="Save file name (*.txt):")
1057
savefilelabel.grid(row=5, column=0, sticky='E', padx=5, pady=2)
1058

    
1059
savelocentry = tk.Entry(Saveutils, textvariable=saveloc)
1060
savelocentry.grid(row=5, column=1, sticky='W', padx=5, pady=2)
1061

    
1062
savebutton = tk.Button(Saveutils, text='Save',command=save)
1063
savebutton.grid(row=5, column=2, sticky='W', padx=5, pady=2)
1064

    
1065
clearbutton = tk.Button(Saveutils, text='Clear',command=clear)
1066
clearbutton.grid(row=6, column=1, sticky='E', padx=5, pady=2)
1067

    
1068
clearallbutton = tk.Button(Saveutils, text='Clear All',command=clearall)
1069
clearallbutton.grid(row=6, column=2, sticky='W', padx=5, pady=2)
1070

    
1071
###############################################################################
1072

    
1073
logfileutils = tk.LabelFrame(GUI, text="Log File Utilities", padx=5, pady=5)
1074
logfileutils.grid(row=4, columnspan=7, sticky='WE', \
1075
                 padx=5, pady=5, ipadx=5, ipady=5)
1076

    
1077
mergetoplabel = tk.Label(Saveutils, text="Select the label to begin recording")
1078
mergetoplabel.grid(row=0, column=0, sticky='E', padx=5, pady=2)
1079

    
1080
mergelabel1 = tk.Label(logfileutils, text="File 1 name (*.txt):")
1081
mergelabel1.grid(row=7, column=0, sticky='E', padx=5, pady=2)
1082

    
1083
mergelabel2 = tk.Label(logfileutils, text="File 2 name (*.txt):")
1084
mergelabel2.grid(row=8, column=0, sticky='E', padx=5, pady=2)
1085

    
1086
moutlabel = tk.Label(logfileutils, text="Merged file name (*.txt):")
1087
moutlabel.grid(row=9, column=0, sticky='E', padx=5, pady=2)
1088

    
1089
ml1entry = tk.Entry(logfileutils, textvariable=mloc1)
1090
ml1entry.grid(row=7, column=1, sticky='E', padx=5, pady=2)
1091

    
1092
ml2entry = tk.Entry(logfileutils, textvariable=mloc2)
1093
ml2entry.grid(row=8, column=1, sticky='E', padx=5, pady=2)
1094

    
1095
moutentry = tk.Entry(logfileutils, textvariable=moutloc)
1096
moutentry.grid(row=9, column=1, sticky='E', padx=5, pady=2)
1097

    
1098
mergebutton = tk.Button(logfileutils, text='Merge files',command=merge)
1099
mergebutton.grid(row=9, column=2, sticky='E', padx=5, pady=2)
1100

    
1101
###############################################################################
1102

    
1103
MLutils = tk.LabelFrame(GUI, text="Supervised Learning Utilities", padx=5, pady=5)
1104
MLutils.grid(row=5, columnspan=7, sticky='WE', \
1105
                 padx=5, pady=5, ipadx=5, ipady=5)
1106

    
1107
Algoselection = tk.IntVar()
1108

    
1109
Algo1 = tk.Radiobutton(MLutils, text="Naive Bayes", variable = Algoselection, value=1).grid(row=3, column=0, sticky='W', padx=5, pady=2)
1110
Algo2 = tk.Radiobutton(MLutils, text="k Nearest Neighbours", variable = Algoselection, value=2).grid(row=3, column=1, sticky='W', padx=5, pady=2)
1111
Algo3 = tk.Radiobutton(MLutils, text="Boosted Decision Tree", variable = Algoselection, value=3).grid(row=4, column=0, sticky='W', padx=5, pady=2)
1112
Algo4 = tk.Radiobutton(MLutils, text="Support Vector Classifier", variable = Algoselection, value=4).grid(row=4, column=1, sticky='W', padx=5, pady=2)
1113
Algo5 = tk.Radiobutton(MLutils, text="Gaussian Process", variable = Algoselection, value=5).grid(row=3, column=2, sticky='W', padx=5, pady=2)
1114
Algo6 = tk.Radiobutton(MLutils, text="Neural Network", variable = Algoselection, value=6).grid(row=4, column=2, sticky='W', padx=5, pady=2)
1115

    
1116
loadfilelabel = tk.Label(MLutils, text="Load datafile name (*.txt):")
1117
loadfilelabel.grid(row=6, column=0, sticky='E', padx=5, pady=2)
1118

    
1119
loadlocentry = tk.Entry(MLutils, textvariable=loadloc)
1120
loadlocentry.grid(row=6, column=1, sticky='W', padx=5, pady=2)
1121

    
1122
netbutton = tk.Button(MLutils, text='Train Predictor',command=choosealgo)
1123
netbutton.grid(row=6, column=2, sticky='E', padx=5, pady=2)
1124

    
1125
savepredictorlabel = tk.Label(MLutils, text="Save predictor as (*.pkl):")
1126
savepredictorlabel.grid(row=7, column=0, sticky='E', padx=5, pady=2)
1127

    
1128
savepredictorlocentry = tk.Entry(MLutils, textvariable=savepredictorloc)
1129
savepredictorlocentry.grid(row=7, column=1, sticky='W', padx=5, pady=2)
1130

    
1131
savepredictornetbutton = tk.Button(MLutils, text='Save Predictor',command=savepredictor)
1132
savepredictornetbutton.grid(row=7, column=2, sticky='E', padx=5, pady=2)
1133

    
1134
loadpredictorlabel = tk.Label(MLutils, text="Load predictor name (*.pkl):")
1135
loadpredictorlabel.grid(row=8, column=0, sticky='E', padx=5, pady=2)
1136

    
1137
loadpredictorlocentry = tk.Entry(MLutils, textvariable=loadpredictorloc)
1138
loadpredictorlocentry.grid(row=8, column=1, sticky='W', padx=5, pady=2)
1139

    
1140
loadpredictornetbutton = tk.Button(MLutils, text='Load Predictor',command=loadpredictor)
1141
loadpredictornetbutton.grid(row=8, column=2, sticky='E', padx=5, pady=2)
1142

    
1143
runpredictornetbutton = tk.Button(MLutils, text='Run Predictor',command=runningpredictor)
1144
runpredictornetbutton.grid(row=10, column=1, sticky='E', padx=5, pady=2)
1145

    
1146
stoppredictornetbutton = tk.Button(MLutils, text='Stop Predictor',command=stoppredictor)
1147
stoppredictornetbutton.grid(row=10, column=2, sticky='E', padx=5, pady=2)
1148

    
1149
###############################################################################
1150

    
1151
uMLutils = tk.LabelFrame(GUI, text="Unsupervised Learning Utilities", padx=5, pady=5)
1152
uMLutils.grid(row=5, column=7, columnspan=7, sticky='WE', \
1153
                 padx=5, pady=5, ipadx=5, ipady=5)
1154

    
1155
uAlgoselection = tk.IntVar()
1156

    
1157
uAlgo1 = tk.Radiobutton(uMLutils, text="One-Class SVM", variable = Algoselection, value=1).grid(row=3, column=0, sticky='W', padx=5, pady=2)
1158
uAlgo2 = tk.Radiobutton(uMLutils, text="Elliptic Envelope", variable = Algoselection, value=2).grid(row=3, column=1, sticky='W', padx=5, pady=2)
1159
uAlgo3 = tk.Radiobutton(uMLutils, text="Isolation Forest", variable = Algoselection, value=3).grid(row=4, column=0, sticky='W', padx=5, pady=2)
1160
uAlgo4 = tk.Radiobutton(uMLutils, text="Local Outlier Factor", variable = Algoselection, value=4).grid(row=4, column=1, sticky='W', padx=5, pady=2)
1161

    
1162
uloadfilelabel = tk.Label(uMLutils, text="Load datafile name (*.txt):")
1163
uloadfilelabel.grid(row=6, column=0, sticky='E', padx=5, pady=2)
1164

    
1165
uloadlocentry = tk.Entry(uMLutils, textvariable=uloadloc)
1166
uloadlocentry.grid(row=6, column=1, sticky='W', padx=5, pady=2)
1167

    
1168
unetbutton = tk.Button(uMLutils, text='Train Predictor',command=uchoosealgo)
1169
unetbutton.grid(row=6, column=2, sticky='E', padx=5, pady=2)
1170

    
1171
usavepredictorlabel = tk.Label(uMLutils, text="Save predictor as (*.pkl):")
1172
usavepredictorlabel.grid(row=7, column=0, sticky='E', padx=5, pady=2)
1173

    
1174
usavepredictorlocentry = tk.Entry(uMLutils, textvariable=usavepredictorloc)
1175
usavepredictorlocentry.grid(row=7, column=1, sticky='W', padx=5, pady=2)
1176

    
1177
usavepredictornetbutton = tk.Button(uMLutils, text='Save Predictor',command=usavepredictor)
1178
usavepredictornetbutton.grid(row=7, column=2, sticky='E', padx=5, pady=2)
1179

    
1180
uloadpredictorlabel = tk.Label(uMLutils, text="Load predictor name (*.pkl):")
1181
uloadpredictorlabel.grid(row=8, column=0, sticky='E', padx=5, pady=2)
1182

    
1183
uloadpredictorlocentry = tk.Entry(uMLutils, textvariable=uloadpredictorloc)
1184
uloadpredictorlocentry.grid(row=8, column=1, sticky='W', padx=5, pady=2)
1185

    
1186
uloadpredictornetbutton = tk.Button(uMLutils, text='Load Predictor',command=uloadpredictor)
1187
uloadpredictornetbutton.grid(row=8, column=2, sticky='E', padx=5, pady=2)
1188

    
1189
urunpredictornetbutton = tk.Button(uMLutils, text='Run Predictor',command=urunningpredictor)
1190
urunpredictornetbutton.grid(row=10, column=1, sticky='E', padx=5, pady=2)
1191

    
1192
ustoppredictornetbutton = tk.Button(uMLutils, text='Stop Predictor',command=ustoppredictor)
1193
ustoppredictornetbutton.grid(row=10, column=2, sticky='E', padx=5, pady=2)
1194

    
1195
###############################################################################
1196

    
1197
Oututils = tk.LabelFrame(GUI, text="Output Utilities", padx=5, pady=5)
1198
Oututils.grid(row=0, column=9, columnspan=2, rowspan=5, \
1199
                sticky='NS', padx=5, pady=5)
1200

    
1201
textbox = tk.Text(master=Oututils)
1202
scr=tk.Scrollbar(Oututils, orient='vertical', command=textbox.yview)
1203
scr.grid(row=2, column=6, rowspan=10 , columnspan=1, sticky='NS')
1204
textbox.grid(row=2, column=5, rowspan=10, columnspan=1, sticky='W')
1205
textbox.config(yscrollcommand=scr.set, font=('Arial', 8, 'bold', 'italic'))
1206

    
1207
#button1.pack()
1208
getparams()
1209
GUI.mainloop()