Update U180-ToolKit.py

AnalyzeU180Data.py

@@ -0,0 +1,32 @@
+import os
+import json
+import time
+import csv
+import GPIBPrologix
+import pandas as pd
+from plotly.subplots import make_subplots
+import plotly.graph_objects as go
+from datetime import datetime
+import numpy
+
+def readRunningConfigs():
+    searchPath = os.path.realpath(os.path.dirname(__file__))+'/RunningConfigs/'
+    filenames = next(os.walk(searchPath), (None, None, []))[2]
+    filenames = [item for item in filenames if str.endswith(item,'.json')]
+    return filenames
+
+for file in readRunningConfigs():
+    dataPath = os.path.realpath(os.path.dirname(__file__))+'/RunningConfigs/'+file
+    with open(dataPath, "r") as read_file:
+        configData = json.load(read_file)
+        serialPath = os.path.realpath(os.path.dirname(__file__))+'/data/'+configData['serial']+'/'
+        df = pd.read_csv(serialPath+configData['serial']+'.csv')
+
+        time_x = pd.to_datetime(df['DateTime'],format='%d-%m-%y %H:%M:%S')
+        coefficients = numpy.polyfit(df['TEMP'], df['G10V'], 1, rcond=None, full=False, w=None, cov=False)
+        polynomial = numpy.poly1d(coefficients)
+
+        fig = make_subplots(rows=1, cols=2)
+        fig.add_trace(go.Scatter(x=time_x, y=df['G10V']-polynomial(df['TEMP']),mode='lines+markers',name='time vs cal72 w tempcomp'),row=1, col=1)
+        fig.add_trace(go.Scatter(x=df['TEMP'], y=df['G10V'],mode='lines+markers',name='temp vs cal72'),row=1, col=2)
+        fig.write_html(serialPath+configData['serial']+'.html')

U180-ToolKit.py

@@ -0,0 +1,168 @@
+#import gpibprologix
+import os
+import json
+import time
+import csv
+import matplotlib.dates as mdates
+import pandas as pd
+from plotly.subplots import make_subplots
+import plotly.graph_objects as go
+import datetime
+import numpy
+import GPIBPrologix
+import git
+import shutil
+import bme280
+import smbus2
+
+GPIB = GPIBPrologix.ResourceManager("/dev/ttyACM0")
+instObj = GPIB.open_resource(22)
+bus = smbus2.SMBus(1)
+calibration_params = bme280.load_calibration_params(bus, 0x76)
+
+## user customisable functions
+def readFunc(instObj):
+    return instObj.read()
+
+def writeFunc(instObj, input):
+    instObj.write(input)
+    time.sleep(1)
+def queryFunc(instObj, input):
+    output = instObj.query(input)
+    time.sleep(1)
+    print(input, output)
+    return output
+
+def getEnvironment(instObj, i2cbus):
+    value = instObj.sample(i2cbus, 0x76, calibration_params)
+    return [str(round(value.humidity,2)),str(round(value.temperature,2)),str(round(value.pressure,2))]
+
+## the main program functionalities, should not be adjusted
+def readRunningConfigs():
+    searchPath = os.path.realpath(os.path.dirname(__file__))+'/RunningConfigs/'
+    filenames = next(os.walk(searchPath), (None, None, []))[2]
+    filenames = [item for item in filenames if str.endswith(item,'.json')]
+    return filenames
+
+def doACAL(dcACAL, ohmACAL, acACAL):
+    writeFunc(instObj,"END 2")
+    writeFunc(instObj,"OFORMAT ASCII")
+    writeFunc(instObj,"BEEP")
+    if dcACAL == "yes":
+        print("Starting DCV cal")
+        writeFunc(instObj, "ACAL DCV")
+        time.sleep(140+60)
+    if acACAL == "yes":
+        print("Starting AC cal")
+        writeFunc(instObj, "ACAL AC")
+        time.sleep(240+60)
+    if ohmACAL == "yes":
+        print("Starting Ohm cal")
+        writeFunc(instObj, "ACAL OHMS")
+        time.sleep(720+60)
+    writeFunc(instObj, "DISP OFF,  ;")
+    writeFunc(instObj,"DISP MSG,\"             \"")
+    return 0
+
+def interrogate3458A(instObj):
+    output = []
+    d = datetime.datetime.now()
+    dx = d - datetime.timedelta(microseconds=d.microsecond)
+    output.append(dx.strftime("%d-%m-%y %H:%M:%S"))
+    output.append(queryFunc(instObj,"TEMP?"))    # get temperature in device
+    output.append(queryFunc(instObj,"CAL? 1,1")) # get RREF cal value
+    output.append(queryFunc(instObj,"CAL? 2,1")) # get VREF cal value
+    output.append(queryFunc(instObj,"CAL? 78"))  # get 10kohm ACAL gain constrant
+    output.append(queryFunc(instObj,"CAL? 79"))  # get 100kohm ACAL gain constrant
+    output.append(queryFunc(instObj,"CAL? 71"))  # get 1v0 ACAL gain constrant
+    output.append(queryFunc(instObj,"CAL? 70"))  # get 0v1 ACAL gain constrant
+    output.append(queryFunc(instObj,"CAL? 86"))  # get 1kohm ACAL ocomp constrant
+    output.append(queryFunc(instObj,"CAL? 87"))  # get 10kohm ACAL ocomp constrant
+    output.append(queryFunc(instObj,"CAL? 176")) # get acal temperature for ohms
+    output.append(queryFunc(instObj,"CAL? 59"))  # get temperature from during calibration
+    output.append(queryFunc(instObj,"CAL? 97"))  # get 1mamp ACAL gain constrant
+    output.append(queryFunc(instObj,"CAL? 72"))  # get 10v ACAL gain constrant
+    print(output)
+    return output
+
+## main program
+#  read the config file and do the folder setup
+for file in readRunningConfigs():
+    dataPath = os.path.realpath(os.path.dirname(__file__))+'/RunningConfigs/'+file
+    with open(dataPath, "r") as read_file:
+        configData = json.load(read_file)
+        serialPath = os.path.realpath(os.path.dirname(__file__))+'/data/'+configData['serial']+'/'
+        if not os.path.exists(serialPath): 
+            os.makedirs(serialPath)
+            with open(serialPath+configData['serial']+'.csv', 'a') as g:
+                heading = ["DateTime","TEMP","CAL RREF","CAL VREF","G10K","G100K", "G1V0","G0V1","OCOMP1K","OCOMP10K", "ACALTEMP", "CALTEMP", "G1mA", "G10V"]
+                if(configData['useBME']):
+                    heading = output.append(['EnvHumidity', 'EnvTemp', 'EnvPressure'])
+                writer = csv.writer(g)
+                writer.writerow(heading)
+        doACAL(configData['ACAL-DCV'],configData['ACAL-OHMS'],configData['ACAL-ACV'])
+        with open(serialPath+configData['serial']+'.csv', 'a') as f:
+            output = interrogate3458A(instObj)
+            if(configData['useBME']):
+                data = getEnvironment(bme280, bus)
+                output += data
+            writer = csv.writer(f)
+            writer.writerow(output)
+
+## do the plotting
+for file in readRunningConfigs():
+    dataPath = os.path.realpath(os.path.dirname(__file__))+'/RunningConfigs/'+file
+    with open(dataPath, "r") as read_file:
+        configData = json.load(read_file)
+        serialPath = os.path.realpath(os.path.dirname(__file__))+'/data/'+configData['serial']+'/'
+        df = pd.read_csv(serialPath+configData['serial']+'.csv',delimiter=',', encoding="utf-8-sig")
+        df.columns = df.columns.str.strip()
+
+        time_x = pd.to_datetime(df['DateTime'],format='%d-%m-%y %H:%M:%S')
+        coefficients = numpy.polyfit(df['TEMP'], df['G10V'], 1, rcond=None, full=False, w=None, cov=False)
+        polynomial = numpy.poly1d(coefficients)
+        #polynom_estimate = numpy.polyfit(mdates.date2num(time_x),df['G10V']-polynomial(df['TEMP']), 1, rcond=None, full=False, w=None, cov=False)
+        polynom_estimate = numpy.polyfit(mdates.date2num(time_x),df['G10V'], 1, rcond=None, full=False, w=None, cov=False)
+        p_est = numpy.poly1d(polynom_estimate)
+
+        fig = make_subplots(rows=1, cols=2)
+        #fig.add_trace(go.Scatter(x=time_x, y=df['G10V']-polynomial(df['TEMP']),mode='lines+markers',name='time vs cal72 w tempcomp'),row=1, col=1)
+        fig.add_trace(go.Scatter(x=time_x, y=df['G10V'],mode='lines+markers',name='time vs cal72 w tempcomp'),row=1, col=1)
+        fig.add_trace(go.Scatter(x=time_x, y=p_est(mdates.date2num(time_x)),mode='lines',line = dict(color='gray', dash='dash'),name='Estimated 1st order'), row=1, col=1)
+        fig.add_trace(go.Scatter(x=df['TEMP'], y=df['G10V'],mode='lines+markers',name='temp vs cal72'),row=1, col=2)
+        fig.add_trace(go.Scatter(x=df['TEMP'], y=polynomial(df['TEMP']),mode='lines',line = dict(color='gray', dash='dash'),name='Tempco 1st order'), row=1, col=2)
+
+        datenow = datetime.datetime.now()
+        date = datenow + datetime.timedelta(days=1)
+        timeframe = (time_x.max()-time_x.min()).total_seconds()/(3600*24)
+        alphaday = (p_est(mdates.date2num(date))-p_est(mdates.date2num(datenow)))
+        driftrate = round((alphaday*1000000000)/(df.loc[:, 'G10V'].mean()*timeframe),2)
+        annotation = f'Estimated drift:  {driftrate} ppb/day'
+        fig.add_annotation(dict(showarrow=False, text=annotation, xanchor='left', xref="paper", yref="paper", x=0, y=0))
+        tempcorate = round(((polynomial(1)-polynomial(0))*1000000000)/df.loc[:, 'G10V'].mean(),2)
+        annotation = f'Estimated tempco:  {tempcorate} ppb/degC'
+        fig.add_annotation(dict(showarrow=False, text=annotation, xanchor='left', xref="paper", yref="paper", x=1, y=0))
+
+
+        fig.write_html(serialPath+configData['serial']+'.html')
+        fig.write_image(serialPath+configData['serial']+'.png',width=1280, height=720)
+
+## do the saving
+for file in readRunningConfigs():
+    localDataPath = os.path.realpath(os.path.dirname(__file__))+'/data/'
+    gitDataRepo = os.path.realpath(os.path.dirname(__file__))+'/CommitDataRepo/'
+    with open(dataPath, "r") as read_file:
+        configData = json.load(read_file)
+        if(configData['useGit']):
+            remote = f"https://{configData['gitUser']}:{configData['gitPassword']}@{configData['gitAddress']}"
+            if os.path.exists(gitDataRepo):
+                shutil.rmtree(gitDataRepo, ignore_errors=True)
+            git.Repo.clone_from(remote, gitDataRepo, b=configData['gitBranch'])
+            shutil.copytree(localDataPath, gitDataRepo+'data/',dirs_exist_ok=True)
+            repo = git.Repo(gitDataRepo)
+            repo.git.add(gitDataRepo)
+            repo.index.commit("Update")
+            repo.remotes[0].push()
+
+## if all ok, blamk screen to save VFD
+queryFunc(instObj, 'DISP OFF,""')

data/2823A14043/2823A14043.csv

@@ -1,25 +0,0 @@
-DateTime, TEMP, CALRREF, CALVREF, G10K, G100K, G1V0, G0V1, OCOMP1K, OCOMP10K, ACALTEMP, CALTEMP, G1mA, G10V, EnvHum, EnvTemp, EnvPres
-09-09-24 23:13:47,37.6, 39.9969936E+03, 7.08255474E+00, 1.00332823E+00, 1.00307763E+00, 986.542173E-03, 986.996064E-03, 1.00272787E+00, 1.00332835E+00, 37.5403889E+00, 37.4595735E+00, 986.624479E-03, 986.756649E-03,55.56,25.21,1008.94
-09-09-24 23:38:10,37.7, 39.9969936E+03, 7.08255474E+00, 1.00332853E+00, 1.00307783E+00, 986.542069E-03, 986.996038E-03, 1.00272798E+00, 1.00332866E+00, 37.6689424E+00, 37.4595735E+00, 986.625104E-03, 986.756514E-03,55.1,24.87,1008.44
-10-09-24 00:16:32,37.8, 39.9969936E+03, 7.08255474E+00, 1.00332864E+00, 1.00307787E+00, 986.542014E-03, 986.995956E-03, 1.00272773E+00, 1.00332877E+00, 37.7193828E+00, 37.4595735E+00, 986.626194E-03, 986.756503E-03,54.78,25.76,1010.03
-10-09-24 00:33:53,37.8, 39.9969936E+03, 7.08255474E+00, 1.00332872E+00, 1.00307797E+00, 986.542032E-03, 986.995825E-03, 1.00272784E+00, 1.00332884E+00, 37.7138974E+00, 37.4595735E+00, 986.626118E-03, 986.756431E-03,54.62,24.79,1008.75
-10-09-24 02:16:32,37.3, 39.9969936E+03, 7.08255474E+00, 1.00332869E+00, 1.00307773E+00, 986.542017E-03, 986.995664E-03, 1.00272784E+00, 1.00332883E+00, 37.3048018E+00, 37.4595735E+00, 986.625955E-03, 986.756457E-03,54.6,25.38,1010.2
-10-09-24 04:16:29,37.2, 39.9969936E+03, 7.08255474E+00, 1.00332869E+00, 1.00307790E+00, 986.541997E-03, 986.995635E-03, 1.00272765E+00, 1.00332882E+00, 37.1218449E+00, 37.4595735E+00, 986.626177E-03, 986.756570E-03,54.02,25.19,1010.03
-10-09-24 06:16:30,36.9, 39.9969936E+03, 7.08255474E+00, 1.00332883E+00, 1.00307811E+00, 986.542104E-03, 986.995570E-03, 1.00272764E+00, 1.00332896E+00, 36.8904663E+00, 37.4595735E+00, 986.626584E-03, 986.756773E-03,52.04,25.02,1009.89
-10-09-24 16:16:32,35.7, 39.9969936E+03, 7.08255474E+00, 1.00332981E+00, 1.00307909E+00, 986.542886E-03, 986.995996E-03, 1.00272645E+00, 1.00332992E+00, 35.5547899E+00, 37.4595735E+00, 986.620363E-03, 986.757775E-03,48.68,24.88,1006.36
-10-09-24 18:16:30,36.6, 39.9969936E+03, 7.08255474E+00, 1.00333038E+00, 1.00307984E+00, 986.542267E-03, 986.995886E-03, 1.00272724E+00, 1.00333051E+00, 36.5370863E+00, 37.4595735E+00, 986.625668E-03, 986.757308E-03,49.86,24.6,1005.2
-10-09-24 20:16:30,36.2, 39.9969936E+03, 7.08255474E+00, 1.00332956E+00, 1.00307913E+00, 986.542065E-03, 986.995399E-03, 1.00272636E+00, 1.00332969E+00, 36.1491491E+00, 37.4595735E+00, 986.625923E-03, 986.757248E-03,51.95,23.8,1004.6
-10-09-24 22:16:29,36.7, 39.9969936E+03, 7.08255474E+00, 1.00333021E+00, 1.00308009E+00, 986.541805E-03, 986.995665E-03, 1.00272658E+00, 1.00333034E+00, 36.6455949E+00, 37.4595735E+00, 986.627099E-03, 986.757151E-03,51.99,24.7,1003.05
-11-09-24 00:16:30,36.7, 39.9969936E+03, 7.08255474E+00, 1.00333026E+00, 1.00308015E+00, 986.541992E-03, 986.995727E-03, 1.00272686E+00, 1.00333039E+00, 36.6793016E+00, 37.4595735E+00, 986.627403E-03, 986.757209E-03,52.15,24.53,1000.47
-11-09-24 02:16:29,36.7, 39.9969936E+03, 7.08255474E+00, 1.00333014E+00, 1.00308020E+00, 986.541789E-03, 986.995423E-03, 1.00272650E+00, 1.00333027E+00, 36.6576869E+00, 37.4595735E+00, 986.627799E-03, 986.757174E-03,51.15,24.66,1000.99
-11-09-24 04:16:30,36.6, 39.9969936E+03, 7.08255474E+00, 1.00333001E+00, 1.00308002E+00, 986.541740E-03, 986.995073E-03, 1.00272658E+00, 1.00333015E+00, 36.5383552E+00, 37.4595735E+00, 986.627592E-03, 986.757203E-03,51.2,24.54,1002.46
-11-09-24 06:16:29,36.4, 39.9969936E+03, 7.08255474E+00, 1.00332978E+00, 1.00307965E+00, 986.541739E-03, 986.995139E-03, 1.00272611E+00, 1.00332992E+00, 36.3297243E+00, 37.4595735E+00, 986.627635E-03, 986.757236E-03,50.71,24.39,1003.33
-11-09-24 08:16:29,36.4, 39.9969936E+03, 7.08255474E+00, 1.00332989E+00, 1.00307995E+00, 986.541724E-03, 986.995147E-03, 1.00272633E+00, 1.00333003E+00, 36.3938184E+00, 37.4595735E+00, 986.627326E-03, 986.757322E-03,52.74,24.58,1004.06
-14-09-24 12:16:33,33.6, 39.9969936E+03, 7.08255474E+00, 1.00332778E+00, 1.00307699E+00, 986.542908E-03, 986.996100E-03, 1.00272282E+00, 1.00332789E+00, 33.5552661E+00, 37.4595735E+00, 986.617013E-03, 986.758379E-03,34.04,22.2,1028.35
-14-09-24 14:16:30,34.7, 39.9969936E+03, 7.08255474E+00, 1.00332905E+00, 1.00307866E+00, 986.542358E-03, 986.995486E-03, 1.00272370E+00, 1.00332916E+00, 34.5979829E+00, 37.4595735E+00, 986.622020E-03, 986.757914E-03,35.55,23.19,1027.89
-14-09-24 16:16:30,35.1, 39.9969936E+03, 7.08255474E+00, 1.00332929E+00, 1.00307902E+00, 986.542230E-03, 986.995456E-03, 1.00272364E+00, 1.00332941E+00, 35.0250820E+00, 37.4595735E+00, 986.624721E-03, 986.757707E-03,38.61,23.38,1027.46
-14-09-24 18:16:30,35.0, 39.9969936E+03, 7.08255474E+00, 1.00332916E+00, 1.00307890E+00, 986.542049E-03, 986.995304E-03, 1.00272374E+00, 1.00332928E+00, 34.9938567E+00, 37.4595735E+00, 986.625454E-03, 986.757546E-03,40.32,23.09,1027.2
-14-09-24 20:16:30,34.7, 39.9969936E+03, 7.08255474E+00, 1.00332873E+00, 1.00307836E+00, 986.541916E-03, 986.994884E-03, 1.00272299E+00, 1.00332885E+00, 34.6883306E+00, 37.4595735E+00, 986.625673E-03, 986.757459E-03,41.43,22.85,1027.46
-14-09-24 22:16:30,34.7, 39.9969936E+03, 7.08255474E+00, 1.00332872E+00, 1.00307841E+00, 986.541912E-03, 986.994503E-03, 1.00272312E+00, 1.00332884E+00, 34.6547834E+00, 37.4595735E+00, 986.625099E-03, 986.757454E-03,42.93,22.82,1027.17
-15-09-24 00:16:29,34.8, 39.9969936E+03, 7.08255474E+00, 1.00332862E+00, 1.00307856E+00, 986.541749E-03, 986.994479E-03, 1.00272287E+00, 1.00332873E+00, 34.8045795E+00, 37.4595735E+00, 986.625672E-03, 986.757460E-03,43.95,22.73,1026.82
-15-09-24 02:16:30,34.8, 39.9969936E+03, 7.08255474E+00, 1.00332881E+00, 1.00307871E+00, 986.541837E-03, 986.994637E-03, 1.00272313E+00, 1.00332893E+00, 34.8110347E+00, 37.4595735E+00, 986.626162E-03, 986.757464E-03,43.96,22.74,1026.6

data/ABCDEFGHIKLMNOPQRSTUVXYZ/ABCDEFGHIKLMNOPQRSTUVXYZ.csv

@@ -1,5 +0,0 @@
-DateTime,TEMP,CAL RREF,CAL VREF,G10K,G100K,G1V0,G0V1,OCOMP1K,OCOMP10K,ACALTEMP,CALTEMP,G1mA,G10V
-32.1, 6.090536316E-07, 39.9962225E+03, 8.097244814E-07, 7.08619112E+00, 4.048622407E-07, 1.00329203E+00, 6.354576908E-07, 1.00303954E+00, 5.439236190E-07, 987.039509E-03, 3.819787227E-07
-32.5,-3.851803980E-01,-3.896509499E-01,34.5,-6.778178235E-01,-6.903253579E-01, 39.9962225E+03,-7.241444465E-01, 7.08619112E+00,-7.516644467E-01, 1.00329991E+00,-7.764174681E-01, 1.00304798E+00
--9.821717549E-01,-1.003709328E+00,36.3,-6.775814974E-01,-6.909099962E-01, 39.9962225E+03,-7.267166260E-01, 7.08619112E+00,-7.553570378E-01, 1.00330024E+00,-7.808196047E-01, 1.00304872E+00
-