"""
desispec.desi_dashboard
=======================
"""
import argparse
import os
import fitsio
import astropy.io.fits as pyfits
import subprocess
import pandas as pd
import time
import numpy as np
import psycopg2
import hashlib
import pdb
[docs]class DESI_DASHBOARD(object):
""" Code to generate the statistic of desi_pipe production status
Usage:
python3 desi_dashboard.py --prod realtime10 --prod_dir /global/cscratch1/sd/zhangkai/desi/ --output_dir /global/project/projectdirs/desi/www/users/zhangkai/desi_dashboard/ --output_url https://portal.nersc.gov/project/desi/users/zhangkai/desi_dashboard/
"""
def __init__(self):
############
## Input ###
############
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser = self._init_parser(parser)
args = parser.parse_args()
prod=args.prod #product name
prod_dir=args.prod_dir # base directory of product
self.output_dir=args.output_dir+prod+"/" # Portal directory for output html files
self.output_url=args.output_url+prod+"/" # corresponding URL
######################################################################
## sub directories. Should not change if generated by the same code ##
## that follows the same directory strucure ##
######################################################################
self.data_dir=prod_dir+prod+"/spectro/data/"
self.log_dir=prod_dir+prod+"/spectro/redux/daily/run/logs"
self.redux_dir=prod_dir+prod+"/spectro/redux/daily"
#############
## Setup ####
#############
self.conn=self.get_db_conn(host="nerscdb03.nersc.gov",database="desidev",user="desidev_admin")
self.cur=self.conn.cursor()
self.schema=self._compute_schema(self.redux_dir)
self.tasktype_arr=['preproc','psf','psfnight','traceshift','extract','fiberflat','fiberflatnight','sky','starfit','fluxcalib','cframe','spectra','redshift']
self.tasktype_arr_nonight=['spectra','redshift']
############
# Load data
############
self.file_count=self.count_files()
for tasktype in self.tasktype_arr:
cmd="self.get_table(tasktype='"+tasktype+"')"
exec(cmd)
nights=np.unique(self.df_preproc['night'])
strTable=self._initialize_page()
strTable=strTable+"<button class='regular' id='b1'>Display All Nights</button><button class='regular' id='b2'>Hide All Nights</button>"
strTable=strTable+"<h2>Data Dir: "+self.redux_dir+"</h2>"
#########################
#### Overall Table ######
#########################
table=self._compute_night_statistic("all")
strTable=strTable+self._add_html_table_with_link(table,"Overall")
####################################
#### Table for individual night ####
####################################
for night in nights:
# Create Statistic table for each night
table=self._compute_night_statistic(night)
strTable=strTable+self._add_html_table(table,str(night))
timestamp=time.strftime("%Y-%m-%d %H:%M:%S",time.localtime())
print(timestamp)
strTable=strTable+"<div style='color:#00FF00'>"+timestamp+"</div>"
strTable=strTable+self._add_js_script1()
strTable=strTable+"</html>"
hs=open(self.output_dir+"desi_pipe_dashboard.html",'w')
hs.write(strTable)
hs.close()
##########################
#### Fix Permission ######
##########################
cmd="chmod -R a+xr "+self.output_dir
os.system(cmd)
def _init_parser(self,parser):
parser.add_argument('-p','--prod', type=str, default = None, required = True, help="product name")
parser.add_argument('-pd','--prod_dir', type=str, default = None, required = True, help="product base directory")
parser.add_argument('-od','--output_dir', type=str, default = None, required = True, help="output portal directory for the html pages ")
parser.add_argument('-ou','--output_url', type=str, default = None, required = True, help="output portal directory url ")
return parser
def _initialize_page(self):
#strTable="<html><style> table {font-family: arial, sans-serif;border-collapse: collapse;width: 100%;}"
#strTable=strTable+"td, th {border: 1px solid #dddddd;text-align: left;padding: 8px;}"
#strTable=strTable+"tr:nth-child(even) {background-color: #dddddd;}</style>"
strTable="""<html><style>
h1 {font-family: 'sans-serif';font-size:50px;color:#4CAF50}
#c {font-family: 'Trebuchet MS', Arial, Helvetica, sans-serif;border-collapse: collapse;width: 100%;}
#c td, #c th {border: 1px solid #ddd;padding: 8px;}
#c tr:nth-child(even){background-color: #f2f2f2;}
#c tr:hover {background-color: #ddd;}
#c th {padding-top: 12px; padding-bottom: 12px; text-align: left; background-color: #4CAF50; color: white;}
.collapsible {background-color: #eee;color: #444;cursor: pointer;padding: 18px;width: 100%;border: none;text-align: left;outline: none;font-size: 25px;}
.regular {background-color: #eee;color: #444; cursor: pointer; padding: 18px; width: 25%; border: 18px; text-align: left; outline: none; font-size: 25px;}
.active, .collapsible:hover { background-color: #ccc;}
.content {padding: 0 18px;display: table;overflow: hidden;background-color: #f1f1f1;maxHeight:0px;}
/* The Modal (background) */
.modal {
display: none; /* Hidden by default */
position: fixed; /* Stay in place */
z-index: 1; /* Sit on top */
padding-top: 100px; /* Location of the box */
left: 0;
top: 0;
width: 100%; /* Full width */
height: 90%; /* Full height */
overflow: auto; /* Enable scroll if needed */
background-color: rgb(0,0,0); /* Fallback color */
background-color: rgba(0,0,0,0.4); /* Black w/ opacity */
}
/* Modal Content */
.modal-content {
background-color: #fefefe;
margin: auto;
padding: 20px;
border: 1px solid #888;
width: 80%;
}
/* The Close Button */
.close {
color: #aaaaaa;
float: right;
font-size: 28px;
font-weight: bold;
}
.close:hover,
.close:focus {
color: #000;
text-decoration: none;
cursor: pointer;
}
</style>
<h1>DESI PIPELINE STATUS MONITOR</h1>"""
return strTable
def _add_html_table(self,table,night):
heading="Night "+night
strTable="<button class='collapsible'>"+heading+"</button><div class='content' style='display:inline-block;min-height:0%;'>"
strTable = strTable+"<table id='c'><tr><th>Tasktype</th><th>waiting</th><th>ready</th><th>running</th><th>done</th><th>failed</th><th>submit</th></tr>"
for i in range(len(table)):
n_expected=0
try:
loc=locals()
cmd="n_expected=self.file_count['"+night+"']['"+self.tasktype_arr[i]+"']"
a=exec(cmd)
n_expected=loc['n_expected']
fraction=np.array(float(table[i][3]))/np.array(float(n_expected))
color="green"
if fraction<1.0:
color="red"
str_row="<tr><td>"+self.tasktype_arr[i]+"</td><td>"+str(table[i][0])+"</td><td>"+str(table[i][1])+"</td><td>"+str(table[i][2])+"</td><td>"+str(table[i][3])+"/"+str(n_expected)+" <font color='"+color+"'>("+str(fraction)[0:5]+")</font>"+"</td><td>"+str(table[i][4])+"</td><td>"+str(table[i][5])+"</td></tr>"
strTable=strTable+str_row
except:
pass
strTable=strTable+"</table></div>"
return strTable
def _add_html_table_with_link(self,table,heading):
strTable="<h2>"+heading+"</h2>"
strTable = strTable+"<table id='c'><tr><th>Tasktype</th><th>waiting</th><th>ready</th><th>running</th><th>done</th><th>failed</th><th>submit</th></tr>"
for i in range(len(table)):
tasktype=self.tasktype_arr[i]
n_expected=0
try:
loc=locals()
cmd="n_expected=self.file_count['overall']['"+tasktype+"']"
a=exec(cmd)
n_expected=loc['n_expected']
fraction=np.array(float(table[i][3]))/np.array(float(n_expected))
color="green"
if fraction<1.0:
color="red"
except:
color="red"
if table[i][4]==0:
str_row="<tr><td>"+self.tasktype_arr[i]+"</td><td>"+str(table[i][0])+"</td><td>"+str(table[i][1])+"</td><td>"+str(table[i][2])+"</td><td>"+str(table[i][3])+"/"+str(n_expected)+" <font color='"+color+"'>("+str(fraction)[0:5]+")</font>"+"</td><td>"+str(table[i][4])+"</td><td>"+str(table[i][5])+"</td></tr>"
else: # Add href link here:
str_row="<tr><td>"+self.tasktype_arr[i]+"</td><td>"+str(table[i][0])+"</td><td>"+str(table[i][1])+"</td><td>"+str(table[i][2])+"</td><td>"+str(table[i][3])+"/"+str(n_expected)+" <font color='"+color+"'>("+str(fraction)[0:5]+")</font>"+"</td><td><a href='"+self.output_url+"failed_"+tasktype+"_list.html'><font color='red'>"+str(table[i][4])+"</font></a></td><td>"+str(table[i][5])+"</td></tr>"
loc=locals()
cmd='df = self.df_'+tasktype
exec(cmd)
df=loc['df']
ind=np.where(df['state'] ==4)[0]
##~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Create new html pages to list failed exposures.
# Add Modal 20191007
## ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
strFailed=self._initialize_page()
# Add failed exposure tables
strFailed=strFailed+"<h2>Failed "+tasktype+"</h2><table id='c'><tr><th>Name</th></tr>"
for j in range(len(ind)):
name=str(df['name'][ind[j]])
strFailed=strFailed+"<tr><td>"+name+"</td><td><button id='Btn"+str(j)+"'>Show Log</button></td></tr>"
strFailed=strFailed+"</table>"
# Add modals
for j in range(len(ind)):
name=str(df['name'][ind[j]])
if tasktype == "spectra" or tasktype == "redshift":
parts=name.split('_')
logfile=self.log_dir+'/healpix/'+parts[2][0:3]+'/'+parts[2]+'/'+name+'.log'
else:
parts=name.split('_')
night=parts[1]
logfile=self.log_dir+'/night/'+night+'/'+name+'.log'
try:
f_log=open(logfile,"r")
log=f_log.read()
f_log.close()
except:
log="Can not find log file "+logfile
strFailed=strFailed+"""
<!-- The Modal -->
<div id='modal"""+str(j)+"""' class='modal'>
<!-- Modal content -->
<div class='modal-content'>
<span class='close' id='span"""+str(j)+"""'>×</span>
<p><pre>"""+log+"""</pre></p>
</div>
</div>"""
strFailed=strFailed+self._add_js_script2(len(ind))
hs=open(self.output_dir+"failed_"+tasktype+"_list.html",'w')
hs.write(strFailed)
hs.close()
strTable=strTable+str_row
strTable=strTable+"</table>"
return strTable
def _add_js_script1(self):
s="""<script>
var coll = document.getElementsByClassName('collapsible');
var i;
for (i = 0; i < coll.length; i++) {
coll[i].nextElementSibling.style.maxHeight='0px';
coll[i].addEventListener('click', function() {
this.classList.toggle('active');
var content = this.nextElementSibling;
if (content.style.maxHeight){
content.style.maxHeight = null;
} else {
content.style.maxHeight = '0px';
}
});
};
var b1 = document.getElementById('b1');
b1.addEventListener('click',function() {
for (i = 0; i < coll.length; i++) {
coll[i].nextElementSibling.style.maxHeight=null;
}});
var b2 = document.getElementById('b2');
b2.addEventListener('click',function() {
for (i = 0; i < coll.length; i++) {
coll[i].nextElementSibling.style.maxHeight='0px'
}});
</script>"""
return s
def _add_js_script2(self,n_modal):
s="""<script>"""
for i in range(n_modal):
s=s+"""
var modal"""+str(i)+""" = document.getElementById('modal"""+str(i)+"""');
var l"""+str(i)+""" = document.getElementById('Btn"""+str(i)+"""');
l"""+str(i)+""".addEventListener('click',function() {
modal"""+str(i)+""".style.display = "block";
})
span"""+str(i)+""".addEventListener('click',function() {
modal"""+str(i)+""".style.display = "none";
})"""
s=s+"""</script>"""
return s
def count_files(self):
from os import listdir
nights=listdir(self.data_dir)
output={}
dict_all={'night':'overall','n_arc':0,'n_flat':0,'n_science':0,'preproc':0,'psf':0,'psfnight':0,'traceshift':0,'extract':0,'fiberflat':0,'fiberflatnight':0,'sky':0,'starfit':0,'fluxcalib':0,'cframe':0}
for night in nights:
dict_t={'night':night,'n_arc':0,'n_flat':0,'n_science':0,'preproc':0,'psf':0,'psfnight':0,'traceshift':0,'extract':0,'fiberflat':0,'fiberflatnight':0,'sky':0,'starfit':0,'fluxcalib':0,'cframe':0}
expids=listdir(self.data_dir+'/'+night+'/')
for expid in expids:
filename=self.data_dir+'/'+night+'/'+expid+'/desi-'+expid+'.fits.fz'
h=fitsio.read_header(filename,1)
flavor=h['flavor'].strip()
if flavor=='arc':
dict_t['n_arc']=dict_t['n_arc']+1
elif flavor=='flat':
dict_t['n_flat']=dict_t['n_flat']+1
elif flavor=='science':
dict_t['n_science']=dict_t['n_science']+1
n_arc=dict_t['n_arc']
n_flat=dict_t['n_flat']
n_science=dict_t['n_science']
dict_t['preproc']=30*(n_arc+n_flat+n_science)
dict_t['psf']=30*(n_arc)
dict_t['psfnight']=30
dict_t['traceshift']=30*(n_flat+n_science)
dict_t['extract']=30*(n_flat+n_science)
dict_t['fiberflat']=30*n_flat
dict_t['fiberflatnight']=30
dict_t['sky']=30*n_science
dict_t['starfit']=10*n_science
dict_t['fluxcalib']=30*n_science
dict_t['cframe']=30*n_science
dict_all['n_arc']+=dict_t['n_arc']
dict_all['n_flat']+=dict_t['n_flat']
dict_all['n_science']+=dict_t['n_science']
dict_all['preproc']+=dict_t['preproc']
dict_all['psf']+=dict_t['psf']
dict_all['psfnight']+=dict_t['psfnight']
dict_all['traceshift']+=dict_t['traceshift']
dict_all['extract']+=dict_t['extract']
dict_all['fiberflat']+=dict_t['fiberflat']
dict_all['fiberflatnight']+=dict_t['fiberflatnight']
dict_all['sky']+=dict_t['sky']
dict_all['starfit']+=dict_t['starfit']
dict_all['fluxcalib']+=dict_t['fluxcalib']
dict_all['cframe']+=dict_t['cframe']
cmd="output['"+night+"']=dict_t"
a=exec(cmd)
output['overall']=dict_all
return output
def get_table(self,tasktype=None):
if tasktype=='preproc':
columns=['name','night','band','spec','expid','flavor','state','submitted']
elif tasktype=='psf' or tasktype=='traceshift' or tasktype=='extract' or tasktype=='fiberflat' or tasktype=='sky' or tasktype=='fluxcalib' or tasktype=='cframe':
columns=['name','night','band','spec','expid','state','submitted']
elif tasktype=='psfnight' or tasktype=='fiberflatnight' or tasktype=='starfit':
columns=['name','night','band','spec','state','submitted']
elif tasktype=='spectra' or tasktype=='redshift':
columns=['name','nside','pixel','state','submitted']
self.cur.execute("select * from "+self.schema+"."+tasktype+";")
status=self.cur.fetchall()
cmd="self.df_"+tasktype+"=pd.DataFrame(status,columns=columns)"
exec(cmd)
def _compute_night_statistic(self,night):
n_tasktype=len(self.tasktype_arr)
n_states=5 # waiting, ready, running, done, failed, submit
a=[0]*n_states
output=[a]*n_tasktype
if night=="all":
n_loop=n_tasktype
else:
n_loop=n_tasktype-2
for i in range(n_loop):
df=0
tasktype=self.tasktype_arr[i]
loc=locals()
cmd='df = self.df_'+tasktype
exec(cmd)
df=loc['df']
temp=[]
try:
ind=np.where(df['night']==night)
except:
pass
if night=="all":
df_this=df
else:
try:
df_this=df.iloc[ind[0].tolist()]
except:
print(night)
import pdb;pdb.set_trace()
try:
temp.append(len(np.where(df_this['state'] ==0)[0]))
except:
temp.append(0)
pass
try:
temp.append(len(np.where(df_this['state'] ==1)[0]))
except:
temp.append(0)
pass
try:
temp.append(len(np.where(df_this['state'] ==2)[0]))
except:
temp.append(0)
pass
try:
temp.append(len(np.where(df_this['state'] ==3)[0]))
except:
temp.append(0)
pass
try:
temp.append(len(np.where(df_this['state'] ==4)[0]))
except:
temp.append(0)
pass
try:
temp.append(len(np.where(df_this['submitted'] ==1)[0]))
except:
temp.append(0)
pass
output[i]=temp
return output
def _compute_schema(self,s):
import hashlib
md=hashlib.md5()
md.update(s.encode())
return 'pipe_'+md.hexdigest()
def get_db_conn(self,host=None,database=None,user=None):
conn=psycopg2.connect(host=host,database=database,user=user)
conn.autocommit=True
return conn
if __name__=="__main__":
process=DESI_DASHBOARD()