"""
desispec.desi_proc_time_distribution
====================================
"""
import argparse
import os,glob
import astropy
from astropy.io import fits
from astropy.table import QTable,vstack
import pandas as pd
import time,datetime
import numpy as np
from os import listdir
import matplotlib.pyplot as plt
#import desispec.io as desi_io
[docs]class DESI_PROC_TIME_DISTRIBUTION(object):
""" Code to generate a webpage for monitoring of desi_dailyproc production status
Normal Mode::
desi_proc_time_distribution -n all --n_night 3 --output_dir /global/project/projectdirs/desi/www/users/zhangkai/desi_proc_dashboard/ --output_url https://portal.nersc.gov/project/desi/users/zhangkai/desi_proc_dashboard/
python3 desi_proc_time_distribution.py -n all --n_night 3 --output_dir /global/project/projectdirs/desi/www/users/zhangkai/desi_proc_dashboard/ --output_url https://portal.nersc.gov/project/desi/users/zhangkai/desi_proc_dashboard/
Cron job script::
*/30 * * * * /global/common/software/desi/cori/desiconda/20190804-1.3.0-spec/conda/bin/python3 /global/project/projectdirs/desi/users/zhangkai/desi/code/desispec/py/desispec/workflow/proc_dashboard_funcs.py -n all --n_night 30 --output_dir /global/project/projectdirs/desi/www/users/zhangkai/desi_proc_dashboard/ --output_url https://portal.nersc.gov/project/desi/users/zhangkai/desi_proc_dashboard/ >/global/project/projectdirs/desi/users/zhangkai/desi_proc_dashboard.log 2>/global/project/projectdirs/desi/users/zhangkai/desi_proc_dashboard.err &
"""
def __init__(self):
if not os.getenv('DESI_SPECTRO_REDUX'): # these are not set by default in cronjob mode.
os.environ['DESI_SPECTRO_REDUX']='/global/cfs/cdirs/desi/spectro/redux/'
os.environ['DESI_SPECTRO_DATA']='/global/cfs/cdirs/desi/spectro/data/'
os.environ['SPECPROD']='daily'
############
## Input ###
############
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser = self._init_parser(parser)
args = parser.parse_args()
self.prod_dir=args.prod_dir
self.output_dir=args.output_dir # Portal directory for output html files
self.output_url=args.output_url # corresponding URL
if args.nights=='all':
nights=listdir(os.path.join(args.prod_dir,'run','scripts','night'))
print(nights)
nights=[int(x) for x in nights]
else:
try:
loc=locals()
cmd='nights='+args.nights
a=exec(cmd)
nights=loc['nights']
except:
nights=[]
print('Searching '+self.prod_dir)
tonight=self.what_night_is_it()
if not tonight in nights:
nights.append(tonight)
nights.sort(reverse=True)
if int(args.n_night)<=len(nights):
nights=nights[0:int(args.n_night)-1]
######################################################################
## sub directories. Should not change if generated by the same code ##
## that follows the same directory strucure ##
######################################################################
strTable=self._initialize_page()
table_all=QTable([[],[],[],[],[]],names=('night','flavor','jobid','expid','time'),dtype=('S10','S10','S10','S10','float'))
for night in nights:
print(night)
stat=self.calculate_one_night(night)
print(stat)
table_all=vstack([table_all,stat])
ind_arc = np.where(table_all['flavor']=='arc')
ind_flat = np.where(table_all['flavor']=='flat')
ind_science = np.where(table_all['flavor']=='science')
time_arc=table_all[ind_arc]['time']
time_flat=table_all[ind_flat]['time']
time_science=table_all[ind_science]['time']
plt.figure(0,figsize=(20,6))
font = {'family' : 'sans-serif',
'weight' : 'normal',
'size' : 10}
plt.rc('font', **font)
plt.subplot(131)
plt.hist(time_arc,bins=20)
plt.xlabel('Time to reduce arcs')
plt.ylabel('N')
plt.subplot(132)
plt.hist(time_flat,bins=20)
plt.xlabel('Time to reduce flats')
plt.ylabel('N')
plt.subplot(133)
plt.hist(time_science,bins=20)
plt.xlabel('Time to reduce sciences')
plt.ylabel('N')
plt.show()
import pdb;pdb.set_trace()
timestamp=time.strftime("%Y-%m-%d %H:%M:%S",time.localtime())
#print(timestamp)
running=self.check_running()
strTable=strTable+"<div style='color:#00FF00'>"+timestamp+" "+"desi_dailyproc running: "+running+"</div>"
strTable=strTable+self._add_js_script1()
strTable=strTable+"</html>"
hs=open(os.path.join(self.output_dir,"desi_proc_dashboard.html"),'w')
hs.write(strTable)
hs.close()
##########################
#### Fix Permission ######
##########################
cmd="chmod -R a+xr "+self.output_dir
os.system(cmd)
[docs] def _init_parser(self,parser):
"""
Initialize the parser to read input
"""
parser.add_argument('-n','--nights', type=str, default = None, required = False, help="nights to monitor")
parser.add_argument('--n_night', type=str, default = None, required = False, help="all:all nights. ifdigit: the last n nights.")
parser.add_argument('--prod_dir', type=str, default = os.path.join(os.environ['DESI_SPECTRO_REDUX'],os.environ['SPECPROD']), required = False, help="Product directory, point to desispec.io.specprod_root() by default ")
parser.add_argument('--output_dir', type=str, default = None, required = True, help="output portal directory for the html pages ")
parser.add_argument('--output_url', type=str, default = None, required = True, help="output portal directory url ")
return parser
[docs] def calculate_one_night(self,night):
"""
For a given night, return the file counts and other other information for each exposure taken on that night
Args:
night
Returns:
A dictionary containing the statistics with expid as key name::
FLAVOR: FLAVOR of this exposure
OBSTYPE: OBSTYPE of this exposure
EXPTIME: Exposure time
SPECTROGRAPHS: a list of spectrographs used
n_spectrographs: number of spectrographs
n_psf: number of PSF files
n_ff: number of fiberflat files
n_frame: number of frame files
n_sframe: number of sframe files
n_cframe: number of cframe files
n_sky: number of sky files
"""
output_arc={}
output_flat={}
output_science={}
fileglob_arc=os.path.join(self.prod_dir,'run/scripts/night',str(night),'arc*.log')
fileglob_flat=os.path.join(self.prod_dir,'run/scripts/night',str(night),'flat*.log')
fileglob_science=os.path.join(self.prod_dir,'run/scripts/night',str(night),'science*.log')
file_arc=sorted(glob.glob(fileglob_arc))
file_flat=sorted(glob.glob(fileglob_flat))
file_science=sorted(glob.glob(fileglob_science))
table_output=QTable([[],[],[],[],[]],names=('night','flavor','jobid','expid','time'),dtype=('S10','S10','S10','S10','float'))
for file_this in file_arc:
jobid_this=file_this.split('.')[0].split('-')[-1]
expid_this=file_this.split('-')[2]
result=os.popen('sacct -j '+jobid_this+' --format=Elapsed').read()
time=result.split('\n')[-2].split(':')
time_this=float(time[0])*60.+float(time[1])+float(time[2])/60.
table_output.add_row([night,'arc',jobid_this,expid_this,time_this])
#output_arc[jobid_this]={'expid':expid_this,'time':time_this}
for file_this in file_flat:
jobid_this=file_this.split('.')[0].split('-')[-1]
expid_this=file_this.split('-')[2]
result=os.popen('sacct -j '+jobid_this+' --format=Elapsed').read()
time=result.split('\n')[-2].split(':')
time_this=float(time[0])*60.+float(time[1])+float(time[2])/60.
table_output.add_row([night,'flat',jobid_this,expid_this,time_this])
#output_flat[jobid_this]={'expid':expid_this,'time':time_this}
for file_this in file_science:
jobid_this=file_this.split('.')[0].split('-')[-1]
expid_this=file_this.split('-')[2]
result=os.popen('sacct -j '+jobid_this+' --format=Elapsed').read()
time=result.split('\n')[-2].split(':')
time_this=float(time[0])*60.+float(time[1])+float(time[2])/60.
table_output.add_row([night,'science',jobid_this,expid_this,time_this])
#output_science[jobid_this]={'expid':expid_this,'time':time_this}
return(table_output)
[docs] def _initialize_page(self):
"""
Initialize the html file for showing the statistics, giving all the headers and CSS setups.
"""
#strTable="<html><style> table {font-family: arial, sans-serif;border-collapse: collapse;width: 100%;}"
#strTable=strTable+"td, th {border: 1px solid #dddddd;text-align: left;padding: 8px;}"
#strTable=strTable+"tr:nth-child(even) {background-color: #dddddd;}</style>"
strTable="""<html><style>
h1 {font-family: 'sans-serif';font-size:50px;color:#4CAF50}
#c {font-family: 'Trebuchet MS', Arial, Helvetica, sans-serif;border-collapse: collapse;width: 100%;}
#c td, #c th {border: 1px solid #ddd;padding: 8px;}
#c tr:nth-child(even){background-color: #f2f2f2;}
#c tr:hover {background-color: #ddd;}
#c th {padding-top: 12px; padding-bottom: 12px; text-align: left; background-color: #4CAF50; color: white;}
.collapsible {background-color: #eee;color: #444;cursor: pointer;padding: 18px;width: 100%;border: none;text-align: left;outline: none;font-size: 25px;}
.regular {background-color: #eee;color: #444; cursor: pointer; padding: 18px; width: 25%; border: 18px; text-align: left; outline: none; font-size: 25px;}
.active, .collapsible:hover { background-color: #ccc;}
.content {padding: 0 18px;display: table;overflow: hidden;background-color: #f1f1f1;maxHeight:0px;}
/* The Modal (background) */
.modal {
display: none; /* Hidden by default */
position: fixed; /* Stay in place */
z-index: 1; /* Sit on top */
padding-top: 100px; /* Location of the box */
left: 0;
top: 0;
width: 100%; /* Full width */
height: 90%; /* Full height */
overflow: auto; /* Enable scroll if needed */
background-color: rgb(0,0,0); /* Fallback color */
background-color: rgba(0,0,0,0.4); /* Black w/ opacity */
}
/* Modal Content */
.modal-content {
background-color: #fefefe;
margin: auto;
padding: 20px;
border: 1px solid #888;
width: 80%;
}
/* The Close Button */
.close {
color: #aaaaaa;
float: right;
font-size: 28px;
font-weight: bold;
}
.close:hover,
.close:focus {
color: #000;
text-decoration: none;
cursor: pointer;
}
</style>
<h1>DESI PROC STATUS MONITOR</h1>"""
return strTable
[docs] def _add_html_table(self,table,night):
"""
Add a collapsible and extendable table to the html file for one specific night
Args:
table: the table generated by 'calculate_one_night'
night: like 20200131
Returns:
The string to be added to the html file
"""
heading="Night "+night
strTable="<button class='collapsible'>"+heading+"</button><div class='content' style='display:inline-block;min-height:0%;'>"
strTable = strTable+"<table id='c'><tr><th>Expid</th><th>FLAVOR</th><th>OBSTYPE</th><th>EXPTIME</th><th>SPECTROGRAGHS</th><th>PSF File</th><th>FFlat file</th><th>frame file</th><th>sframe file</th><th>sky file</th><th>cframe file</th></tr>"
for i in range(len(table)):
expid=list(table.keys())[i]
n_expected=0
if True:
obstype=str(table[expid]['OBSTYPE']).upper().strip()
n_spectrographs=int(table[expid]['n_spectrographs'])
if obstype=='ZERO':
n_ref=[str(n_spectrographs*0),str(n_spectrographs*0),str(n_spectrographs*0),str(n_spectrographs*0),str(n_spectrographs*0),str(n_spectrographs*0)]
elif obstype=='ARC':
n_ref=[str(n_spectrographs*3),str(n_spectrographs*0),str(n_spectrographs*0),str(n_spectrographs*0),str(n_spectrographs*0),str(n_spectrographs*0)]
elif obstype=='FLAT':
n_ref=[str(n_spectrographs*3),str(n_spectrographs*3),str(n_spectrographs*3),str(n_spectrographs*0),str(n_spectrographs*0),str(n_spectrographs*0)]
elif obstype=='SKY' or obstype=='SCIENCE' or obstype=='NONE':
n_ref=[str(n_spectrographs*3),str(n_spectrographs*0),str(n_spectrographs*3),str(n_spectrographs*3),str(n_spectrographs*3),str(n_spectrographs*3)]
elif obstype=='TWILIGHT':
n_ref=[str(n_spectrographs*3),str(n_spectrographs*0),str(n_spectrographs*3),str(n_spectrographs*0),str(n_spectrographs*0),str(n_spectrographs*0)]
else:
n_ref=[str(n_spectrographs*0),str(n_spectrographs*0),str(n_spectrographs*0),str(n_spectrographs*0),str(n_spectrographs*0),str(n_spectrographs*0)]
color="green"
str_row="<tr><td>"+expid+"</td><td>"+str(table[expid]['FLAVOR'])+"</td><td>"+str(table[expid]['OBSTYPE'])+"</td><td>"+str(table[expid]['EXPTIME'])+"</td><td>"+table[expid]['SPECTROGRAPHS']+"</td><td>"+str(table[expid]['n_psf'])+'/'+n_ref[0]+"</td><td>"+str(table[expid]['n_ff'])+'/'+n_ref[1]+"</td><td>"+str(table[expid]['n_frame'])+'/'+n_ref[2]+"</td><td>"+str(table[expid]['n_sframe'])+'/'+n_ref[3]+"</td><td>"+str(table[expid]['n_sky'])+'/'+n_ref[4]+"</td><td>"+str(table[expid]['n_cframe'])+'/'+n_ref[5]+"</td></tr>"
strTable=strTable+str_row
else:
pass
strTable=strTable+"</table></div>"
return strTable
[docs] def _add_js_script1(self):
"""
Return the javascript script to be added to the html file
"""
s="""<script>
var coll = document.getElementsByClassName('collapsible');
var i;
for (i = 0; i < coll.length; i++) {
coll[i].nextElementSibling.style.maxHeight='0px';
coll[i].addEventListener('click', function() {
this.classList.toggle('active');
var content = this.nextElementSibling;
if (content.style.maxHeight){
content.style.maxHeight = null;
} else {
content.style.maxHeight = '0px';
}
});
};
var b1 = document.getElementById('b1');
b1.addEventListener('click',function() {
for (i = 0; i < coll.length; i++) {
coll[i].nextElementSibling.style.maxHeight=null;
}});
var b2 = document.getElementById('b2');
b2.addEventListener('click',function() {
for (i = 0; i < coll.length; i++) {
coll[i].nextElementSibling.style.maxHeight='0px'
}});
</script>"""
return s
[docs] def _add_js_script2(self,n_modal):
"""
Return the another javascript script to be added to the html file
"""
s="""<script>"""
for i in range(n_modal):
s=s+"""
var modal"""+str(i)+""" = document.getElementById('modal"""+str(i)+"""');
var l"""+str(i)+""" = document.getElementById('Btn"""+str(i)+"""');
l"""+str(i)+""".addEventListener('click',function() {
modal"""+str(i)+""".style.display = "block";
})
span"""+str(i)+""".addEventListener('click',function() {
modal"""+str(i)+""".style.display = "none";
})"""
s=s+"""</script>"""
return s
[docs] def what_night_is_it(self):
"""
Return the current night
"""
d = datetime.datetime.utcnow() - datetime.timedelta(7/24+0.5)
tonight = int(d.strftime('%Y%m%d'))
return tonight
[docs] def find_newexp(self,night, fileglob, known_exposures):
"""
Check the path given for new exposures
"""
datafiles = sorted(glob.glob(fileglob))
newexp = list()
for filepath in datafiles:
expid = int(os.path.basename(os.path.dirname(filepath)))
if (night, expid) not in known_exposures:
newexp.append( (night, expid) )
return set(newexp)
[docs] def check_running(self):
"""
Check if the desi_dailyproc process is running
"""
import psutil
a=psutil.process_iter()
running='No'
for p in a:
if 'desi_dailyproc' in ' '.join(p.cmdline()):
running='Yes'
return running
if __name__=="__main__":
process=DESI_PROC_TIME_DISTRIBUTION()