Project

General

Profile

RE: converting .csv to .nc to perform time-based statistics ยป kyle.py

Ralf Mueller, 2020-05-06 09:30

 
from cdo import Cdo
import sys
import glob
from multiprocessing import Pool

NTASKS = 6
cdo = Cdo()
ifile = sys.argv[1]
print("ifile = "+ifile)
inputs = [line.rstrip().split(' ') for line in open(ifile).readlines()]

def process_input_line(input):
year, month, day, hour, *values = input
month = month.rjust(2,'0')
day = day.rjust(2,'0')
hour = hour.rjust(2,'0')
print("##############################:%s-%s-%sT%s:00:00"%(year,month,day,hour))
outputFiles = []
for index,value in enumerate(values):
ofile = cdo.settaxis('{year}-{month}-{day},{hour}:00:00,1hours'.format(**locals()),
input= '-setname,var{index} -const,{value},r1x1'.format(**locals()),
output= 'var{index}_{year}-{month}-{day}_{hour}.nc'.format(**locals()),
options= '-f nc',debug=True)
outputFiles.append(ofile)
return outputFiles

pool = Pool(NTASKS)
results = dict()
for input in inputs:
ofiles = pool.apply_async(process_input_line, ([input]))
results[''.join(input)] = ofiles
pool.close()
pool.join()


for i in range(0,6):
inputFiles = sorted(glob.glob("var{0}_*.nc".format(i)))
cdo.cat(input= ' '.join(inputFiles), output= "VAR{0}.nc".format(i))

# cleanup with:
# rm var*nc
    (1-1/1)