GeezerVison

import numpy as np

import matplotlib.pyplot as plt

import datetime as dt

import os, sys, json, re

import os, sys, json, re, shutil

import multiprocessing as mp

from copy import copy

def __init__(self, working_dir, stage):

self.working_dir = working_dir

self.stage = stage

self.metadata_json_paths = self._get_optr_jsons(working_dir, stage)

self.metadata_json_paths = self._get_optr_jsons(working_dir, stage)

self.total_sims = self._count_total_sims()

self.statvar_name = self._get_statvar_name(stage)

self.measured = self._get_measured(stage)

self.input_dir = self._get_input_dir(stage)

params with their corresponding output sims.""".format(stage))

print('\nThis optimization stage used the\

following input parameter files:\n{}'.format('\n'.join(self.input_params)))

def _count_total_sims(self):

# total number of simulations of given stage in working directory

tracked_dirs = []

for f in self.metadata_json_paths[self.stage]:

with open(f) as fh:

json_data = json.load(fh)

tracked_dirs.extend(json_data.get('sim_dirs'))

return len(tracked_dirs)

def _get_optr_jsons(self, work_dir, stage):

"""

return var_name

def result_table(self, freq='daily', top_n=5, latex=False):

def result_table(self, freq='daily', top_n=5, latex=False):

##TODO: add stats for freq options annual (means or sum)

sim_dirs = self._get_sim_dirs(self.stage)

orig_mo = orig_statvar.groupby(orig_statvar.index.month).mean()

for i, sim in enumerate(sim_dirs):

sim_out = load_statvar(OPJ(sim, 'outputs', 'statvar.dat'))\

try:

sim_out = load_statvar(OPJ(sim, 'outputs', 'statvar.dat'))\

['{}'.format(statvar_name)]

except: # simulation might have been removed or missing

continue

sim_out = sim_out[idx]

if freq == 'daily':

result_df.loc[sim_names[i]] = [\

'params_adjusted' : []

}

json_paths = self.metadata_json_paths[self.stage]

json_paths = self.metadata_json_paths[self.stage]

for i,el in enumerate(sorted_df.index):

for el in sorted_df.drop('orig_params').index:

ret['dir_name'].append(el)

ret['param_path'].append(OPJ(self.working_dir,el,'inputs',\

'parameters'))

return ret

def archive(self, remove_sims=True, metric_freq='daily'):

"""

# create output archive directory

archive_dir = OPJ(self.working_dir,"{}_archived".format(self.stage))

if not os.path.isdir(archive_dir):

os.mkdir(archive_dir)

# create table and used to make mapping dic

table = self.result_table(freq=metric_freq,\

top_n=self.total_sims, latex=False)

map_dic = self.get_top_ranked_sims(table)

# get measured optimization variable path

first_json = self.metadata_json_paths[self.stage][0]

with open(first_json) as json_file:

json_data = json.load(json_file)

measured_path = json_data.get('measured')

# record info for each simulation and archive to JSONs

# pandas series and numpy arrays are converted to Python lists

# for JSON serialization

for i, sim in enumerate(map_dic.get('dir_name')):

json_path = OPJ(archive_dir, '{sim}.json'.format(sim=sim))

try:

output_series = load_statvar(OPJ(self.working_dir, sim,\

'outputs', 'statvar.dat'))\

[self.statvar_name]

except: # simulation directory was already removed

continue

json_data = {

'param_names' : [],

'param_values' : [],

'original_param_path' : self.input_params,

'measured_path' : measured_path,

'output_name' : self.statvar_name,

'output_date_index' : output_series.\

index.astype(str).tolist(),

'output_values' : output_series.values.tolist(),

'metric_freq' : metric_freq,

'NSE' : table.loc[sim, 'NSE'],

'RMSE' : table.loc[sim, 'RMSE'],

'PBIAS' : table.loc[sim, 'PBIAS'],

'COEF_DET' : table.loc[sim, 'COEF_DET']

}

for param in map_dic.get('params_adjusted')[i]:

json_data['param_names'].append(param)

json_data['param_values'].append(Parameters(\

map_dic.get('param_path')[i])[param]\

.tolist())

# save JSON file into archive directory

with open(json_path, 'w') as outf:

json.dump(json_data, outf, sort_keys = True, indent = 4,\

ensure_ascii = False)

# recursively delete all simulation directories after archiving

if remove_sims:

path = OPJ(self.working_dir, sim)

for dirpath, dirnames, filenames in os.walk(path,\

topdown=False):

shutil.rmtree(dirpath, ignore_errors=True)

else:

continue

# after archiving all simulations delete the original JSON metadata

for meta_file in self.metadata_json_paths[self.stage]:

try:

os.remove(meta_file)

except:

continue

import numpy as np

import pandas as pd

from .optimizer import OptimizationResult

def remove_all_optimization_sims_of_other_stage(work_directory,stage):

"""

def remove_all_optimization_sims_of_other_stage(work_directory, stage):

"""

result = OptimizationResult(work_directory,stage=stage)

tracked_dirs = []

from .optimizer import OptimizationResult # avoid circular import

result = OptimizationResult(work_directory,stage=stage)

tracked_dirs = []

for f in result.metadata_json_paths[stage]:

with open(f) as fh:

json_data = json.load(fh)

tracked_dirs.extend(json_data.get('sim_dirs'))

# track number of simulation directories not tracked by certain stage

# and recursively delete them and their contents

count = 0

for d in os.listdir(result.working_dir):

path = os.path.join(result.working_dir, d)

elif os.path.isdir(path):

count+=1

for dirpath, dirnames, filenames in os.walk(path, topdown=False):

shutil.rmtree(dirpath, ignore_errors=True)

shutil.rmtree(dirpath, ignore_errors=True)

print('deleted {} simulations that were either not tracked by a JSON file'\

.format(count) + ' or were not part of {} optimization stage'\

.format(stage))

def delete_files(work_directory, file_name=''):

"""

def load_data_file(data_file):

"""

# valid input time series that can be put into a data file

valid_input_variables = ('gate_ht',