Commit e850d1ce authored by David Trudgian's avatar David Trudgian
Browse files

PEP8 fixes

parent 557a6a99
from abc import ABCMeta, abstractmethod
from subprocess import call
from pathos.multiprocessing import Pool
import os
import datetime
import logging
import os
import re
import sys
from abc import ABCMeta, abstractmethod
from subprocess import call
from pathos.multiprocessing import Pool
logger = logging.getLogger(__name__)
class BaseExecutor:
metaclass = ABCMeta
commands = []
logdir =""
logdir = ""
params = None
def __init__(self, commands, cwd, params):
......@@ -24,11 +24,15 @@ class BaseExecutor:
self.commands = commands
self.params = params
logdir = os.path.join(cwd, "param_runner_%s" % datetime.datetime.now().strftime('%Y%m%d-%H%M%s') )
logdir = os.path.join(cwd,
"param_runner_%s" % datetime.datetime.now().strftime(
'%Y%m%d-%H%M%s'))
try:
os.mkdir(logdir)
logger.info(" - Trace file and output will be at: %s" % os.path.relpath(logdir))
logger.info(
" - Trace file and output will be at: %s" % os.path.relpath(
logdir))
except IOError:
raise IOError("Could not create log directory: %s" % logdir)
......@@ -36,11 +40,10 @@ class BaseExecutor:
self.start_trace()
def start_trace(self):
trace_path = os.path.join(self.logdir, 'trace.txt')
trace_file = open(trace_path, 'w', buffering = False)
trace_file = open(trace_path, 'w', buffering=False)
trace_file.write("Index\t")
......@@ -54,17 +57,15 @@ class BaseExecutor:
trace_file.write("\n")
def write_trace(self, msg):
trace_path = os.path.join(self.logdir, 'trace.txt')
trace_file = open(trace_path, 'a', buffering = False)
trace_file = open(trace_path, 'a', buffering=False)
trace_file.write(msg)
trace_file.close()
def wrap_cmd(self, cmd_idx):
logger.debug(" - Task %d is running" % cmd_idx)
......@@ -104,35 +105,30 @@ class BaseExecutor:
return summary_line
@staticmethod
def __pygrep_first(regex, text_file):
def __pygrep_first(self, regex, file):
for i, line in enumerate(open(file)):
for i, line in enumerate(open(text_file)):
for match in re.finditer(regex, line):
return match.group(1)
return ''
@abstractmethod
def run(self):
pass
@abstractmethod
def run_cmd(self):
def run_cmd(self, cmd, stderr_file, stdout_file):
pass
class LocalExecutor(BaseExecutor):
def run(self):
for cmd_idx, cmd in enumerate(self.commands):
self.write_trace(self.wrap_cmd(cmd_idx))
def run_cmd(self, cmd, stderr_file, stdout_file):
ret = call(cmd['__command']['value'], shell=True, stderr=stderr_file,
stdout=stdout_file)
......@@ -140,15 +136,16 @@ class LocalExecutor(BaseExecutor):
class SrunExecutor(BaseExecutor):
def run(self):
if 'SLURM_JOB_CPUS_PER_NODE' not in os.environ:
raise EnvironmentError('The srun executor can only be run inside a SLURM allocation.')
raise EnvironmentError(
'The srun executor can only be run inside a SLURM allocation.')
logger.info(" - Running in SLURM JOB %s" % os.environ.get('SLURM_JOB_ID' ))
cpus_per_node = int(re.match('\d+', os.environ.get('SLURM_JOB_CPUS_PER_NODE') ).group(0))
logger.info(
" - Running in SLURM JOB %s" % os.environ.get('SLURM_JOB_ID'))
cpus_per_node = int(
re.match('\d+', os.environ.get('SLURM_JOB_CPUS_PER_NODE')).group(0))
logger.info(" - %d CPUs per node" % cpus_per_node)
nodes = int(os.environ.get('SLURM_NNODES'))
logger.info(" - %d nodes" % nodes)
......@@ -159,28 +156,21 @@ class SrunExecutor(BaseExecutor):
p = Pool(conc_tasks)
for cmd_idx, cmd in enumerate(self.commands):
logger.debug(" - Task %d is waiting in the pool" % cmd_idx)
p.apply_async( self.wrap_cmd, [cmd_idx], callback=self.write_trace )
p.apply_async(self.wrap_cmd, [cmd_idx], callback=self.write_trace)
p.close()
p.join()
logger.info(" - All tasks have completed")
def run_cmd(self, cmd, stderr_file, stdout_file):
cpus_per_task = self.params.vals['cpus_per_task']
srun_cmd = "srun -N1 -n1 -c%d %s" % ( cpus_per_task, cmd['__command']['value'])
srun_cmd = "srun -N1 -n1 -c%d %s" % (
cpus_per_task, cmd['__command']['value'])
ret = call(srun_cmd, shell=True, stderr=stderr_file,
stdout=stdout_file, env=os.environ)
stdout=stdout_file, env=os.environ)
return ret
import anyconfig
import os
import logging
import os
import re
from copy import deepcopy
from collections import OrderedDict
from yamllint.config import YamlLintConfig, YamlLintConfigError
from copy import deepcopy
import anyconfig
from yamllint import linter
from yamllint.cli import Format
from yamllint.config import YamlLintConfig
logger = logging.getLogger(__name__)
......@@ -81,7 +82,7 @@ class ParamFile(object):
if err:
message = "Parameter file invalid: " + \
err.splitlines()[0]
err.splitlines()[0]
raise ParamFileException(message)
logging.info("Parameter file is valid")
......@@ -120,7 +121,8 @@ class ParamFile(object):
"""Read in metadata from the astrocyte_pkg.yml to populate vals dict"""
if self.loaded:
raise ParamFileException("Parameter information was already loaded.")
raise ParamFileException(
"Parameter information was already loaded.")
self.validate()
metadata = anyconfig.load(self.param_file)
......@@ -131,7 +133,6 @@ class ParamFile(object):
self.commands = self.__compute_commands()
self.loaded = True
def __check_summary_regexes(self):
for summary in self.vals['summary']:
......@@ -140,13 +141,17 @@ class ParamFile(object):
compiled_re = re.compile(summary['regex'])
if compiled_re.groups != 1:
raise(TypeError("Regular expression for summary %s must contain exactly 1 matching group." % summary['id']))
raise (TypeError(
"Regular expression for summary %s must contain exactly 1 matching group." %
summary['id']))
logging.info( '- Summary %s using regex %s' % (summary['id'], summary['regex']))
logging.info('- Summary %s using regex %s' % (
summary['id'], summary['regex']))
except re.error:
raise(re.error("Regular expression for summary %s is invalid" % summary['id']))
raise (re.error(
"Regular expression for summary %s is invalid" % summary[
'id']))
def __compute_param_ranges(self):
"""Compute the parameter ranges as lists, from their definitions"""
......@@ -174,7 +179,8 @@ class ParamFile(object):
logger.info(" - Parameter '%s' will explore values: %s " %
(param['id'], param['range']))
logger.info(" - Total combinations to explore: %d" % self.combination_count)
logger.info(
" - Total combinations to explore: %d" % self.combination_count)
def __compute_commands(self):
"""Compute the commands to be run across parameter ranges
......@@ -194,16 +200,15 @@ class ParamFile(object):
assert len(arg_combns) == self.combination_count
for arg_combn in arg_combns:
arg_combn['__command'] = { 'value': self.__cmd_from_args(arg_combn) }
arg_combn['__command'] = {'value': self.__cmd_from_args(arg_combn)}
logger.info(" - Generated %d commands" % len(arg_combns))
return arg_combns
def __recurse_arg_combns(self, param_idx=0, current_combination=OrderedDict(), all_combinations=[]):
def __recurse_arg_combns(self, param_idx=0,
current_combination=OrderedDict(),
all_combinations=[]):
"""Recursively find a combinations of arguments for a execution
Args:
......@@ -230,17 +235,19 @@ class ParamFile(object):
current_param = params[param_idx]
current_id = current_param['id']
current_flag = current_param.get('flag', None)
num_values = len(current_param['range'])
num_values = len(current_param['range'])
for val_idx in range(num_values):
current_value = current_param['range'][val_idx]
current_combination[current_id] = {'flag': current_flag, 'value': current_value }
all_combinations = self.__recurse_arg_combns(param_idx+1, deepcopy(current_combination), all_combinations)
current_combination[current_id] = {'flag': current_flag,
'value': current_value}
all_combinations = self.__recurse_arg_combns(param_idx + 1,
deepcopy(
current_combination),
all_combinations)
return all_combinations
def __cmd_from_args(self, arg_combn):
"""Convert a dict of arguments into a full command string
......@@ -265,9 +272,7 @@ class ParamFile(object):
return cmd_str.rstrip(' ')
class IntRange(object):
"""Compute integer parameters from an int_range definition
Attributes:
......@@ -307,7 +312,6 @@ class IntRange(object):
class RealRange(IntRange):
"""Compute real-value parameters from an real_range definition
Attributes:
......@@ -320,7 +324,6 @@ class RealRange(IntRange):
class ChoiceRange(object):
"""Compute choice parameters from an choice definition
Attributes:
......@@ -335,7 +338,6 @@ class ChoiceRange(object):
self.param = param
def to_list(self):
vals = []
if self.param.get('optional', False):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment