From 9830963ae7a45daefd28a32f31e7383403f40914 Mon Sep 17 00:00:00 2001 From: Nitin Madnani Date: Fri, 10 Apr 2020 17:01:51 -0400 Subject: [PATCH 01/20] Convert `config.py` into `config` package - Split into `config/__init__.py` and `config/utils.py` --- skll/{config.py => config/__init__.py} | 312 ++++++------------------- skll/config/utils.py | 187 +++++++++++++++ 2 files changed, 253 insertions(+), 246 deletions(-) rename skll/{config.py => config/__init__.py} (82%) create mode 100644 skll/config/utils.py diff --git a/skll/config.py b/skll/config/__init__.py similarity index 82% rename from skll/config.py rename to skll/config/__init__.py index 5eed3e75..ba06abf2 100644 --- a/skll/config.py +++ b/skll/config/__init__.py @@ -1,6 +1,6 @@ # License: BSD 3 clause """ -Functions related to parsing configuration files. +The main class and functions used to parse SKLL configuration files. :author: Nitin Madnani (nmadnani@ets.org) :author: Dan Blanchard (dblanchard@ets.org) @@ -8,46 +8,37 @@ :author: Chee Wee Leong (cleong@ets.org) """ -import csv import errno import itertools import logging import os -from os.path import (basename, dirname, exists, - isabs, join, normpath, realpath) +from os.path import basename, dirname, exists, join, realpath import configparser import numpy as np import ruamel.yaml as yaml -from sklearn.metrics import SCORERS -from skll import get_skll_logger from skll.data.readers import safe_float -from skll.metrics import _PROBABILISTIC_METRICS +from skll.utils.constants import (PROBABILISTIC_METRICS, + VALID_TASKS, + VALID_SAMPLERS, + VALID_FEATURE_SCALING_OPTIONS) +from skll.utils.logging import get_skll_logger -_VALID_TASKS = frozenset(['cross_validate', - 'evaluate', - 'learning_curve', - 'predict', - 'train']) +from .utils import (fix_json, + load_cv_folds, + locate_file, + _munge_featureset_name, + _parse_and_validate_metrics) -_VALID_SAMPLERS = frozenset(['Nystroem', - 'RBFSampler', - 'SkewedChi2Sampler', - 'AdditiveChi2Sampler', - '']) - -_VALID_FEATURE_SCALING_OPTIONS = frozenset(['both', - 'none', - 'with_std', - 'with_mean']) +__all__ = ['SKLLConfigParser', 'fix_json', 'load_cv_folds', 'locate_file'] class SKLLConfigParser(configparser.ConfigParser): """ - A custom configuration file parser for SKLL + A custom configuration file parser for SKLL. """ def __init__(self): @@ -248,77 +239,7 @@ def validate(self): incorrectly_specified_options])) -def _locate_file(file_path, config_dir): - """ - Locate a file, given a file path and configuration directory. - - Parameters - ---------- - file_path : str - The file to locate. Path may be absolute or relative. - config_dir : str - The path to the configuration file directory. - - Returns - ------- - path_to_check : str - The normalized absolute path, if it exists. - - Raises - ------ - IOError - If the file does not exist. - """ - if not file_path: - return '' - path_to_check = file_path if isabs(file_path) else normpath(join(config_dir, - file_path)) - ans = exists(path_to_check) - if not ans: - raise IOError(errno.ENOENT, "File does not exist", path_to_check) - else: - return path_to_check - - -def _setup_config_parser(config_path, validate=True): - """ - Returns a config parser at a given path. Only implemented as a separate - function to simplify testing. - - Parameters - ---------- - config_path : str - The path to the configuration file. - validate : bool, optional - Whether to validate the configuration file. - Defaults to ``True``. - - Returns - ------- - config : SKLLConfigParser - A SKLL configuration object. - - Raises - ------ - IOError - If the configuration file does not exist. - """ - # initialize config parser with the given defaults - config = SKLLConfigParser() - - # Read file if it exists - if not exists(config_path): - raise IOError(errno.ENOENT, "Configuration file does not exist", - config_path) - config.read(config_path) - - if validate: - config.validate() - - return config - - -def _parse_config_file(config_path, log_level=logging.INFO): +def parse_config_file(config_path, log_level=logging.INFO): """ Parses a SKLL experiment configuration file with the given path. Log messages with the given log level (default: INFO). @@ -470,7 +391,7 @@ def _parse_config_file(config_path, log_level=logging.INFO): # save all logging messages to a log file in addition to displaying # them on the console try: - log_path = _locate_file(config.get("Output", "log"), config_dir) + log_path = locate_file(config.get("Output", "log"), config_dir) except IOError as e: if e.errno == errno.ENOENT: log_path = e.filename @@ -492,18 +413,18 @@ def _parse_config_file(config_path, log_level=logging.INFO): else: raise ValueError("Configuration file does not contain task in the " "[General] section.") - if task not in _VALID_TASKS: + if task not in VALID_TASKS: raise ValueError('An invalid task was specified: {}. Valid tasks are:' - ' {}'.format(task, ', '.join(_VALID_TASKS))) + ' {}'.format(task, ', '.join(VALID_TASKS))) #################### # 2. Input section # #################### sampler = config.get("Input", "sampler") - if sampler not in _VALID_SAMPLERS: + if sampler not in VALID_SAMPLERS: raise ValueError('An invalid sampler was specified: {}. Valid ' 'samplers are: {}'.format(sampler, - ', '.join(_VALID_SAMPLERS))) + ', '.join(VALID_SAMPLERS))) # produce warnings if feature_hasher is set but hasher_features # is less than or equal to zero. @@ -526,7 +447,7 @@ def _parse_config_file(config_path, log_level=logging.INFO): else: raise ValueError("Configuration file does not contain list of learners " "in [Input] section.") - learners = yaml.safe_load(_fix_json(learners_string)) + learners = yaml.safe_load(fix_json(learners_string)) if len(learners) == 0: raise ValueError("Configuration file contains an empty list of learners" @@ -537,12 +458,12 @@ def _parse_config_file(config_path, log_level=logging.INFO): ' times, which is not currently supported. Please use' ' param_grids with tuning to find the optimal settings' ' for the learner.') - custom_learner_path = _locate_file(config.get("Input", "custom_learner_path"), - config_dir) + custom_learner_path = locate_file(config.get("Input", "custom_learner_path"), + config_dir) # get the featuresets featuresets_string = config.get("Input", "featuresets") - featuresets = yaml.safe_load(_fix_json(featuresets_string)) + featuresets = yaml.safe_load(fix_json(featuresets_string)) # ensure that featuresets is either a list of features or a list of lists # of features @@ -552,8 +473,8 @@ def _parse_config_file(config_path, log_level=logging.INFO): "features or a list of lists of features. You " "specified: {}".format(featuresets)) - featureset_names = yaml.safe_load(_fix_json(config.get("Input", - "featureset_names"))) + featureset_names = yaml.safe_load(fix_json(config.get("Input", + "featureset_names"))) # ensure that featureset_names is a list of strings, if specified if featureset_names: @@ -570,7 +491,7 @@ def _parse_config_file(config_path, log_level=logging.INFO): # that we are using 10 folds for each learner. learning_curve_cv_folds_list_string = config.get("Input", "learning_curve_cv_folds_list") - learning_curve_cv_folds_list = yaml.safe_load(_fix_json(learning_curve_cv_folds_list_string)) + learning_curve_cv_folds_list = yaml.safe_load(fix_json(learning_curve_cv_folds_list_string)) if len(learning_curve_cv_folds_list) == 0: learning_curve_cv_folds_list = [10] * len(learners) else: @@ -587,7 +508,7 @@ def _parse_config_file(config_path, log_level=logging.INFO): # floats (proportions). If it's not specified, then we just # assume that we are using np.linspace(0.1, 1.0, 5). learning_curve_train_sizes_string = config.get("Input", "learning_curve_train_sizes") - learning_curve_train_sizes = yaml.safe_load(_fix_json(learning_curve_train_sizes_string)) + learning_curve_train_sizes = yaml.safe_load(fix_json(learning_curve_train_sizes_string)) if len(learning_curve_train_sizes) == 0: learning_curve_train_sizes = np.linspace(0.1, 1.0, 5).tolist() else: @@ -601,12 +522,12 @@ def _parse_config_file(config_path, log_level=logging.INFO): # do we need to shuffle the training data do_shuffle = config.getboolean("Input", "shuffle") - fixed_parameter_list = yaml.safe_load(_fix_json(config.get("Input", - "fixed_parameters"))) - fixed_sampler_parameters = _fix_json(config.get("Input", - "sampler_parameters")) + fixed_parameter_list = yaml.safe_load(fix_json(config.get("Input", + "fixed_parameters"))) + fixed_sampler_parameters = fix_json(config.get("Input", + "sampler_parameters")) fixed_sampler_parameters = yaml.safe_load(fixed_sampler_parameters) - param_grid_list = yaml.safe_load(_fix_json(config.get("Tuning", "param_grids"))) + param_grid_list = yaml.safe_load(fix_json(config.get("Tuning", "param_grids"))) # read and normalize the value of `pos_label_str` pos_label_str = safe_float(config.get("Tuning", "pos_label_str")) @@ -616,7 +537,7 @@ def _parse_config_file(config_path, log_level=logging.INFO): # ensure that feature_scaling is specified only as one of the # four available choices feature_scaling = config.get("Input", "feature_scaling") - if feature_scaling not in _VALID_FEATURE_SCALING_OPTIONS: + if feature_scaling not in VALID_FEATURE_SCALING_OPTIONS: raise ValueError("Invalid value for feature_scaling parameter: {}" .format(feature_scaling)) @@ -626,12 +547,12 @@ def _parse_config_file(config_path, log_level=logging.INFO): ids_to_floats = config.getboolean("Input", "ids_to_floats") # if an external folds file is specified, then read it into a dictionary - folds_file = _locate_file(config.get("Input", "folds_file"), config_dir) + folds_file = locate_file(config.get("Input", "folds_file"), config_dir) num_cv_folds = config.getint("Input", "num_cv_folds") specified_folds_mapping = None specified_num_folds = None if folds_file: - specified_folds_mapping = _load_cv_folds(folds_file, ids_to_floats=ids_to_floats) + specified_folds_mapping = load_cv_folds(folds_file, ids_to_floats=ids_to_floats) else: # if no file is specified, then set the number of folds for cross-validation specified_num_folds = num_cv_folds if num_cv_folds else 10 @@ -696,12 +617,12 @@ def _parse_config_file(config_path, log_level=logging.INFO): featuresets[0][0] += '_test_{}'.format(basename(test_file)) # make sure all the specified paths/files exist - train_path = _locate_file(train_path, config_dir) - test_path = _locate_file(test_path, config_dir) + train_path = locate_file(train_path, config_dir) + test_path = locate_file(test_path, config_dir) # Get class mapping dictionary if specified class_map_string = config.get("Input", "class_map") - original_class_map = yaml.safe_load(_fix_json(class_map_string)) + original_class_map = yaml.safe_load(fix_json(class_map_string)) if original_class_map: # Change class_map to map from originals to replacements instead of # from replacement to list of originals @@ -722,8 +643,8 @@ def _parse_config_file(config_path, log_level=logging.INFO): # do we want to keep the predictions? # make sure the predictions path exists and if not create it try: - prediction_dir = _locate_file(config.get("Output", "predictions"), - config_dir) + prediction_dir = locate_file(config.get("Output", "predictions"), + config_dir) except IOError as e: if e.errno == errno.ENOENT: prediction_dir = e.filename @@ -731,7 +652,7 @@ def _parse_config_file(config_path, log_level=logging.INFO): # make sure model path exists and if not, create it try: - model_path = _locate_file(config.get("Output", "models"), config_dir) + model_path = locate_file(config.get("Output", "models"), config_dir) except IOError as e: if e.errno == errno.ENOENT: model_path = e.filename @@ -739,7 +660,7 @@ def _parse_config_file(config_path, log_level=logging.INFO): # make sure results path exists try: - results_path = _locate_file(config.get("Output", "results"), config_dir) + results_path = locate_file(config.get("Output", "results"), config_dir) except IOError as e: if e.errno == errno.ENOENT: results_path = e.filename @@ -845,7 +766,7 @@ def _parse_config_file(config_path, log_level=logging.INFO): # if any of the objectives or metrics require probabilities to be output, # probability must be specified as true - specified_probabilistic_metrics = _PROBABILISTIC_METRICS.intersection(grid_objectives + output_metrics) + specified_probabilistic_metrics = PROBABILISTIC_METRICS.intersection(grid_objectives + output_metrics) if specified_probabilistic_metrics and not probability: raise ValueError("The 'probability' option must be 'true' " " to compute the following: " @@ -907,140 +828,39 @@ def _parse_config_file(config_path, log_level=logging.INFO): learning_curve_train_sizes, output_metrics) -def _munge_featureset_name(featureset): - """ - Joins features in featureset by '+' if featureset is not a string, and just - returns featureset otherwise. - - Parameters - ---------- - featureset : SKLL.FeatureSet - A SKLL feature_set object. - - Returns - ------- - res : str - feature_set names joined with '+', if feature_set is not a string. - """ - if isinstance(featureset, str): - return featureset - - res = '+'.join(sorted(featureset)) - return res - - -def _fix_json(json_string): - """ - Fixes incorrectly formatted quotes and capitalized booleans in the given - JSON string. - - Parameters - ---------- - json_string : str - A JSON-style string. - - Returns - ------- - json_string : str - The normalized JSON string. - """ - json_string = json_string.replace('True', 'true') - json_string = json_string.replace('False', 'false') - json_string = json_string.replace("'", '"') - return json_string - - -def _parse_and_validate_metrics(metrics, option_name, logger=None): +def _setup_config_parser(config_path, validate=True): """ - Given a string containing a list of metrics, this function - parses that string into a list and validates the list. + Returns a config parser at a given path. Only implemented as a separate + function to simplify testing. Parameters ---------- - metrics : str - A string containing a list of metrics - option_name : str - The name of the option with which the metrics are associated. - logger : logging.Logger, optional - A logging object - Defaults to ``None``. + config_path : str + The path to the configuration file. + validate : bool, optional + Whether to validate the configuration file. + Defaults to ``True``. Returns ------- - metrics : list of str - A list of metrics for the given option. + config : SKLLConfigParser + A SKLL configuration object. Raises ------ - TypeError - If the given string cannot be converted to a list. - ValueError - If there are any invalid metrics specified. - """ - - # create a logger if one was not passed in - if not logger: - logger = logging.getLogger(__name__) - - # make sure the given metrics data type is a list - # and parse it correctly - metrics = yaml.safe_load(_fix_json(metrics)) - if not isinstance(metrics, list): - raise TypeError("{} should be a list, not a {}.".format(option_name, - type(metrics))) - - # `mean_squared_error` is no more supported. - # It is replaced by `neg_mean_squared_error` - if 'mean_squared_error' in metrics: - raise ValueError("The metric \"mean_squared_error\" " - "is no longer supported." - " please use the metric " - "\"neg_mean_squared_error\" instead.") - - invalid_metrics = [metric for metric in metrics if metric not in SCORERS] - if invalid_metrics: - raise ValueError('Invalid metric(s) {} ' - 'specified for {}'.format(invalid_metrics, - option_name)) - - return metrics - - -def _load_cv_folds(folds_file, ids_to_floats=False): + IOError + If the configuration file does not exist. """ - Loads CV folds from a CSV file with columns for example ID and fold ID (and - a header). + # initialize config parser with the given defaults + config = SKLLConfigParser() - Parameters - ---------- - folds_file : str - The path to a folds file to read. - ids_to_floats : bool, optional - Whether to convert IDs to floats. - Defaults to ``False``. + # Read file if it exists + if not exists(config_path): + raise IOError(errno.ENOENT, "Configuration file does not exist", + config_path) + config.read(config_path) - Returns - ------- - res : dict - A dictionary with example IDs as the keys and fold IDs as the values. + if validate: + config.validate() - Raises - ------ - ValueError - If example IDs cannot be converted to floats and `ids_to_floats` is `True`. - """ - with open(folds_file, 'r') as f: - reader = csv.reader(f) - next(reader) # discard the header - res = {} - for row in reader: - if ids_to_floats: - try: - row[0] = float(row[0]) - except ValueError: - raise ValueError('You set ids_to_floats to true, but ID {}' - ' could not be converted to float' - .format(row[0])) - res[row[0]] = row[1] - - return res + return config diff --git a/skll/config/utils.py b/skll/config/utils.py new file mode 100644 index 00000000..cad5923c --- /dev/null +++ b/skll/config/utils.py @@ -0,0 +1,187 @@ +# License: BSD 3 clause +""" +Utility classes and functions to parse SKLL configuration files. + +:author: Nitin Madnani (nmadnani@ets.org) +:author: Dan Blanchard (dblanchard@ets.org) +:author: Michael Heilman (mheilman@ets.org) +""" + +import csv +import errno +import logging +from os.path import exists, isabs, join, normpath + +import ruamel.yaml as yaml +from sklearn.metrics import SCORERS + + +def fix_json(json_string): + """ + Fixes incorrectly formatted quotes and capitalized booleans in the given + JSON string. + + Parameters + ---------- + json_string : str + A JSON-style string. + + Returns + ------- + json_string : str + The normalized JSON string. + """ + json_string = json_string.replace('True', 'true') + json_string = json_string.replace('False', 'false') + json_string = json_string.replace("'", '"') + return json_string + + +def load_cv_folds(folds_file, ids_to_floats=False): + """ + Loads cross-validation folds from a CSV file with two columns for example + ID and fold ID (and a header). + + Parameters + ---------- + folds_file : str + The path to a folds file to read. + ids_to_floats : bool, optional + Whether to convert IDs to floats. + Defaults to ``False``. + + Returns + ------- + res : dict + A dictionary with example IDs as the keys and fold IDs as the values. + + Raises + ------ + ValueError + If example IDs cannot be converted to floats and `ids_to_floats` is `True`. + """ + with open(folds_file, 'r') as f: + reader = csv.reader(f) + next(reader) # discard the header + res = {} + for row in reader: + if ids_to_floats: + try: + row[0] = float(row[0]) + except ValueError: + raise ValueError('You set ids_to_floats to true, but ID {}' + ' could not be converted to float' + .format(row[0])) + res[row[0]] = row[1] + + return res + + +def locate_file(file_path, config_dir): + """ + Locate a file, given a file path and configuration directory. + + Parameters + ---------- + file_path : str + The file to locate. Path may be absolute or relative. + config_dir : str + The path to the configuration file directory. + + Returns + ------- + path_to_check : str + The normalized absolute path, if it exists. + + Raises + ------ + IOError + If the file does not exist. + """ + if not file_path: + return '' + path_to_check = file_path if isabs(file_path) else normpath(join(config_dir, + file_path)) + ans = exists(path_to_check) + if not ans: + raise IOError(errno.ENOENT, "File does not exist", path_to_check) + else: + return path_to_check + + +def _munge_featureset_name(featureset): + """ + Joins features in featureset by '+' if featureset is not a string, and just + returns featureset otherwise. + + Parameters + ---------- + featureset : SKLL.FeatureSet + A SKLL feature_set object. + + Returns + ------- + res : str + feature_set names joined with '+', if feature_set is not a string. + """ + if isinstance(featureset, str): + return featureset + + res = '+'.join(sorted(featureset)) + return res + + +def _parse_and_validate_metrics(metrics, option_name, logger=None): + """ + Given a string containing a list of metrics, this function + parses that string into a list and validates the list. + + Parameters + ---------- + metrics : str + A string containing a list of metrics + option_name : str + The name of the option with which the metrics are associated. + logger : logging.Logger, optional + A logging object + Defaults to ``None``. + + Returns + ------- + metrics : list of str + A list of metrics for the given option. + + Raises + ------ + TypeError + If the given string cannot be converted to a list. + ValueError + If there are any invalid metrics specified. + """ + + # create a logger if one was not passed in + if not logger: + logger = logging.getLogger(__name__) + + # make sure the given metrics data type is a list + # and parse it correctly + metrics = yaml.safe_load(fix_json(metrics)) + if not isinstance(metrics, list): + raise TypeError("{} should be a list, not a {}.".format(option_name, + type(metrics))) + + # `mean_squared_error` is no more supported. + # It is replaced by `neg_mean_squared_error` + if 'mean_squared_error' in metrics: + raise ValueError("The metric \"mean_squared_error\" " + "is no longer supported." + " please use the metric " + "\"neg_mean_squared_error\" instead.") + + invalid_metrics = [metric for metric in metrics if metric not in SCORERS] + if invalid_metrics: + raise ValueError('Invalid metric(s) {} ' + 'specified for {}'.format(invalid_metrics, + option_name)) + + return metrics From cd74ad8b20780b5bb99afa7bfe25b233fc06423d Mon Sep 17 00:00:00 2001 From: Nitin Madnani Date: Fri, 10 Apr 2020 17:02:44 -0400 Subject: [PATCH 02/20] Convert `experiments.py` into `experiments` package Split into `experiments/__init__.py`, `experiments/input.py`, `experiments/output.py`, and `experiments/utils.py`. --- .../__init__.py} | 810 ++---------------- skll/experiments/input.py | 112 +++ skll/experiments/output.py | 429 ++++++++++ skll/experiments/utils.py | 239 ++++++ 4 files changed, 830 insertions(+), 760 deletions(-) rename skll/{experiments.py => experiments/__init__.py} (52%) create mode 100644 skll/experiments/input.py create mode 100644 skll/experiments/output.py create mode 100644 skll/experiments/utils.py diff --git a/skll/experiments.py b/skll/experiments/__init__.py similarity index 52% rename from skll/experiments.py rename to skll/experiments/__init__.py index 4671c68a..6a38b500 100644 --- a/skll/experiments.py +++ b/skll/experiments/__init__.py @@ -1,40 +1,43 @@ # License: BSD 3 clause """ -Functions related to running experiments and parsing configuration files. +Functions for running and interacting with SKLL experiments. +:author: Nitin Madnani (nmadnani@ets.org) :author: Dan Blanchard (dblanchard@ets.org) :author: Michael Heilman (mheilman@ets.org) -:author: Nitin Madnani (nmadnani@ets.org) :author: Chee Wee Leong (cleong@ets.org) """ -import csv import datetime import json import logging -import math -import sys -import matplotlib import matplotlib.pyplot as plt import numpy as np -import pandas as pd -import ruamel.yaml as yaml -import seaborn as sns -from collections import defaultdict from itertools import combinations -from os.path import exists, isfile, join, getsize +from os.path import exists, join, getsize from sklearn import __version__ as SCIKIT_VERSION -from skll import close_and_remove_logger_handlers, get_skll_logger -from skll.config import _munge_featureset_name, _parse_config_file -from skll.data.readers import Reader -from skll.learner import (Learner, MAX_CONCURRENT_PROCESSES, - _import_custom_learner) +from skll.config import parse_config_file +from skll.config.utils import _munge_featureset_name +from skll.learner import (import_custom_learner, + Learner, + MAX_CONCURRENT_PROCESSES) +from skll.utils.logging import (close_and_remove_logger_handlers, + get_skll_logger) from skll.version import __version__ -from tabulate import tabulate + +from .input import load_featureset +from .output import (generate_learning_curve_plots, + _print_fancy_output, + _write_learning_curve_file, + _write_skll_folds, + _write_summary_file) +from .utils import (_check_job_results, + _create_learner_result_dicts, + NumpyTypeEncoder) # Check if gridmap is available try: @@ -48,402 +51,9 @@ plt.ioff() -_VALID_TASKS = frozenset(['predict', 'train', 'evaluate', 'cross_validate']) -_VALID_SAMPLERS = frozenset(['Nystroem', 'RBFSampler', 'SkewedChi2Sampler', - 'AdditiveChi2Sampler', '']) - - -class NumpyTypeEncoder(json.JSONEncoder): - """ - This class is used when serializing results, particularly the input label - values if the input has int-valued labels. Numpy int64 objects can't - be serialized by the json module, so we must convert them to int objects. - - A related issue where this was adapted from: - https://stackoverflow.com/questions/11561932/why-does-json-dumpslistnp-arange5-fail-while-json-dumpsnp-arange5-tolis - """ - - def default(self, obj): - if isinstance(obj, (np.int32, np.int64)): - return int(obj) - elif isinstance(obj, np.ndarray): - return obj.tolist() - return json.JSONEncoder.default(self, obj) - - -def _get_stat_float(label_result_dict, stat): - """ - A helper function to get output for the precision, recall, and f-score - columns in the confusion matrix. - - Parameters - ---------- - label_result_dict : dict - Dictionary containing the stat we'd like - to retrieve for a particular label. - stat : str - The statistic we're looking for in the dictionary. - - Returns - ------- - stat_float : float - The value of the stat if it's in the dictionary, and NaN - otherwise. - """ - if stat in label_result_dict and label_result_dict[stat] is not None: - return label_result_dict[stat] - else: - return float('nan') - - -def _write_skll_folds(skll_fold_ids, skll_fold_ids_file): - """ - Function to take a dictionary of id->test-fold-number and - write it to a file. - - Parameters - ---------- - skll_fold_ids : dict - Dictionary with ids as keys and test-fold-numbers as values. - skll_fold_ids_file : file buffer - An open file handler to write to. - """ - - f = csv.writer(skll_fold_ids_file) - f.writerow(['id', 'cv_test_fold']) - for example_id in skll_fold_ids: - f.writerow([example_id, skll_fold_ids[example_id]]) - - skll_fold_ids_file.flush() - - -def _write_summary_file(result_json_paths, output_file, ablation=0): - """ - Function to take a list of paths to individual result - json files and returns a single file that summarizes - all of them. - - Parameters - ---------- - result_json_paths : list of str - A list of paths to the individual result JSON files. - output_file : str - The path to the output file (TSV format). - ablation : int, optional - The number of features to remove when doing ablation experiment. - Defaults to 0. - """ - learner_result_dicts = [] - # Map from feature set names to all features in them - all_features = defaultdict(set) - logger = get_skll_logger('experiment') - for json_path in result_json_paths: - if not exists(json_path): - logger.error(('JSON results file %s not found. Skipping summary ' - 'creation. You can manually create the summary file' - ' after the fact by using the summarize_results ' - 'script.'), json_path) - return - else: - with open(json_path, 'r') as json_file: - obj = json.load(json_file) - featureset_name = obj[0]['featureset_name'] - if ablation != 0 and '_minus_' in featureset_name: - parent_set = featureset_name.split('_minus_', 1)[0] - all_features[parent_set].update( - yaml.safe_load(obj[0]['featureset'])) - learner_result_dicts.extend(obj) - - # Build and write header - header = set(learner_result_dicts[0].keys()) - {'result_table', - 'descriptive'} - if ablation != 0: - header.add('ablated_features') - header = sorted(header) - writer = csv.DictWriter(output_file, - header, - extrasaction='ignore', - dialect=csv.excel_tab) - writer.writeheader() - - # Build "ablated_features" list and fix some backward compatible things - for lrd in learner_result_dicts: - featureset_name = lrd['featureset_name'] - if ablation != 0: - parent_set = featureset_name.split('_minus_', 1)[0] - ablated_features = all_features[parent_set].difference( - yaml.safe_load(lrd['featureset'])) - lrd['ablated_features'] = '' - if ablated_features: - lrd['ablated_features'] = json.dumps(sorted(ablated_features)) - - # write out the new learner dict with the readable fields - writer.writerow(lrd) - - output_file.flush() - - -def _write_learning_curve_file(result_json_paths, output_file): - """ - Function to take a list of paths to individual learning curve - results json files and writes out a single TSV file with the - learning curve data. - - Parameters - ---------- - result_json_paths : list of str - A list of paths to the individual result JSON files. - output_file : str - The path to the output file (TSV format). - """ - - learner_result_dicts = [] - - # Map from feature set names to all features in them - logger = get_skll_logger('experiment') - for json_path in result_json_paths: - if not exists(json_path): - logger.error(('JSON results file %s not found. Skipping summary ' - 'creation. You can manually create the summary file' - ' after the fact by using the summarize_results ' - 'script.'), json_path) - return - else: - with open(json_path, 'r') as json_file: - obj = json.load(json_file) - learner_result_dicts.extend(obj) - - # Build and write header - header = ['featureset_name', 'learner_name', 'metric', - 'train_set_name', 'training_set_size', 'train_score_mean', - 'test_score_mean', 'train_score_std', 'test_score_std', - 'scikit_learn_version', 'version'] - writer = csv.DictWriter(output_file, - header, - extrasaction='ignore', - dialect=csv.excel_tab) - writer.writeheader() - - # write out the fields we need for the learning curve file - # specifically, we need to separate out the curve sizes - # and scores into individual entries. - for lrd in learner_result_dicts: - training_set_sizes = lrd['computed_curve_train_sizes'] - train_scores_means_by_size = lrd['learning_curve_train_scores_means'] - test_scores_means_by_size = lrd['learning_curve_test_scores_means'] - train_scores_stds_by_size = lrd['learning_curve_train_scores_stds'] - test_scores_stds_by_size = lrd['learning_curve_test_scores_stds'] - - # rename `grid_objective` to `metric` since the latter name can be confusing - lrd['metric'] = lrd['grid_objective'] - - for (size, - train_score_mean, - test_score_mean, - train_score_std, - test_score_std) in zip(training_set_sizes, - train_scores_means_by_size, - test_scores_means_by_size, - train_scores_stds_by_size, - test_scores_stds_by_size): - lrd['training_set_size'] = size - lrd['train_score_mean'] = train_score_mean - lrd['test_score_mean'] = test_score_mean - lrd['train_score_std'] = train_score_std - lrd['test_score_std'] = test_score_std - - writer.writerow(lrd) - - output_file.flush() - - -def _print_fancy_output(learner_result_dicts, output_file=sys.stdout): - """ - Function to take all of the results from all of the folds and print - nice tables with the results. - - Parameters - ---------- - learner_result_dicts : list of str - A list of paths to the individual result JSON files. - output_file : file buffer, optional - The file buffer to print to. - Defaults to ``sys.stdout``. - """ - if not learner_result_dicts: - raise ValueError('Result dictionary list is empty!') - - lrd = learner_result_dicts[0] - print('Experiment Name: {}'.format(lrd['experiment_name']), - file=output_file) - print('SKLL Version: {}'.format(lrd['version']), file=output_file) - print('Training Set: {}'.format(lrd['train_set_name']), file=output_file) - print('Training Set Size: {}'.format( - lrd['train_set_size']), file=output_file) - print('Test Set: {}'.format(lrd['test_set_name']), file=output_file) - print('Test Set Size: {}'.format(lrd['test_set_size']), file=output_file) - print('Shuffle: {}'.format(lrd['shuffle']), file=output_file) - print('Feature Set: {}'.format(lrd['featureset']), file=output_file) - print('Learner: {}'.format(lrd['learner_name']), file=output_file) - print('Task: {}'.format(lrd['task']), file=output_file) - if lrd['folds_file']: - print('Specified Folds File: {}'.format(lrd['folds_file']), - file=output_file) - if lrd['task'] == 'cross_validate': - print('Number of Folds: {}'.format(lrd['cv_folds']), - file=output_file) - if not lrd['cv_folds'].endswith('folds file'): - print('Stratified Folds: {}'.format(lrd['stratified_folds']), - file=output_file) - print('Feature Scaling: {}'.format(lrd['feature_scaling']), - file=output_file) - print('Grid Search: {}'.format(lrd['grid_search']), file=output_file) - if lrd['grid_search']: - print('Grid Search Folds: {}'.format(lrd['grid_search_folds']), - file=output_file) - print('Grid Objective Function: {}'.format(lrd['grid_objective']), - file=output_file) - if (lrd['task'] == 'cross_validate' and - lrd['grid_search'] and - lrd['cv_folds'].endswith('folds file')): - print('Using Folds File for Grid Search: {}'.format(lrd['use_folds_file_for_grid_search']), - file=output_file) - if lrd['task'] in ['evaluate', 'cross_validate'] and lrd['additional_scores']: - print('Additional Evaluation Metrics: {}'.format(list(lrd['additional_scores'].keys())), - file=output_file) - print('Scikit-learn Version: {}'.format(lrd['scikit_learn_version']), - file=output_file) - print('Start Timestamp: {}'.format( - lrd['start_timestamp']), file=output_file) - print('End Timestamp: {}'.format(lrd['end_timestamp']), file=output_file) - print('Total Time: {}'.format(lrd['total_time']), file=output_file) - print('\n', file=output_file) - - for lrd in learner_result_dicts: - print('Fold: {}'.format(lrd['fold']), file=output_file) - print('Model Parameters: {}'.format(lrd.get('model_params', '')), - file=output_file) - print('Grid Objective Score (Train) = {}'.format(lrd.get('grid_score', - '')), - file=output_file) - if 'result_table' in lrd: - print(lrd['result_table'], file=output_file) - print('Accuracy = {}'.format(lrd['accuracy']), - file=output_file) - if 'descriptive' in lrd: - print('Descriptive statistics:', file=output_file) - for desc_stat in ['min', 'max', 'avg', 'std']: - actual = lrd['descriptive']['actual'][desc_stat] - predicted = lrd['descriptive']['predicted'][desc_stat] - print((' {} = {: .4f} (actual), {: .4f} ' - '(predicted)').format(desc_stat.title(), actual, - predicted), - file=output_file) - print('Pearson = {: f}'.format(lrd['pearson']), - file=output_file) - print('Objective Function Score (Test) = {}'.format(lrd.get('score', '')), - file=output_file) - - # now print the additional metrics, if there were any - if lrd['additional_scores']: - print('', file=output_file) - print('Additional Evaluation Metrics (Test):', file=output_file) - for metric, score in lrd['additional_scores'].items(): - score = '' if np.isnan(score) else score - print(' {} = {}'.format(metric, score), file=output_file) - print('', file=output_file) - - -def _load_featureset(dir_path, feat_files, suffix, id_col='id', label_col='y', - ids_to_floats=False, quiet=False, class_map=None, - feature_hasher=False, num_features=None, logger=None): - """ - Load a list of feature files and merge them. - - Parameters - ---------- - dir_path : str - Path to the directory that contains the feature files. - feat_files : list of str - A list of feature file prefixes. - suffix : str - The suffix to add to feature file prefixes to get the full filenames. - id_col : str, optional - Name of the column which contains the instance IDs. - If no column with that name exists, or `None` is - specified, example IDs will be automatically generated. - Defaults to ``'id'``. - label_col : str, optional - Name of the column which contains the class labels. - If no column with that name exists, or `None` is - specified, the data is considered to be unlabeled. - Defaults to ``'y'``. - ids_to_floats : bool, optional - Whether to convert the IDs to floats to save memory. Will raise error - if we encounter non-numeric IDs. - Defaults to ``False``. - quiet : bool, optional - Do not print "Loading..." status message to stderr. - Defaults to ``False``. - class_map : dict, optional - Mapping from original class labels to new ones. This is - mainly used for collapsing multiple labels into a single - class. Anything not in the mapping will be kept the same. - Defaults to ``None``. - feature_hasher : bool, optional - Should we use a FeatureHasher when vectorizing - features? - Defaults to ``False``. - num_features : int, optional - The number of features to use with the ``FeatureHasher``. - This should always be set to the power of 2 greater - than the actual number of features you're using. - Defaults to ``None``. - logger : logging.Logger, optional - A logger instance to use to log messages instead of creating - a new one by default. - Defaults to ``None``. - - Returns - ------- - merged_set : skll.FeatureSet - A ``FeatureSet`` instance containing the specified labels, IDs, features, - and feature vectorizer. - """ - # if the training file is specified via train_file, then dir_path - # actually contains the entire file name - if isfile(dir_path): - return Reader.for_path(dir_path, - label_col=label_col, - id_col=id_col, - ids_to_floats=ids_to_floats, - quiet=quiet, - class_map=class_map, - feature_hasher=feature_hasher, - num_features=num_features, - logger=logger).read() - else: - if len(feat_files) > 1 and feature_hasher: - logger.warning("Since there are multiple feature files, " - "feature hashing applies to each specified " - "feature file separately.") - merged_set = None - for file_name in sorted(join(dir_path, featfile + suffix) for - featfile in feat_files): - fs = Reader.for_path(file_name, - label_col=label_col, - id_col=id_col, - ids_to_floats=ids_to_floats, - quiet=quiet, - class_map=class_map, - feature_hasher=feature_hasher, - num_features=num_features, - logger=logger).read() - if merged_set is None: - merged_set = fs - else: - merged_set += fs - return merged_set +__all__ = ['generate_learning_curve_plots', + 'load_featureset', + 'run_configuration'] def _classify_featureset(args): @@ -573,17 +183,17 @@ def _classify_featureset(args): if (task in ['cross_validate', 'learning_curve'] or not exists(modelfile) or overwrite): - train_examples = _load_featureset(train_path, - featureset, - suffix, - label_col=label_col, - id_col=id_col, - ids_to_floats=ids_to_floats, - quiet=quiet, - class_map=class_map, - feature_hasher=feature_hasher, - num_features=hasher_features, - logger=logger) + train_examples = load_featureset(train_path, + featureset, + suffix, + label_col=label_col, + id_col=id_col, + ids_to_floats=ids_to_floats, + quiet=quiet, + class_map=class_map, + feature_hasher=feature_hasher, + num_features=hasher_features, + logger=logger) train_set_size = len(train_examples.ids) if not train_examples.has_labels: @@ -606,7 +216,7 @@ def _classify_featureset(args): # import the custom learner path here in case we are reusing a # saved model if custom_learner_path: - _import_custom_learner(custom_learner_path, learner_name) + import_custom_learner(custom_learner_path, learner_name) train_set_size = 'unknown' if exists(modelfile) and not overwrite: logger.info("Loading pre-existing {} model: {}".format(learner_name, @@ -618,16 +228,16 @@ def _classify_featureset(args): # Load test set if there is one if task == 'evaluate' or task == 'predict': - test_examples = _load_featureset(test_path, - featureset, - suffix, - label_col=label_col, - id_col=id_col, - ids_to_floats=ids_to_floats, - quiet=quiet, - class_map=class_map, - feature_hasher=feature_hasher, - num_features=hasher_features) + test_examples = load_featureset(test_path, + featureset, + suffix, + label_col=label_col, + id_col=id_col, + ids_to_floats=ids_to_floats, + quiet=quiet, + class_map=class_map, + feature_hasher=feature_hasher, + num_features=hasher_features) test_set_size = len(test_examples.ids) else: test_set_size = 'n/a' @@ -832,167 +442,6 @@ def _classify_featureset(args): return res -def _create_learner_result_dicts(task_results, - grid_scores, - grid_search_cv_results_dicts, - learner_result_dict_base): - """ - Create the learner result dictionaries that are used to create JSON and - plain-text results files. - - Parameters - ---------- - task_results : list - The task results list. - grid_scores : list - The grid scores list. - grid_search_cv_results_dicts : list of dicts - A list of dictionaries of grid search CV results, one per fold, - with keys such as "params", "mean_test_score", etc, that are - mapped to lists of values associated with each hyperparameter set - combination. - learner_result_dict_base : dict - Base dictionary for all learner results. - - Returns - ------- - res : list of dicts - The results of the learners, as a list of - dictionaries. - """ - res = [] - - num_folds = len(task_results) - accuracy_sum = 0.0 - pearson_sum = 0.0 - additional_metric_score_sums = {} - score_sum = None - prec_sum_dict = defaultdict(float) - recall_sum_dict = defaultdict(float) - f_sum_dict = defaultdict(float) - result_table = None - - for (k, - ((conf_matrix, - fold_accuracy, - result_dict, - model_params, - score, - additional_scores), - grid_score, - grid_search_cv_results)) in enumerate(zip(task_results, - grid_scores, - grid_search_cv_results_dicts), - start=1): - - # create a new dict for this fold - learner_result_dict = {} - learner_result_dict.update(learner_result_dict_base) - - # initialize some variables to blanks so that the - # set of columns is fixed. - learner_result_dict['result_table'] = '' - learner_result_dict['accuracy'] = '' - learner_result_dict['pearson'] = '' - learner_result_dict['score'] = '' - learner_result_dict['fold'] = '' - - if learner_result_dict_base['task'] == 'cross_validate': - learner_result_dict['fold'] = k - - learner_result_dict['model_params'] = json.dumps(model_params) - if grid_score is not None: - learner_result_dict['grid_score'] = grid_score - learner_result_dict['grid_search_cv_results'] = grid_search_cv_results - - if conf_matrix: - labels = sorted(task_results[0][2].keys()) - headers = [""] + labels + ["Precision", "Recall", "F-measure"] - rows = [] - for i, actual_label in enumerate(labels): - conf_matrix[i][i] = "[{}]".format(conf_matrix[i][i]) - label_prec = _get_stat_float(result_dict[actual_label], - "Precision") - label_recall = _get_stat_float(result_dict[actual_label], - "Recall") - label_f = _get_stat_float(result_dict[actual_label], - "F-measure") - if not math.isnan(label_prec): - prec_sum_dict[actual_label] += float(label_prec) - if not math.isnan(label_recall): - recall_sum_dict[actual_label] += float(label_recall) - if not math.isnan(label_f): - f_sum_dict[actual_label] += float(label_f) - result_row = ([actual_label] + conf_matrix[i] + - [label_prec, label_recall, label_f]) - rows.append(result_row) - - result_table = tabulate(rows, - headers=headers, - stralign="right", - floatfmt=".3f", - tablefmt="grid") - result_table_str = '{}'.format(result_table) - result_table_str += '\n(row = reference; column = predicted)' - learner_result_dict['result_table'] = result_table_str - learner_result_dict['accuracy'] = fold_accuracy - accuracy_sum += fold_accuracy - - # if there is no confusion matrix, then we must be dealing - # with a regression model - else: - learner_result_dict.update(result_dict) - pearson_sum += float(learner_result_dict['pearson']) - - # get the scores for all the metrics and compute the sums - if score is not None: - if score_sum is None: - score_sum = score - else: - score_sum += score - learner_result_dict['score'] = score - learner_result_dict['additional_scores'] = additional_scores - for metric, score in additional_scores.items(): - if score is not None: - additional_metric_score_sums[metric] = \ - additional_metric_score_sums.get(metric, 0) + score - res.append(learner_result_dict) - - if num_folds > 1: - learner_result_dict = {} - learner_result_dict.update(learner_result_dict_base) - - learner_result_dict['fold'] = 'average' - - if result_table: - headers = ["Label", "Precision", "Recall", "F-measure"] - rows = [] - for actual_label in labels: - # Convert sums to means - prec_mean = prec_sum_dict[actual_label] / num_folds - recall_mean = recall_sum_dict[actual_label] / num_folds - f_mean = f_sum_dict[actual_label] / num_folds - rows.append([actual_label] + [prec_mean, recall_mean, f_mean]) - - result_table = tabulate(rows, - headers=headers, - floatfmt=".3f", - tablefmt="psql") - learner_result_dict['result_table'] = '{}'.format(result_table) - learner_result_dict['accuracy'] = accuracy_sum / num_folds - else: - learner_result_dict['pearson'] = pearson_sum / num_folds - - if score_sum is not None: - learner_result_dict['score'] = score_sum / num_folds - scoredict = {} - for metric, score_sum in additional_metric_score_sums.items(): - scoredict[metric] = score_sum / num_folds - learner_result_dict['additional_scores'] = scoredict - res.append(learner_result_dict) - return res - - def run_configuration(config_file, local=False, overwrite=True, queue='all.q', hosts=None, write_summary=True, quiet=False, ablation=0, resume=False, log_level=logging.INFO): @@ -1066,8 +515,8 @@ def run_configuration(config_file, local=False, overwrite=True, queue='all.q', do_stratified_folds, fixed_parameter_list, param_grid_list, featureset_names, learners, prediction_dir, log_path, train_path, test_path, ids_to_floats, class_map, custom_learner_path, learning_curve_cv_folds_list, - learning_curve_train_sizes, output_metrics) = _parse_config_file(config_file, - log_level=log_level) + learning_curve_train_sizes, output_metrics) = parse_config_file(config_file, + log_level=log_level) # get the main experiment logger that will already have been # created by the configuration parser so we don't need anything @@ -1299,9 +748,9 @@ def run_configuration(config_file, local=False, overwrite=True, queue='all.q', _write_learning_curve_file(result_json_paths, output_file) # generate the actual plot if we have the requirements installed - _generate_learning_curve_plots(experiment_name, - results_path, - output_file_path) + generate_learning_curve_plots(experiment_name, + results_path, + output_file_path) finally: @@ -1309,162 +758,3 @@ def run_configuration(config_file, local=False, overwrite=True, queue='all.q', close_and_remove_logger_handlers(get_skll_logger('experiment')) return result_json_paths - - -def _check_job_results(job_results): - """ - See if we have a complete results dictionary for every job. - - Parameters - ---------- - job_results : list of dicts - A list of job result dictionaries. - """ - logger = get_skll_logger('experiment') - logger.info('Checking job results') - for result_dicts in job_results: - if not result_dicts or 'task' not in result_dicts[0]: - logger.error('There was an error running the experiment:\n%s', - result_dicts) - - -def _compute_ylimits_for_featureset(df, metrics): - """ - Compute the y-limits for learning curve plots. - - Parameters - ---------- - df : pd.DataFrame - A data_frame with relevant metric information for - train and test. - metrics : list of str - A list of metrics for learning curve plots. - - Returns - ------- - ylimits : dict - A dictionary, with metric names as keys - and a tuple of (lower_limit, upper_limit) as values. - """ - - # set the y-limits of the curves depending on what kind - # of values the metric produces - ylimits = {} - for metric in metrics: - # get the real min and max for the values that will be plotted - df_train = df[(df['variable'] == 'train_score_mean') & (df['metric'] == metric)] - df_test = df[(df['variable'] == 'test_score_mean') & (df['metric'] == metric)] - train_values_lower = df_train['value'].values - df_train['train_score_std'].values - test_values_lower = df_test['value'].values - df_test['test_score_std'].values - min_score = np.min(np.concatenate([train_values_lower, - test_values_lower])) - train_values_upper = df_train['value'].values + df_train['train_score_std'].values - test_values_upper = df_test['value'].values + df_test['test_score_std'].values - max_score = np.max(np.concatenate([train_values_upper, - test_values_upper])) - - # squeeze the limits to hide unnecessary parts of the graph - # set the limits with a little buffer on either side but not too much - if min_score < 0: - lower_limit = max(min_score - 0.1, math.floor(min_score) - 0.05) - else: - lower_limit = 0 - - if max_score > 0: - upper_limit = min(max_score + 0.1, math.ceil(max_score) + 0.05) - else: - upper_limit = 0 - - ylimits[metric] = (lower_limit, upper_limit) - - return ylimits - - -def _generate_learning_curve_plots(experiment_name, - output_dir, - learning_curve_tsv_file): - """ - Generate the learning curve plots given the TSV output - file from a learning curve experiment. - - Parameters - ---------- - experiment_name : str - The name of the experiment. - output_dir : str - Path to the output directory for the plots. - learning_curve_tsv_file : str - The path to the learning curve TSV file. - """ - - # use pandas to read in the TSV file into a data frame - # and massage it from wide to long format for plotting - df = pd.read_csv(learning_curve_tsv_file, sep='\t') - num_learners = len(df['learner_name'].unique()) - num_metrics = len(df['metric'].unique()) - df_melted = pd.melt(df, id_vars=[c for c in df.columns - if c not in ['train_score_mean', 'test_score_mean']]) - - # if there are any training sizes greater than 1000, - # then we should probably rotate the tick labels - # since otherwise the labels are not clearly rendered - rotate_labels = np.any([size >= 1000 for size in df['training_set_size'].unique()]) - - # set up and draw the actual learning curve figures, one for - # each of the featuresets - for fs_name, df_fs in df_melted.groupby('featureset_name'): - fig = plt.figure() - fig.set_size_inches(2.5 * num_learners, 2.5 * num_metrics) - - # compute ylimits for this feature set for each objective - with sns.axes_style('whitegrid', {"grid.linestyle": ':', - "xtick.major.size": 3.0}): - g = sns.FacetGrid(df_fs, row="metric", col="learner_name", - hue="variable", height=2.5, aspect=1, - margin_titles=True, despine=True, sharex=False, - sharey=False, legend_out=False, palette="Set1") - colors = train_color, test_color = sns.color_palette("Set1")[:2] - g = g.map_dataframe(sns.pointplot, "training_set_size", "value", - scale=.5, ci=None) - ylimits = _compute_ylimits_for_featureset(df_fs, g.row_names) - for ax in g.axes.flat: - plt.setp(ax.texts, text="") - g = (g.set_titles(row_template='', col_template='{col_name}') - .set_axis_labels('Training Examples', 'Score')) - if rotate_labels: - g = g.set_xticklabels(rotation=60) - - for i, row_name in enumerate(g.row_names): - for j, col_name in enumerate(g.col_names): - ax = g.axes[i][j] - ax.set(ylim=ylimits[row_name]) - df_ax_train = df_fs[(df_fs['learner_name'] == col_name) & - (df_fs['metric'] == row_name) & - (df_fs['variable'] == 'train_score_mean')] - df_ax_test = df_fs[(df_fs['learner_name'] == col_name) & - (df_fs['metric'] == row_name) & - (df_fs['variable'] == 'test_score_mean')] - ax.fill_between(list(range(len(df_ax_train))), - df_ax_train['value'] - df_ax_train['train_score_std'], - df_ax_train['value'] + df_ax_train['train_score_std'], - alpha=0.1, - color=train_color) - ax.fill_between(list(range(len(df_ax_test))), - df_ax_test['value'] - df_ax_test['test_score_std'], - df_ax_test['value'] + df_ax_test['test_score_std'], - alpha=0.1, - color=test_color) - if j == 0: - ax.set_ylabel(row_name) - if i == 0: - ax.legend(handles=[matplotlib.lines.Line2D([], [], color=c, label=l, linestyle='-') - for c, l in zip(colors, ['Training', 'Cross-validation'])], - loc=4, - fancybox=True, - fontsize='x-small', - ncol=1, - frameon=True) - g.fig.tight_layout(w_pad=1) - plt.savefig(join(output_dir, '{}_{}.png'.format(experiment_name, fs_name)), dpi=300) - # explicitly close figure to save memory - plt.close(fig) diff --git a/skll/experiments/input.py b/skll/experiments/input.py new file mode 100644 index 00000000..cf66bd28 --- /dev/null +++ b/skll/experiments/input.py @@ -0,0 +1,112 @@ +# License: BSD 3 clause +""" +Functions for reading inputs for SKLL experiments. + +:author: Nitin Madnani (nmadnani@ets.org) +:author: Dan Blanchard (dblanchard@ets.org) +:author: Michael Heilman (mheilman@ets.org) +""" + +from os.path import isfile, join + +from skll.data.readers import Reader + + +def load_featureset(dir_path, + feat_files, + suffix, + id_col='id', + label_col='y', + ids_to_floats=False, + quiet=False, + class_map=None, + feature_hasher=False, + num_features=None, + logger=None): + """ + Load a list of feature files and merge them. + + Parameters + ---------- + dir_path : str + Path to the directory that contains the feature files. + feat_files : list of str + A list of feature file prefixes. + suffix : str + The suffix to add to feature file prefixes to get the full filenames. + id_col : str, optional + Name of the column which contains the instance IDs. + If no column with that name exists, or `None` is + specified, example IDs will be automatically generated. + Defaults to ``'id'``. + label_col : str, optional + Name of the column which contains the class labels. + If no column with that name exists, or `None` is + specified, the data is considered to be unlabeled. + Defaults to ``'y'``. + ids_to_floats : bool, optional + Whether to convert the IDs to floats to save memory. Will raise error + if we encounter non-numeric IDs. + Defaults to ``False``. + quiet : bool, optional + Do not print "Loading..." status message to stderr. + Defaults to ``False``. + class_map : dict, optional + Mapping from original class labels to new ones. This is + mainly used for collapsing multiple labels into a single + class. Anything not in the mapping will be kept the same. + Defaults to ``None``. + feature_hasher : bool, optional + Should we use a FeatureHasher when vectorizing + features? + Defaults to ``False``. + num_features : int, optional + The number of features to use with the ``FeatureHasher``. + This should always be set to the power of 2 greater + than the actual number of features you're using. + Defaults to ``None``. + logger : logging.Logger, optional + A logger instance to use to log messages instead of creating + a new one by default. + Defaults to ``None``. + + Returns + ------- + merged_set : skll.FeatureSet + A ``FeatureSet`` instance containing the specified labels, IDs, features, + and feature vectorizer. + """ + # if the training file is specified via train_file, then dir_path + # actually contains the entire file name + if isfile(dir_path): + return Reader.for_path(dir_path, + label_col=label_col, + id_col=id_col, + ids_to_floats=ids_to_floats, + quiet=quiet, + class_map=class_map, + feature_hasher=feature_hasher, + num_features=num_features, + logger=logger).read() + else: + if len(feat_files) > 1 and feature_hasher: + logger.warning("Since there are multiple feature files, " + "feature hashing applies to each specified " + "feature file separately.") + merged_set = None + for file_name in sorted(join(dir_path, featfile + suffix) for + featfile in feat_files): + fs = Reader.for_path(file_name, + label_col=label_col, + id_col=id_col, + ids_to_floats=ids_to_floats, + quiet=quiet, + class_map=class_map, + feature_hasher=feature_hasher, + num_features=num_features, + logger=logger).read() + if merged_set is None: + merged_set = fs + else: + merged_set += fs + return merged_set diff --git a/skll/experiments/output.py b/skll/experiments/output.py new file mode 100644 index 00000000..64421453 --- /dev/null +++ b/skll/experiments/output.py @@ -0,0 +1,429 @@ +# License: BSD 3 clause +""" +Functions related to running experiments and parsing configuration files. + +:author: Dan Blanchard (dblanchard@ets.org) +:author: Michael Heilman (mheilman@ets.org) +:author: Nitin Madnani (nmadnani@ets.org) +:author: Chee Wee Leong (cleong@ets.org) +""" + +import csv +import json +import math +import sys + +import matplotlib +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import ruamel.yaml as yaml +import seaborn as sns + +from collections import defaultdict +from os.path import exists, join + +from skll.utils.logging import get_skll_logger + +# Turn off interactive plotting for matplotlib +plt.ioff() + + +def _compute_ylimits_for_featureset(df, metrics): + """ + Compute the y-limits for learning curve plots. + + Parameters + ---------- + df : pd.DataFrame + A data_frame with relevant metric information for + train and test. + metrics : list of str + A list of metrics for learning curve plots. + + Returns + ------- + ylimits : dict + A dictionary, with metric names as keys + and a tuple of (lower_limit, upper_limit) as values. + """ + + # set the y-limits of the curves depending on what kind + # of values the metric produces + ylimits = {} + for metric in metrics: + # get the real min and max for the values that will be plotted + df_train = df[(df['variable'] == 'train_score_mean') & (df['metric'] == metric)] + df_test = df[(df['variable'] == 'test_score_mean') & (df['metric'] == metric)] + train_values_lower = df_train['value'].values - df_train['train_score_std'].values + test_values_lower = df_test['value'].values - df_test['test_score_std'].values + min_score = np.min(np.concatenate([train_values_lower, + test_values_lower])) + train_values_upper = df_train['value'].values + df_train['train_score_std'].values + test_values_upper = df_test['value'].values + df_test['test_score_std'].values + max_score = np.max(np.concatenate([train_values_upper, + test_values_upper])) + + # squeeze the limits to hide unnecessary parts of the graph + # set the limits with a little buffer on either side but not too much + if min_score < 0: + lower_limit = max(min_score - 0.1, math.floor(min_score) - 0.05) + else: + lower_limit = 0 + + if max_score > 0: + upper_limit = min(max_score + 0.1, math.ceil(max_score) + 0.05) + else: + upper_limit = 0 + + ylimits[metric] = (lower_limit, upper_limit) + + return ylimits + + +def generate_learning_curve_plots(experiment_name, + output_dir, + learning_curve_tsv_file): + """ + Generate the learning curve plots given the TSV output + file from a learning curve experiment. + + Parameters + ---------- + experiment_name : str + The name of the experiment. + output_dir : str + Path to the output directory for the plots. + learning_curve_tsv_file : str + The path to the learning curve TSV file. + """ + + # use pandas to read in the TSV file into a data frame + # and massage it from wide to long format for plotting + df = pd.read_csv(learning_curve_tsv_file, sep='\t') + num_learners = len(df['learner_name'].unique()) + num_metrics = len(df['metric'].unique()) + df_melted = pd.melt(df, id_vars=[c for c in df.columns + if c not in ['train_score_mean', 'test_score_mean']]) + + # if there are any training sizes greater than 1000, + # then we should probably rotate the tick labels + # since otherwise the labels are not clearly rendered + rotate_labels = np.any([size >= 1000 for size in df['training_set_size'].unique()]) + + # set up and draw the actual learning curve figures, one for + # each of the featuresets + for fs_name, df_fs in df_melted.groupby('featureset_name'): + fig = plt.figure() + fig.set_size_inches(2.5 * num_learners, 2.5 * num_metrics) + + # compute ylimits for this feature set for each objective + with sns.axes_style('whitegrid', {"grid.linestyle": ':', + "xtick.major.size": 3.0}): + g = sns.FacetGrid(df_fs, row="metric", col="learner_name", + hue="variable", height=2.5, aspect=1, + margin_titles=True, despine=True, sharex=False, + sharey=False, legend_out=False, palette="Set1") + colors = train_color, test_color = sns.color_palette("Set1")[:2] + g = g.map_dataframe(sns.pointplot, "training_set_size", "value", + scale=.5, ci=None) + ylimits = _compute_ylimits_for_featureset(df_fs, g.row_names) + for ax in g.axes.flat: + plt.setp(ax.texts, text="") + g = (g.set_titles(row_template='', col_template='{col_name}') + .set_axis_labels('Training Examples', 'Score')) + if rotate_labels: + g = g.set_xticklabels(rotation=60) + + for i, row_name in enumerate(g.row_names): + for j, col_name in enumerate(g.col_names): + ax = g.axes[i][j] + ax.set(ylim=ylimits[row_name]) + df_ax_train = df_fs[(df_fs['learner_name'] == col_name) & + (df_fs['metric'] == row_name) & + (df_fs['variable'] == 'train_score_mean')] + df_ax_test = df_fs[(df_fs['learner_name'] == col_name) & + (df_fs['metric'] == row_name) & + (df_fs['variable'] == 'test_score_mean')] + ax.fill_between(list(range(len(df_ax_train))), + df_ax_train['value'] - df_ax_train['train_score_std'], + df_ax_train['value'] + df_ax_train['train_score_std'], + alpha=0.1, + color=train_color) + ax.fill_between(list(range(len(df_ax_test))), + df_ax_test['value'] - df_ax_test['test_score_std'], + df_ax_test['value'] + df_ax_test['test_score_std'], + alpha=0.1, + color=test_color) + if j == 0: + ax.set_ylabel(row_name) + if i == 0: + ax.legend(handles=[matplotlib.lines.Line2D([], [], color=c, label=l, linestyle='-') + for c, l in zip(colors, ['Training', 'Cross-validation'])], + loc=4, + fancybox=True, + fontsize='x-small', + ncol=1, + frameon=True) + g.fig.tight_layout(w_pad=1) + plt.savefig(join(output_dir, '{}_{}.png'.format(experiment_name, fs_name)), dpi=300) + # explicitly close figure to save memory + plt.close(fig) + + +def _print_fancy_output(learner_result_dicts, output_file=sys.stdout): + """ + Function to take all of the results from all of the folds and print + nice tables with the results. + + Parameters + ---------- + learner_result_dicts : list of str + A list of paths to the individual result JSON files. + output_file : file buffer, optional + The file buffer to print to. + Defaults to ``sys.stdout``. + """ + if not learner_result_dicts: + raise ValueError('Result dictionary list is empty!') + + lrd = learner_result_dicts[0] + print('Experiment Name: {}'.format(lrd['experiment_name']), + file=output_file) + print('SKLL Version: {}'.format(lrd['version']), file=output_file) + print('Training Set: {}'.format(lrd['train_set_name']), file=output_file) + print('Training Set Size: {}'.format( + lrd['train_set_size']), file=output_file) + print('Test Set: {}'.format(lrd['test_set_name']), file=output_file) + print('Test Set Size: {}'.format(lrd['test_set_size']), file=output_file) + print('Shuffle: {}'.format(lrd['shuffle']), file=output_file) + print('Feature Set: {}'.format(lrd['featureset']), file=output_file) + print('Learner: {}'.format(lrd['learner_name']), file=output_file) + print('Task: {}'.format(lrd['task']), file=output_file) + if lrd['folds_file']: + print('Specified Folds File: {}'.format(lrd['folds_file']), + file=output_file) + if lrd['task'] == 'cross_validate': + print('Number of Folds: {}'.format(lrd['cv_folds']), + file=output_file) + if not lrd['cv_folds'].endswith('folds file'): + print('Stratified Folds: {}'.format(lrd['stratified_folds']), + file=output_file) + print('Feature Scaling: {}'.format(lrd['feature_scaling']), + file=output_file) + print('Grid Search: {}'.format(lrd['grid_search']), file=output_file) + if lrd['grid_search']: + print('Grid Search Folds: {}'.format(lrd['grid_search_folds']), + file=output_file) + print('Grid Objective Function: {}'.format(lrd['grid_objective']), + file=output_file) + if (lrd['task'] == 'cross_validate' and + lrd['grid_search'] and + lrd['cv_folds'].endswith('folds file')): + print('Using Folds File for Grid Search: {}'.format(lrd['use_folds_file_for_grid_search']), + file=output_file) + if lrd['task'] in ['evaluate', 'cross_validate'] and lrd['additional_scores']: + print('Additional Evaluation Metrics: {}'.format(list(lrd['additional_scores'].keys())), + file=output_file) + print('Scikit-learn Version: {}'.format(lrd['scikit_learn_version']), + file=output_file) + print('Start Timestamp: {}'.format( + lrd['start_timestamp']), file=output_file) + print('End Timestamp: {}'.format(lrd['end_timestamp']), file=output_file) + print('Total Time: {}'.format(lrd['total_time']), file=output_file) + print('\n', file=output_file) + + for lrd in learner_result_dicts: + print('Fold: {}'.format(lrd['fold']), file=output_file) + print('Model Parameters: {}'.format(lrd.get('model_params', '')), + file=output_file) + print('Grid Objective Score (Train) = {}'.format(lrd.get('grid_score', + '')), + file=output_file) + if 'result_table' in lrd: + print(lrd['result_table'], file=output_file) + print('Accuracy = {}'.format(lrd['accuracy']), + file=output_file) + if 'descriptive' in lrd: + print('Descriptive statistics:', file=output_file) + for desc_stat in ['min', 'max', 'avg', 'std']: + actual = lrd['descriptive']['actual'][desc_stat] + predicted = lrd['descriptive']['predicted'][desc_stat] + print((' {} = {: .4f} (actual), {: .4f} ' + '(predicted)').format(desc_stat.title(), actual, + predicted), + file=output_file) + print('Pearson = {: f}'.format(lrd['pearson']), + file=output_file) + print('Objective Function Score (Test) = {}'.format(lrd.get('score', '')), + file=output_file) + + # now print the additional metrics, if there were any + if lrd['additional_scores']: + print('', file=output_file) + print('Additional Evaluation Metrics (Test):', file=output_file) + for metric, score in lrd['additional_scores'].items(): + score = '' if np.isnan(score) else score + print(' {} = {}'.format(metric, score), file=output_file) + print('', file=output_file) + + +def _write_learning_curve_file(result_json_paths, output_file): + """ + Function to take a list of paths to individual learning curve + results json files and writes out a single TSV file with the + learning curve data. + + Parameters + ---------- + result_json_paths : list of str + A list of paths to the individual result JSON files. + output_file : str + The path to the output file (TSV format). + """ + + learner_result_dicts = [] + + # Map from feature set names to all features in them + logger = get_skll_logger('experiment') + for json_path in result_json_paths: + if not exists(json_path): + logger.error(('JSON results file %s not found. Skipping summary ' + 'creation. You can manually create the summary file' + ' after the fact by using the summarize_results ' + 'script.'), json_path) + return + else: + with open(json_path, 'r') as json_file: + obj = json.load(json_file) + learner_result_dicts.extend(obj) + + # Build and write header + header = ['featureset_name', 'learner_name', 'metric', + 'train_set_name', 'training_set_size', 'train_score_mean', + 'test_score_mean', 'train_score_std', 'test_score_std', + 'scikit_learn_version', 'version'] + writer = csv.DictWriter(output_file, + header, + extrasaction='ignore', + dialect=csv.excel_tab) + writer.writeheader() + + # write out the fields we need for the learning curve file + # specifically, we need to separate out the curve sizes + # and scores into individual entries. + for lrd in learner_result_dicts: + training_set_sizes = lrd['computed_curve_train_sizes'] + train_scores_means_by_size = lrd['learning_curve_train_scores_means'] + test_scores_means_by_size = lrd['learning_curve_test_scores_means'] + train_scores_stds_by_size = lrd['learning_curve_train_scores_stds'] + test_scores_stds_by_size = lrd['learning_curve_test_scores_stds'] + + # rename `grid_objective` to `metric` since the latter name can be confusing + lrd['metric'] = lrd['grid_objective'] + + for (size, + train_score_mean, + test_score_mean, + train_score_std, + test_score_std) in zip(training_set_sizes, + train_scores_means_by_size, + test_scores_means_by_size, + train_scores_stds_by_size, + test_scores_stds_by_size): + lrd['training_set_size'] = size + lrd['train_score_mean'] = train_score_mean + lrd['test_score_mean'] = test_score_mean + lrd['train_score_std'] = train_score_std + lrd['test_score_std'] = test_score_std + + writer.writerow(lrd) + + output_file.flush() + + +def _write_skll_folds(skll_fold_ids, skll_fold_ids_file): + """ + Function to take a dictionary of id->test-fold-number and + write it to a file. + + Parameters + ---------- + skll_fold_ids : dict + Dictionary with ids as keys and test-fold-numbers as values. + skll_fold_ids_file : file buffer + An open file handler to write to. + """ + + f = csv.writer(skll_fold_ids_file) + f.writerow(['id', 'cv_test_fold']) + for example_id in skll_fold_ids: + f.writerow([example_id, skll_fold_ids[example_id]]) + + skll_fold_ids_file.flush() + + +def _write_summary_file(result_json_paths, output_file, ablation=0): + """ + Function to take a list of paths to individual result + json files and returns a single file that summarizes + all of them. + + Parameters + ---------- + result_json_paths : list of str + A list of paths to the individual result JSON files. + output_file : str + The path to the output file (TSV format). + ablation : int, optional + The number of features to remove when doing ablation experiment. + Defaults to 0. + """ + learner_result_dicts = [] + # Map from feature set names to all features in them + all_features = defaultdict(set) + logger = get_skll_logger('experiment') + for json_path in result_json_paths: + if not exists(json_path): + logger.error(('JSON results file %s not found. Skipping summary ' + 'creation. You can manually create the summary file' + ' after the fact by using the summarize_results ' + 'script.'), json_path) + return + else: + with open(json_path, 'r') as json_file: + obj = json.load(json_file) + featureset_name = obj[0]['featureset_name'] + if ablation != 0 and '_minus_' in featureset_name: + parent_set = featureset_name.split('_minus_', 1)[0] + all_features[parent_set].update( + yaml.safe_load(obj[0]['featureset'])) + learner_result_dicts.extend(obj) + + # Build and write header + header = set(learner_result_dicts[0].keys()) - {'result_table', + 'descriptive'} + if ablation != 0: + header.add('ablated_features') + header = sorted(header) + writer = csv.DictWriter(output_file, + header, + extrasaction='ignore', + dialect=csv.excel_tab) + writer.writeheader() + + # Build "ablated_features" list and fix some backward compatible things + for lrd in learner_result_dicts: + featureset_name = lrd['featureset_name'] + if ablation != 0: + parent_set = featureset_name.split('_minus_', 1)[0] + ablated_features = all_features[parent_set].difference( + yaml.safe_load(lrd['featureset'])) + lrd['ablated_features'] = '' + if ablated_features: + lrd['ablated_features'] = json.dumps(sorted(ablated_features)) + + # write out the new learner dict with the readable fields + writer.writerow(lrd) + + output_file.flush() diff --git a/skll/experiments/utils.py b/skll/experiments/utils.py new file mode 100644 index 00000000..da0593c9 --- /dev/null +++ b/skll/experiments/utils.py @@ -0,0 +1,239 @@ +# License: BSD 3 clause +""" +Utility classes and functions for running SKLL experiments. + +:author: Nitin Madnani (nmadnani@ets.org) +:author: Dan Blanchard (dblanchard@ets.org) +:author: Michael Heilman (mheilman@ets.org) +""" + +import json +import math + +import numpy as np + +from collections import defaultdict +from tabulate import tabulate + +from skll.utils.logging import get_skll_logger + + +class NumpyTypeEncoder(json.JSONEncoder): + """ + This class is used when serializing results, particularly the input label + values if the input has int-valued labels. Numpy int64 objects can't + be serialized by the json module, so we must convert them to int objects. + + A related issue where this was adapted from: + https://stackoverflow.com/questions/11561932/why-does-json-dumpslistnp-arange5-fail-while-json-dumpsnp-arange5-tolis + """ + + def default(self, obj): + if isinstance(obj, (np.int32, np.int64)): + return int(obj) + elif isinstance(obj, np.ndarray): + return obj.tolist() + return json.JSONEncoder.default(self, obj) + + +def _check_job_results(job_results): + """ + See if we have a complete results dictionary for every job. + + Parameters + ---------- + job_results : list of dicts + A list of job result dictionaries. + """ + logger = get_skll_logger('experiment') + logger.info('Checking job results') + for result_dicts in job_results: + if not result_dicts or 'task' not in result_dicts[0]: + logger.error('There was an error running the experiment:\n%s', + result_dicts) + + +def _create_learner_result_dicts(task_results, + grid_scores, + grid_search_cv_results_dicts, + learner_result_dict_base): + """ + Create the learner result dictionaries that are used to create JSON and + plain-text results files. + + Parameters + ---------- + task_results : list + The task results list. + grid_scores : list + The grid scores list. + grid_search_cv_results_dicts : list of dicts + A list of dictionaries of grid search CV results, one per fold, + with keys such as "params", "mean_test_score", etc, that are + mapped to lists of values associated with each hyperparameter set + combination. + learner_result_dict_base : dict + Base dictionary for all learner results. + + Returns + ------- + res : list of dicts + The results of the learners, as a list of + dictionaries. + """ + res = [] + + num_folds = len(task_results) + accuracy_sum = 0.0 + pearson_sum = 0.0 + additional_metric_score_sums = {} + score_sum = None + prec_sum_dict = defaultdict(float) + recall_sum_dict = defaultdict(float) + f_sum_dict = defaultdict(float) + result_table = None + + for (k, + ((conf_matrix, + fold_accuracy, + result_dict, + model_params, + score, + additional_scores), + grid_score, + grid_search_cv_results)) in enumerate(zip(task_results, + grid_scores, + grid_search_cv_results_dicts), + start=1): + + # create a new dict for this fold + learner_result_dict = {} + learner_result_dict.update(learner_result_dict_base) + + # initialize some variables to blanks so that the + # set of columns is fixed. + learner_result_dict['result_table'] = '' + learner_result_dict['accuracy'] = '' + learner_result_dict['pearson'] = '' + learner_result_dict['score'] = '' + learner_result_dict['fold'] = '' + + if learner_result_dict_base['task'] == 'cross_validate': + learner_result_dict['fold'] = k + + learner_result_dict['model_params'] = json.dumps(model_params) + if grid_score is not None: + learner_result_dict['grid_score'] = grid_score + learner_result_dict['grid_search_cv_results'] = grid_search_cv_results + + if conf_matrix: + labels = sorted(task_results[0][2].keys()) + headers = [""] + labels + ["Precision", "Recall", "F-measure"] + rows = [] + for i, actual_label in enumerate(labels): + conf_matrix[i][i] = "[{}]".format(conf_matrix[i][i]) + label_prec = _get_stat_float(result_dict[actual_label], + "Precision") + label_recall = _get_stat_float(result_dict[actual_label], + "Recall") + label_f = _get_stat_float(result_dict[actual_label], + "F-measure") + if not math.isnan(label_prec): + prec_sum_dict[actual_label] += float(label_prec) + if not math.isnan(label_recall): + recall_sum_dict[actual_label] += float(label_recall) + if not math.isnan(label_f): + f_sum_dict[actual_label] += float(label_f) + result_row = ([actual_label] + conf_matrix[i] + + [label_prec, label_recall, label_f]) + rows.append(result_row) + + result_table = tabulate(rows, + headers=headers, + stralign="right", + floatfmt=".3f", + tablefmt="grid") + result_table_str = '{}'.format(result_table) + result_table_str += '\n(row = reference; column = predicted)' + learner_result_dict['result_table'] = result_table_str + learner_result_dict['accuracy'] = fold_accuracy + accuracy_sum += fold_accuracy + + # if there is no confusion matrix, then we must be dealing + # with a regression model + else: + learner_result_dict.update(result_dict) + pearson_sum += float(learner_result_dict['pearson']) + + # get the scores for all the metrics and compute the sums + if score is not None: + if score_sum is None: + score_sum = score + else: + score_sum += score + learner_result_dict['score'] = score + learner_result_dict['additional_scores'] = additional_scores + for metric, score in additional_scores.items(): + if score is not None: + additional_metric_score_sums[metric] = \ + additional_metric_score_sums.get(metric, 0) + score + res.append(learner_result_dict) + + if num_folds > 1: + learner_result_dict = {} + learner_result_dict.update(learner_result_dict_base) + + learner_result_dict['fold'] = 'average' + + if result_table: + headers = ["Label", "Precision", "Recall", "F-measure"] + rows = [] + for actual_label in labels: + # Convert sums to means + prec_mean = prec_sum_dict[actual_label] / num_folds + recall_mean = recall_sum_dict[actual_label] / num_folds + f_mean = f_sum_dict[actual_label] / num_folds + rows.append([actual_label] + [prec_mean, recall_mean, f_mean]) + + result_table = tabulate(rows, + headers=headers, + floatfmt=".3f", + tablefmt="psql") + learner_result_dict['result_table'] = '{}'.format(result_table) + learner_result_dict['accuracy'] = accuracy_sum / num_folds + else: + learner_result_dict['pearson'] = pearson_sum / num_folds + + if score_sum is not None: + learner_result_dict['score'] = score_sum / num_folds + scoredict = {} + for metric, score_sum in additional_metric_score_sums.items(): + scoredict[metric] = score_sum / num_folds + learner_result_dict['additional_scores'] = scoredict + res.append(learner_result_dict) + return res + + +def _get_stat_float(label_result_dict, stat): + """ + A helper function to get output for the precision, recall, and f-score + columns in the confusion matrix. + + Parameters + ---------- + label_result_dict : dict + Dictionary containing the stat we'd like + to retrieve for a particular label. + stat : str + The statistic we're looking for in the dictionary. + + Returns + ------- + stat_float : float + The value of the stat if it's in the dictionary, and NaN + otherwise. + """ + if stat in label_result_dict and label_result_dict[stat] is not None: + return label_result_dict[stat] + else: + return float('nan') From 1446f7d65345fb42f7f329b60f5dfb98017dddfd Mon Sep 17 00:00:00 2001 From: Nitin Madnani Date: Fri, 10 Apr 2020 17:03:19 -0400 Subject: [PATCH 03/20] Convert `learner.py` into `learner` package Split into `learner/__init__.py` and `learner/utils.py`. --- skll/{learner.py => learner/__init__.py} | 923 ++++------------------- skll/learner/utils.py | 582 ++++++++++++++ 2 files changed, 723 insertions(+), 782 deletions(-) rename skll/{learner.py => learner/__init__.py} (77%) create mode 100644 skll/learner/utils.py diff --git a/skll/learner.py b/skll/learner/__init__.py similarity index 77% rename from skll/learner.py rename to skll/learner/__init__.py index dceaf4e1..a38e1533 100644 --- a/skll/learner.py +++ b/skll/learner/__init__.py @@ -1,23 +1,18 @@ -# License: BSD 3 clause """ -Provides easy-to-use wrapper around scikit-learn. +An easy-to-use class that wraps scikit-learn estimators. -:author: Michael Heilman (mheilman@ets.org) :author: Nitin Madnani (nmadnani@ets.org) +:author: Michael Heilman (mheilman@ets.org) :author: Dan Blanchard (dblanchard@ets.org) :author: Aoife Cahill (acahill@ets.org) :organization: ETS """ -# pylint: disable=F0401,W0622,E1002,E1101 import copy -import inspect import logging import os -import sys from collections import Counter, defaultdict -from functools import wraps from math import floor, log10 from importlib import import_module from itertools import combinations @@ -26,13 +21,11 @@ import joblib import numpy as np import scipy.sparse as sp -from sklearn.base import BaseEstimator, TransformerMixin from sklearn.model_selection import (GridSearchCV, KFold, - LeaveOneGroupOut, ShuffleSplit, StratifiedKFold) -from sklearn.dummy import DummyClassifier, DummyRegressor +from sklearn.dummy import DummyClassifier from sklearn.ensemble import (AdaBoostClassifier, AdaBoostRegressor, GradientBoostingClassifier, @@ -41,11 +34,9 @@ RandomForestRegressor) from sklearn.feature_extraction import FeatureHasher from sklearn.feature_extraction import DictVectorizer as OldDictVectorizer -from sklearn.feature_selection import SelectKBest from sklearn.metrics import make_scorer from sklearn.pipeline import Pipeline from sklearn.utils.multiclass import type_of_target -# AdditiveChi2Sampler is used indirectly, so ignore linting message from sklearn.kernel_approximation import (Nystroem, RBFSampler, SkewedChi2Sampler) @@ -67,7 +58,7 @@ confusion_matrix, precision_recall_fscore_support) from sklearn.naive_bayes import MultinomialNB -from sklearn.neighbors import KNeighborsClassifier, KNeighborsRegressor +from sklearn.neighbors import KNeighborsRegressor from sklearn.neural_network import MLPClassifier, MLPRegressor from sklearn.preprocessing import StandardScaler from sklearn.svm import LinearSVC, SVC, LinearSVR, SVR @@ -77,768 +68,35 @@ from skll.data import FeatureSet from skll.data.dict_vectorizer import DictVectorizer from skll.data.readers import safe_float -from skll.metrics import (_CLASSIFICATION_ONLY_METRICS, - _CORRELATION_METRICS, - _REGRESSION_ONLY_METRICS, - _UNWEIGHTED_KAPPA_METRICS, - _WEIGHTED_KAPPA_METRICS, - SCORERS, - use_score_func) +from skll.metrics import SCORERS, use_score_func +from skll.utils.constants import (CORRELATION_METRICS, + KNOWN_REQUIRES_DENSE, + KNOWN_DEFAULT_PARAM_GRIDS, + MAX_CONCURRENT_PROCESSES) from skll.version import VERSION -# Constants # -_DEFAULT_PARAM_GRIDS = {AdaBoostClassifier: - [{'learning_rate': [0.01, 0.1, 1.0, 10.0, 100.0]}], - AdaBoostRegressor: - [{'learning_rate': [0.01, 0.1, 1.0, 10.0, 100.0]}], - BayesianRidge: - [{'alpha_1': [1e-6, 1e-4, 1e-2, 1, 10], - 'alpha_2': [1e-6, 1e-4, 1e-2, 1, 10], - 'lambda_1': [1e-6, 1e-4, 1e-2, 1, 10], - 'lambda_2': [1e-6, 1e-4, 1e-2, 1, 10]}], - DecisionTreeClassifier: - [{'max_features': ["auto", None]}], - DecisionTreeRegressor: - [{'max_features': ["auto", None]}], - DummyClassifier: - [{}], - DummyRegressor: - [{}], - ElasticNet: - [{'alpha': [0.01, 0.1, 1.0, 10.0, 100.0]}], - GradientBoostingClassifier: - [{'max_depth': [1, 3, 5]}], - GradientBoostingRegressor: - [{'max_depth': [1, 3, 5]}], - HuberRegressor: - [{'epsilon': [1.05, 1.35, 1.5, 2.0, 2.5, 5.0], - 'alpha': [1e-4, 1e-3, 1e-2, 1e-1, 1, 10, 100, 1000]}], - KNeighborsClassifier: - [{'n_neighbors': [1, 5, 10, 100], - 'weights': ['uniform', 'distance']}], - KNeighborsRegressor: - [{'n_neighbors': [1, 5, 10, 100], - 'weights': ['uniform', 'distance']}], - MLPClassifier: - [{'activation': ['logistic', 'tanh', 'relu'], - 'alpha': [1e-4, 1e-3, 1e-2, 1e-1, 1], - 'learning_rate_init': [0.001, 0.01, 0.1]}], - MLPRegressor: - [{'activation': ['logistic', 'tanh', 'relu'], - 'alpha': [1e-4, 1e-3, 1e-2, 1e-1, 1], - 'learning_rate_init': [0.001, 0.01, 0.1]}], - MultinomialNB: - [{'alpha': [0.1, 0.25, 0.5, 0.75, 1.0]}], - Lars: - [{}], - Lasso: - [{'alpha': [0.01, 0.1, 1.0, 10.0, 100.0]}], - LinearRegression: - [{}], - LinearSVC: - [{'C': [0.01, 0.1, 1.0, 10.0, 100.0]}], - LogisticRegression: - [{'C': [0.01, 0.1, 1.0, 10.0, 100.0]}], - SVC: - [{'C': [0.01, 0.1, 1.0, 10.0, 100.0], - 'gamma': ['auto', 'scale', 0.01, 0.1, 1.0, 10.0, 100.0]}], - RandomForestClassifier: - [{'max_depth': [1, 5, 10, None]}], - RandomForestRegressor: - [{'max_depth': [1, 5, 10, None]}], - RANSACRegressor: - [{}], - Ridge: - [{'alpha': [0.01, 0.1, 1.0, 10.0, 100.0]}], - RidgeClassifier: - [{'alpha': [0.01, 0.1, 1.0, 10.0, 100.0]}], - SGDClassifier: - [{'alpha': [0.000001, 0.00001, 0.0001, 0.001, 0.01], - 'penalty': ['l1', 'l2', 'elasticnet']}], - SGDRegressor: - [{'alpha': [0.000001, 0.00001, 0.0001, 0.001, 0.01], - 'penalty': ['l1', 'l2', 'elasticnet']}], - LinearSVR: - [{'C': [0.01, 0.1, 1.0, 10.0, 100.0]}], - SVR: - [{'C': [0.01, 0.1, 1.0, 10.0, 100.0], - 'gamma': ['auto', 'scale', 0.01, 0.1, 1.0, 10.0, 100.0]}], - TheilSenRegressor: - [{}]} - -_REQUIRES_DENSE = (BayesianRidge, Lars, TheilSenRegressor) - -MAX_CONCURRENT_PROCESSES = int(os.getenv('SKLL_MAX_CONCURRENT_PROCESSES', '3')) - - -# pylint: disable=W0223,R0903 -class Densifier(BaseEstimator, TransformerMixin): - """ - A custom pipeline stage that will be inserted into the - learner pipeline attribute to accommodate the situation - when SKLL needs to manually convert feature arrays from - sparse to dense. For example, when features are being hashed - but we are also doing centering using the feature means. - """ - - def fit(self, X, y=None): - return self - - def fit_transform(self, X, y=None): - return self - - def transform(self, X): - return X.todense() - - -class FilteredLeaveOneGroupOut(LeaveOneGroupOut): - - """ - Version of ``LeaveOneGroupOut`` cross-validation iterator that only outputs - indices of instances with IDs in a prespecified set. - - Parameters - ---------- - keep : set of str - A set of IDs to keep. - example_ids : list of str, of length n_samples - A list of example IDs. - """ - - def __init__(self, keep, example_ids, logger=None): - super(FilteredLeaveOneGroupOut, self).__init__() - self.keep = keep - self.example_ids = example_ids - self._warned = False - self.logger = logger if logger else logging.getLogger(__name__) - - def split(self, X, y, groups): - """ - Generate indices to split data into training and test set. - - Parameters - ---------- - X : array-like, with shape (n_samples, n_features) - Training data, where n_samples is the number of samples - and n_features is the number of features. - y : array-like, of length n_samples - The target variable for supervised learning problems. - groups : array-like, with shape (n_samples,) - Group labels for the samples used while splitting the dataset into - train/test set. - - Yields - ------- - train_index : np.array - The training set indices for that split. - test_index : np.array - The testing set indices for that split. - """ - for train_index, test_index in super(FilteredLeaveOneGroupOut, - self).split(X, y, groups): - train_len = len(train_index) - test_len = len(test_index) - train_index = [i for i in train_index if self.example_ids[i] in - self.keep] - test_index = [i for i in test_index if self.example_ids[i] in - self.keep] - if not self._warned and (train_len != len(train_index) or - test_len != len(test_index)): - self.logger.warning('Feature set contains IDs that are not ' + - 'in folds dictionary. Skipping those IDs.') - self._warned = True - - yield train_index, test_index - - -def _contiguous_ints_or_floats(numbers): - """ - Check whether the given list of numbers contains - contiguous integers or contiguous integer-like - floats. For example, [1, 2, 3] or [4.0, 5.0, 6.0] - are both contiguous but [1.1, 1.2, 1.3] is not. - - Parameters - ---------- - numbers : array-like of ints or floats - The numbers we want to check. - - Returns - ------- - answer : bool - True if the numbers are contiguous integers - or contiguous integer-like floats (1.0, 2.0, etc.) - - Raises - ------ - TypeError - If ``numbers`` does not contain integers or floating - point values. - ValueError - If ``numbers`` is empty. - """ - - try: - - # make sure that number is not empty - assert len(numbers) > 0 - - # first check that the numbers are all integers - # or integer-like floats (e.g., 1.0, 2.0 etc.) - ints_or_int_like_floats = np.all(np.mod(numbers, 1) == 0) - - # next check that the successive differences between - # the numbers are all 1, i.e., they are nuermicontiguous - contiguous = np.all(np.diff(numbers) == 1) - - except AssertionError: - raise ValueError('Input cannot be empty.') - - except TypeError: - raise TypeError('Input should only contain numbers.') - - # we need both conditions to be true - return ints_or_int_like_floats and contiguous - - -def _find_default_param_grid(cls): - """ - Finds the default parameter grid for the specified classifier. - - Parameters - ---------- - cls - A parent classifier class to check, and find the - default param grid. - - Returns - ------- - grid : list of dicts or None - The parameters grid for a given classifier. - """ - for key_cls, grid in _DEFAULT_PARAM_GRIDS.items(): - if issubclass(cls, key_cls): - return grid - return None - - -def _import_custom_learner(custom_learner_path, custom_learner_name): - """ - Does the gruntwork of adding the custom model's module to globals. - - Parameters - ---------- - custom_learner_path : str - The path to a custom learner. - custom_learner_name : str - The name of a custom learner. - - Raises - ------ - ValueError - If the custom learner path is None. - ValueError - If the custom learner path does not end in '.py'. - """ - if not custom_learner_path: - raise ValueError('custom_learner_path was not set and learner {} ' - 'was not found.'.format(custom_learner_name)) - - if not custom_learner_path.endswith('.py'): - raise ValueError('custom_learner_path must end in .py ({})' - .format(custom_learner_path)) - - custom_learner_module_name = os.path.basename(custom_learner_path)[:-3] - sys.path.append(os.path.dirname(os.path.abspath(custom_learner_path))) - import_module(custom_learner_module_name) - globals()[custom_learner_name] = \ - getattr(sys.modules[custom_learner_module_name], custom_learner_name) - - -def _train_and_score(learner, - train_examples, - test_examples, - metric): - """ - A utility method to train a given learner instance on the given training examples, - generate predictions on the training set itself and also the given - test set, and score those predictions using the given metric. - The method returns the train and test scores. - - Note that this method needs to be a top-level function since it is - called from within ``joblib.Parallel()`` and, therefore, needs to be - picklable which it would not be as an instancemethod of the ``Learner`` - class. - - Parameters - ---------- - learner : skll.Learner - A SKLL ``Learner`` instance. - train_examples : array-like, with shape (n_samples, n_features) - The training examples. - test_examples : array-like, of length n_samples - The test examples. - metric : str - The scoring function passed to ``use_score_func()``. - - Returns - ------- - train_score : float - Output of the score function applied to predictions of - ``learner`` on ``train_examples``. - test_score : float - Output of the score function applied to predictions of - ``learner`` on ``test_examples``. - """ - - _ = learner.train(train_examples, grid_search=False, shuffle=False) - train_predictions = learner.predict(train_examples) - test_predictions = learner.predict(test_examples) - if learner.model_type._estimator_type == 'classifier': - test_label_list = np.unique(test_examples.labels).tolist() - unseen_test_label_list = [label for label in test_label_list - if label not in learner.label_list] - unseen_label_dict = {label: i for i, label in enumerate(unseen_test_label_list, - start=len(learner.label_list))} - # combine the two dictionaries - train_and_test_label_dict = learner.label_dict.copy() - train_and_test_label_dict.update(unseen_label_dict) - train_labels = np.array([train_and_test_label_dict[label] - for label in train_examples.labels]) - test_labels = np.array([train_and_test_label_dict[label] - for label in test_examples.labels]) - else: - train_labels = train_examples.labels - test_labels = test_examples.labels - - train_score = use_score_func(metric, train_labels, train_predictions) - test_score = use_score_func(metric, test_labels, test_predictions) - return train_score, test_score - - -def _get_acceptable_regression_metrics(): - """ - Return the set of metrics that are acceptable for regression. - """ - - # it's fairly straightforward for regression since - # we do not have to check the labels - acceptable_metrics = (_REGRESSION_ONLY_METRICS | - _UNWEIGHTED_KAPPA_METRICS | - _WEIGHTED_KAPPA_METRICS | - _CORRELATION_METRICS) - return acceptable_metrics - - -def _get_acceptable_classification_metrics(label_array): - """ - Return the set of metrics that are acceptable given the - the unique set of labels that we are classifying. - - Parameters - ---------- - label_array : numpy.ndarray - A sorted numpy array containing the unique labels - that we are trying to predict. Optional for regressors - but required for classifiers. - - Returns - ------- - acceptable_metrics : set - A set of metric names that are acceptable - for the given classification scenario. - """ - - # this is a classifier so the acceptable objective - # functions definitely include those metrics that - # are specifically for classification and also - # the unweighted kappa metrics - acceptable_metrics = _CLASSIFICATION_ONLY_METRICS | _UNWEIGHTED_KAPPA_METRICS - - # now let us consider which other metrics may also - # be acceptable depending on whether the labels - # are strings or (contiguous) integers/floats - label_type = label_array.dtype.type - - # CASE 1: labels are strings, then no other metrics - # are acceptable - if issubclass(label_type, (np.object_, str)): - pass - - # CASE 2: labels are integers or floats; the way - # it works in SKLL, it's guaranteed that - # class indices will be sorted in the same order - # as the class labels therefore, ranking metrics - # such as various correlations should work fine. - elif issubclass(label_type, (int, - np.int32, - np.int64, - float, - np.float32, - np.float64)): - acceptable_metrics.update(_CORRELATION_METRICS) - - # CASE 3: labels are numerically contiguous integers - # this is a special sub-case of CASE 2 which - # represents ordinal classification. Only in this - # case, weighted kappas -- where the distance - # between the class labels has a special - # meaning -- can be allowed. This is because - # class indices are always contiguous and all - # metrics in SKLL are computed in the index - # space, not the label space. Note that floating - # point numbers that are equivalent to integers - # (e.g., [1.0, 2.0, 3.0]) are also acceptable. - if _contiguous_ints_or_floats(label_array): - acceptable_metrics.update(_WEIGHTED_KAPPA_METRICS) - - return acceptable_metrics - - -class SelectByMinCount(SelectKBest): - - """ - Select features occurring in more (and/or fewer than) than a specified - number of examples in the training data (or a CV training fold). - - Parameters - ---------- - min_count : int, optional - The minimum feature count to select. - Defaults to 1. - """ - - def __init__(self, min_count=1): - self.min_count = min_count - self.scores_ = None - - def fit(self, X, y=None): - """ - Fit the SelectByMinCount model. - - Parameters - ---------- - X : array-like, with shape (n_samples, n_features) - The training data to fit. - y : Ignored - - Returns - ------- - self - """ - - # initialize a list of counts of times each feature appears - col_counts = [0 for _ in range(X.shape[1])] - - if sp.issparse(X): - # find() is scipy.sparse's equivalent of nonzero() - _, col_indices, _ = sp.find(X) - else: - # assume it's a numpy array (not a numpy matrix) - col_indices = X.nonzero()[1].tolist() - - for i in col_indices: - col_counts[i] += 1 - - self.scores_ = np.array(col_counts) - return self - - def _get_support_mask(self): - """ - Returns an indication of which features to keep. - Adapted from ``SelectKBest``. - - Returns - ------- - mask : np.array - The mask with features to keep set to True. - """ - mask = np.zeros(self.scores_.shape, dtype=bool) - mask[self.scores_ >= self.min_count] = True - return mask - - -def rescaled(cls): - """ - Decorator to create regressors that store a min and a max for the training - data and make sure that predictions fall within that range. It also stores - the means and SDs of the gold standard and the predictions on the training - set to rescale the predictions (e.g., as in e-rater). - - Parameters - ---------- - cls : BaseEstimator - An estimator class to add rescaling to. - - Returns - ------- - cls : BaseEstimator - Modified version of estimator class with rescaled functions added. - - Raises - ------ - ValueError - If classifier cannot be rescaled (i.e. is not a regressor). - """ - # If this class has already been run through the decorator, return it - if hasattr(cls, 'rescale'): - return cls - - # Save original versions of functions to use later. - orig_init = cls.__init__ - orig_fit = cls.fit - orig_predict = cls.predict - - if cls._estimator_type == 'classifier': - raise ValueError('Classifiers cannot be rescaled. ' + - 'Only regressors can.') - - # Define all new versions of functions - @wraps(cls.fit) - def fit(self, X, y=None): - """ - Fit a model, then store the mean, SD, max and min of the training set - and the mean and SD of the predictions on the training set. - - Parameters - ---------- - X : array-like, with shape (n_samples, n_features) - The data to fit. - y : Ignored - - Returns - ------- - self - """ - - # fit a regular regression model - orig_fit(self, X, y=y) - - if self.constrain: - # also record the training data min and max - self.y_min = min(y) - self.y_max = max(y) - - if self.rescale: - # also record the means and SDs for the training set - y_hat = orig_predict(self, X) - self.yhat_mean = np.mean(y_hat) - self.yhat_sd = np.std(y_hat) - self.y_mean = np.mean(y) - self.y_sd = np.std(y) - - return self - - @wraps(cls.predict) - def predict(self, X): - """ - Make predictions with the super class, and then adjust them using the - stored min, max, means, and standard deviations. - - Parameters - ---------- - X : array-like, with shape (n_samples,) - The data to predict. - - Returns - ------- - res : array-like - The prediction results. - """ - # get the unconstrained predictions - res = orig_predict(self, X) - - if self.rescale: - # convert the predictions to z-scores, - # then rescale to match the training set distribution - res = (((res - self.yhat_mean) / self.yhat_sd) * self.y_sd) + self.y_mean - - if self.constrain: - # apply min and max constraints - res = np.array([max(self.y_min, min(self.y_max, pred)) - for pred in res]) - - return res - - @classmethod - @wraps(cls._get_param_names) - def _get_param_names(class_x): - """ - This is adapted from scikit-learns's ``BaseEstimator`` class. - It gets the kwargs for the superclass's init method and adds the - kwargs for newly added ``__init__()`` method. - - Parameters - ---------- - class_x - The the superclass from which to retrieve param names. - - Returns - ------- - args : list - A list of parameter names for the class's init method. - - Raises - ------ - RunTimeError - If `varargs` exist in the scikit-learn estimator. - """ - try: - init = getattr(orig_init, 'deprecated_original', orig_init) - - args, varargs, _, _ = inspect.getargspec(init) - if varargs is not None: - raise RuntimeError('scikit-learn estimators should always ' - 'specify their parameters in the signature' - ' of their init (no varargs).') - # Remove 'self' - args.pop(0) - except TypeError: - args = [] +from .utils import (Densifier, + FilteredLeaveOneGroupOut, + get_acceptable_classification_metrics, + get_acceptable_regression_metrics, + import_custom_learner, + rescaled, + SelectByMinCount, + train_and_score) - rescale_args = inspect.getargspec(class_x.__init__)[0] - # Remove 'self' - rescale_args.pop(0) +# we need a list of learners requiring dense input and a dictionary of +# default parameter grids that we can dynamically update in case we +# import a custom learner +_REQUIRES_DENSE = copy.copy(KNOWN_REQUIRES_DENSE) +_DEFAULT_PARAM_GRIDS = copy.deepcopy(KNOWN_DEFAULT_PARAM_GRIDS) - args += rescale_args - args.sort() - - return args - - @wraps(cls.__init__) - def init(self, constrain=True, rescale=True, **kwargs): - """ - This special init function is used by the decorator to make sure - that things get initialized in the right order. - - Parameters - ---------- - constrain : bool, optional - Whether to constrain predictions within min and max values. - Defaults to True. - rescale : bool, optional - Whether to rescale prediction values using z-scores. - Defaults to True. - kwargs : dict, optional - Arguments for base class. - """ - # pylint: disable=W0201 - self.constrain = constrain - self.rescale = rescale - self.y_min = None - self.y_max = None - self.yhat_mean = None - self.yhat_sd = None - self.y_mean = None - self.y_sd = None - orig_init(self, **kwargs) - - # Override original functions with new ones - cls.__init__ = init - cls.fit = fit - cls.predict = predict - cls._get_param_names = _get_param_names - cls.rescale = True - - # Return modified class - return cls - - -# Rescaled regressors -@rescaled -class RescaledBayesianRidge(BayesianRidge): - pass - - -@rescaled -class RescaledAdaBoostRegressor(AdaBoostRegressor): - pass - - -@rescaled -class RescaledDecisionTreeRegressor(DecisionTreeRegressor): - pass - - -@rescaled -class RescaledElasticNet(ElasticNet): - pass - - -@rescaled -class RescaledGradientBoostingRegressor(GradientBoostingRegressor): - pass - - -@rescaled -class RescaledHuberRegressor(HuberRegressor): - pass - - -@rescaled -class RescaledKNeighborsRegressor(KNeighborsRegressor): - pass - - -@rescaled -class RescaledLars(Lars): - pass - - -@rescaled -class RescaledLasso(Lasso): - pass - - -@rescaled -class RescaledLinearRegression(LinearRegression): - pass - - -@rescaled -class RescaledLinearSVR(LinearSVR): - pass - - -@rescaled -class RescaledMLPRegressor(MLPRegressor): - pass - - -@rescaled -class RescaledRandomForestRegressor(RandomForestRegressor): - pass - - -@rescaled -class RescaledRANSACRegressor(RANSACRegressor): - pass - - -@rescaled -class RescaledRidge(Ridge): - pass - - -@rescaled -class RescaledSGDRegressor(SGDRegressor): - pass - - -@rescaled -class RescaledSVR(SVR): - pass - - -@rescaled -class RescaledTheilSenRegressor(TheilSenRegressor): - pass +__all__ = ['Learner', 'MAX_CONCURRENT_PROCESSES', 'import_custom_learner'] class Learner(object): """ A simpler learner interface around many scikit-learn classification - and regression functions. + and regression estimators. Parameters ---------- @@ -902,10 +160,18 @@ class Learner(object): Defaults to ``None``. """ - def __init__(self, model_type, probability=False, pipeline=False, - feature_scaling='none', model_kwargs=None, pos_label_str=None, - min_feature_count=1, sampler=None, sampler_kwargs=None, - custom_learner_path=None, logger=None): + def __init__(self, + model_type, + probability=False, + pipeline=False, + feature_scaling='none', + model_kwargs=None, + pos_label_str=None, + min_feature_count=1, + sampler=None, + sampler_kwargs=None, + custom_learner_path=None, + logger=None): """ Initializes a learner object with the specified settings. """ @@ -928,7 +194,7 @@ def __init__(self, model_type, probability=False, pipeline=False, if model_type not in globals(): # here, we need to import the custom model and add it # to the appropriate lists of models. - _import_custom_learner(custom_learner_path, model_type) + import_custom_learner(custom_learner_path, model_type) model_class = globals()[model_type] default_param_grid = (model_class.default_param_grid() @@ -1355,7 +621,10 @@ def _create_estimator(self): If there is no default parameter grid for estimator. """ estimator = None - default_param_grid = _find_default_param_grid(self._model_type) + default_param_grid = None + for key_class, grid in _DEFAULT_PARAM_GRIDS.items(): + if issubclass(self._model_type, key_class): + default_param_grid = grid if default_param_grid is None: raise ValueError("%s is not a valid learner type." % (self._model_type.__name__,)) @@ -1570,9 +839,9 @@ def train(self, examples, param_grid=None, grid_search_folds=3, label_type = examples.labels.dtype.type if estimator_type == 'classifier': sorted_unique_labels = np.unique(examples.labels) - allowed_objectives = _get_acceptable_classification_metrics(sorted_unique_labels) + allowed_objectives = get_acceptable_classification_metrics(sorted_unique_labels) else: - allowed_objectives = _get_acceptable_regression_metrics() + allowed_objectives = get_acceptable_regression_metrics() if grid_objective not in allowed_objectives: raise ValueError("'{}' is not a valid objective " @@ -1585,7 +854,7 @@ def train(self, examples, param_grid=None, grid_search_folds=3, # classification and probability is set to true, we assume # that the user actually wants the `_with_probabilities` # version of the metric - if (grid_objective in _CORRELATION_METRICS and + if (grid_objective in CORRELATION_METRICS and estimator_type == 'classifier' and self.probability): self.logger.info('You specified "{}" as the objective with ' @@ -1899,9 +1168,9 @@ def evaluate(self, examples, prediction_prefix=None, append=False, label_type = examples.labels.dtype.type if estimator_type == 'classifier': sorted_unique_labels = np.unique(examples.labels) - acceptable_metrics = _get_acceptable_classification_metrics(sorted_unique_labels) + acceptable_metrics = get_acceptable_classification_metrics(sorted_unique_labels) else: - acceptable_metrics = _get_acceptable_regression_metrics() + acceptable_metrics = get_acceptable_regression_metrics() # check that all of the output metrics are acceptable unacceptable_metrics = set(output_metrics).difference(acceptable_metrics) @@ -1949,7 +1218,7 @@ def evaluate(self, examples, prediction_prefix=None, append=False, # probabilities via argmax and use those # for all other metrics if (len(self.label_list) == 2 and - (metric in _CORRELATION_METRICS or + (metric in CORRELATION_METRICS or metric in ['average_precision', 'roc_auc']) and metric != grid_objective): self.logger.info('using probabilities for the positive class to ' @@ -2590,10 +1859,10 @@ def learning_curve(self, # Run jobs in parallel that train the model on each subset # of the training data and compute train and test scores parallel = joblib.Parallel(n_jobs=n_jobs, pre_dispatch=n_jobs) - out = parallel(joblib.delayed(_train_and_score)(self, - train_fs[:n_train_samples], - test_fs, - metric) + out = parallel(joblib.delayed(train_and_score)(self, + train_fs[:n_train_samples], + test_fs, + metric) for train_fs, test_fs in featureset_iter for n_train_samples in train_sizes_abs) @@ -2604,3 +1873,93 @@ def learning_curve(self, out = np.asarray(out).transpose((2, 1, 0)) return list(out[0]), list(out[1]), list(train_sizes_abs) + +# Rescaled regressors +@rescaled +class RescaledBayesianRidge(BayesianRidge): + pass + + +@rescaled +class RescaledAdaBoostRegressor(AdaBoostRegressor): + pass + + +@rescaled +class RescaledDecisionTreeRegressor(DecisionTreeRegressor): + pass + + +@rescaled +class RescaledElasticNet(ElasticNet): + pass + + +@rescaled +class RescaledGradientBoostingRegressor(GradientBoostingRegressor): + pass + + +@rescaled +class RescaledHuberRegressor(HuberRegressor): + pass + + +@rescaled +class RescaledKNeighborsRegressor(KNeighborsRegressor): + pass + + +@rescaled +class RescaledLars(Lars): + pass + + +@rescaled +class RescaledLasso(Lasso): + pass + + +@rescaled +class RescaledLinearRegression(LinearRegression): + pass + + +@rescaled +class RescaledLinearSVR(LinearSVR): + pass + + +@rescaled +class RescaledMLPRegressor(MLPRegressor): + pass + + +@rescaled +class RescaledRandomForestRegressor(RandomForestRegressor): + pass + + +@rescaled +class RescaledRANSACRegressor(RANSACRegressor): + pass + + +@rescaled +class RescaledRidge(Ridge): + pass + + +@rescaled +class RescaledSGDRegressor(SGDRegressor): + pass + + +@rescaled +class RescaledSVR(SVR): + pass + + +@rescaled +class RescaledTheilSenRegressor(TheilSenRegressor): + pass diff --git a/skll/learner/utils.py b/skll/learner/utils.py new file mode 100644 index 00000000..dcff0ff1 --- /dev/null +++ b/skll/learner/utils.py @@ -0,0 +1,582 @@ +""" +Utility classes and functions for SKLL learners. + +:author: Nitin Madnani (nmadnani@ets.org) +:author: Michael Heilman (mheilman@ets.org) +:author: Dan Blanchard (dblanchard@ets.org) +:author: Aoife Cahill (acahill@ets.org) +:organization: ETS +""" + +import inspect +import logging +import os +import sys + +from functools import wraps +from importlib import import_module + +import numpy as np +import scipy.sparse as sp +from sklearn.base import BaseEstimator, TransformerMixin +from sklearn.model_selection import LeaveOneGroupOut +from sklearn.feature_selection import SelectKBest +from skll.metrics import use_score_func +from skll.utils.constants import (CLASSIFICATION_ONLY_METRICS, + CORRELATION_METRICS, + REGRESSION_ONLY_METRICS, + UNWEIGHTED_KAPPA_METRICS, + WEIGHTED_KAPPA_METRICS) + + +class Densifier(BaseEstimator, TransformerMixin): + """ + A custom pipeline stage that will be inserted into the + learner pipeline attribute to accommodate the situation + when SKLL needs to manually convert feature arrays from + sparse to dense. For example, when features are being hashed + but we are also doing centering using the feature means. + """ + + def fit(self, X, y=None): + return self + + def fit_transform(self, X, y=None): + return self + + def transform(self, X): + return X.todense() + + +class FilteredLeaveOneGroupOut(LeaveOneGroupOut): + + """ + Version of ``LeaveOneGroupOut`` cross-validation iterator that only outputs + indices of instances with IDs in a prespecified set. + + Parameters + ---------- + keep : set of str + A set of IDs to keep. + example_ids : list of str, of length n_samples + A list of example IDs. + """ + + def __init__(self, keep, example_ids, logger=None): + super(FilteredLeaveOneGroupOut, self).__init__() + self.keep = keep + self.example_ids = example_ids + self._warned = False + self.logger = logger if logger else logging.getLogger(__name__) + + def split(self, X, y, groups): + """ + Generate indices to split data into training and test set. + + Parameters + ---------- + X : array-like, with shape (n_samples, n_features) + Training data, where n_samples is the number of samples + and n_features is the number of features. + y : array-like, of length n_samples + The target variable for supervised learning problems. + groups : array-like, with shape (n_samples,) + Group labels for the samples used while splitting the dataset into + train/test set. + + Yields + ------- + train_index : np.array + The training set indices for that split. + test_index : np.array + The testing set indices for that split. + """ + for train_index, test_index in super(FilteredLeaveOneGroupOut, + self).split(X, y, groups): + train_len = len(train_index) + test_len = len(test_index) + train_index = [i for i in train_index if self.example_ids[i] in + self.keep] + test_index = [i for i in test_index if self.example_ids[i] in + self.keep] + if not self._warned and (train_len != len(train_index) or + test_len != len(test_index)): + self.logger.warning('Feature set contains IDs that are not ' + + 'in folds dictionary. Skipping those IDs.') + self._warned = True + + yield train_index, test_index + + +class SelectByMinCount(SelectKBest): + + """ + Select features occurring in more (and/or fewer than) than a specified + number of examples in the training data (or a CV training fold). + + Parameters + ---------- + min_count : int, optional + The minimum feature count to select. + Defaults to 1. + """ + + def __init__(self, min_count=1): + self.min_count = min_count + self.scores_ = None + + def fit(self, X, y=None): + """ + Fit the SelectByMinCount model. + + Parameters + ---------- + X : array-like, with shape (n_samples, n_features) + The training data to fit. + y : Ignored + + Returns + ------- + self + """ + + # initialize a list of counts of times each feature appears + col_counts = [0 for _ in range(X.shape[1])] + + if sp.issparse(X): + # find() is scipy.sparse's equivalent of nonzero() + _, col_indices, _ = sp.find(X) + else: + # assume it's a numpy array (not a numpy matrix) + col_indices = X.nonzero()[1].tolist() + + for i in col_indices: + col_counts[i] += 1 + + self.scores_ = np.array(col_counts) + return self + + def _get_support_mask(self): + """ + Returns an indication of which features to keep. + Adapted from ``SelectKBest``. + + Returns + ------- + mask : np.array + The mask with features to keep set to True. + """ + mask = np.zeros(self.scores_.shape, dtype=bool) + mask[self.scores_ >= self.min_count] = True + return mask + + +def contiguous_ints_or_floats(numbers): + """ + Check whether the given list of numbers contains + contiguous integers or contiguous integer-like + floats. For example, [1, 2, 3] or [4.0, 5.0, 6.0] + are both contiguous but [1.1, 1.2, 1.3] is not. + + Parameters + ---------- + numbers : array-like of ints or floats + The numbers we want to check. + + Returns + ------- + answer : bool + True if the numbers are contiguous integers + or contiguous integer-like floats (1.0, 2.0, etc.) + + Raises + ------ + TypeError + If ``numbers`` does not contain integers or floating + point values. + ValueError + If ``numbers`` is empty. + """ + + try: + + # make sure that number is not empty + assert len(numbers) > 0 + + # first check that the numbers are all integers + # or integer-like floats (e.g., 1.0, 2.0 etc.) + ints_or_int_like_floats = np.all(np.mod(numbers, 1) == 0) + + # next check that the successive differences between + # the numbers are all 1, i.e., they are nuermicontiguous + contiguous = np.all(np.diff(numbers) == 1) + + except AssertionError: + raise ValueError('Input cannot be empty.') + + except TypeError: + raise TypeError('Input should only contain numbers.') + + # we need both conditions to be true + return ints_or_int_like_floats and contiguous + + +def get_acceptable_regression_metrics(): + """ + Return the set of metrics that are acceptable for regression. + """ + + # it's fairly straightforward for regression since + # we do not have to check the labels + acceptable_metrics = (REGRESSION_ONLY_METRICS | + UNWEIGHTED_KAPPA_METRICS | + WEIGHTED_KAPPA_METRICS | + CORRELATION_METRICS) + return acceptable_metrics + + +def get_acceptable_classification_metrics(label_array): + """ + Return the set of metrics that are acceptable given the + the unique set of labels that we are classifying. + + Parameters + ---------- + label_array : numpy.ndarray + A sorted numpy array containing the unique labels + that we are trying to predict. Optional for regressors + but required for classifiers. + + Returns + ------- + acceptable_metrics : set + A set of metric names that are acceptable + for the given classification scenario. + """ + + # this is a classifier so the acceptable objective + # functions definitely include those metrics that + # are specifically for classification and also + # the unweighted kappa metrics + acceptable_metrics = CLASSIFICATION_ONLY_METRICS | UNWEIGHTED_KAPPA_METRICS + + # now let us consider which other metrics may also + # be acceptable depending on whether the labels + # are strings or (contiguous) integers/floats + label_type = label_array.dtype.type + + # CASE 1: labels are strings, then no other metrics + # are acceptable + if issubclass(label_type, (np.object_, str)): + pass + + # CASE 2: labels are integers or floats; the way + # it works in SKLL, it's guaranteed that + # class indices will be sorted in the same order + # as the class labels therefore, ranking metrics + # such as various correlations should work fine. + elif issubclass(label_type, (int, + np.int32, + np.int64, + float, + np.float32, + np.float64)): + acceptable_metrics.update(CORRELATION_METRICS) + + # CASE 3: labels are numerically contiguous integers + # this is a special sub-case of CASE 2 which + # represents ordinal classification. Only in this + # case, weighted kappas -- where the distance + # between the class labels has a special + # meaning -- can be allowed. This is because + # class indices are always contiguous and all + # metrics in SKLL are computed in the index + # space, not the label space. Note that floating + # point numbers that are equivalent to integers + # (e.g., [1.0, 2.0, 3.0]) are also acceptable. + if contiguous_ints_or_floats(label_array): + acceptable_metrics.update(WEIGHTED_KAPPA_METRICS) + + return acceptable_metrics + + +def import_custom_learner(custom_learner_path, custom_learner_name): + """ + Does the gruntwork of adding the custom model's module to globals. + + Parameters + ---------- + custom_learner_path : str + The path to a custom learner. + custom_learner_name : str + The name of a custom learner. + + Raises + ------ + ValueError + If the custom learner path is None. + ValueError + If the custom learner path does not end in '.py'. + """ + if not custom_learner_path: + raise ValueError('custom_learner_path was not set and learner {} ' + 'was not found.'.format(custom_learner_name)) + + if not custom_learner_path.endswith('.py'): + raise ValueError('custom_learner_path must end in .py ({})' + .format(custom_learner_path)) + + custom_learner_module_name = os.path.basename(custom_learner_path)[:-3] + sys.path.append(os.path.dirname(os.path.abspath(custom_learner_path))) + import_module(custom_learner_module_name) + globals()[custom_learner_name] = \ + getattr(sys.modules[custom_learner_module_name], custom_learner_name) + + +def rescaled(cls): + """ + Decorator to create regressors that store a min and a max for the training + data and make sure that predictions fall within that range. It also stores + the means and SDs of the gold standard and the predictions on the training + set to rescale the predictions (e.g., as in e-rater). + + Parameters + ---------- + cls : BaseEstimator + An estimator class to add rescaling to. + + Returns + ------- + cls : BaseEstimator + Modified version of estimator class with rescaled functions added. + + Raises + ------ + ValueError + If classifier cannot be rescaled (i.e. is not a regressor). + """ + # If this class has already been run through the decorator, return it + if hasattr(cls, 'rescale'): + return cls + + # Save original versions of functions to use later. + orig_init = cls.__init__ + orig_fit = cls.fit + orig_predict = cls.predict + + if cls._estimator_type == 'classifier': + raise ValueError('Classifiers cannot be rescaled. ' + + 'Only regressors can.') + + # Define all new versions of functions + @wraps(cls.fit) + def fit(self, X, y=None): + """ + Fit a model, then store the mean, SD, max and min of the training set + and the mean and SD of the predictions on the training set. + + Parameters + ---------- + X : array-like, with shape (n_samples, n_features) + The data to fit. + y : Ignored + + Returns + ------- + self + """ + + # fit a regular regression model + orig_fit(self, X, y=y) + + if self.constrain: + # also record the training data min and max + self.y_min = min(y) + self.y_max = max(y) + + if self.rescale: + # also record the means and SDs for the training set + y_hat = orig_predict(self, X) + self.yhat_mean = np.mean(y_hat) + self.yhat_sd = np.std(y_hat) + self.y_mean = np.mean(y) + self.y_sd = np.std(y) + + return self + + @wraps(cls.predict) + def predict(self, X): + """ + Make predictions with the super class, and then adjust them using the + stored min, max, means, and standard deviations. + + Parameters + ---------- + X : array-like, with shape (n_samples,) + The data to predict. + + Returns + ------- + res : array-like + The prediction results. + """ + # get the unconstrained predictions + res = orig_predict(self, X) + + if self.rescale: + # convert the predictions to z-scores, + # then rescale to match the training set distribution + res = (((res - self.yhat_mean) / self.yhat_sd) * self.y_sd) + self.y_mean + + if self.constrain: + # apply min and max constraints + res = np.array([max(self.y_min, min(self.y_max, pred)) + for pred in res]) + + return res + + @classmethod + @wraps(cls._get_param_names) + def _get_param_names(class_x): + """ + This is adapted from scikit-learns's ``BaseEstimator`` class. + It gets the kwargs for the superclass's init method and adds the + kwargs for newly added ``__init__()`` method. + + Parameters + ---------- + class_x + The the superclass from which to retrieve param names. + + Returns + ------- + args : list + A list of parameter names for the class's init method. + + Raises + ------ + RunTimeError + If `varargs` exist in the scikit-learn estimator. + """ + try: + init = getattr(orig_init, 'deprecated_original', orig_init) + + args, varargs, _, _ = inspect.getargspec(init) + if varargs is not None: + raise RuntimeError('scikit-learn estimators should always ' + 'specify their parameters in the signature' + ' of their init (no varargs).') + # Remove 'self' + args.pop(0) + except TypeError: + args = [] + + rescale_args = inspect.getargspec(class_x.__init__)[0] + # Remove 'self' + rescale_args.pop(0) + + args += rescale_args + args.sort() + + return args + + @wraps(cls.__init__) + def init(self, constrain=True, rescale=True, **kwargs): + """ + This special init function is used by the decorator to make sure + that things get initialized in the right order. + + Parameters + ---------- + constrain : bool, optional + Whether to constrain predictions within min and max values. + Defaults to True. + rescale : bool, optional + Whether to rescale prediction values using z-scores. + Defaults to True. + kwargs : dict, optional + Arguments for base class. + """ + # pylint: disable=W0201 + self.constrain = constrain + self.rescale = rescale + self.y_min = None + self.y_max = None + self.yhat_mean = None + self.yhat_sd = None + self.y_mean = None + self.y_sd = None + orig_init(self, **kwargs) + + # Override original functions with new ones + cls.__init__ = init + cls.fit = fit + cls.predict = predict + cls._get_param_names = _get_param_names + cls.rescale = True + + # Return modified class + return cls + + +def train_and_score(learner, + train_examples, + test_examples, + metric): + """ + A utility method to train a given learner instance on the given training examples, + generate predictions on the training set itself and also the given + test set, and score those predictions using the given metric. + The method returns the train and test scores. + + Note that this method needs to be a top-level function since it is + called from within ``joblib.Parallel()`` and, therefore, needs to be + picklable which it would not be as an instancemethod of the ``Learner`` + class. + + Parameters + ---------- + learner : skll.Learner + A SKLL ``Learner`` instance. + train_examples : array-like, with shape (n_samples, n_features) + The training examples. + test_examples : array-like, of length n_samples + The test examples. + metric : str + The scoring function passed to ``use_score_func()``. + + Returns + ------- + train_score : float + Output of the score function applied to predictions of + ``learner`` on ``train_examples``. + test_score : float + Output of the score function applied to predictions of + ``learner`` on ``test_examples``. + """ + + _ = learner.train(train_examples, grid_search=False, shuffle=False) + train_predictions = learner.predict(train_examples) + test_predictions = learner.predict(test_examples) + if learner.model_type._estimator_type == 'classifier': + test_label_list = np.unique(test_examples.labels).tolist() + unseen_test_label_list = [label for label in test_label_list + if label not in learner.label_list] + unseen_label_dict = {label: i for i, label in enumerate(unseen_test_label_list, + start=len(learner.label_list))} + # combine the two dictionaries + train_and_test_label_dict = learner.label_dict.copy() + train_and_test_label_dict.update(unseen_label_dict) + train_labels = np.array([train_and_test_label_dict[label] + for label in train_examples.labels]) + test_labels = np.array([train_and_test_label_dict[label] + for label in test_examples.labels]) + else: + train_labels = train_examples.labels + test_labels = test_examples.labels + + train_score = use_score_func(metric, train_labels, train_predictions) + test_score = use_score_func(metric, test_labels, test_predictions) + return train_score, test_score + + From 9ea9e248b994899e1a6908d471a7c5b65c760066 Mon Sep 17 00:00:00 2001 From: Nitin Madnani Date: Fri, 10 Apr 2020 17:05:56 -0400 Subject: [PATCH 04/20] Create `utils` package Move all `utilities/*.py` into `utils/commadline/*.py`. Move skll constants from all across codebase into `utils/constants.py`. Move `logutils.py` into `utils/logging.py`. --- skll/metrics.py | 46 +---- .../commandline}/__init__.py | 0 .../compute_eval_from_predictions.py | 0 .../commandline}/filter_features.py | 0 .../commandline}/generate_predictions.py | 0 .../commandline}/join_features.py | 0 .../commandline}/plot_learning_curves.py | 19 +- .../commandline}/print_model_weights.py | 0 .../commandline}/run_experiment.py | 0 .../commandline}/skll_convert.py | 7 +- .../commandline}/summarize_results.py | 2 +- skll/utils/constants.py | 179 ++++++++++++++++++ skll/{logutils.py => utils/logging.py} | 3 +- 13 files changed, 201 insertions(+), 55 deletions(-) rename skll/{utilities => utils/commandline}/__init__.py (100%) rename skll/{utilities => utils/commandline}/compute_eval_from_predictions.py (100%) rename skll/{utilities => utils/commandline}/filter_features.py (100%) rename skll/{utilities => utils/commandline}/generate_predictions.py (100%) rename skll/{utilities => utils/commandline}/join_features.py (100%) rename skll/{utilities => utils/commandline}/plot_learning_curves.py (83%) rename skll/{utilities => utils/commandline}/print_model_weights.py (100%) rename skll/{utilities => utils/commandline}/run_experiment.py (100%) rename skll/{utilities => utils/commandline}/skll_convert.py (96%) rename skll/{utilities => utils/commandline}/summarize_results.py (97%) create mode 100644 skll/utils/constants.py rename skll/{logutils.py => utils/logging.py} (96%) diff --git a/skll/metrics.py b/skll/metrics.py index 67b5d974..0ab18335 100644 --- a/skll/metrics.py +++ b/skll/metrics.py @@ -1,7 +1,6 @@ # License: BSD 3 clause """ -This module contains a bunch of evaluation metrics that can be used to -evaluate the performance of learners. +Metrics that can be used to evaluate the performance of learners. :author: Michael Heilman (mheilman@ets.org) :author: Nitin Madnani (nmadnani@ets.org) @@ -16,41 +15,6 @@ SCORERS) -# Useful constants -_CORRELATION_METRICS = set(['kendall_tau', 'pearson', 'spearman']) - -_REGRESSION_ONLY_METRICS = set(['explained_variance', - 'max_error', - 'neg_mean_squared_error', - 'neg_mean_absolute_error', - 'r2']) - -_CLASSIFICATION_ONLY_METRICS = set(['accuracy', - 'average_precision', - 'balanced_accuracy', - 'f1', - 'f1_score_least_frequent', - 'f1_score_macro', - 'f1_score_micro', - 'f1_score_weighted', - 'neg_log_loss', - 'precision', - 'recall', - 'roc_auc']) - -_PROBABILISTIC_METRICS = frozenset(['average_precision', - 'neg_log_loss', - 'roc_auc']) - -_UNWEIGHTED_KAPPA_METRICS = set(['unweighted_kappa', - 'uwk_off_by_one']) - -_WEIGHTED_KAPPA_METRICS = set(['linear_weighted_kappa', - 'lwk_off_by_one', - 'quadratic_weighted_kappa', - 'qwk_off_by_one']) - - def kappa(y_true, y_pred, weights=None, allow_off_by_one=False): """ Calculates the kappa inter-rater agreement between two the gold standard @@ -253,10 +217,10 @@ def f1_score_least_frequent(y_true, y_pred): def use_score_func(func_name, y_true, y_pred): """ - Call the scoring function in ``sklearn.metrics.SCORERS`` with the given name. - This takes care of handling keyword arguments that were pre-specified when - creating the scorer. This applies any sign-flipping that was specified by - ``make_scorer()`` when the scorer was created. + Call the scoring function in ``sklearn.metrics.SCORERS`` with the given + name. This takes care of handling keyword arguments that were pre-specified + when creating the scorer. This applies any sign-flipping that was specified + by ``make_scorer()`` when the scorer was created. Parameters ---------- diff --git a/skll/utilities/__init__.py b/skll/utils/commandline/__init__.py similarity index 100% rename from skll/utilities/__init__.py rename to skll/utils/commandline/__init__.py diff --git a/skll/utilities/compute_eval_from_predictions.py b/skll/utils/commandline/compute_eval_from_predictions.py similarity index 100% rename from skll/utilities/compute_eval_from_predictions.py rename to skll/utils/commandline/compute_eval_from_predictions.py diff --git a/skll/utilities/filter_features.py b/skll/utils/commandline/filter_features.py similarity index 100% rename from skll/utilities/filter_features.py rename to skll/utils/commandline/filter_features.py diff --git a/skll/utilities/generate_predictions.py b/skll/utils/commandline/generate_predictions.py similarity index 100% rename from skll/utilities/generate_predictions.py rename to skll/utils/commandline/generate_predictions.py diff --git a/skll/utilities/join_features.py b/skll/utils/commandline/join_features.py similarity index 100% rename from skll/utilities/join_features.py rename to skll/utils/commandline/join_features.py diff --git a/skll/utilities/plot_learning_curves.py b/skll/utils/commandline/plot_learning_curves.py similarity index 83% rename from skll/utilities/plot_learning_curves.py rename to skll/utils/commandline/plot_learning_curves.py index d8928112..776f19b3 100755 --- a/skll/utilities/plot_learning_curves.py +++ b/skll/utils/commandline/plot_learning_curves.py @@ -1,15 +1,16 @@ #!/usr/bin/env python # License: BSD 3 clause """ -A Helper script to generate learning plots from the learning curve output TSV file. +A Helper script to generate learning plots from the learning curve output TSV +file. -This is necessary in scenarios where the plots were not generated as part of the original -learning curve experiment, e.g. the experiment was run on a remote server where plots -may not have been generated either due to a crash or incorrect setting of the DISPLAY -environment variable. +This is necessary in scenarios where the plots were not generated as part of +the original learning curve experiment, e.g. the experiment was run on a remote +server where plots may not have been generated either due to a crash or +incorrect setting of the DISPLAY environment variable. -In these cases, the summary file should always be generated and this script can then be used -to generate the plots later. +In these cases, the summary file should always be generated and this script can +then be used to generate the plots later. :author: Nitin Madnani :organization: ETS @@ -22,7 +23,7 @@ from os import makedirs from os.path import basename, exists -from skll.experiments import _generate_learning_curve_plots +from skll.experiments import generate_learning_curve_plots from skll.version import __version__ @@ -68,7 +69,7 @@ def main(argv=None): # output_file_name = experiment_name + '_summary.tsv' experiment_name = basename(args.tsv_file).rstrip('_summary.tsv') logging.info("Generating learning curve(s)") - _generate_learning_curve_plots(experiment_name, args.output_dir, args.tsv_file) + generate_learning_curve_plots(experiment_name, args.output_dir, args.tsv_file) if __name__ == '__main__': diff --git a/skll/utilities/print_model_weights.py b/skll/utils/commandline/print_model_weights.py similarity index 100% rename from skll/utilities/print_model_weights.py rename to skll/utils/commandline/print_model_weights.py diff --git a/skll/utilities/run_experiment.py b/skll/utils/commandline/run_experiment.py similarity index 100% rename from skll/utilities/run_experiment.py rename to skll/utils/commandline/run_experiment.py diff --git a/skll/utilities/skll_convert.py b/skll/utils/commandline/skll_convert.py similarity index 96% rename from skll/utilities/skll_convert.py rename to skll/utils/commandline/skll_convert.py index 73fd33e1..813ef083 100755 --- a/skll/utilities/skll_convert.py +++ b/skll/utils/commandline/skll_convert.py @@ -16,8 +16,11 @@ from skll.data.dict_vectorizer import DictVectorizer from skll.data.readers import EXT_TO_READER -from skll.data.writers import (ARFFWriter, CSVWriter, TSVWriter, LibSVMWriter, - EXT_TO_WRITER) +from skll.data.writers import (ARFFWriter, + CSVWriter, + EXT_TO_WRITER, + LibSVMWriter, + TSVWriter) from skll.version import __version__ diff --git a/skll/utilities/summarize_results.py b/skll/utils/commandline/summarize_results.py similarity index 97% rename from skll/utilities/summarize_results.py rename to skll/utils/commandline/summarize_results.py index 50cc618d..c89cfac6 100755 --- a/skll/utilities/summarize_results.py +++ b/skll/utils/commandline/summarize_results.py @@ -11,7 +11,7 @@ import argparse import logging -from skll.experiments import _write_summary_file +from skll.experiments.output import _write_summary_file from skll.version import __version__ diff --git a/skll/utils/constants.py b/skll/utils/constants.py new file mode 100644 index 00000000..c52d5c5f --- /dev/null +++ b/skll/utils/constants.py @@ -0,0 +1,179 @@ +""" +Constants useful for SKLL learners. + +:author: Nitin Madnani (nmadnani@ets.org) +:author: Michael Heilman (mheilman@ets.org) +:author: Dan Blanchard (dblanchard@ets.org) +:author: Aoife Cahill (acahill@ets.org) +:organization: ETS +""" + +import os + +from sklearn.dummy import DummyClassifier, DummyRegressor +from sklearn.ensemble import (AdaBoostClassifier, + AdaBoostRegressor, + GradientBoostingClassifier, + GradientBoostingRegressor, + RandomForestClassifier, + RandomForestRegressor) + +from sklearn.linear_model import (BayesianRidge, + ElasticNet, + HuberRegressor, + Lars, + Lasso, + LinearRegression, + LogisticRegression, + RANSACRegressor, + Ridge, + RidgeClassifier, + SGDClassifier, + SGDRegressor, + TheilSenRegressor) +from sklearn.naive_bayes import MultinomialNB +from sklearn.neighbors import KNeighborsClassifier, KNeighborsRegressor +from sklearn.neural_network import MLPClassifier, MLPRegressor +from sklearn.svm import LinearSVC, SVC, LinearSVR, SVR +from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor + +KNOWN_DEFAULT_PARAM_GRIDS = {AdaBoostClassifier: + [{'learning_rate': [0.01, 0.1, 1.0, 10.0, 100.0]}], + AdaBoostRegressor: + [{'learning_rate': [0.01, 0.1, 1.0, 10.0, 100.0]}], + BayesianRidge: + [{'alpha_1': [1e-6, 1e-4, 1e-2, 1, 10], + 'alpha_2': [1e-6, 1e-4, 1e-2, 1, 10], + 'lambda_1': [1e-6, 1e-4, 1e-2, 1, 10], + 'lambda_2': [1e-6, 1e-4, 1e-2, 1, 10]}], + DecisionTreeClassifier: + [{'max_features': ["auto", None]}], + DecisionTreeRegressor: + [{'max_features': ["auto", None]}], + DummyClassifier: + [{}], + DummyRegressor: + [{}], + ElasticNet: + [{'alpha': [0.01, 0.1, 1.0, 10.0, 100.0]}], + GradientBoostingClassifier: + [{'max_depth': [1, 3, 5]}], + GradientBoostingRegressor: + [{'max_depth': [1, 3, 5]}], + HuberRegressor: + [{'epsilon': [1.05, 1.35, 1.5, 2.0, 2.5, 5.0], + 'alpha': [1e-4, 1e-3, 1e-2, 1e-1, 1, 10, 100, 1000]}], + KNeighborsClassifier: + [{'n_neighbors': [1, 5, 10, 100], + 'weights': ['uniform', 'distance']}], + KNeighborsRegressor: + [{'n_neighbors': [1, 5, 10, 100], + 'weights': ['uniform', 'distance']}], + MLPClassifier: + [{'activation': ['logistic', 'tanh', 'relu'], + 'alpha': [1e-4, 1e-3, 1e-2, 1e-1, 1], + 'learning_rate_init': [0.001, 0.01, 0.1]}], + MLPRegressor: + [{'activation': ['logistic', 'tanh', 'relu'], + 'alpha': [1e-4, 1e-3, 1e-2, 1e-1, 1], + 'learning_rate_init': [0.001, 0.01, 0.1]}], + MultinomialNB: + [{'alpha': [0.1, 0.25, 0.5, 0.75, 1.0]}], + Lars: + [{}], + Lasso: + [{'alpha': [0.01, 0.1, 1.0, 10.0, 100.0]}], + LinearRegression: + [{}], + LinearSVC: + [{'C': [0.01, 0.1, 1.0, 10.0, 100.0]}], + LogisticRegression: + [{'C': [0.01, 0.1, 1.0, 10.0, 100.0]}], + SVC: + [{'C': [0.01, 0.1, 1.0, 10.0, 100.0], + 'gamma': ['auto', 'scale', 0.01, 0.1, 1.0, 10.0, 100.0]}], + RandomForestClassifier: + [{'max_depth': [1, 5, 10, None]}], + RandomForestRegressor: + [{'max_depth': [1, 5, 10, None]}], + RANSACRegressor: + [{}], + Ridge: + [{'alpha': [0.01, 0.1, 1.0, 10.0, 100.0]}], + RidgeClassifier: + [{'alpha': [0.01, 0.1, 1.0, 10.0, 100.0]}], + SGDClassifier: + [{'alpha': [0.000001, 0.00001, 0.0001, 0.001, 0.01], + 'penalty': ['l1', 'l2', 'elasticnet']}], + SGDRegressor: + [{'alpha': [0.000001, 0.00001, 0.0001, 0.001, 0.01], + 'penalty': ['l1', 'l2', 'elasticnet']}], + LinearSVR: + [{'C': [0.01, 0.1, 1.0, 10.0, 100.0]}], + SVR: + [{'C': [0.01, 0.1, 1.0, 10.0, 100.0], + 'gamma': ['auto', 'scale', 0.01, 0.1, 1.0, 10.0, 100.0]}], + TheilSenRegressor: + [{}] + } + +KNOWN_REQUIRES_DENSE = (BayesianRidge, Lars, TheilSenRegressor) + +MAX_CONCURRENT_PROCESSES = int(os.getenv('SKLL_MAX_CONCURRENT_PROCESSES', '3')) + +VALID_FEATURE_SCALING_OPTIONS = frozenset(['both', + 'none', + 'with_std', + 'with_mean']) + +VALID_SAMPLERS = frozenset(['Nystroem', + 'RBFSampler', + 'SkewedChi2Sampler', + 'AdditiveChi2Sampler', + '']) + +VALID_TASKS = frozenset(['cross_validate', + 'evaluate', + 'learning_curve', + 'predict', + 'train']) + +#: Set of evaluation metrics only used for classification tasks +CLASSIFICATION_ONLY_METRICS = set(['accuracy', + 'average_precision', + 'balanced_accuracy', + 'f1', + 'f1_score_least_frequent', + 'f1_score_macro', + 'f1_score_micro', + 'f1_score_weighted', + 'neg_log_loss', + 'precision', + 'recall', + 'roc_auc']) + + +#: Set of evaluation metrics based on correlation +CORRELATION_METRICS = set(['kendall_tau', 'pearson', 'spearman']) + +#: Set of evaluation metrics that can use prediction probabilities +PROBABILISTIC_METRICS = frozenset(['average_precision', + 'neg_log_loss', + 'roc_auc']) + +#: Set of evaluation metrics only used for regression tasks +REGRESSION_ONLY_METRICS = set(['explained_variance', + 'max_error', + 'neg_mean_squared_error', + 'neg_mean_absolute_error', + 'r2']) + +#: Set of unweighted kappa agreement metrics +UNWEIGHTED_KAPPA_METRICS = set(['unweighted_kappa', + 'uwk_off_by_one']) + +#: Set of weighed kappa agreement metrics +WEIGHTED_KAPPA_METRICS = set(['linear_weighted_kappa', + 'lwk_off_by_one', + 'quadratic_weighted_kappa', + 'qwk_off_by_one']) diff --git a/skll/logutils.py b/skll/utils/logging.py similarity index 96% rename from skll/logutils.py rename to skll/utils/logging.py index 1210646c..c838e774 100644 --- a/skll/logutils.py +++ b/skll/utils/logging.py @@ -6,7 +6,6 @@ :organization: ETS """ import logging -from logging import FileHandler from functools import partial from os.path import sep import re @@ -69,7 +68,7 @@ def get_skll_logger(name, filepath=None, log_level=logging.INFO): # have a file handler for this file, then add one. if filepath: def is_file_handler(handler): - return isinstance(handler, FileHandler) and handler.stream.name == filepath + return isinstance(handler, logging.FileHandler) and handler.stream.name == filepath need_file_handler = not any([is_file_handler(handler) for handler in logger.handlers]) if need_file_handler: formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') From adcd2796eac74a2ea46f763e7a136f59c42775bf Mon Sep 17 00:00:00 2001 From: Nitin Madnani Date: Fri, 10 Apr 2020 17:06:50 -0400 Subject: [PATCH 05/20] Update main `__init__.py` - Remove unnecessary functions and classes from top-level namespace. - Remove some unnecessary newlines. --- skll/__init__.py | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/skll/__init__.py b/skll/__init__.py index ae51731d..d2829bd3 100644 --- a/skll/__init__.py +++ b/skll/__init__.py @@ -3,43 +3,37 @@ This package provides a number of utilities to make it simpler to run common scikit-learn experiments with pre-generated features. +:author: Nitin Madnani (nmadnani@ets.org) :author: Dan Blanchard (dblanchard@ets.org) :author: Michael Heilman (mheilman@ets.org) -:author: Nitin Madnani (nmadnani@ets.org) :organization: ETS """ from sklearn.metrics import f1_score, make_scorer, SCORERS -from .logutils import (close_and_remove_logger_handlers, - get_skll_logger, orig_showwarning) from .data import FeatureSet, Reader, Writer from .experiments import run_configuration from .learner import Learner from .metrics import correlation, f1_score_least_frequent, kappa -__all__ = ['FeatureSet', 'Learner', 'Reader', 'get_skll_logger', - 'orig_showwarning', 'close_and_remove_logger_handlers', - 'run_configuration', 'Writer'] +__all__ = ['FeatureSet', 'Learner', 'Reader', 'run_configuration', 'Writer'] # Add our scorers to the sklearn dictionary here so that they will always be # available if you import anything from skll -_scorers = {'f1_score_micro': make_scorer(f1_score, - average='micro'), - 'f1_score_macro': make_scorer(f1_score, - average='macro'), - 'f1_score_weighted': make_scorer(f1_score, - average='weighted'), +_scorers = {'f1_score_micro': make_scorer(f1_score, average='micro'), + 'f1_score_macro': make_scorer(f1_score, average='macro'), + 'f1_score_weighted': make_scorer(f1_score, average='weighted'), 'f1_score_least_frequent': make_scorer(f1_score_least_frequent), 'pearson': make_scorer(correlation, corr_type='pearson'), 'spearman': make_scorer(correlation, corr_type='spearman'), 'kendall_tau': make_scorer(correlation, corr_type='kendall_tau'), 'unweighted_kappa': make_scorer(kappa), - 'quadratic_weighted_kappa': make_scorer(kappa, - weights='quadratic'), + 'quadratic_weighted_kappa': make_scorer(kappa, weights='quadratic'), 'linear_weighted_kappa': make_scorer(kappa, weights='linear'), - 'qwk_off_by_one': make_scorer(kappa, weights='quadratic', + 'qwk_off_by_one': make_scorer(kappa, + weights='quadratic', allow_off_by_one=True), - 'lwk_off_by_one': make_scorer(kappa, weights='linear', + 'lwk_off_by_one': make_scorer(kappa, + weights='linear', allow_off_by_one=True), 'uwk_off_by_one': make_scorer(kappa, allow_off_by_one=True)} From e319810f88cc03bdd279a7615ceb76e38df477d7 Mon Sep 17 00:00:00 2001 From: Nitin Madnani Date: Fri, 10 Apr 2020 17:07:11 -0400 Subject: [PATCH 06/20] Update imports in all tests. --- tests/test_ablation.py | 4 +- tests/test_classification.py | 136 +++++++++++++++++------------------ tests/test_custom_learner.py | 5 +- tests/test_cv.py | 17 ++--- tests/test_featureset.py | 44 +++++++----- tests/test_input.py | 73 ++++++++++--------- tests/test_logutils.py | 5 +- tests/test_metrics.py | 4 +- tests/test_output.py | 14 ++-- tests/test_preprocessing.py | 7 +- tests/test_regression.py | 13 ++-- tests/test_utilities.py | 31 ++++---- 12 files changed, 182 insertions(+), 171 deletions(-) diff --git a/tests/test_ablation.py b/tests/test_ablation.py index bc3e9b55..b3db08df 100644 --- a/tests/test_ablation.py +++ b/tests/test_ablation.py @@ -17,13 +17,13 @@ from nose.tools import eq_ from skll.experiments import run_configuration -from skll.learner import _DEFAULT_PARAM_GRIDS +from skll.utils.constants import KNOWN_DEFAULT_PARAM_GRIDS from tests.utils import (create_jsonlines_feature_files, fill_in_config_paths) -_ALL_MODELS = list(_DEFAULT_PARAM_GRIDS.keys()) +_ALL_MODELS = list(KNOWN_DEFAULT_PARAM_GRIDS.keys()) _my_dir = abspath(dirname(__file__)) diff --git a/tests/test_classification.py b/tests/test_classification.py index 2ddef416..e73ca56a 100644 --- a/tests/test_classification.py +++ b/tests/test_classification.py @@ -34,19 +34,19 @@ from sklearn.utils import shuffle as sk_shuffle from skll import run_configuration -from skll.config import _parse_config_file +from skll.config import parse_config_file from skll.data import FeatureSet, NDJReader, NDJWriter -from skll.learner import (_DEFAULT_PARAM_GRIDS, - FilteredLeaveOneGroupOut, - Learner, - _contiguous_ints_or_floats, - _train_and_score) -from skll.metrics import (_CORRELATION_METRICS, - _PROBABILISTIC_METRICS, - _REGRESSION_ONLY_METRICS, - _UNWEIGHTED_KAPPA_METRICS, - _WEIGHTED_KAPPA_METRICS, - use_score_func) +from skll.learner import Learner +from skll.learner.utils import (contiguous_ints_or_floats, + FilteredLeaveOneGroupOut, + train_and_score) +from skll.utils.constants import (CORRELATION_METRICS, + KNOWN_DEFAULT_PARAM_GRIDS, + PROBABILISTIC_METRICS, + REGRESSION_ONLY_METRICS, + UNWEIGHTED_KAPPA_METRICS, + WEIGHTED_KAPPA_METRICS) +from skll.metrics import use_score_func from tests.utils import (make_classification_data, make_regression_data, @@ -55,7 +55,7 @@ fill_in_config_paths_for_single_file) -_ALL_MODELS = list(_DEFAULT_PARAM_GRIDS.keys()) +_ALL_MODELS = list(KNOWN_DEFAULT_PARAM_GRIDS.keys()) _my_dir = abspath(dirname(__file__)) @@ -126,23 +126,23 @@ def test_contiguous_int_or_float_labels(): """ Test that we can accurately detect contiguous int/float labels """ - eq_(_contiguous_ints_or_floats([1, 2, 3, 4]), True) - eq_(_contiguous_ints_or_floats([0, 1]), True) - eq_(_contiguous_ints_or_floats([1.0, 2.0]), True) - eq_(_contiguous_ints_or_floats([0, 1.0]), True) - eq_(_contiguous_ints_or_floats([-2, -1, 0, 1, 2]), True) - eq_(_contiguous_ints_or_floats([1.0, 2.0, 3.0, 4.0]), True) - eq_(_contiguous_ints_or_floats([4, 5, 6]), True) - eq_(_contiguous_ints_or_floats([1, 2, 3, 4.0]), True) - eq_(_contiguous_ints_or_floats([-1, 1]), False) - eq_(_contiguous_ints_or_floats([2, 4, 6]), False) - eq_(_contiguous_ints_or_floats([3, 6, 11]), False) - eq_(_contiguous_ints_or_floats([-2.0, -1.0, 1.0, 2.0]), False) - eq_(_contiguous_ints_or_floats([1.0, 1.1, 1.2]), False) - assert_raises(TypeError, _contiguous_ints_or_floats, ['a', 'b', 'c']) - assert_raises(TypeError, _contiguous_ints_or_floats, np.array([1, 2, 3, 'a'])) - assert_raises(ValueError, _contiguous_ints_or_floats, []) - assert_raises(ValueError, _contiguous_ints_or_floats, np.array([])) + eq_(contiguous_ints_or_floats([1, 2, 3, 4]), True) + eq_(contiguous_ints_or_floats([0, 1]), True) + eq_(contiguous_ints_or_floats([1.0, 2.0]), True) + eq_(contiguous_ints_or_floats([0, 1.0]), True) + eq_(contiguous_ints_or_floats([-2, -1, 0, 1, 2]), True) + eq_(contiguous_ints_or_floats([1.0, 2.0, 3.0, 4.0]), True) + eq_(contiguous_ints_or_floats([4, 5, 6]), True) + eq_(contiguous_ints_or_floats([1, 2, 3, 4.0]), True) + eq_(contiguous_ints_or_floats([-1, 1]), False) + eq_(contiguous_ints_or_floats([2, 4, 6]), False) + eq_(contiguous_ints_or_floats([3, 6, 11]), False) + eq_(contiguous_ints_or_floats([-2.0, -1.0, 1.0, 2.0]), False) + eq_(contiguous_ints_or_floats([1.0, 1.1, 1.2]), False) + assert_raises(TypeError, contiguous_ints_or_floats, ['a', 'b', 'c']) + assert_raises(TypeError, contiguous_ints_or_floats, np.array([1, 2, 3, 'a'])) + assert_raises(ValueError, contiguous_ints_or_floats, []) + assert_raises(ValueError, contiguous_ints_or_floats, np.array([])) def test_label_index_order(): @@ -420,7 +420,7 @@ def test_default_param_grids_no_duplicates(): """ Verify that the default parameter grids don't contain duplicate values. """ - for learner, param_list in _DEFAULT_PARAM_GRIDS.items(): + for learner, param_list in KNOWN_DEFAULT_PARAM_GRIDS.items(): param_dict = param_list[0] for param_name, values in param_dict.items(): assert(len(set(values)) == len(values)) @@ -798,7 +798,7 @@ def test_train_file_and_train_directory(): 'test_single_file.' 'jsonlines'), train_directory='foo') - _parse_config_file(config_path) + parse_config_file(config_path) @raises(ValueError) @@ -817,7 +817,7 @@ def test_test_file_and_test_directory(): 'test_single_file.' 'jsonlines'), test_directory='foo') - _parse_config_file(config_path) + parse_config_file(config_path) def check_adaboost_predict(base_estimator, algorithm, expected_score): @@ -994,7 +994,7 @@ def check_train_and_score_function(model_type): estimator_name = 'LogisticRegression' if model_type == 'classifier' else 'Ridge' metric = 'accuracy' if model_type == 'classifier' else 'pearson' learner1 = Learner(estimator_name) - train_score1, test_score1 = _train_and_score(learner1, train_fs, test_fs, metric) + train_score1, test_score1 = train_and_score(learner1, train_fs, test_fs, metric) # this should yield identical results when training another instance # of the same learner without grid search and shuffling and evaluating @@ -1112,11 +1112,11 @@ def test_invalid_classification_grid_objective(): np.array(['yes', 'no']), np.array([1, 2, 4.0]), np.array(['A', 'B', 1, 2])], - [_CORRELATION_METRICS | _REGRESSION_ONLY_METRICS | _WEIGHTED_KAPPA_METRICS, - _REGRESSION_ONLY_METRICS | _WEIGHTED_KAPPA_METRICS, - _CORRELATION_METRICS | _REGRESSION_ONLY_METRICS | _WEIGHTED_KAPPA_METRICS, - _REGRESSION_ONLY_METRICS | _WEIGHTED_KAPPA_METRICS, - _CORRELATION_METRICS | _REGRESSION_ONLY_METRICS | _WEIGHTED_KAPPA_METRICS])): + [CORRELATION_METRICS | REGRESSION_ONLY_METRICS | WEIGHTED_KAPPA_METRICS, + REGRESSION_ONLY_METRICS | WEIGHTED_KAPPA_METRICS, + CORRELATION_METRICS | REGRESSION_ONLY_METRICS | WEIGHTED_KAPPA_METRICS, + REGRESSION_ONLY_METRICS | WEIGHTED_KAPPA_METRICS, + CORRELATION_METRICS | REGRESSION_ONLY_METRICS | WEIGHTED_KAPPA_METRICS])): # check each bad objective for metric in bad_objectives: @@ -1162,11 +1162,11 @@ def test_invalid_classification_metric(): np.array(['yes', 'no']), np.array([1, 2, 4.0]), np.array(['A', 'B', 1, 2])], - [_CORRELATION_METRICS | _REGRESSION_ONLY_METRICS | _WEIGHTED_KAPPA_METRICS, - _REGRESSION_ONLY_METRICS | _WEIGHTED_KAPPA_METRICS, - _CORRELATION_METRICS | _REGRESSION_ONLY_METRICS | _WEIGHTED_KAPPA_METRICS, - _REGRESSION_ONLY_METRICS | _WEIGHTED_KAPPA_METRICS, - _CORRELATION_METRICS | _REGRESSION_ONLY_METRICS | _WEIGHTED_KAPPA_METRICS])): + [CORRELATION_METRICS | REGRESSION_ONLY_METRICS | WEIGHTED_KAPPA_METRICS, + REGRESSION_ONLY_METRICS | WEIGHTED_KAPPA_METRICS, + CORRELATION_METRICS | REGRESSION_ONLY_METRICS | WEIGHTED_KAPPA_METRICS, + REGRESSION_ONLY_METRICS | WEIGHTED_KAPPA_METRICS, + CORRELATION_METRICS | REGRESSION_ONLY_METRICS | WEIGHTED_KAPPA_METRICS])): # check each bad objective for metric in bad_objectives: @@ -1323,15 +1323,15 @@ def check_objective_values_for_classification(metric_name, elif metric_name == 'accuracy': metric_value = accuracy_score(y_fold_test_indices, sklearn_fold_test_labels) - elif metric_name in _UNWEIGHTED_KAPPA_METRICS: + elif metric_name in UNWEIGHTED_KAPPA_METRICS: metric_value = use_score_func(metric_name, y_fold_test_indices, sklearn_fold_test_labels) # 5. The only ones left are the weighted kapps; # these require contiguous ints or floats - elif metric_name in _WEIGHTED_KAPPA_METRICS: - if _contiguous_ints_or_floats(label_array): + elif metric_name in WEIGHTED_KAPPA_METRICS: + if contiguous_ints_or_floats(label_array): metric_value = use_score_func(metric_name, y_fold_test_indices, sklearn_fold_test_labels) @@ -1384,10 +1384,10 @@ def test_objective_values_for_classification(): # 4. We then compare values in 2 and 3 to verify that they are equal. metrics_to_test = set(['accuracy']) - metrics_to_test.update(_CORRELATION_METRICS, - _PROBABILISTIC_METRICS, - _UNWEIGHTED_KAPPA_METRICS, - _WEIGHTED_KAPPA_METRICS) + metrics_to_test.update(CORRELATION_METRICS, + PROBABILISTIC_METRICS, + UNWEIGHTED_KAPPA_METRICS, + WEIGHTED_KAPPA_METRICS) metrics_to_test = sorted(metrics_to_test) for (metric, @@ -1418,12 +1418,12 @@ def test_objective_values_for_classification(): # (d) probabilistic metrics and no probabilities skipped_conditions = ((metric in ['average_precision', 'roc_auc'] and len(label_array) != 2) or - ((metric in _WEIGHTED_KAPPA_METRICS or - metric in _CORRELATION_METRICS) and + ((metric in WEIGHTED_KAPPA_METRICS or + metric in CORRELATION_METRICS) and issubclass(label_array.dtype.type, str)) or - (metric in _WEIGHTED_KAPPA_METRICS and - not _contiguous_ints_or_floats(label_array)) or - (metric in _PROBABILISTIC_METRICS and + (metric in WEIGHTED_KAPPA_METRICS and + not contiguous_ints_or_floats(label_array)) or + (metric in PROBABILISTIC_METRICS and not use_probabilities)) if skipped_conditions: continue @@ -1575,7 +1575,7 @@ def check_metric_values_for_classification(metric_name, elif metric_name == 'accuracy': sklearn_metric_value = accuracy_score(y_test, sklearn_test_labels) - elif metric_name in _UNWEIGHTED_KAPPA_METRICS: + elif metric_name in UNWEIGHTED_KAPPA_METRICS: sklearn_metric_value = use_score_func(metric_name, y_test, sklearn_test_labels) @@ -1583,8 +1583,8 @@ def check_metric_values_for_classification(metric_name, # 5. The only ones left are the weighted kappas and they are not in sklearn # so we are forced to use the SKLL implementations; both types # require integer labels - elif metric_name in _WEIGHTED_KAPPA_METRICS: - if _contiguous_ints_or_floats(label_array): + elif metric_name in WEIGHTED_KAPPA_METRICS: + if contiguous_ints_or_floats(label_array): sklearn_metric_value = use_score_func(metric_name, y_test, sklearn_test_labels) @@ -1628,10 +1628,10 @@ def test_metric_values_for_classification(): # 4. We then compare values in 2 and 3 to verify that they are equal. metrics_to_test = set(['accuracy']) - metrics_to_test.update(_CORRELATION_METRICS, - _PROBABILISTIC_METRICS, - _UNWEIGHTED_KAPPA_METRICS, - _WEIGHTED_KAPPA_METRICS) + metrics_to_test.update(CORRELATION_METRICS, + PROBABILISTIC_METRICS, + UNWEIGHTED_KAPPA_METRICS, + WEIGHTED_KAPPA_METRICS) metrics_to_test = sorted(metrics_to_test) for (metric, @@ -1662,12 +1662,12 @@ def test_metric_values_for_classification(): # (d) probabilistic metrics and no probabilities skipped_conditions = ((metric in ['average_precision', 'roc_auc'] and len(label_array) != 2) or - ((metric in _WEIGHTED_KAPPA_METRICS or - metric in _CORRELATION_METRICS) and + ((metric in WEIGHTED_KAPPA_METRICS or + metric in CORRELATION_METRICS) and issubclass(label_array.dtype.type, str)) or - (metric in _WEIGHTED_KAPPA_METRICS and - not _contiguous_ints_or_floats(label_array)) or - (metric in _PROBABILISTIC_METRICS and + (metric in WEIGHTED_KAPPA_METRICS and + not contiguous_ints_or_floats(label_array)) or + (metric in PROBABILISTIC_METRICS and not use_probabilities)) if skipped_conditions: continue diff --git a/tests/test_custom_learner.py b/tests/test_custom_learner.py index e69dad61..c07056ee 100644 --- a/tests/test_custom_learner.py +++ b/tests/test_custom_learner.py @@ -19,11 +19,12 @@ from numpy.testing import assert_array_equal from skll.data import NDJWriter from skll.experiments import run_configuration -from skll.learner import _DEFAULT_PARAM_GRIDS, Learner +from skll.learner import Learner +from skll.utils.constants import KNOWN_DEFAULT_PARAM_GRIDS from tests.utils import fill_in_config_paths, make_classification_data -_ALL_MODELS = list(_DEFAULT_PARAM_GRIDS.keys()) +_ALL_MODELS = list(KNOWN_DEFAULT_PARAM_GRIDS.keys()) _my_dir = abspath(dirname(__file__)) diff --git a/tests/test_cv.py b/tests/test_cv.py index 93f77f0d..7a02b9d9 100644 --- a/tests/test_cv.py +++ b/tests/test_cv.py @@ -25,16 +25,17 @@ from sklearn.feature_extraction import FeatureHasher from sklearn.datasets import make_classification +from sklearn.model_selection import StratifiedKFold -from skll.config import _load_cv_folds +from skll.config import load_cv_folds from skll.data import FeatureSet -from skll.experiments import _load_featureset, run_configuration -from skll.learner import _DEFAULT_PARAM_GRIDS, Learner -from sklearn.model_selection import StratifiedKFold +from skll.experiments import load_featureset, run_configuration +from skll.learner import Learner +from skll.utils.constants import KNOWN_DEFAULT_PARAM_GRIDS from tests.utils import (create_jsonlines_feature_files, fill_in_config_paths_for_single_file) -_ALL_MODELS = list(_DEFAULT_PARAM_GRIDS.keys()) +_ALL_MODELS = list(KNOWN_DEFAULT_PARAM_GRIDS.keys()) _my_dir = abspath(dirname(__file__)) @@ -199,7 +200,7 @@ def test_load_cv_folds(): w.writerow([example_id, fold_label]) # now read the CSV file using _load_cv_folds - custom_cv_folds_loaded = _load_cv_folds(fold_file_path) + custom_cv_folds_loaded = load_cv_folds(fold_file_path) eq_(custom_cv_folds_loaded, custom_cv_folds) @@ -222,7 +223,7 @@ def test_load_cv_folds_non_float_ids(): w.writerow([example_id, fold_label]) # now read the CSV file using _load_cv_folds, which should raise ValueError - _load_cv_folds(fold_file_path, ids_to_floats=True) + load_cv_folds(fold_file_path, ids_to_floats=True) def test_retrieve_cv_folds(): @@ -397,7 +398,7 @@ def test_cross_validate_task(): # Check that the fold ids were saved correctly expected_skll_ids = {} - examples = _load_featureset(train_path, '', suffix, quiet=True) + examples = load_featureset(train_path, '', suffix, quiet=True) kfold = StratifiedKFold(n_splits=10) for fold_num, (_, test_indices) in enumerate(kfold.split(examples.features, examples.labels)): for index in test_indices: diff --git a/tests/test_featureset.py b/tests/test_featureset.py index 0701a4ea..4723f598 100644 --- a/tests/test_featureset.py +++ b/tests/test_featureset.py @@ -23,17 +23,23 @@ from sklearn.datasets import make_classification import skll -from skll.data import (FeatureSet, Writer, Reader, - CSVReader, TSVReader, NDJReader, NDJWriter) +from skll.data import (CSVReader, + FeatureSet, + NDJReader, + NDJWriter, + Reader, + TSVReader, + Writer) + from skll.data.readers import DictListReader -from skll.experiments import _load_featureset -from skll.learner import _DEFAULT_PARAM_GRIDS -from skll.utilities import skll_convert +from skll.experiments import load_featureset +from skll.utils.constants import DEFAULT_PARAM_GRIDS +from skll.utils.commandline import skll_convert from tests.utils import make_classification_data, make_regression_data -_ALL_MODELS = list(_DEFAULT_PARAM_GRIDS.keys()) +_ALL_MODELS = list(DEFAULT_PARAM_GRIDS.keys()) _my_dir = abspath(dirname(__file__)) @@ -723,12 +729,12 @@ def check_load_featureset(suffix, numeric_ids): # Load unmerged data and merge it dirpath = join(_my_dir, 'train', 'test_merging') featureset = ['{}'.format(i) for i in range(num_feat_files)] - merged_exs = _load_featureset(dirpath, featureset, suffix, quiet=True) + merged_exs = load_featureset(dirpath, featureset, suffix, quiet=True) # Load pre-merged data featureset = ['all'] - premerged_exs = _load_featureset(dirpath, featureset, suffix, - quiet=True) + premerged_exs = load_featureset(dirpath, featureset, suffix, + quiet=True) assert_array_equal(merged_exs.ids, premerged_exs.ids) assert_array_equal(merged_exs.labels, premerged_exs.labels) @@ -922,19 +928,19 @@ def check_convert_featureset(from_suffix, to_suffix, with_labels=True): featureset = ['{}_{}{}'.format(feature_name_prefix, i, with_labels_part) for i in range(num_feat_files)] label_col = 'y' if with_labels else None - merged_exs = _load_featureset(dirpath, - featureset, - to_suffix, - label_col=label_col, - quiet=True) + merged_exs = load_featureset(dirpath, + featureset, + to_suffix, + label_col=label_col, + quiet=True) # Load pre-merged data in the `to_suffix` format featureset = ['{}{}_all'.format(feature_name_prefix, with_labels_part)] - premerged_exs = _load_featureset(dirpath, - featureset, - to_suffix, - label_col=label_col, - quiet=True) + premerged_exs = load_featureset(dirpath, + featureset, + to_suffix, + label_col=label_col, + quiet=True) # make sure that the pre-generated merged data in the to_suffix format # is the same as the converted, merged data in the to_suffix format diff --git a/tests/test_input.py b/tests/test_input.py index a7472b6f..de3e1b79 100644 --- a/tests/test_input.py +++ b/tests/test_input.py @@ -21,13 +21,11 @@ from nose.tools import eq_, ok_, raises -from skll.config import (_parse_config_file, - _load_cv_folds, - _locate_file) +from skll.config import locate_file, load_cv_folds, parse_config_file from skll.data.readers import safe_float from skll.experiments import _load_featureset -from skll.logutils import (close_and_remove_logger_handlers, - get_skll_logger) +from skll.utils.logging import (close_and_remove_logger_handlers, + get_skll_logger) from tests.utils import (create_jsonlines_feature_files, fill_in_config_options) @@ -94,7 +92,7 @@ def test_locate_file_valid_paths1(): config_abs_path = join(_my_dir, 'configs', 'test_config_parsing_relative_path1.cfg') open(config_abs_path, 'w').close() - eq_(_locate_file(config_abs_path, _my_dir), + eq_(locate_file(config_abs_path, _my_dir), join(_my_dir, 'configs', 'test_config_parsing_relative_path1.cfg')) @@ -107,7 +105,7 @@ def test_locate_file_valid_paths2(): 'test_config_parsing_relative_path2.cfg') config_rel_path = 'configs/test_config_parsing_relative_path2.cfg' open(config_abs_path, 'w').close() - eq_(_locate_file(config_rel_path, _my_dir), config_abs_path) + eq_(locate_file(config_rel_path, _my_dir), config_abs_path) def test_locate_file_valid_paths3(): @@ -119,8 +117,8 @@ def test_locate_file_valid_paths3(): 'test_config_parsing_relative_path3.cfg') config_rel_path = 'configs/test_config_parsing_relative_path3.cfg' open(config_abs_path, 'w').close() - eq_(_locate_file(config_abs_path, _my_dir), - _locate_file(config_rel_path, _my_dir)) + eq_(locate_file(config_abs_path, _my_dir), + locate_file(config_rel_path, _my_dir)) @raises(IOError) @@ -130,7 +128,7 @@ def test_locate_file_invalid_path(): exist. """ - _locate_file('test/does_not_exist.cfg', _my_dir) + locate_file('test/does_not_exist.cfg', _my_dir) @raises(ValueError) @@ -185,7 +183,7 @@ def check_config_parsing_value_error(config_path): """ Assert that calling `_parse_config_file` on `config_path` raises ValueError """ - _parse_config_file(config_path) + parse_config_file(config_path) @raises(TypeError) @@ -193,7 +191,7 @@ def check_config_parsing_type_error(config_path): """ Assert that calling `_parse_config_file` on `config_path` raises TypeError """ - _parse_config_file(config_path) + parse_config_file(config_path) @raises(KeyError) @@ -201,7 +199,7 @@ def check_config_parsing_key_error(config_path): """ Assert that calling `_parse_config_file` on `config_path` raises KeyError """ - _parse_config_file(config_path) + parse_config_file(config_path) @raises(IOError) @@ -209,7 +207,7 @@ def check_config_parsing_file_not_found_error(config_path): """ Assert that calling `_parse_config_file` on `config_path` raises FileNotFoundError """ - _parse_config_file(config_path) + parse_config_file(config_path) @raises(IOError) @@ -217,7 +215,7 @@ def test_empty_config_name_raises_file_not_found_error(): """ Assert that calling _parse_config_file on an empty string raises IOError """ - _parse_config_file("") + parse_config_file("") def test_config_parsing_no_name(): @@ -1063,7 +1061,7 @@ def test_config_parsing_mse_throws_exception(): values_to_fill_dict, 'mse_to_neg_mse') - _parse_config_file(config_path) + parse_config_file(config_path) def test_config_parsing_no_grid_objectives_needed_for_learning_curve(): @@ -1100,7 +1098,7 @@ def test_config_parsing_no_grid_objectives_needed_for_learning_curve(): fixed_parameter_list, param_grid_list, featureset_names, learners, prediction_dir, log_path, train_path, test_path, ids_to_floats, class_map, custom_learner_path, learning_curve_cv_folds_list, - learning_curve_train_sizes, output_metrics) = _parse_config_file(config_path) + learning_curve_train_sizes, output_metrics) = parse_config_file(config_path) eq_(do_grid_search, False) eq_(grid_objectives, []) @@ -1139,7 +1137,7 @@ def test_config_parsing_relative_input_path(): fixed_parameter_list, param_grid_list, featureset_names, learners, prediction_dir, log_path, train_path, test_path, ids_to_floats, class_map, custom_learner_path, learning_curve_cv_folds_list, - learning_curve_train_sizes, output_metrics) = _parse_config_file(config_path) + learning_curve_train_sizes, output_metrics) = parse_config_file(config_path) # we need to use normcase here for Azure package builds to pass eq_(normcase(normpath(train_path)), normcase(join(_my_dir, 'train'))) @@ -1168,7 +1166,7 @@ def test_config_parsing_relative_input_paths(): values_to_fill_dict, 'relative_paths') - _parse_config_file(config_path) + parse_config_file(config_path) def test_config_parsing_automatic_output_directory_creation(): @@ -1207,7 +1205,7 @@ def test_config_parsing_automatic_output_directory_creation(): values_to_fill_dict, 'auto_dir_creation') - _parse_config_file(config_path) + parse_config_file(config_path) ok_(exists(new_log_path)) ok_(exists(new_results_path)) @@ -1367,7 +1365,8 @@ def check_cv_folds_and_grid_search_folds(task, # read in the folds file into a dictionary and replace the string # 'fold_mapping' with this dictionary. - fold_mapping = _load_cv_folds(join(_my_dir, 'train/folds_file_test.csv'), ids_to_floats=False) + fold_mapping = load_cv_folds(join(_my_dir, 'train/folds_file_test.csv'), + ids_to_floats=False) if chosen_grid_search_folds == 'fold_mapping': chosen_grid_search_folds = fold_mapping if chosen_cv_folds == 'fold_mapping': @@ -1421,7 +1420,7 @@ def check_cv_folds_and_grid_search_folds(task, fixed_parameter_list, param_grid_list, featureset_names, learners, prediction_dir, log_path, train_path, test_path, ids_to_floats, class_map, custom_learner_path, learning_curve_cv_folds_list, - learning_curve_train_sizes, output_metrics) = _parse_config_file(config_path) + learning_curve_train_sizes, output_metrics) = parse_config_file(config_path) eq_(cv_folds, chosen_cv_folds) eq_(grid_search_folds, chosen_grid_search_folds) @@ -1460,7 +1459,7 @@ def test_default_number_of_cv_folds(): fixed_parameter_list, param_grid_list, featureset_names, learners, prediction_dir, log_path, train_path, test_path, ids_to_floats, class_map, custom_learner_path, learning_curve_cv_folds_list, - learning_curve_train_sizes, output_metrics) = _parse_config_file(config_path) + learning_curve_train_sizes, output_metrics) = parse_config_file(config_path) eq_(cv_folds, 10) @@ -1498,7 +1497,7 @@ def test_setting_number_of_cv_folds(): fixed_parameter_list, param_grid_list, featureset_names, learners, prediction_dir, log_path, train_path, test_path, ids_to_floats, class_map, custom_learner_path, learning_curve_cv_folds_list, - learning_curve_train_sizes, output_metrics) = _parse_config_file(config_path) + learning_curve_train_sizes, output_metrics) = parse_config_file(config_path) eq_(cv_folds, 5) @@ -1539,7 +1538,7 @@ def test_setting_param_grids(): fixed_parameter_list, param_grid_list, featureset_names, learners, prediction_dir, log_path, train_path, test_path, ids_to_floats, class_map, custom_learner_path, learning_curve_cv_folds_list, - learning_curve_train_sizes, output_metrics) = _parse_config_file(config_path) + learning_curve_train_sizes, output_metrics) = parse_config_file(config_path) eq_(param_grid_list[0]['C'][0], 1e-6) eq_(param_grid_list[0]['C'][1], 1e-3) @@ -1585,7 +1584,7 @@ def test_setting_fixed_parameters(): fixed_parameter_list, param_grid_list, featureset_names, learners, prediction_dir, log_path, train_path, test_path, ids_to_floats, class_map, custom_learner_path, learning_curve_cv_folds_list, - learning_curve_train_sizes, output_metrics) = _parse_config_file(config_path) + learning_curve_train_sizes, output_metrics) = parse_config_file(config_path) eq_(fixed_parameter_list[0]['C'][0], 1e-6) eq_(fixed_parameter_list[0]['C'][1], 1e-3) @@ -1620,7 +1619,7 @@ def test_learning_curve_objectives_unsupported_error(): values_to_fill_dict, 'default_learning_curve') - _parse_config_file(config_path) + parse_config_file(config_path) def test_default_learning_curve_options(): @@ -1654,7 +1653,7 @@ def test_default_learning_curve_options(): fixed_parameter_list, param_grid_list, featureset_names, learners, prediction_dir, log_path, train_path, test_path, ids_to_floats, class_map, custom_learner_path, learning_curve_cv_folds_list, - learning_curve_train_sizes, output_metrics) = _parse_config_file(config_path) + learning_curve_train_sizes, output_metrics) = parse_config_file(config_path) eq_(learning_curve_cv_folds_list, [10, 10]) ok_(np.all(learning_curve_train_sizes == np.linspace(0.1, 1.0, 5))) @@ -1692,7 +1691,7 @@ def test_setting_learning_curve_options(): fixed_parameter_list, param_grid_list, featureset_names, learners, prediction_dir, log_path, train_path, test_path, ids_to_floats, class_map, custom_learner_path, learning_curve_cv_folds_list, - learning_curve_train_sizes, output_metrics) = _parse_config_file(config_path) + learning_curve_train_sizes, output_metrics) = parse_config_file(config_path) eq_(learning_curve_cv_folds_list, [100, 10]) eq_(learning_curve_train_sizes, [10, 50, 100, 200, 500]) @@ -1730,7 +1729,7 @@ def test_learning_curve_metrics_and_objectives_throw_error(): fixed_parameter_list, param_grid_list, featureset_names, learners, prediction_dir, log_path, train_path, test_path, ids_to_floats, class_map, custom_learner_path, learning_curve_cv_folds_list, - learning_curve_train_sizes, output_metrics) = _parse_config_file(config_path) + learning_curve_train_sizes, output_metrics) = parse_config_file(config_path) eq_(output_metrics, ["accuracy", "f1_score_micro"]) @@ -1765,7 +1764,7 @@ def test_learning_curve_metrics_and_no_objectives(): fixed_parameter_list, param_grid_list, featureset_names, learners, prediction_dir, log_path, train_path, test_path, ids_to_floats, class_map, custom_learner_path, learning_curve_cv_folds_list, - learning_curve_train_sizes, output_metrics) = _parse_config_file(config_path) + learning_curve_train_sizes, output_metrics) = parse_config_file(config_path) eq_(output_metrics, ["accuracy", "unweighted_kappa"]) @@ -1800,7 +1799,7 @@ def test_learning_curve_metrics(): fixed_parameter_list, param_grid_list, featureset_names, learners, prediction_dir, log_path, train_path, test_path, ids_to_floats, class_map, custom_learner_path, learning_curve_cv_folds_list, - learning_curve_train_sizes, output_metrics) = _parse_config_file(config_path) + learning_curve_train_sizes, output_metrics) = parse_config_file(config_path) eq_(output_metrics, ["accuracy"]) eq_(grid_objectives, []) @@ -1837,7 +1836,7 @@ def test_learning_curve_pipeline_option(): fixed_parameter_list, param_grid_list, featureset_names, learners, prediction_dir, log_path, train_path, test_path, ids_to_floats, class_map, custom_learner_path, learning_curve_cv_folds_list, - learning_curve_train_sizes, output_metrics) = _parse_config_file(config_path) + learning_curve_train_sizes, output_metrics) = parse_config_file(config_path) eq_(pipeline, True) @@ -1935,7 +1934,7 @@ def test_config_parsing_param_grids_no_grid_search(): values_to_fill_dict, 'param_grids_no_grid_search') - _parse_config_file(config_path) + parse_config_file(config_path) log_path = join(output_dir, "config_parsing_param_grids_no_grid_search.log") with open(log_path) as f: warning_pattern = re.compile('Since "grid_search" is set to False, ' @@ -1980,7 +1979,7 @@ def test_config_parsing_no_grid_search_but_objectives_specified(): fixed_parameter_list, param_grid_list, featureset_names, learners, prediction_dir, log_path, train_path, test_path, ids_to_floats, class_map, custom_learner_path, learning_curve_cv_folds_list, - learning_curve_train_sizes, output_metrics) = _parse_config_file(config_path) + learning_curve_train_sizes, output_metrics) = parse_config_file(config_path) eq_(do_grid_search, False) eq_(grid_objectives, []) @@ -2021,7 +2020,7 @@ def test_config_parsing_param_grids_fixed_parameters_conflict(): values_to_fill_dict, 'param_grids_no_grid_search') - _parse_config_file(config_path) + parse_config_file(config_path) log_path = join(output_dir, "config_parsing_param_grids_fixed_parameters_conflict.log") with open(log_path) as f: @@ -2071,6 +2070,6 @@ def test_config_parsing_default_pos_label_str_value(): fixed_parameter_list, param_grid_list, featureset_names, learners, prediction_dir, log_path, train_path, test_path, ids_to_floats, class_map, custom_learner_path, learning_curve_cv_folds_list, - learning_curve_train_sizes, output_metrics) = _parse_config_file(config_path) + learning_curve_train_sizes, output_metrics) = parse_config_file(config_path) eq_(pos_label_str, None) diff --git a/tests/test_logutils.py b/tests/test_logutils.py index 0fe72b42..a2fa24e3 100644 --- a/tests/test_logutils.py +++ b/tests/test_logutils.py @@ -7,8 +7,9 @@ from sklearn.metrics import roc_curve from tempfile import NamedTemporaryFile -from skll import (close_and_remove_logger_handlers, get_skll_logger, - orig_showwarning) +from skll.logutils import (close_and_remove_logger_handlers, + get_skll_logger, + orig_showwarning) TEMP_FILES = [] TEMP_FILE_PATHS = [] diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 8b0fdf63..77a8e5fa 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -16,10 +16,10 @@ from nose.tools import raises from numpy.testing import assert_almost_equal -from skll.learner import _DEFAULT_PARAM_GRIDS +from skll.utils.constants import KNOWN_DEFAULT_PARAM_GRIDS from skll.metrics import kappa -_ALL_MODELS = list(_DEFAULT_PARAM_GRIDS.keys()) +_ALL_MODELS = list(KNOWN_DEFAULT_PARAM_GRIDS.keys()) # Inputs derived from Ben Hamner's unit tests for his # kappa implementation as part of the ASAP competition diff --git a/tests/test_output.py b/tests/test_output.py index daeab684..9ac60eac 100644 --- a/tests/test_output.py +++ b/tests/test_output.py @@ -34,10 +34,10 @@ from sklearn.naive_bayes import MultinomialNB from skll.data import FeatureSet, NDJWriter, Reader -from skll.config import _VALID_TASKS -from skll.experiments import (_compute_ylimits_for_featureset, - run_configuration) -from skll.learner import Learner, _DEFAULT_PARAM_GRIDS +from skll.experiments import run_configuration +from skll.experiments.output import _compute_ylimits_for_featureset +from skll.learner import Learner +from skll.utils.constants import KNOWN_DEFAULT_PARAM_GRIDS, VALID_TASKS from tests.utils import (create_jsonlines_feature_files, fill_in_config_options, @@ -47,7 +47,7 @@ make_regression_data) -_ALL_MODELS = list(_DEFAULT_PARAM_GRIDS.keys()) +_ALL_MODELS = list(KNOWN_DEFAULT_PARAM_GRIDS.keys()) _my_dir = abspath(dirname(__file__)) @@ -95,7 +95,7 @@ def tearDown(): + glob(join(output_dir, 'test_majority_class_custom_learner_*')): os.unlink(output_file) - for suffix in _VALID_TASKS: + for suffix in VALID_TASKS: config_files = ['test_cv_results_{}.cfg'.format(suffix)] for cf in config_files: if exists(join(config_dir, cf)): @@ -581,7 +581,7 @@ def time_field(x): def test_grid_search_cv_results(): - for task in _VALID_TASKS: + for task in VALID_TASKS: for do_grid_search in [True, False]: yield check_grid_search_cv_results, task, do_grid_search diff --git a/tests/test_preprocessing.py b/tests/test_preprocessing.py index ec1c227c..9644b9f1 100644 --- a/tests/test_preprocessing.py +++ b/tests/test_preprocessing.py @@ -22,13 +22,14 @@ from skll.data import FeatureSet, NDJWriter from skll.experiments import run_configuration -from skll.learner import Learner, SelectByMinCount -from skll.learner import _DEFAULT_PARAM_GRIDS +from skll.learner import Learner +from skll.learner.utils import SelectByMinCount +from skll.utils.constants import KNOWN_DEFAULT_PARAM_GRIDS from tests.utils import fill_in_config_paths -_ALL_MODELS = list(_DEFAULT_PARAM_GRIDS.keys()) +_ALL_MODELS = list(KNOWN_DEFAULT_PARAM_GRIDS.keys()) SCORE_OUTPUT_RE = re.compile(r'Objective Function Score \(Test\) = ' r'([\-\d\.]+)') _my_dir = abspath(dirname(__file__)) diff --git a/tests/test_regression.py b/tests/test_regression.py index 58cb0b65..05128e5e 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -33,14 +33,15 @@ from skll.data import FeatureSet, NDJWriter from skll.config import _setup_config_parser from skll.experiments import run_configuration -from skll.learner import Learner, rescaled -from skll.learner import _DEFAULT_PARAM_GRIDS -from skll.metrics import _CLASSIFICATION_ONLY_METRICS +from skll.learner import Learner +from skll.learner.utils import rescaled +from skll.utils.constants import (CLASSIFICATION_ONLY_METRICS, + KNOWN_DEFAULT_PARAM_GRIDS) from tests.utils import (make_regression_data, fill_in_config_paths_for_fancy_output) -_ALL_MODELS = list(_DEFAULT_PARAM_GRIDS.keys()) +_ALL_MODELS = list(KNOWN_DEFAULT_PARAM_GRIDS.keys()) _my_dir = abspath(dirname(__file__)) @@ -713,7 +714,7 @@ def test_invalid_regression_grid_objective(): 'RandomForestRegressor', 'RANSACRegressor', 'Ridge', 'LinearSVR', 'SVR', 'SGDRegressor', 'TheilSenRegressor']: - for metric in _CLASSIFICATION_ONLY_METRICS: + for metric in CLASSIFICATION_ONLY_METRICS: yield check_invalid_regression_grid_objective, learner, metric @@ -738,6 +739,6 @@ def test_invalid_regression_metric(): 'RandomForestRegressor', 'RANSACRegressor', 'Ridge', 'LinearSVR', 'SVR', 'SGDRegressor', 'TheilSenRegressor']: - for metric in _CLASSIFICATION_ONLY_METRICS: + for metric in CLASSIFICATION_ONLY_METRICS: yield check_invalid_regression_metric, learner, metric, True yield check_invalid_regression_metric, learner, metric, False diff --git a/tests/test_utilities.py b/tests/test_utilities.py index 6406c203..4678fe98 100644 --- a/tests/test_utilities.py +++ b/tests/test_utilities.py @@ -36,16 +36,16 @@ from sklearn.linear_model import SGDClassifier, SGDRegressor import skll -import skll.utilities.compute_eval_from_predictions as cefp -from skll.utilities.compute_eval_from_predictions import get_prediction_from_probabilities -import skll.utilities.filter_features as ff -import skll.utilities.generate_predictions as gp -import skll.utilities.print_model_weights as pmw -import skll.utilities.run_experiment as rex -import skll.utilities.skll_convert as sk -import skll.utilities.summarize_results as sr -import skll.utilities.join_features as jf -import skll.utilities.plot_learning_curves as plc +import skll.utils.commandline.compute_eval_from_predictions as cefp +from skll.utils.commandline.compute_eval_from_predictions import get_prediction_from_probabilities +import skll.utils.commandline.filter_features as ff +import skll.utils.commandline.generate_predictions as gp +import skll.utils.commandline.print_model_weights as pmw +import skll.utils.commandline.run_experiment as rex +import skll.utils.commandline.skll_convert as sk +import skll.utils.commandline.summarize_results as sr +import skll.utils.commandline.join_features as jf +import skll.utils.commandline.plot_learning_curves as plc from skll.data import (FeatureSet, NDJWriter, @@ -55,15 +55,16 @@ safe_float) from skll.data.readers import EXT_TO_READER from skll.data.writers import EXT_TO_WRITER -from skll.experiments import (_generate_learning_curve_plots, - _write_summary_file, +from skll.experiments import (generate_learning_curve_plots, run_configuration) -from skll.learner import Learner, _DEFAULT_PARAM_GRIDS +from skll.experiments.output import _write_summary_file +from skll.learner import Learner +from skll.utils.constants import KNOWN_DEFAULT_PARAM_GRIDS from tests.utils import make_classification_data, make_regression_data -_ALL_MODELS = list(_DEFAULT_PARAM_GRIDS.keys()) +_ALL_MODELS = list(KNOWN_DEFAULT_PARAM_GRIDS.keys()) _my_dir = abspath(dirname(__file__)) @@ -1223,7 +1224,7 @@ def test_plot_learning_curves_argparse(): # replace the _generate_learning_curve_plots function that's called # by the main() in plot_learning_curves with a mocked up version - generate_learning_curve_plots_mock = create_autospec(_generate_learning_curve_plots) + generate_learning_curve_plots_mock = create_autospec(generate_learning_curve_plots) plc._generate_learning_curve_plots = generate_learning_curve_plots_mock # now call main with some arguments From 2b55d5a824300522da35a4b7ad5d9b5734cf453c Mon Sep 17 00:00:00 2001 From: Nitin Madnani Date: Fri, 10 Apr 2020 17:08:20 -0400 Subject: [PATCH 07/20] Update `setup.py` with new command-line utility paths. --- setup.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index a8daa4c0..1d7877d0 100644 --- a/setup.py +++ b/setup.py @@ -31,15 +31,15 @@ def requirements(): license='BSD 3 clause', packages=find_packages(exclude=['tests', 'examples']), entry_points={'console_scripts': - ['filter_features = skll.utilities.filter_features:main', - 'generate_predictions = skll.utilities.generate_predictions:main', - 'join_features = skll.utilities.join_features:main', - 'print_model_weights = skll.utilities.print_model_weights:main', - 'run_experiment = skll.utilities.run_experiment:main', - 'skll_convert = skll.utilities.skll_convert:main', - 'summarize_results = skll.utilities.summarize_results:main', - 'compute_eval_from_predictions = skll.utilities.compute_eval_from_predictions:main', - 'plot_learning_curves = skll.utilities.plot_learning_curves:main']}, + ['filter_features = skll.utils.commandline.filter_features:main', + 'generate_predictions = skll.utils.commandline.generate_predictions:main', + 'join_features = skll.utils.commandline.join_features:main', + 'print_model_weights = skll.utils.commandline.print_model_weights:main', + 'run_experiment = skll.utils.commandline.run_experiment:main', + 'skll_convert = skll.utils.commandline.skll_convert:main', + 'summarize_results = skll.utils.commandline.summarize_results:main', + 'compute_eval_from_predictions = skll.utils.commandline.compute_eval_from_predictions:main', + 'plot_learning_curves = skll.utils.commandline.plot_learning_curves:main']}, install_requires=requirements(), classifiers=['Intended Audience :: Science/Research', 'Intended Audience :: Developers', From 76d5848e16be063002d115c377bb1fc45da5c80f Mon Sep 17 00:00:00 2001 From: Nitin Madnani Date: Fri, 10 Apr 2020 17:09:32 -0400 Subject: [PATCH 08/20] Update API documentation - Add new sections for `config` and `utils` packages. - Replace "module" with "Package" in various places. - Remove documentation for functions that are no longer in top-level skll namespace. --- doc/api.rst | 2 ++ doc/api/config.rst | 6 ++++++ doc/api/experiments.rst | 4 ++-- doc/api/learner.rst | 4 ++-- doc/api/skll.rst | 19 +++++++------------ doc/api/utils.rst | 15 +++++++++++++++ 6 files changed, 34 insertions(+), 16 deletions(-) create mode 100644 doc/api/config.rst create mode 100644 doc/api/utils.rst diff --git a/doc/api.rst b/doc/api.rst index fe17a3f8..24adb678 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -8,7 +8,9 @@ API Documentation api/quickstart api/skll + api/config api/data api/experiments api/learner api/metrics + api/utils diff --git a/doc/api/config.rst b/doc/api/config.rst new file mode 100644 index 00000000..1a01fbea --- /dev/null +++ b/doc/api/config.rst @@ -0,0 +1,6 @@ +:mod:`config` Package +===================== + +.. autofunction:: skll.config.fix_json +.. autofunction:: skll.config.load_cv_folds +.. autofunction:: skll.config.locate_file diff --git a/doc/api/experiments.rst b/doc/api/experiments.rst index 7d98a9a5..ac945775 100644 --- a/doc/api/experiments.rst +++ b/doc/api/experiments.rst @@ -1,5 +1,5 @@ -:mod:`experiments` Module -========================= +:mod:`experiments` Package +========================== .. automodule:: skll.experiments :members: diff --git a/doc/api/learner.rst b/doc/api/learner.rst index 135e5eef..c5586d31 100644 --- a/doc/api/learner.rst +++ b/doc/api/learner.rst @@ -1,5 +1,5 @@ -:mod:`learner` Module -===================== +:mod:`learner` Package +====================== .. automodule:: skll.learner :members: diff --git a/doc/api/skll.rst b/doc/api/skll.rst index b462e0f8..abeee037 100644 --- a/doc/api/skll.rst +++ b/doc/api/skll.rst @@ -1,8 +1,9 @@ :mod:`skll` Package =================== -The most useful parts of our API are available at the package level in addition -to the module level. They are documented in both places for convenience. +We have made the most useful parts of our API are available in the top level +``skll`` namespace even though some of them actually live in subpackages. +They are documented in both places for convenience. From :py:mod:`~skll.data` Package --------------------------------- @@ -16,18 +17,12 @@ From :py:mod:`~skll.data` Package :members: :show-inheritance: -From :py:mod:`~skll.experiments` Module ---------------------------------------- +From :py:mod:`~skll.experiments` Package +---------------------------------------- .. autofunction:: skll.run_configuration -From :py:mod:`~skll.learner` Module ------------------------------------ +From :py:mod:`~skll.learner` Package +------------------------------------ .. autoclass:: skll.Learner :members: :show-inheritance: - -From :py:mod:`~skll.metrics` Module ------------------------------------ -.. autofunction:: skll.f1_score_least_frequent -.. autofunction:: skll.kappa -.. autofunction:: skll.correlation diff --git a/doc/api/utils.rst b/doc/api/utils.rst new file mode 100644 index 00000000..54348aff --- /dev/null +++ b/doc/api/utils.rst @@ -0,0 +1,15 @@ +:mod:`utils` Package +==================== + +Various useful constants defining groups of evaluation metrics. + +.. autodata:: skll.utils.constants.CLASSIFICATION_ONLY_METRICS +.. autodata:: skll.utils.constants.CORRELATION_METRICS +.. autodata:: skll.utils.constants.PROBABILISTIC_METRICS +.. autodata:: skll.utils.constants.REGRESSION_ONLY_METRICS +.. autodata:: skll.utils.constants.UNWEIGHTED_KAPPA_METRICS +.. autodata:: skll.utils.constants.WEIGHTED_KAPPA_METRICS + +A useful logging function for SKLL developers + +.. autodata:: skll.utils.logging.get_skll_logger From 24c6a3e5e7594e1df9f296398e8858aa74b08382 Mon Sep 17 00:00:00 2001 From: Nitin Madnani Date: Fri, 10 Apr 2020 17:36:05 -0400 Subject: [PATCH 09/20] Add missing learners. --- skll/learner/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/skll/learner/__init__.py b/skll/learner/__init__.py index a38e1533..bbcb1a73 100644 --- a/skll/learner/__init__.py +++ b/skll/learner/__init__.py @@ -25,7 +25,7 @@ KFold, ShuffleSplit, StratifiedKFold) -from sklearn.dummy import DummyClassifier +from sklearn.dummy import DummyClassifier, DummyRegressor from sklearn.ensemble import (AdaBoostClassifier, AdaBoostRegressor, GradientBoostingClassifier, @@ -58,7 +58,7 @@ confusion_matrix, precision_recall_fscore_support) from sklearn.naive_bayes import MultinomialNB -from sklearn.neighbors import KNeighborsRegressor +from sklearn.neighbors import KNeighborsClassifier, KNeighborsRegressor from sklearn.neural_network import MLPClassifier, MLPRegressor from sklearn.preprocessing import StandardScaler from sklearn.svm import LinearSVC, SVC, LinearSVR, SVR From 7e7a8cc66bd895d517554b2c8b3e0fc22631346e Mon Sep 17 00:00:00 2001 From: Nitin Madnani Date: Fri, 10 Apr 2020 17:39:13 -0400 Subject: [PATCH 10/20] Fix invalid import. --- tests/test_logutils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_logutils.py b/tests/test_logutils.py index a2fa24e3..13e2d7cd 100644 --- a/tests/test_logutils.py +++ b/tests/test_logutils.py @@ -7,9 +7,9 @@ from sklearn.metrics import roc_curve from tempfile import NamedTemporaryFile -from skll.logutils import (close_and_remove_logger_handlers, - get_skll_logger, - orig_showwarning) +from skll.utils.logging import (close_and_remove_logger_handlers, + get_skll_logger, + orig_showwarning) TEMP_FILES = [] TEMP_FILE_PATHS = [] From b7716b458d4f476e87b839e348e1ddfb5d29f2ec Mon Sep 17 00:00:00 2001 From: Nitin Madnani Date: Fri, 10 Apr 2020 17:44:01 -0400 Subject: [PATCH 11/20] Rename test files to match organization. --- tests/{test_utilities.py => test_commandline_utils.py} | 0 tests/{test_logutils.py => test_logging_utils.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/{test_utilities.py => test_commandline_utils.py} (100%) rename tests/{test_logutils.py => test_logging_utils.py} (100%) diff --git a/tests/test_utilities.py b/tests/test_commandline_utils.py similarity index 100% rename from tests/test_utilities.py rename to tests/test_commandline_utils.py diff --git a/tests/test_logutils.py b/tests/test_logging_utils.py similarity index 100% rename from tests/test_logutils.py rename to tests/test_logging_utils.py From f95538891d8c23a35ca82133aaa0d539cb9cff59 Mon Sep 17 00:00:00 2001 From: Nitin Madnani Date: Fri, 10 Apr 2020 20:40:58 -0400 Subject: [PATCH 12/20] Remove `Reader` and `Writer` from the top-level namespace. --- doc/api/quickstart.rst | 2 +- doc/api/skll.rst | 6 ------ examples/make_titanic_example_data.py | 3 +-- skll/__init__.py | 4 ++-- 4 files changed, 4 insertions(+), 11 deletions(-) diff --git a/doc/api/quickstart.rst b/doc/api/quickstart.rst index ae6676fa..b01f0f49 100644 --- a/doc/api/quickstart.rst +++ b/doc/api/quickstart.rst @@ -5,7 +5,7 @@ Here is a quick run-down of how you accomplish common tasks. Load a ``FeatureSet`` from a file:: - from skll import Reader + from skll.data import Reader example_reader = Reader.for_path('myexamples.megam') train_examples = example_reader.read() diff --git a/doc/api/skll.rst b/doc/api/skll.rst index abeee037..a51513b4 100644 --- a/doc/api/skll.rst +++ b/doc/api/skll.rst @@ -10,12 +10,6 @@ From :py:mod:`~skll.data` Package .. autoclass:: skll.FeatureSet :members: :show-inheritance: -.. autoclass:: skll.Reader - :members: - :show-inheritance: -.. autoclass:: skll.Writer - :members: - :show-inheritance: From :py:mod:`~skll.experiments` Package ---------------------------------------- diff --git a/examples/make_titanic_example_data.py b/examples/make_titanic_example_data.py index 4d90c9b3..c4237fe4 100755 --- a/examples/make_titanic_example_data.py +++ b/examples/make_titanic_example_data.py @@ -10,10 +10,9 @@ import logging import os -import sys from itertools import chain -from skll import Writer, Reader +from skll.data import Reader, Writer def main(): diff --git a/skll/__init__.py b/skll/__init__.py index d2829bd3..031bf051 100644 --- a/skll/__init__.py +++ b/skll/__init__.py @@ -10,12 +10,12 @@ """ from sklearn.metrics import f1_score, make_scorer, SCORERS -from .data import FeatureSet, Reader, Writer +from .data import FeatureSet from .experiments import run_configuration from .learner import Learner from .metrics import correlation, f1_score_least_frequent, kappa -__all__ = ['FeatureSet', 'Learner', 'Reader', 'run_configuration', 'Writer'] +__all__ = ['FeatureSet', 'Learner', 'run_configuration'] # Add our scorers to the sklearn dictionary here so that they will always be # available if you import anything from skll From f4192fa17e19f09ebf0a0161246a56920ace69e3 Mon Sep 17 00:00:00 2001 From: Nitin Madnani Date: Fri, 10 Apr 2020 20:41:19 -0400 Subject: [PATCH 13/20] Rename test files in CI scripts --- .travis.yml | 4 ++-- DistributeTests.ps1 | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index ef2b9a1a..6bc75a1d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,10 +6,10 @@ notifications: slack: secure: d+NSaR+cfRkcJfX/uEVwdyQDlayZO6c8QV96grl1stmWxG3XOXvXg1tM6v7EBeb9VRw5T4VglnnJGNJA62j1ReEJ0XyWr5XtaNWiF6Lc4UOty/TTG36IQdkWS1vQA8v2Hre73YbvOhtBb6biNneVAk+rrfRSgomEa+ec22cjgUo= env: - - TESTFILES="tests/test_featureset.py tests/test_utilities.py" + - TESTFILES="tests/test_featureset.py tests/test_commandline_utils.py" - TESTFILES="tests/test_output.py" - TESTFILES="tests/test_regression.py" - - TESTFILES="tests/test_input.py tests/test_preprocessing.py tests/test_metrics.py tests/test_custom_learner.py tests/test_logutils.py tests/test_examples.py" + - TESTFILES="tests/test_input.py tests/test_preprocessing.py tests/test_metrics.py tests/test_custom_learner.py tests/test_logging_utils.py tests/test_examples.py" - TESTFILES="tests/test_classification.py tests/test_cv.py tests/test_ablation.py" # run on the new Travis infrastructure diff --git a/DistributeTests.ps1 b/DistributeTests.ps1 index d27a78d6..73d47ff6 100644 --- a/DistributeTests.ps1 +++ b/DistributeTests.ps1 @@ -30,7 +30,7 @@ $testsToRun= @() if ($agentNumber -eq 1) { $testsToRun = $testsToRun + "tests/test_featureset.py" - $testsToRun = $testsToRun + "tests/test_utilities.py" + $testsToRun = $testsToRun + "tests/test_commandline_utils.py" } elseif ($agentNumber -eq 2) { $testsToRun = $testsToRun + "tests/test_output.py" @@ -43,7 +43,7 @@ elseif ($agentNumber -eq 4) { $testsToRun = $testsToRun + "tests/test_preprocessing.py" $testsToRun = $testsToRun + "tests/test_metrics.py" $testsToRun = $testsToRun + "tests/test_custom_learner.py" - $testsToRun = $testsToRun + "tests/test_logutils.py" + $testsToRun = $testsToRun + "tests/test_logging_utils.py" $testsToRun = $testsToRun + "tests/test_examples.py" } elseif ($agentNumber -eq 5) { From 6acb7ee1584229bb9dd8908c1b015e80934cf6e5 Mon Sep 17 00:00:00 2001 From: Nitin Madnani Date: Fri, 10 Apr 2020 20:41:30 -0400 Subject: [PATCH 14/20] Fix typoe --- tests/test_input.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_input.py b/tests/test_input.py index de3e1b79..1b64ea3f 100644 --- a/tests/test_input.py +++ b/tests/test_input.py @@ -23,7 +23,7 @@ from skll.config import locate_file, load_cv_folds, parse_config_file from skll.data.readers import safe_float -from skll.experiments import _load_featureset +from skll.experiments import load_featureset from skll.utils.logging import (close_and_remove_logger_handlers, get_skll_logger) @@ -139,7 +139,7 @@ def test_input_checking1(): dirpath = join(_my_dir, 'train') suffix = '.jsonlines' featureset = ['test_input_2examples_1', 'test_input_3examples_1'] - _load_featureset(dirpath, featureset, suffix, quiet=True) + load_featureset(dirpath, featureset, suffix, quiet=True) @raises(ValueError) @@ -150,7 +150,7 @@ def test_input_checking2(): dirpath = join(_my_dir, 'train') suffix = '.jsonlines' featureset = ['test_input_3examples_1', 'test_input_3examples_1'] - _load_featureset(dirpath, featureset, suffix, quiet=True) + load_featureset(dirpath, featureset, suffix, quiet=True) def test_input_checking3(): @@ -160,21 +160,21 @@ def test_input_checking3(): dirpath = join(_my_dir, 'train') suffix = '.jsonlines' featureset = ['test_input_3examples_1', 'test_input_3examples_2'] - examples_tuple = _load_featureset(dirpath, featureset, suffix, quiet=True) + examples_tuple = load_featureset(dirpath, featureset, suffix, quiet=True) eq_(examples_tuple.features.shape[0], 3) def test_one_file_load_featureset(): """ - Test loading a single file with _load_featureset + Test loading a single file with load_featureset """ dirpath = join(_my_dir, 'train') suffix = '.jsonlines' featureset = ['test_input_2examples_1'] - single_file_fs = _load_featureset(join(dirpath, - 'test_input_2examples_1.jsonlines'), - '', '', quiet=True) - single_fs = _load_featureset(dirpath, featureset, suffix, quiet=True) + single_file_fs = load_featureset(join(dirpath, + 'test_input_2examples_1.jsonlines'), + '', '', quiet=True) + single_fs = load_featureset(dirpath, featureset, suffix, quiet=True) eq_(single_file_fs, single_fs) From d258ed53e9bf1df4c9c4eb66591ef3a5af624df3 Mon Sep 17 00:00:00 2001 From: Nitin Madnani Date: Fri, 10 Apr 2020 21:10:23 -0400 Subject: [PATCH 15/20] Fix import typo --- tests/test_featureset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_featureset.py b/tests/test_featureset.py index 4723f598..5b86465f 100644 --- a/tests/test_featureset.py +++ b/tests/test_featureset.py @@ -33,13 +33,13 @@ from skll.data.readers import DictListReader from skll.experiments import load_featureset -from skll.utils.constants import DEFAULT_PARAM_GRIDS from skll.utils.commandline import skll_convert +from skll.utils.constants import KNOWN_DEFAULT_PARAM_GRIDS from tests.utils import make_classification_data, make_regression_data -_ALL_MODELS = list(DEFAULT_PARAM_GRIDS.keys()) +_ALL_MODELS = list(KNOWN_DEFAULT_PARAM_GRIDS.keys()) _my_dir = abspath(dirname(__file__)) From c3a6f0c2ec1c3ce1354280d6514cb9065284dfc1 Mon Sep 17 00:00:00 2001 From: Nitin Madnani Date: Fri, 10 Apr 2020 21:11:33 -0400 Subject: [PATCH 16/20] Change how `import_custom_learner` works and rename - Call it `load_custom_learner()` and have it return the object instead of populating `globals()` directly. - Populate globals() where this function is called instead, --- skll/experiments/__init__.py | 10 +++++----- skll/learner/__init__.py | 9 ++++----- skll/learner/utils.py | 14 ++++++++------ 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/skll/experiments/__init__.py b/skll/experiments/__init__.py index 6a38b500..651f2447 100644 --- a/skll/experiments/__init__.py +++ b/skll/experiments/__init__.py @@ -22,8 +22,8 @@ from skll.config import parse_config_file from skll.config.utils import _munge_featureset_name -from skll.learner import (import_custom_learner, - Learner, +from skll.learner import (Learner, + load_custom_learner, MAX_CONCURRENT_PROCESSES) from skll.utils.logging import (close_and_remove_logger_handlers, get_skll_logger) @@ -213,10 +213,10 @@ def _classify_featureset(args): # load the model if it already exists else: - # import the custom learner path here in case we are reusing a - # saved model + # import custom learner into global namespace if we are reusing + # a saved model if custom_learner_path: - import_custom_learner(custom_learner_path, learner_name) + globals()[learner_name] = load_custom_learner(custom_learner_path, learner_name) train_set_size = 'unknown' if exists(modelfile) and not overwrite: logger.info("Loading pre-existing {} model: {}".format(learner_name, diff --git a/skll/learner/__init__.py b/skll/learner/__init__.py index bbcb1a73..a5202dfe 100644 --- a/skll/learner/__init__.py +++ b/skll/learner/__init__.py @@ -79,7 +79,7 @@ FilteredLeaveOneGroupOut, get_acceptable_classification_metrics, get_acceptable_regression_metrics, - import_custom_learner, + load_custom_learner, rescaled, SelectByMinCount, train_and_score) @@ -90,7 +90,7 @@ _REQUIRES_DENSE = copy.copy(KNOWN_REQUIRES_DENSE) _DEFAULT_PARAM_GRIDS = copy.deepcopy(KNOWN_DEFAULT_PARAM_GRIDS) -__all__ = ['Learner', 'MAX_CONCURRENT_PROCESSES', 'import_custom_learner'] +__all__ = ['Learner', 'MAX_CONCURRENT_PROCESSES', 'load_custom_learner'] class Learner(object): @@ -193,10 +193,9 @@ def __init__(self, if model_type not in globals(): # here, we need to import the custom model and add it - # to the appropriate lists of models. - import_custom_learner(custom_learner_path, model_type) + # to the appropriate lists of models + globals()[model_type] = load_custom_learner(custom_learner_path, model_type) model_class = globals()[model_type] - default_param_grid = (model_class.default_param_grid() if hasattr(model_class, 'default_param_grid') else [{}]) diff --git a/skll/learner/utils.py b/skll/learner/utils.py index dcff0ff1..580fb077 100644 --- a/skll/learner/utils.py +++ b/skll/learner/utils.py @@ -300,9 +300,9 @@ def get_acceptable_classification_metrics(label_array): return acceptable_metrics -def import_custom_learner(custom_learner_path, custom_learner_name): +def load_custom_learner(custom_learner_path, custom_learner_name): """ - Does the gruntwork of adding the custom model's module to globals. + Import and load the custom learner object from the given path. Parameters ---------- @@ -313,10 +313,13 @@ def import_custom_learner(custom_learner_path, custom_learner_name): Raises ------ - ValueError - If the custom learner path is None. ValueError If the custom learner path does not end in '.py'. + + Returns + ------- + custom_learner_obj : skll.Learner object + The SKLL learner object loaded from the given path. """ if not custom_learner_path: raise ValueError('custom_learner_path was not set and learner {} ' @@ -329,8 +332,7 @@ def import_custom_learner(custom_learner_path, custom_learner_name): custom_learner_module_name = os.path.basename(custom_learner_path)[:-3] sys.path.append(os.path.dirname(os.path.abspath(custom_learner_path))) import_module(custom_learner_module_name) - globals()[custom_learner_name] = \ - getattr(sys.modules[custom_learner_module_name], custom_learner_name) + return getattr(sys.modules[custom_learner_module_name], custom_learner_name) def rescaled(cls): From e1e47d7fddaede479171d162a7b71002e80983f3 Mon Sep 17 00:00:00 2001 From: Nitin Madnani Date: Fri, 10 Apr 2020 21:12:01 -0400 Subject: [PATCH 17/20] Fix FQNs in warnings and mock target. --- tests/test_commandline_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_commandline_utils.py b/tests/test_commandline_utils.py index 4678fe98..4215ee6c 100644 --- a/tests/test_commandline_utils.py +++ b/tests/test_commandline_utils.py @@ -193,7 +193,7 @@ def test_warning_when_prediction_method_and_no_probabilities(): sys.stdout = old_stdout sys.stderr = old_stderr - log_msg = ("skll.utilities.compute_eval_from_predictions: WARNING: A prediction " + log_msg = ("skll.utils.commandline.compute_eval_from_predictions: WARNING: A prediction " "method was provided, but the predictions file doesn't contain " "probabilities. Ignoring prediction method 'highest'.") @@ -588,7 +588,7 @@ def test_generate_predictions_console_bad_input_ext(): _ = _run_generate_predictions_and_capture_output(generate_cmd, 'stdout') - expected_log_mssg = ("skll.utilities.generate_predictions: ERROR: Input " + expected_log_mssg = ("skll.utils.commandline.generate_predictions: ERROR: Input " "file must be in either .arff, .csv, .jsonlines, " ".libsvm, .megam, .ndj, or .tsv format. Skipping " "file fake_input_file.txt") @@ -1225,7 +1225,7 @@ def test_plot_learning_curves_argparse(): # replace the _generate_learning_curve_plots function that's called # by the main() in plot_learning_curves with a mocked up version generate_learning_curve_plots_mock = create_autospec(generate_learning_curve_plots) - plc._generate_learning_curve_plots = generate_learning_curve_plots_mock + plc.generate_learning_curve_plots = generate_learning_curve_plots_mock # now call main with some arguments summary_file_name = join(_my_dir, 'other', 'sample_learning_curve_summary.tsv') From 3e755c25278f6b923969ab3c6ae31331eb3c1d60 Mon Sep 17 00:00:00 2001 From: Nitin Madnani Date: Fri, 10 Apr 2020 21:34:02 -0400 Subject: [PATCH 18/20] Add missing init file for subpackage. --- skll/utils/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 skll/utils/__init__.py diff --git a/skll/utils/__init__.py b/skll/utils/__init__.py new file mode 100644 index 00000000..e69de29b From 52b62d2c7dfc4d547a954f108e0938f41728b7bd Mon Sep 17 00:00:00 2001 From: Nitin Madnani Date: Sat, 11 Apr 2020 09:59:11 -0400 Subject: [PATCH 19/20] Fix some pep8 issues. --- skll/experiments/output.py | 11 +++++++++-- skll/learner/utils.py | 2 -- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/skll/experiments/output.py b/skll/experiments/output.py index 64421453..26d5efb9 100644 --- a/skll/experiments/output.py +++ b/skll/experiments/output.py @@ -158,8 +158,15 @@ def generate_learning_curve_plots(experiment_name, if j == 0: ax.set_ylabel(row_name) if i == 0: - ax.legend(handles=[matplotlib.lines.Line2D([], [], color=c, label=l, linestyle='-') - for c, l in zip(colors, ['Training', 'Cross-validation'])], + # set up the legend handles for this plot + plot_handles = [matplotlib.lines.Line2D([], + [], + color=c, + label=l, + linestyle='-') + for c, l in zip(colors, ['Training', + 'Cross-validation'])] + ax.legend(handles=plot_handles, loc=4, fancybox=True, fontsize='x-small', diff --git a/skll/learner/utils.py b/skll/learner/utils.py index 580fb077..82281ec5 100644 --- a/skll/learner/utils.py +++ b/skll/learner/utils.py @@ -580,5 +580,3 @@ def train_and_score(learner, train_score = use_score_func(metric, train_labels, train_predictions) test_score = use_score_func(metric, test_labels, test_predictions) return train_score, test_score - - From 996d44fb2c1e214c40947340fe6925814a524a07 Mon Sep 17 00:00:00 2001 From: Nitin Madnani Date: Tue, 14 Apr 2020 15:36:50 -0400 Subject: [PATCH 20/20] Apply suggestions from code review Co-Authored-By: Matt Mulholland --- doc/api/skll.rst | 4 ++-- doc/api/utils.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/api/skll.rst b/doc/api/skll.rst index a51513b4..7539a2a3 100644 --- a/doc/api/skll.rst +++ b/doc/api/skll.rst @@ -1,8 +1,8 @@ :mod:`skll` Package =================== -We have made the most useful parts of our API are available in the top level -``skll`` namespace even though some of them actually live in subpackages. +We have made the most useful parts of our API available in the top-level +``skll`` namespace even though some of them actually live in subpackages. They are documented in both places for convenience. From :py:mod:`~skll.data` Package diff --git a/doc/api/utils.rst b/doc/api/utils.rst index 54348aff..884ee9c1 100644 --- a/doc/api/utils.rst +++ b/doc/api/utils.rst @@ -12,4 +12,4 @@ Various useful constants defining groups of evaluation metrics. A useful logging function for SKLL developers -.. autodata:: skll.utils.logging.get_skll_logger +.. autofunction:: skll.utils.logging.get_skll_logger