diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 876179e1..f2a9b714 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: rev: 'v0.0.264' hooks: - id: ruff - args: [--line-length=100, --select, "D,E,F,I", --ignore, "D212", --per-file-ignores, "tests/test*.py:D103"] + args: [--line-length=100, --select, "D,E,F,I", --ignore, "D212", --per-file-ignores, "tests/test*.py:D103,skll/data/featureset.py:E501"] - repo: https://github.com/pre-commit/mirrors-mypy rev: 'v1.2.0' hooks: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2393f86c..fb71018e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -155,6 +155,6 @@ README file in the doc/ directory for more information. For building the documentation, you will need [sphinx](http://sphinx.pocoo.org/) as well as the readthedocs sphinx theme. To install both, just run: - $ conda install 'sphinx>=6,<7' sphinx_rtd_theme==1.2.0 + $ conda install 'sphinx<6' sphinx_rtd_theme==1.2.0 in your existing conda environment. diff --git a/doc/api/data.rst b/doc/api/data.rst index 75f6ec43..0ec82673 100644 --- a/doc/api/data.rst +++ b/doc/api/data.rst @@ -12,13 +12,58 @@ :mod:`data.readers` Module -------------------------- -.. automodule:: skll.data.readers +.. autoclass:: skll.data.readers.Reader :members: :show-inheritance: +.. autoclass:: skll.data.readers.CSVReader + :members: + :show-inheritance: + +.. autoclass:: skll.data.readers.TSVReader + :members: + :show-inheritance: + +.. autoclass:: skll.data.readers.NDJReader + :members: + :show-inheritance: + +.. autoclass:: skll.data.readers.DictListReader + :members: + :show-inheritance: + +.. autoclass:: skll.data.readers.ARFFReader + :members: + :show-inheritance: + +.. autoclass:: skll.data.readers.LibSVMReader + :members: + :show-inheritance: + + :mod:`data.writers` Module -------------------------- -.. automodule:: skll.data.writers +.. autoclass:: skll.data.writers.Writer + :members: + :show-inheritance: + +.. autoclass:: skll.data.writers.CSVWriter + :members: + :show-inheritance: + +.. autoclass:: skll.data.writers.TSVWriter + :members: + :show-inheritance: + +.. autoclass:: skll.data.writers.NDJWriter + :members: + :show-inheritance: + +.. autoclass:: skll.data.writers.ARFFWriter + :members: + :show-inheritance: + +.. autoclass:: skll.data.writers.LibSVMWriter :members: :show-inheritance: diff --git a/doc/api/types.rst b/doc/api/types.rst index 4f4b9727..32fd0d04 100644 --- a/doc/api/types.rst +++ b/doc/api/types.rst @@ -40,7 +40,8 @@ data. .. autoclass:: skll.types.LabelType -A float, integer, or a string; this is useful for SKLL labels that can be both. +A float, integer, or a string; this is useful for SKLL labels that can be any +of them. .. autoclass:: skll.types.LearningCurveSizes diff --git a/doc/requirements.txt b/doc/requirements.txt index 3b9555d0..d6b69b01 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -2,7 +2,7 @@ beautifulsoup4 ruamel.yaml scikit-learn>=1.2.0,<1.3.0 seaborn -sphinx>=6,<7 +sphinx>=5.0,<6.0 sphinx_rtd_theme==1.2.0 tabulate typing_extensions diff --git a/skll/data/dict_vectorizer.py b/skll/data/dict_vectorizer.py index d55e14a1..f9255569 100644 --- a/skll/data/dict_vectorizer.py +++ b/skll/data/dict_vectorizer.py @@ -1,4 +1,6 @@ """ +Enhance scikit-learn's ``DictVectorizer`` to add equality checking. + This module is here because the scikit-learn version of `DictVectorizer` does not contain an `__eq__` method for vectorizer equality which we need for SKLL. @@ -13,7 +15,7 @@ class DictVectorizer(OldDictVectorizer): """ - Transforms lists of feature-value mappings to vectors. + Transform lists of feature-value mappings to vectors. This transformer turns lists of mappings (dict-like objects) of feature names to feature values into Numpy arrays or scipy.sparse matrices for use @@ -37,26 +39,26 @@ class DictVectorizer(OldDictVectorizer): Parameters ---------- - dtype : callable, optional + dtype : Optional[Callable] The type of feature values. Passed to Numpy array/scipy.sparse matrix constructors as the dtype argument. - separator : string, optional + separator : Optional[str] Separator string used when constructing new features for one-hot coding. - sparse : boolean, default=True + sparse : bool, default=True Whether transform should produce scipy.sparse matrices. - sort : boolean, default=True + sort : bool, default=True Whether `feature_names_` and `vocabulary_` should be sorted when fitting. Attributes ---------- - vocabulary_ : dict + vocabulary_ : Dict[str, Any] A dictionary mapping feature names to feature indices. - feature_names_ : list + feature_names_ : List[str] A list of length n_features containing the feature names (e.g., "f=ham" and "f=spam"). @@ -75,18 +77,15 @@ class DictVectorizer(OldDictVectorizer): >>> v.transform({'foo': 4, 'unseen_feature': 3}) array([[ 0., 0., 4.]]) - See also - -------- + Notes + ----- FeatureHasher : performs vectorization using only a hash function. sklearn.preprocessing.OneHotEncoder : handles nominal/categorical features encoded as columns of integers. """ def __eq__(self, other): - """ - Check whether two vectorizers are the same, assuming - we are actually comparing to a vectorizer - """ + """Check whether two vectorizers are the same.""" return ( isinstance(other, OldDictVectorizer) and self.dtype == other.dtype diff --git a/skll/data/featureset.py b/skll/data/featureset.py index 06384d02..8c377817 100644 --- a/skll/data/featureset.py +++ b/skll/data/featureset.py @@ -35,12 +35,12 @@ class FeatureSet(object): labels : Optional[Union[List[str], numpy.ndarray], default=None Labels for this set. - features : Optional[Union[FeatureDictList, np.ndarray]], default=None + features : Optional[Union[:class:`skll.types.FeatureDictList`, :class:`numpy.ndarray`]], default=None The features for each instance represented as either a - list of dictionaries or an array-like (if ``vectorizer`` is + list of dictionaries or a numpy array (if ``vectorizer`` is also specified). - vectorizer : Union[DictVectorizer, FeatureHasher], default=None + vectorizer : Optional[Union[:class:`sklearn.feature_extraction.DictVectorizer`, :class:`sklearn.feature_extraction.FeatureHasher`], default=None Vectorizer which will be used to generate the feature matrix. Warnings @@ -134,7 +134,7 @@ def __eq__(self, other): Parameters ---------- - other : skll.data.FeatureSet + other : :class:`skll.data.featureset.FeatureSet` The other ``FeatureSet`` to check equivalence with. Returns @@ -142,8 +142,8 @@ def __eq__(self, other): bool ``True`` if they are the same, ``False`` otherwise. - Note - ---- + Notes + ----- We consider feature values to be equal if any differences are in the sixth decimal place or higher. """ @@ -193,12 +193,12 @@ def __add__(self, other: "FeatureSet") -> "FeatureSet": Parameters ---------- - other : skll.data.FeatureSet + other : :class:`skll.data.featureset.FeatureSet` The other ``FeatureSet`` to add to this one. Returns ------- - skll.data.FeatureSet + :class:`skll.data.featureset.FeatureSet The combined feature set. Raises @@ -301,17 +301,17 @@ def filter( inverse: bool = False, ) -> None: """ - Remove or keep features and/or examples from the ``Featureset``. + Remove or keep features and/or examples from the given feature set. Filtering is done in-place. Parameters ---------- - ids : Optional[List[FloatOrStr]], default=None + ids : Optional[List[:class:`skll.types.IdType`]], default=None Examples to keep in the FeatureSet. If ``None``, no ID filtering takes place. - labels : Optional[List[LabelType]], default=None + labels : Optional[List[:class:`skll.types.LabelType`]], default=None Labels that we want to retain examples for. If ``None``, no label filtering takes place. @@ -386,11 +386,11 @@ def filtered_iter( Parameters ---------- - ids : Optional[List[IdType]], default=None + ids : Optional[List[:class:`skll.types.IdType`]], default=None Examples to keep in the ``FeatureSet``. If ``None``, no ID filtering takes place. - labels : Optional[List[LabelType]], default=None + labels : Optional[List[:class:`skll.types.LabelType`]], default=None Labels that we want to retain examples for. If ``None``, no label filtering takes place. @@ -409,17 +409,18 @@ def filtered_iter( Instead of keeping features and/or examples in lists, remove them. - Yields - ------ - id_ : IdType - The ID of the example. + Returns + ------- + :class:`skll.types.FeatGenerator` - label_ : LabelType - The label of the example. + A generator that yields 3-tuples containing: - feat_dict : FeatureDict - The feature dictionary, with feature name as the key - and example value as the value. + - :class:`skll.types.IdType` - The ID of the example. + + - :class:`skll.types.LabelType` - The label of the example. + + - :class:`skll.types.FeatureDict` - The feature dictionary, with + feature name as the key and example value as the value. Raises ------ @@ -468,13 +469,13 @@ def __sub__(self, other: "FeatureSet") -> "FeatureSet": Parameters ---------- - other : skll.data.FeatureSet + other : :class:`skll.data.featureset.FeatureSet` The other ``FeatureSet`` containing the features that should be removed from this ``FeatureSet``. Returns ------- - FeatureSet: + :class:`skll.data.featureset.FeatureSet` A copy of ``self`` with all features in ``other`` removed. """ new_set = deepcopy(self) @@ -537,7 +538,7 @@ def __getitem__( Returns ------- - Union["FeatureSet", Tuple[IdType, LabelType, FeatureDictList]] + Union[:class:`skll.data.featureset.FeatureSet`, Tuple[:class:`skll.types.IdType`, :class:`skll.types.LabelType`, :class:`skll.types.FeatureDictList`]] # noqa: E501 If `value` is a slice, then return a new ``FeatureSet`` instance containing a subset of the data. If it's an index, return the specific example by row number. @@ -574,7 +575,7 @@ def split( Parameters ---------- - fs : skll.data.FeatureSet + fs : skll.data.featureset.FeatureSet The ``FeatureSet`` instance to split. ids_for_split1 : List[int] @@ -594,11 +595,8 @@ def split( Returns ------- - fs1 : skll.data.FeatureSet - The first ``FeatureSet``. - - fs2 : skll.data.FeatureSet - The second ``FeatureSet``. + Tuple[:class:`skll.data.featureset.FeatureSet`, :class:`skll.data.featureset.FeatureSet`] + A tuple containing the two featureset instances. """ # Note: an alternative way to implement this is to make copies # of the given FeatureSet instance and then use the `filter()` @@ -634,7 +632,7 @@ def from_data_frame( vectorizer: Optional[Union[DictVectorizer, FeatureHasher]] = None, ) -> "FeatureSet": """ - Create a ``FeatureSet`` instance from a `pandas.DataFrame`. + Create a ``FeatureSet`` instance from a pandas data frame. Will raise an Exception if pandas is not installed in your environment. The ``ids`` in the ``FeatureSet`` will be the index from the given frame. @@ -650,12 +648,12 @@ def from_data_frame( labels_column : Optional[str], default=None The name of the column containing the labels (data to predict). - vectorizer : Optional[Union[DictVectorizer, FeatureHasher]], default=None + vectorizer : Optional[Union[:class:`sklearn.feature_extraction.DictVectorizer`, :class:`sklearn.feature_extraction.FeatureHasher`]], default=None Vectorizer which will be used to generate the feature matrix. Returns ------- - feature_set : skll.data.FeatureSet + :class:`skll.data.featureset.FeatureSet` A ``FeatureSet`` instance generated from from the given data frame. """ if labels_column: diff --git a/skll/data/readers.py b/skll/data/readers.py index 78c2fb07..963bf747 100644 --- a/skll/data/readers.py +++ b/skll/data/readers.py @@ -62,16 +62,17 @@ from skll.data.dict_vectorizer import DictVectorizer from skll.types import ClassMap, FeatGenerator, FeatureDictList, IdType, LabelType, PathOrStr -# define some custom types for readability - class Reader(object): """ - Make picklable iterators out of example dictionary generators. + Load FeatureSets from files on disk. + + This is the base class used to create featureset readers for different + file types. Parameters ---------- - path_or_list : Union[PathOrStr, List[Dict[str, Any]] + path_or_list : Union[:class:`skll.types.PathOrStr`, List[Dict[str, Any]] Path or a list of example dictionaries. quiet : bool, default=True @@ -92,7 +93,7 @@ class Reader(object): If no column with that name exists, or ``None`` is specified, example IDs will be automatically generated. - class_map : Optional[ClassMap], default=None + class_map : Optional[:class:`skll.types.ClassMap`], default=None Mapping from original class labels to new ones. This is mainly used for collapsing multiple labels into a single class. Anything not in the mapping will be kept the same. @@ -160,15 +161,15 @@ def for_path(cls, path_or_list: Union[PathOrStr, FeatureDictList], **kwargs) -> Parameters ---------- - path_or_list : Union[PathOrStr, FeatureDictList] + path_or_list : Union[:class:`skll.types.PathOrStr`, :class:`skll.types.FeatureDictList`] A path or list of example dictionaries. - kwargs : Dict[str, Any], optional + kwargs : Optional[Dict[str, Any]] The arguments to the Reader object being instantiated. Returns ------- - reader : skll.data.Reader + reader : :class:`skll.data.readers.Reader` A new instance of the Reader sub-class that is appropriate for the given path. @@ -204,10 +205,8 @@ def _sub_read(self, file): Parameters ---------- - file : file buffer or str - Either a file buffer, if ``_sub_read_rows()`` - is calling this method, or a path to a file, - if it is being read with ``pandas``. + file + Not used. Raises ------ @@ -223,7 +222,7 @@ def _print_progress(self, progress_num: Union[int, str], end="\r"): Parameters ---------- - progress_num: int + progress_num: Union[int, str] Progress indicator value. Usually either a line number or a percentage. Must be able to convert to string. @@ -247,19 +246,19 @@ def _sub_read_rows(self, file: PathOrStr) -> Tuple[np.ndarray, np.ndarray, Featu Parameters ---------- - file : PathOrStr + file : :class:`skll.types.PathOrStr` The path to the input file. Returns ------- - ids : np.array of shape (n_ids,) + ids : numpy.ndarray of shape (n_ids,) The ids array. - labels : np.array of shape (n_labels,) + labels : numpy.ndarray of shape (n_labels,) The labels array. - features : FeatureDictList - The features dictionary. + features : :class:`skll.types.FeatureDictList` + List containing feature dictionaries. Raises ------ @@ -358,14 +357,14 @@ def _parse_dataframe( Returns ------- - ids : np.array of shape (n_ids,) + ids : numpy.ndarray of shape (n_ids,) The ids for the feature set. - labels : np.array of shape (n_labels,) + labels : numpy.ndarray of shape (n_labels,) The labels for the feature set. - features : FeatureDictList - The features for the feature set. + features : :class:`skll.types.FeatureDictList` + List of feature dictionaries. """ if df.empty: raise ValueError("No features found in possibly empty file " f"'{self.path_or_list}'.") @@ -447,8 +446,8 @@ def read(self) -> FeatureSet: Returns ------- - skll.data.FeatureSet - ``FeatureSet`` instance representing the input file. + :class:`skll.data.featureset.FeatureSet` + A ``FeatureSet`` instance representing the input file. Raises ------ @@ -502,6 +501,55 @@ class DictListReader(Reader): objects as input. It iterates over examples in the same way as other ``Reader`` classes, but uses a list of example dictionaries instead of a path to a file. + + Parameters + ---------- + path_or_list : Union[:class:`skll.types.PathOrStr`, List[Dict[str, Any]] + Path or a list of example dictionaries. + + quiet : bool, default=True + Do not print "Loading..." status message to stderr. + + ids_to_floats : bool, default=False + Convert IDs to float to save memory. Will raise error + if we encounter an a non-numeric ID. + + label_col : Optional[str], default='y' + Name of the column which contains the class labels + for ARFF/CSV/TSV files. If no column with that name + exists, or ``None`` is specified, the data is + considered to be unlabelled. + + id_col : str, default='id' + Name of the column which contains the instance IDs. + If no column with that name exists, or ``None`` is + specified, example IDs will be automatically generated. + + class_map : Optional[:class:`skll.types.ClassMap`], default=None + Mapping from original class labels to new ones. This is + mainly used for collapsing multiple labels into a single + class. Anything not in the mapping will be kept the same. + The keys are the new labels and the list of values for each + key is the labels to be collapsed to said new label. + + sparse : bool, default=True + Whether or not to store the features in a numpy CSR + matrix when using a DictVectorizer to vectorize the + features. + + feature_hasher : bool, default=False + Whether or not a FeatureHasher should be used to + vectorize the features. + + num_features : Optional[int], default=None + If using a FeatureHasher, how many features should the + resulting matrix have? You should set this to a power + of 2 greater than the actual number of features to + avoid collisions. + + logger : Optional[logging.Logger], default=None + A logger instance to use to log messages instead of creating + a new one by default. """ def read(self) -> FeatureSet: @@ -510,8 +558,8 @@ def read(self) -> FeatureSet: Returns ------- - feature_set : skll.data.FeatureSet - FeatureSet representing the list of dictionaries we read in. + :class:`skll.data.FeatureSet` + A ``FeatureSet`` representing the list of dictionaries we read in. """ # if we are in this method, `self.path_or_list` must be a # list of dictionaries @@ -572,6 +620,56 @@ class NDJReader(Reader): If example/instance IDs are included in the files, they must be specified as the "id" key in each JSON dictionary. + + Parameters + ---------- + path_or_list : Union[:class:`skll.types.PathOrStr`, List[Dict[str, Any]] + Path or a list of example dictionaries. + + quiet : bool, default=True + Do not print "Loading..." status message to stderr. + + ids_to_floats : bool, default=False + Convert IDs to float to save memory. Will raise error + if we encounter an a non-numeric ID. + + label_col : Optional[str], default='y' + Name of the column which contains the class labels + for ARFF/CSV/TSV files. If no column with that name + exists, or ``None`` is specified, the data is + considered to be unlabelled. + + id_col : str, default='id' + Name of the column which contains the instance IDs. + If no column with that name exists, or ``None`` is + specified, example IDs will be automatically generated. + + class_map : Optional[:class:`skll.types.ClassMap`], default=None + Mapping from original class labels to new ones. This is + mainly used for collapsing multiple labels into a single + class. Anything not in the mapping will be kept the same. + The keys are the new labels and the list of values for each + key is the labels to be collapsed to said new label. + + sparse : bool, default=True + Whether or not to store the features in a numpy CSR + matrix when using a DictVectorizer to vectorize the + features. + + feature_hasher : bool, default=False + Whether or not a FeatureHasher should be used to + vectorize the features. + + num_features : Optional[int], default=None + If using a FeatureHasher, how many features should the + resulting matrix have? You should set this to a power + of 2 greater than the actual number of features to + avoid collisions. + + logger : Optional[logging.Logger], default=None + A logger instance to use to log messages instead of creating + a new one by default. + """ def _sub_read(self, file: IO[str]) -> FeatGenerator: @@ -585,13 +683,13 @@ def _sub_read(self, file: IO[str]) -> FeatGenerator: Yields ------ - curr_id : IdType + curr_id : :class:`skll.types.IdType` The current ID for the example. - class_name : Optional[LabelType] + class_name : Optional[:class:`skll.types.LabelType`] The name of the class label for the example. - example : FeatureDict + example : :class:`skll.types.FeatureDict` The example value in dictionary format, with 'x' as list of features. @@ -642,6 +740,55 @@ class LibSVMReader(Reader): not have names. The comment is structured as follows:: ExampleID | 1=FirstClass | 1=FirstFeature 2=SecondFeature + + Parameters + ---------- + path_or_list : Union[:class:`skll.types.PathOrStr`, List[Dict[str, Any]] + Path or a list of example dictionaries. + + quiet : bool, default=True + Do not print "Loading..." status message to stderr. + + ids_to_floats : bool, default=False + Convert IDs to float to save memory. Will raise error + if we encounter an a non-numeric ID. + + label_col : Optional[str], default='y' + Name of the column which contains the class labels + for ARFF/CSV/TSV files. If no column with that name + exists, or ``None`` is specified, the data is + considered to be unlabelled. + + id_col : str, default='id' + Name of the column which contains the instance IDs. + If no column with that name exists, or ``None`` is + specified, example IDs will be automatically generated. + + class_map : Optional[:class:`skll.types.ClassMap`], default=None + Mapping from original class labels to new ones. This is + mainly used for collapsing multiple labels into a single + class. Anything not in the mapping will be kept the same. + The keys are the new labels and the list of values for each + key is the labels to be collapsed to said new label. + + sparse : bool, default=True + Whether or not to store the features in a numpy CSR + matrix when using a DictVectorizer to vectorize the + features. + + feature_hasher : bool, default=False + Whether or not a FeatureHasher should be used to + vectorize the features. + + num_features : Optional[int], default=None + If using a FeatureHasher, how many features should the + resulting matrix have? You should set this to a power + of 2 greater than the actual number of features to + avoid collisions. + + logger : Optional[logging.Logger], default=None + A logger instance to use to log messages instead of creating + a new one by default. """ line_regex = re.compile( @@ -678,8 +825,7 @@ def _pair_to_tuple(pair: str, feat_map: Dict[str, str]) -> Tuple[str, Union[floa ------- name : str The name of the feature. - - value + value : Union[float, int, str] The value of the example. """ name, value = pair.split(":") @@ -694,18 +840,18 @@ def _sub_read(self, file: IO[str]) -> FeatGenerator: Parameters ---------- - file : file buffer + file : IO[str] A file buffer for an LibSVM file. Yields ------ - curr_id : IdType + curr_id : :class:`skll.types.IdType` The current ID for the example. - class_ : LabelType + class_ : :class:`skll.types.LabelType` The name of the class label for the example. - example : FeatureDict + example : :class:`skll.types.FeatureDict` The example valued in dictionary format, with 'x' as list of features. @@ -783,10 +929,10 @@ class CSVReader(Reader): Parameters ---------- - path_or_list : Union[PathOrStr, List[Dict[str, Any]]] + path_or_list : Union[:class:`skll.types.PathOrStr`, List[Dict[str, Any]]] The path to a comma-delimited file. - replace_blanks_with : Optional[Union[Number, Dict[str, Number]]] + replace_blanks_with : Optional[Union[Number, Dict[str, Number]]], default=None Specifies a new value with which to replace blank values. Options are: @@ -795,17 +941,16 @@ class CSVReader(Reader): - ``None`` : Blank values will be left as blanks, and not replaced. The replacement occurs after the data set is read into a ``pd.DataFrame``. - Defaults to ``None``. drop_blanks : bool, default=False If ``True``, remove lines/rows that have any blank values. These lines/rows are removed after the the data set is read into a ``pd.DataFrame``. - pandas_kwargs : Optional[Dict[str, Any]] + pandas_kwargs : Optional[Dict[str, Any]], default=None Arguments that will be passed directly to the ``pandas`` I/O reader. - kwargs : Dict[str, Any], optional + kwargs : Optional[Dict[str, Any]] Other arguments to the Reader object. """ @@ -832,18 +977,18 @@ def _sub_read(self, file: PathOrStr) -> Tuple[np.ndarray, np.ndarray, FeatureDic Parameters ---------- - file : PathOrStr + file : :class:`skll.types.PathOrStr` The path to the CSV file. Returns ------- - ids : np.array of shape (n_ids,) + ids : numpy.ndarray of shape (n_ids,) The ids for the feature set. - labels : np.array of shape (n_labels,) + labels : numpy.ndarray of shape (n_labels,) The labels for the feature set. - features : FeatureDictList + features : :class:`skll.types.FeatureDictList` The list of feature dictionaries for the feature set. """ df = pd.read_csv(file, sep=self._sep, engine=self._engine, **self._pandas_kwargs) @@ -870,7 +1015,7 @@ class TSVReader(CSVReader): path_or_list : str The path to a comma-delimited file. - replace_blanks_with : Optional[Union[Number, Dict[str, Number]]] + replace_blanks_with : Optional[Union[Number, Dict[str, Number]]], default=None Specifies a new value with which to replace blank values. Options are: @@ -879,18 +1024,16 @@ class TSVReader(CSVReader): - ``None`` : Blank values will be left as blanks, and not replaced. The replacement occurs after the data set is read into a ``pd.DataFrame``. - Defaults to ``None``. drop_blanks : bool, default=False If ``True``, remove lines/rows that have any blank values. These lines/rows are removed after the the data set is read into a ``pd.DataFrame``. - pandas_kwargs : Optional[Dict[str, Any]] + pandas_kwargs : Optional[Dict[str, Any]], default=None Arguments that will be passed directly to the ``pandas`` I/O reader. - Defaults to ``None``. - kwargs : Dict[str, Any], optional + kwargs : Optional[Dict[str, Any]] Other arguments to the Reader object. """ @@ -924,10 +1067,10 @@ class ARFFReader(Reader): Parameters ---------- - path_or_list : Union[PathOrStr, List[Dict[str, Any]]] + path_or_list : Union[:class:`skll.types.PathOrStr`, List[Dict[str, Any]]] The path to the ARFF file. - kwargs : Dict[str, Any], optional + kwargs : Optional[Dict[str, Any]] Other arguments to the Reader object. """ @@ -974,15 +1117,14 @@ def _sub_read(self, file: IO[str]) -> FeatGenerator: Yields ------ - curr_id : IdType + curr_id : :class:`skll.types.IdType` The current ID for the example. - class_name : LabelType + class_name : :class:`skll.types.LabelType` The name of the class label for the example. - example : FeatureDict - The example valued in dictionary format, with 'x' - as list of features. + example : :class:`skll.types.FeatureDict` + The example features in dictionary format. """ field_names = [] # Process ARFF header diff --git a/skll/data/writers.py b/skll/data/writers.py index af131da2..b8dc6b7a 100644 --- a/skll/data/writers.py +++ b/skll/data/writers.py @@ -28,11 +28,14 @@ class Writer(object): """ - Helper class for writing out FeatureSets to files on disk. + Write out FeatureSets to files on disk. + + This is the base class used to create featureset writers for different + file types. Parameters ---------- - path : PathOrStr + path : :class:`skll.types.PathOrStr` A path to the feature file we would like to create. The suffix to this filename must be ``.arff``, ``.csv``, ``.jsonlines``, ``.libsvm``, ``.ndj``, or ``.tsv``. If ``subsets`` @@ -41,7 +44,7 @@ class Writer(object): write the feature files with an additional file extension specifying the file type. For example ``/foo/.csv``. - feature_set : skll.data.FeatureSet + feature_set : :class:`skll.data.featureset.FeatureSet` The ``FeatureSet`` instance to dump to the file. quiet : bool, default=True @@ -60,20 +63,26 @@ class Writer(object): enumerate all of these boolean feature names in your mapping. - logger : logging.Logger, default=None + logger : Optional[logging.Logger], default=None A logger instance to use to log messages instead of creating a new one by default. """ - def __init__(self, path: PathOrStr, feature_set: FeatureSet, **kwargs): + def __init__( + self, + path: PathOrStr, + feature_set: FeatureSet, + quiet: bool = True, + subsets: Optional[Dict[str, List[str]]] = None, + logger: Optional[logging.Logger] = None, + ): """Initialize base Writer class.""" super(Writer, self).__init__() - self.quiet = kwargs.pop("quiet", True) + self.quiet = quiet self.path = Path(path) self.feat_set = feature_set - self.subsets = kwargs.pop("subsets", None) - logger = kwargs.pop("logger", None) + self.subsets = subsets self.logger = logger if logger else logging.getLogger(__name__) # Get prefix & extension for checking file types & writing subset files; @@ -89,8 +98,6 @@ def __init__(self, path: PathOrStr, feature_set: FeatureSet, **kwargs): self.ext = suffix.lower() self._progress_msg = "" self._use_pandas = False - if kwargs: - raise ValueError("Passed extra keyword arguments to Writer " f"constructor: {kwargs}") @classmethod def for_path(cls, path: PathOrStr, feature_set: FeatureSet, **kwargs) -> "Writer": @@ -99,7 +106,7 @@ def for_path(cls, path: PathOrStr, feature_set: FeatureSet, **kwargs) -> "Writer Parameters ---------- - path : PathOrStr + path : :class:`skll.types.PathOrStr` A path to the feature file we would like to create. The suffix to this filename must be ``.arff``, ``.csv``, ``.jsonlines``, ``.libsvm``, ``.ndj``, or @@ -109,16 +116,16 @@ def for_path(cls, path: PathOrStr, feature_set: FeatureSet, **kwargs) -> "Writer files with an additional file extension specifying the file type. For example ``/foo/.csv``. - feature_set : skll.data.FeatureSet + feature_set : :class:`skll.data.featureset.FeatureSet` The ``FeatureSet`` instance to dump to the output file. - kwargs : Dict[str, Any], optional + kwargs : Optional[Dict[str, Any]] The keyword arguments for ``for_path`` are the same as the initializer for the desired ``Writer`` subclass. Returns ------- - writer : skll.data.Writer + writer : :class:`skll.data.Writer` New instance of the Writer sub-class that is appropriate for the given path. """ @@ -161,7 +168,7 @@ def _write_subset( Parameters ---------- - sub_path : PathOrStr + sub_path : :class:`skll.types.PathOrStr` The path to the file we want to create for this subset of our data. @@ -211,16 +218,14 @@ def _write_header(self, feature_set, output_file, filter_features): Parameters ---------- - feature_set : skll.data.FeatureSet - The ``FeatureSet`` instance being written to a file. + feature_set + Not used. - output_file : IO[str] - The file being written to. + output_file + Not used. - filter_features : Set[str] - If only writing a subset of the features in the - FeatureSet to ``output_file``, these are the - features to include in this file. + filter_features + Not used. """ pass @@ -230,22 +235,23 @@ def _write_line(self, id_, label_, feat_dict, output_file): Parameters ---------- - id_ : IdType - The ID for the current instance. + id_ : + Not used. - label_ : str - The label for the current instance. + label_ + Not used. - feat_dict : FeatureDict - The feature dictionary for the current instance. + feat_dict + Not used. - output_file : IO[str] - The file being written to. + output_file + Not used. Raises ------ NotImplementedError """ + __import__("ipdb").sset_trace() raise NotImplementedError def _write_data(self, feature_set, output_file, filter_features): @@ -254,21 +260,20 @@ def _write_data(self, feature_set, output_file, filter_features): Parameters ---------- - feature_set : skll.data.FeatureSet - The ``FeatureSet`` instance being written to a file. + feature_set + Not used. - output_file : IO[str] - The file being written to. + output_file + Not used. - filter_features : Set[str] - If only writing a subset of the features in the - FeatureSet to ``output_file``, these are the - features to include in this file. + filter_features + Not used. Raises ------ NotImplementedError """ + __import__("ipdb").sset_trace() raise NotImplementedError def _get_column_names_and_indexes( @@ -279,7 +284,7 @@ def _get_column_names_and_indexes( Parameters ---------- - feature_set : skll.data.FeatureSet + feature_set : :class:`skll.data.featureset.FeatureSet` The ``FeatureSet`` instance being written to a file. filter_features : Optional[Set[str]], default=None @@ -326,37 +331,66 @@ class CSVWriter(Writer): Parameters ---------- - path : PathOrStr + path : :class:`skll.types.PathOrStr` A path to the feature file we would like to create. If ``subsets`` is not ``None``, this is assumed to be a string containing the path to the directory to write the feature files with an additional file extension specifying the file type. For example ``/foo/.csv``. - feature_set : skll.data.FeatureSet + feature_set : :class:`skll.data.featureset.FeatureSet` The ``FeatureSet`` instance to dump to the output file. + quiet : bool, default=True + Do not print "Writing..." status message to stderr. + + subsets : Optional[Dict[str, List[str]]], default=None + A mapping from subset names to lists of feature names + that are included in those sets. If given, a feature + file will be written for every subset (with the name + containing the subset name as suffix to ``path``). + Note, since string- valued features are automatically + converted into boolean features with names of the form + ``FEATURE_NAME=STRING_VALUE``, when doing the + filtering, the portion before the ``=`` is all that's + used for matching. Therefore, you do not need to + enumerate all of these boolean feature names in your + mapping. + + logger : Optional[logging.Logger], default=None + A logger instance to use to log messages instead of creating + a new one by default. + + label_col : str, default="y" + The column name containing the label. + + id_col : str, default="id" + The column name containing the ID. + pandas_kwargs : Optional[Dict[str], Any], default=None Arguments that will be passed directly to the `pandas` I/O reader. - - kwargs : Optional[Dict[str, Any]], optional - The arguments to the ``Writer`` object being instantiated. """ def __init__( self, path: PathOrStr, feature_set: FeatureSet, + quiet: bool = True, + subsets: Optional[Dict[str, List[str]]] = None, + logger: Optional[logging.Logger] = None, + label_col: str = "y", + id_col: str = "id", pandas_kwargs: Optional[Dict[str, Any]] = None, - **kwargs, ): """Initialize the CSVWriter class.""" - self.label_col = kwargs.pop("label_col", "y") - self.id_col = kwargs.pop("id_col", "id") - super(CSVWriter, self).__init__(path, feature_set, **kwargs) + self.label_col = label_col + self.id_col = id_col self._pandas_kwargs = {} if pandas_kwargs is None else pandas_kwargs self._sep = self._pandas_kwargs.pop("sep", ",") self._index = self._pandas_kwargs.pop("index", False) + super(CSVWriter, self).__init__( + path, feature_set, quiet=quiet, subsets=subsets, logger=logger + ) self._use_pandas = True def _build_dataframe_with_features( @@ -367,7 +401,7 @@ def _build_dataframe_with_features( Parameters ---------- - feature_set : skll.data.FeatureSet + feature_set : :class:`skll.data.featureset.FeatureSet` The ``FeatureSet`` instance being written to a file. filter_features : Optional[Set[str]], default=None @@ -421,7 +455,7 @@ def _build_dataframe( Parameters ---------- - feature_set : skll.data.FeatureSet + feature_set : :class:`skll.data.featureset.FeatureSet` The ``FeatureSet`` instance being written to a file. filter_features : Optional[Set[str]], default=None @@ -436,7 +470,7 @@ def _build_dataframe( Returns ------- - df_features : pd.DataFrame + df_features : pandas.DataFrame The data frame constructed from the feature set. Raises @@ -478,10 +512,10 @@ def _write_data( Parameters ---------- - feature_set : skll.data.FeatureSet + feature_set : :class:`skll.data.featureset.FeatureSet` The ``FeatureSet`` instance being written to a file. - output_file : PathOrStr + output_file : :class:`skll.types.PathOrStr` The path of the file being written to filter_features : Optional[Set[str]], default=None @@ -499,33 +533,68 @@ class TSVWriter(CSVWriter): Parameters ---------- - path : PathOrStr + path : :class:`skll.types.PathOrStr` A path to the feature file we would like to create. If ``subsets`` is not ``None``, this is assumed to be a string containing the path to the directory to write the feature files with an additional file extension specifying the file type. For example ``/foo/.tsv``. - feature_set : skll.data.FeatureSet + feature_set : :class:`skll.data.featureset.FeatureSet` The ``FeatureSet`` instance to dump to the output file. - pandas_kwargs : Optional[Dict[str, Any]], default=None - Arguments that will be passed directly - to the `pandas` I/O reader. + quiet : bool, default=True + Do not print "Writing..." status message to stderr. - kwargs : Optional[Dict[str, Any]], optional - The arguments to the ``Writer`` object being instantiated. + subsets : Optional[Dict[str, List[str]]], default=None + A mapping from subset names to lists of feature names + that are included in those sets. If given, a feature + file will be written for every subset (with the name + containing the subset name as suffix to ``path``). + Note, since string- valued features are automatically + converted into boolean features with names of the form + ``FEATURE_NAME=STRING_VALUE``, when doing the + filtering, the portion before the ``=`` is all that's + used for matching. Therefore, you do not need to + enumerate all of these boolean feature names in your + mapping. + + logger : Optional[logging.Logger], default=None + A logger instance to use to log messages instead of creating + a new one by default. + + label_col: str, default="y" + The column name containing the label. + + id_col: str, default="id" + The column name containing the ID. + + pandas_kwargs : Optional[Dict[str, Any]], default=None + Arguments that will be passed directly to the `pandas` I/O reader. """ def __init__( self, path: PathOrStr, feature_set: FeatureSet, + quiet: bool = True, + subsets: Optional[Dict[str, List[str]]] = None, + logger: Optional[logging.Logger] = None, + label_col: str = "y", + id_col: str = "id", pandas_kwargs: Optional[Dict[str, Any]] = None, - **kwargs, ): """Initialize the TSVWriter class.""" - super(TSVWriter, self).__init__(path, feature_set, pandas_kwargs, **kwargs) + super(TSVWriter, self).__init__( + path, + feature_set, + quiet=quiet, + subsets=subsets, + logger=logger, + label_col=label_col, + id_col=id_col, + pandas_kwargs=pandas_kwargs, + ) self._sep = str("\t") @@ -535,34 +604,68 @@ class ARFFWriter(Writer): Parameters ---------- - path : PathOrStr + path : :class:`skll.types.PathOrStr` A path to the feature file we would like to create. If ``subsets`` is not ``None``, this is assumed to be a string containing the path to the directory to write the feature files with an additional file extension specifying the file type. For example ``/foo/.arff``. - feature_set : skll.data.FeatureSet + feature_set : :class:`skll.data.featureset.FeatureSet` The ``FeatureSet`` instance to dump to the output file. + quiet : bool, default=True + Do not print "Writing..." status message to stderr. + + subsets : Optional[Dict[str, List[str]]], default=None + A mapping from subset names to lists of feature names + that are included in those sets. If given, a feature + file will be written for every subset (with the name + containing the subset name as suffix to ``path``). + Note, since string- valued features are automatically + converted into boolean features with names of the form + ``FEATURE_NAME=STRING_VALUE``, when doing the + filtering, the portion before the ``=`` is all that's + used for matching. Therefore, you do not need to + enumerate all of these boolean feature names in your + mapping. + + logger : Optional[logging.Logger], default=None + A logger instance to use to log messages instead of creating + a new one by default. + relation : str, default='skll_relation' The name of the relation in the ARFF file. regression : bool, default=False Is this an ARFF file to be used for regression? - kwargs : Optional[Dict[str, Any]], optional + kwargs : Optional[Dict[str, Any]] The arguments to the ``Writer`` object being instantiated. """ - def __init__(self, path: PathOrStr, feature_set: FeatureSet, **kwargs): + def __init__( + self, + path: PathOrStr, + feature_set: FeatureSet, + quiet: bool = True, + subsets: Optional[Dict[str, List[str]]] = None, + logger: Optional[logging.Logger] = None, + relation="skll_relation", + regression=False, + dialect="excel-tab", + label_col="y", + id_col="id", + ): """Initialize the ARFFWRiter class.""" - self.relation = kwargs.pop("relation", "skll_relation") - self.regression = kwargs.pop("regression", False) - self.dialect = kwargs.pop("dialect", "excel-tab") - self.label_col = kwargs.pop("label_col", "y") - self.id_col = kwargs.pop("id_col", "id") - super(ARFFWriter, self).__init__(path, feature_set, **kwargs) + self.relation = relation + self.regression = regression + self.dialect = dialect + self.label_col = label_col + self.id_col = id_col + super(ARFFWriter, self).__init__( + path, feature_set, quiet=quiet, subsets=subsets, logger=logger + ) self._dict_writer: Optional[DictWriter[str]] = None def _write_header( @@ -576,8 +679,8 @@ def _write_header( Parameters ---------- - feature_set : skll.data.FeatureSet - The FeatureSet being written to a file. + feature_set + Not used. output_file : IO[str] The file being written to. @@ -627,17 +730,17 @@ def _write_line( Parameters ---------- - id_ : IdType + id_ : :class:`skll.types.IdType` The ID for the current instance. - label_ : LabelType + label_ : :class:`skll.types.LabelType` The label for the current instance. - feat_dict : FeatureDict + feat_dict : :class:`skll.types.FeatureDict` The feature dictionary for the current instance. - output_file : IO[str] - The file being written to. + output_file + Not used. Raises ------ @@ -672,23 +775,49 @@ class NDJWriter(Writer): Parameters ---------- - path : PathOrStr + path : :class:`skll.types.PathOrStr` A path to the feature file we would like to create. If ``subsets`` is not ``None``, this is assumed to be a string containing the path to the directory to write the feature files with an additional file extension specifying the file type. For example ``/foo/.ndj``. - feature_set : skll.data.FeatureSet + feature_set : :class:`skll.data.featureset.FeatureSet` The ``FeatureSet`` instance to dump to the output file. - kwargs : Optional[Dict[str, Any]], optional - The arguments to the ``Writer`` object being instantiated. + quiet : bool, default=True + Do not print "Writing..." status message to stderr. + + subsets : Optional[Dict[str, List[str]]], default=None + A mapping from subset names to lists of feature names + that are included in those sets. If given, a feature + file will be written for every subset (with the name + containing the subset name as suffix to ``path``). + Note, since string- valued features are automatically + converted into boolean features with names of the form + ``FEATURE_NAME=STRING_VALUE``, when doing the + filtering, the portion before the ``=`` is all that's + used for matching. Therefore, you do not need to + enumerate all of these boolean feature names in your + mapping. + + logger : Optional[logging.Logger], default=None + A logger instance to use to log messages instead of creating + a new one by default. """ - def __init__(self, path: PathOrStr, feature_set: FeatureSet, **kwargs): + def __init__( + self, + path: PathOrStr, + feature_set: FeatureSet, + quiet: bool = True, + subsets: Optional[Dict[str, List[str]]] = None, + logger: Optional[logging.Logger] = None, + ): """Initialize the NDJWriter class.""" - super(NDJWriter, self).__init__(path, feature_set, **kwargs) + super(NDJWriter, self).__init__( + path, feature_set, quiet=quiet, subsets=subsets, logger=logger + ) def _write_line( self, @@ -702,13 +831,13 @@ def _write_line( Parameters ---------- - id_ : IdType + id_ : :class:`skll.types.IdType` The ID for the current instance. - label_ : LabelType + label_ : :class:`skll.types.LabelType` The label for the current instance. - feat_dict : FeatureDict + feat_dict : :class:`skll.types.FeatureDict` The feature dictionary for the current instance. output_file : IO[str] @@ -741,18 +870,38 @@ class LibSVMWriter(Writer): Parameters ---------- - path : PathOrStr + path : :class:`skll.types.PathOrStr` A path to the feature file we would like to create. If ``subsets`` is not ``None``, this is assumed to be a string containing the path to the directory to write the feature files with an additional file extension specifying the file type. For example ``/foo/.libsvm``. - feature_set : skll.data.FeatureSet + feature_set : :class:`skll.data.featureset.FeatureSet` The ``FeatureSet`` instance to dump to the output file. - kwargs : Optional[Dict[str, Any]], optional - The arguments to the ``Writer`` object being instantiated. + quiet : bool, default=True + Do not print "Writing..." status message to stderr. + + subsets : Optional[Dict[str, List[str]]], default=None + A mapping from subset names to lists of feature names + that are included in those sets. If given, a feature + file will be written for every subset (with the name + containing the subset name as suffix to ``path``). + Note, since string- valued features are automatically + converted into boolean features with names of the form + ``FEATURE_NAME=STRING_VALUE``, when doing the + filtering, the portion before the ``=`` is all that's + used for matching. Therefore, you do not need to + enumerate all of these boolean feature names in your + mapping. + + logger : Optional[logging.Logger], default=None + A logger instance to use to log messages instead of creating + a new one by default. + + label_map : Optional[Dict[str, int]], default=None + A mapping from label strings to integers. """ LIBSVM_REPLACE_DICT = { @@ -763,10 +912,20 @@ class LibSVMWriter(Writer): "|": "\u2223", } - def __init__(self, path: PathOrStr, feature_set: FeatureSet, **kwargs): + def __init__( + self, + path: PathOrStr, + feature_set: FeatureSet, + quiet: bool = True, + subsets: Optional[Dict[str, List[str]]] = None, + logger: Optional[logging.Logger] = None, + label_map: Optional[Dict[Any, Any]] = None, + ): """Initialize the LibSVMWriter class.""" - self.label_map = kwargs.pop("label_map", None) - super(LibSVMWriter, self).__init__(path, feature_set, **kwargs) + self.label_map = label_map + super(LibSVMWriter, self).__init__( + path, feature_set, quiet=quiet, subsets=subsets, logger=logger + ) if self.label_map is None: fs_labels = feature_set.labels if feature_set.has_labels else np.array([]) self.label_map = { @@ -795,13 +954,13 @@ def _sanitize(name: Union[IdType, LabelType]) -> Union[IdType, LabelType]: Parameters ---------- - name : Union[IdType, LabelType] + name : Union[:class:`skll.types.IdType`, :class:`skll.types.LabelType`] Input name in which special characters are replaced with unicode equivalents. Returns ------- - Union[IdType, LabelType] + Union[:class:`skll.types.IdType`, :class:`skll.types.LabelType`] The sanitized name with special characters replaced. """ sanitized_name = name @@ -818,13 +977,13 @@ def _write_line( Parameters ---------- - id_ : IdType + id_ : :class:`skll.types.IdType` The ID for the current instance. - label_ : LabelType + label_ : :class:`skll.types.LabelType` The label for the current instance. - feat_dict : FeatureDict + feat_dict : :class:`skll.types.FeatureDict` The feature dictionary for the current instance. output_file : IO[str] @@ -843,10 +1002,11 @@ def _write_line( ) # Print label - if label_ in self.label_map: - print(self.label_map[label_], end=" ", file=output_file) - else: - print(label_, end=" ", file=output_file) + if self.label_map: + if label_ in self.label_map: + print(self.label_map[label_], end=" ", file=output_file) + else: + print(label_, end=" ", file=output_file) # Print features print( @@ -860,18 +1020,19 @@ def _write_line( print(self._sanitize(id_), end="", file=output_file) print(" |", end=" ", file=output_file) - if label_ in self.label_map: - print( - f"{self._sanitize(self.label_map[label_])}=" f"{self._sanitize(label_)}", - end=" | ", - file=output_file, - ) - else: - print(" |", end=" ", file=output_file) + if self.label_map: + if label_ in self.label_map: + print( + f"{self._sanitize(self.label_map[label_])}={self._sanitize(label_)}", + end=" | ", + file=output_file, + ) + else: + print(" |", end=" ", file=output_file) line = ( " ".join( - f"{self.feat_set.vectorizer.vocabulary_[field] + 1}=" f"{self._sanitize(field)}" + f"{self.feat_set.vectorizer.vocabulary_[field] + 1}={self._sanitize(field)}" for field, value in feat_dict.items() if Decimal(value) != 0 )