Add W&B logging

.pre-commit-config.yaml

@@ -33,8 +33,9 @@ repos:
       - id: ruff
         args: [--line-length=100, --select, "D,E,F,I", --ignore, "D212", --per-file-ignores, "tests/test*.py:D102,tests/test*.py:D103,tests/test_input.py:E501,skll/data/featureset.py:E501,skll/learner/__init__.py:E501,skll/learner/voting.py:E501,skll/learner/utils.py:E501"]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v1.2.0'
+    rev: 'v1.8.0'
     hooks:
       - id: mypy
         args: [--ignore-missing-imports]
+        additional_dependencies: ["wandb"]
         exclude: tests/

doc/run_experiment.rst

@@ -1392,6 +1392,8 @@ Whether to save each of the K :ref:`model files <output_model_files>` trained du
 each step of a K-fold cross-validation experiment.
 Defaults to ``False``.
 
+.. _save_votes:
+
 save_votes *(Optional)*
 """""""""""""""""""""""
 
@@ -1400,6 +1402,8 @@ Whether to save the predictions from the individual estimators underlying a
 :ref:`predictions <predictions>` must be set.
 Defaults to ``False``.
 
+.. _wandb_credentials:
+
 wandb_credentials *(Optional)*
 """"""""""""""""""""""""""""""
 To enable logging metrics and artifacts to `Weights & Biases <https://wandb.ai/>`__, specify
@@ -1539,6 +1543,8 @@ For most of the SKLL tasks the various output files generated by :ref:`run_exper
     contains only a single value, the job can be disambiguated using only
     the featuresets and the learners since the objective is fixed. Therefore,
     the output files will have the prefix ``<EXPERIMENT>_<FEATURESET>_<LEARNER>``.
+    Similarly, if a task has a single :ref:`feature set <featuresets>`, the output
+    files prefix will not include the ``<FEATURESET>`` component.
 
 The following types of output files can be generated after running an experiment
 configuration file through :ref:`run_experiment <run_experiment>`. Note that
@@ -1676,6 +1682,26 @@ Here's an example of this plot.
 You can also generate the plots from the learning curve summary
 file using the :ref:`plot_learning_curves <plot_learning_curves>` utility script.
 
+.. _output_wandb:
+
+Integration with Weights & Biases
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+The output of any SKLL experiment can be automatically logged to `Weights & Biases <https://wandb.ai>`. 
+Once the logging is :ref:`enabled<wandb_credentials>`, a new
+run will be created under the specified W&B project. The following is logged
+for _all_ tasks: 
+- The SKLL configuration file, including default values for fields that were left unspecified
+- The learner, feature set, and size of training and testing sets for each job in the experiment
+
+There are additional items logged depending on the task type: 
+  *   **train**: The full path to the generated model file is logged in the project summary.
+  *   **predict**: The predictions file is logged as a table, separately for each job in the experiment.
+  *   **evaluate**: The task summary file is logged as a table. For classification experiments, 
+      the confusion matrix as well as a table that shows per-label precision, recall and f-measure
+      are logged for each job.
+  *   **cross_validate**: Similar output logged as the `evaluate` task, with a separate job per CV fold.
+  *   **learning_curve** The summary file is logged as a table, and all learning curve plots
+      are logged as media artifacts.
 .. rubric:: Footnotes
 
 .. [#] We are considering adding support for YAML configuration files in the

examples/california/cross_val.cfg

@@ -1,5 +1,5 @@
 [General]
-experiment_name = Example_CV
+experiment_name = California_CV
 task = cross_validate
 
 [Input]

examples/california/evaluate.cfg

@@ -1,5 +1,5 @@
 [General]
-experiment_name = Example_Evaluate
+experiment_name = California_Evaluate
 task = evaluate
 
 [Input]

examples/titanic/predict_train+dev_tuned.cfg

@@ -22,3 +22,4 @@ objectives = ['accuracy']
 logs = output
 predictions = output
 models = output
+results = output

examples/titanic/predict_train_only_tuned.cfg

@@ -22,3 +22,4 @@ objectives = ['accuracy']
 logs = output
 predictions = output
 models = output
+results = output

examples/titanic/train.cfg

@@ -20,3 +20,4 @@ objectives = ['accuracy']
 # again, these can be absolute paths
 logs = output
 models = output
+results = output

skll/experiments/__init__.py

@@ -245,6 +245,7 @@ def _classify_featureset(args: Dict[str, Any]) -> List[Dict[str, Any]]:
             )
 
             train_set_size = len(train_examples.ids)
+
             if not train_examples.has_labels:
                 raise ValueError("Training examples do not have labels")
 
@@ -345,6 +346,7 @@ def _classify_featureset(args: Dict[str, Any]) -> List[Dict[str, Any]]:
         # create a list of dictionaries of the results information
         learner_result_dict_base = {
             "experiment_name": experiment_name,
+            "job_name": job_name,
             "train_set_name": train_set_name,
             "train_set_size": train_set_size,
             "test_set_name": test_set_name,
@@ -491,6 +493,7 @@ def _classify_featureset(args: Dict[str, Any]) -> List[Dict[str, Any]]:
                 # save model, if asked
                 if model_path:
                     learner.save(modelfile)
+                    learner_result_dict_base["model_file"] = str(modelfile)
 
             # print out the model parameters; note that for
             # voting learners, we exclude the parameters for
@@ -533,15 +536,18 @@ def _classify_featureset(args: Dict[str, Any]) -> List[Dict[str, Any]]:
                     class_labels=False,
                     **extra_kwargs,
                 )
+                learner_result_dict_base[
+                    "predictions_file"
+                ] = f"{prediction_prefix}_predictions.tsv"
 
         end_timestamp = datetime.datetime.now()
         learner_result_dict_base["end_timestamp"] = end_timestamp.strftime("%d %b %Y %H:%M:%S.%f")
         total_time = end_timestamp - start_timestamp
         learner_result_dict_base["total_time"] = str(total_time)
 
-        if task == "cross_validate" or task == "evaluate":
-            results_json_path = Path(results_path) / f"{job_name}.results.json"
+        results_json_path = Path(results_path) / f"{job_name}.results.json"
 
+        if task == "cross_validate" or task == "evaluate":
             res = _create_learner_result_dicts(
                 task_results, grid_scores, grid_search_cv_results_dicts, learner_result_dict_base
             )
@@ -554,7 +560,6 @@ def _classify_featureset(args: Dict[str, Any]) -> List[Dict[str, Any]]:
                 _print_fancy_output(res, output_file)
 
         elif task == "learning_curve":
-            results_json_path = Path(results_path) / f"{job_name}.results.json"
             result_dict = {}
             result_dict.update(learner_result_dict_base)
             result_dict.update(
@@ -581,14 +586,12 @@ def _classify_featureset(args: Dict[str, Any]) -> List[Dict[str, Any]]:
         # For all other tasks, i.e. train or predict
         else:
             if results_path:
-                results_json_path = Path(results_path) / f"{job_name}.results.json"
-
                 assert len(grid_scores) == 1
                 assert len(grid_search_cv_results_dicts) == 1
-                grid_search_cv_results_dict: Dict[str, Any] = {"grid_score": grid_scores[0]}
-                grid_search_cv_results_dict[
-                    "grid_search_cv_results"
-                ] = grid_search_cv_results_dicts[0]
+                grid_search_cv_results_dict: Dict[str, Any] = {
+                    "grid_score": grid_scores[0],
+                    "grid_search_cv_results": grid_search_cv_results_dicts[0],
+                }
                 grid_search_cv_results_dict.update(learner_result_dict_base)
                 # write out the result dictionary to a json file
                 with open(results_json_path, "w") as json_file:
@@ -724,10 +727,9 @@ def run_configuration(
         # created by the configuration parser so we don't need anything
         # except the name `experiment`.
         logger = get_skll_logger("experiment")
-        wandb_logger = WandbLogger(wandb_credentials)
-        wandb_logger.log_configuration(
-            {"experiment_name": experiment_name, "task": task, "learners": learners}
-        )
+        # init WandbLogger. If W&B credentials are not specified,
+        # logging to wandb will not be performed.
+        wandb_logger = WandbLogger(wandb_credentials, str(config_file))
 
         # Check if we have gridmap
         if not local and not _HAVE_GRIDMAP:
@@ -823,30 +825,37 @@ def run_configuration(
             grid_objectives = output_metrics
 
         # if there were no grid objectives provided, just set it to
-        # a list containing a single None so as to allow the parallelization
-        # to proceeed and to pass the correct default value of grid_objective
+        # a list containing a single None to allow the parallelization
+        # to proceed and to pass the correct default value of grid_objective
         # down to _classify_featureset().
         grid_objectives_extra: Union[List[str], List[None]]
         grid_objectives_extra = [None] if not grid_objectives else grid_objectives
 
+        job_results = []
         # Run each featureset-learner-objective combination
         for featureset, featureset_name in zip(featuresets, featureset_names):
             for learner_num, learner_name in enumerate(learners):
                 for grid_objective in grid_objectives_extra:
-                    # for the individual job name, we need to add the feature set name
-                    # and the learner name
-                    if grid_objective is None or len(grid_objectives_extra) == 1:
-                        job_name_components = [experiment_name, featureset_name, learner_name]
-                    else:
-                        job_name_components = [
-                            experiment_name,
-                            featureset_name,
-                            learner_name,
-                            grid_objective,
-                        ]
+                    # The individual job name is built from the experiment name, featureset,
+                    # learner and objective. To keep the job name as compact as possible,
+                    # some of the components are only included if they have multiple values
+                    # in this run.
+                    job_name_components = [experiment_name]
+                    if len(featuresets) > 1:
+                        job_name_components.append(featureset_name)
+
+                    job_name_components.append(learner_name)
+
+                    if len(grid_objectives_extra) > 1 and grid_objective is not None:
+                        job_name_components.append(grid_objective)
 
                     job_name = "_".join(job_name_components)
 
+                    logger.info(f"Job name: {job_name}")
+                    wandb_logger.log_to_summary(job_name, "featureset name", featureset_name)
+                    wandb_logger.log_to_summary(job_name, "learner name", learner_name)
+                    wandb_logger.log_to_summary(job_name, "grid objective", grid_objective)
+
                     # change the prediction prefix to include the feature set
                     prediction_prefix = str(Path(prediction_dir) / job_name)
 
@@ -869,7 +878,7 @@ def run_configuration(
                         )
                         continue
 
-                    # create job if we're doing things on the grid
+                    # create args for classification job
                     job_args: Dict[str, Any] = {}
                     job_args["experiment_name"] = experiment_name
                     job_args["task"] = task
@@ -930,6 +939,7 @@ def run_configuration(
                     job_args["learning_curve_train_sizes"] = learning_curve_train_sizes
 
                     if not local:
+                        # add to job list if we're doing things on the grid
                         jobs.append(
                             Job(
                                 _classify_featureset,
@@ -944,7 +954,7 @@ def run_configuration(
                             )
                         )
                     else:
-                        _classify_featureset(job_args)
+                        job_results.append(_classify_featureset(job_args))
 
         # Call get_skll_logger again after _classify_featureset
         # calls are finished so that any warnings that may
@@ -960,19 +970,43 @@ def run_configuration(
                 job_results = process_jobs(jobs, white_list=hosts)
             _check_job_results(job_results)
 
-        # write out the summary results file
-        if (task == "cross_validate" or task == "evaluate") and write_summary:
-            summary_file_name = f"{experiment_name}_summary.tsv"
-            with open(Path(results_path) / summary_file_name, "w", newline="") as output_file:
-                _write_summary_file(result_json_paths, output_file, ablation=ablation)
+        # process output and log to wandb
+        if task == "predict":
+            for job_result in job_results:
+                task_result = job_result[0]  # predict outputs a single dict
+                wandb_logger.log_predict_results(task_result)
+
+        elif task == "train":
+            for job_result in job_results:
+                # train outputs a single dict
+                wandb_logger.log_train_results(job_result[0])
+
+        elif task == "cross_validate" or task == "evaluate":
+            if write_summary:
+                # write out the summary results file
+                summary_file_path = Path(results_path) / f"{experiment_name}_summary.tsv"
+                with open(summary_file_path, "w", newline="") as output_file:
+                    _write_summary_file(result_json_paths, output_file, ablation=ablation)
+                wandb_logger.log_summary_file(summary_file_path)
+
+            for job_result_list in job_results:
+                for result_dict in job_result_list:
+                    wandb_logger.log_evaluation_results(result_dict)
+            wandb_logger.log_label_metric_table()
+
         elif task == "learning_curve":
-            output_file_name = f"{experiment_name}_summary.tsv"
-            output_file_path = Path(results_path) / output_file_name
-            with open(output_file_path, "w", newline="") as output_file:
+            # write out the summary file
+            summary_file_path = Path(results_path) / f"{experiment_name}_summary.tsv"
+            with open(summary_file_path, "w", newline="") as output_file:
                 _write_learning_curve_file(result_json_paths, output_file)
+            wandb_logger.log_summary_file(summary_file_path)
 
             # generate the actual plot if we have the requirements installed
-            generate_learning_curve_plots(experiment_name, results_path, output_file_path)
+            plot_paths = generate_learning_curve_plots(
+                experiment_name, results_path, summary_file_path
+            )
+            for plot_path in plot_paths:
+                wandb_logger.log_plot(plot_path)
 
     finally:
         # Close/remove any logger handlers