Enhance clustering evaluation and update benchmark scripts

script/benchmark/clustering_utilities.py

@@ -10,40 +10,48 @@
 
 # Modified from:
 # https://gist.github.com/lmcinnes/24ed5c22c80125be5133811d677eae7b
-def eval_clusters(cluster_labels, true_labels, singleton_cluster_to_noise_points=False):
+def eval_clusters(cluster_labels, true_labels, singleton_cluster_to_noise_points=False, ignore_true_noise_points=False):
+    print(f"Assigning singleton clusters to noise points: {singleton_cluster_to_noise_points}")
+    print(f"Ignore true noise points in the ground truth: {ignore_true_noise_points}")
     max_cluster_id = max(np.max(cluster_labels), np.max(true_labels))
 
     if np.any(true_labels < 0):
         print("Ground truth labels contain noise points")
         pct_clustered_gt = (np.sum(true_labels >= 0) / cluster_labels.shape[0])
-        print(f"GT clustered Points: {pct_clustered_gt * 100:.2f}%")
-        print(
-            "Assigning a singleton cluster to each noise point in the ground truth labels")
-        true_labels = assign_singleton_cluster_to_noise_points(true_labels, max_cluster_id)
-        max_cluster_id = max(max_cluster_id, np.max(true_labels))
-
-    if np.any(cluster_labels < 0):  # Has noise points
-        clustered_points = (cluster_labels >= 0)
-        pct_clustered = (np.sum(clustered_points) / cluster_labels.shape[0])
-        print(f"Cluster coverage (%): {pct_clustered * 100:.2f}")
-
+        print(f"Ground truth cluster coverage (%): {pct_clustered_gt * 100:.2f}%")
+
+        if ignore_true_noise_points:
+            print("Remove noise points in the ground truth from the evaluation")
+            print(f"Before filtering: {len(true_labels)} ground truth points")
+            mask = true_labels >= 0
+            true_labels = true_labels[mask]
+            cluster_labels = cluster_labels[mask]
+            max_cluster_id = max(np.max(cluster_labels), np.max(true_labels))
+            print(f"After filtering: {len(true_labels)} ground truth points")
+
+    non_noise_points_mask = (cluster_labels >= 0)
+    pct_clustered = (np.sum(non_noise_points_mask) / cluster_labels.shape[0])
+    print(f"Cluster coverage (%): {pct_clustered * 100:.2f}")
+
+    if len(non_noise_points_mask) < cluster_labels.shape[0]:  # Has noise points
         if singleton_cluster_to_noise_points:
             print(
                 "Assigning a singleton cluster to each noise point in the clustering result")
             cluster_labels = assign_singleton_cluster_to_noise_points(cluster_labels, max_cluster_id)
             ari = adjusted_rand_score(true_labels, cluster_labels)
             ami = adjusted_mutual_info_score(true_labels, cluster_labels)
         else:
-            ari = adjusted_rand_score(true_labels[clustered_points],
-                                    cluster_labels[clustered_points])
-            ami = adjusted_mutual_info_score(true_labels[clustered_points],
-                                            cluster_labels[clustered_points])
-            # sil = silhouette_score(raw_data[clustered_points], cluster_labels[clustered_points])
+            print("Noise points are ignored in the evaluation")
+            ari = adjusted_rand_score(true_labels[non_noise_points_mask],
+                                    cluster_labels[non_noise_points_mask])
+            ami = adjusted_mutual_info_score(true_labels[non_noise_points_mask],
+                                            cluster_labels[non_noise_points_mask])
+            # sil = silhouette_score(raw_data[non_noise_points_mask], cluster_labels[non_noise_points_mask])
     else:
+        print(f"No noise points in the clustering result")
         ari = adjusted_rand_score(true_labels, cluster_labels)
         ami = adjusted_mutual_info_score(true_labels, cluster_labels)
         # sil = silhouette_score(raw_data, cluster_labels)
-        print(f"No noise points in the clustering result")
 
     print(f"ARI: {ari:.4f}")
     print(f"AMI: {ami:.4f}")
@@ -58,6 +66,9 @@ def assign_singleton_cluster_to_noise_points(cluster_labels, noise_id_offset):
         if label == -1:
             cnt_noise += 1
             new_labels[i] = cnt_noise + noise_id_offset
+            new_labels[i] = cnt_noise + noise_id_offset
+
+    print(f"Assigned singleton clusters to {cnt_noise} noise points")
 
     return new_labels
 

script/benchmark/evaluate_clustering_quality.py

@@ -41,7 +41,7 @@
 
 def parse_options():
     parser = argparse.ArgumentParser(
-        description='Evaluate kNN index')
+        description='Evaluate clustering quality')
 
     parser.add_argument('-c', '--cluster',
                         dest='cluster_path',
@@ -52,7 +52,10 @@ def parse_options():
                         required=True, action='store', type=str,
                         help='Path to file or directory containing ground truth labels')
     parser.add_argument('-s', '--singleton_noise_points', action='store_true',
-                        help='Assign a singleton cluster to each noise point in the clustering result.')
+                        help='Assign a singleton cluster to each noise point in the clustering labels to evaluate.')
+    parser.add_argument('-T', '--ignore_true_noise_points', action='store_true',
+                        help='Remove true noise points from the evaluation.')
+
     args = parser.parse_args()
     return args
 
@@ -64,10 +67,7 @@ def main():
     true_labels = read_label_data(opt.gt_path)
 
     try:
-        eval_clusters(cluster_labels, true_labels, False)
-        contains_noise_points = np.any(cluster_labels == -1)
-        if opt.singleton_noise_points and contains_noise_points:
-            eval_clusters(cluster_labels, true_labels, True)
+        eval_clusters(cluster_labels, true_labels, opt.singleton_noise_points, opt.ignore_true_noise_points)
     except Exception as e:
         print(f"Error: {e}")
         exit(1)

script/benchmark/extract_clams_clustering_qualities.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+# Copyright 2023-2026 Lawrence Livermore National Security, LLC and other ClaMS
+# Project Developers. See the top-level COPYRIGHT file for details.
+#
+# Extracts clustering parameters and results (ARI/AMI) from ClaMS benchmark
+# log files (e.g., out.log). Values are tracked as a running state while
+# scanning the log top to bottom, so parameter lines that appear less often
+# than ARI/AMI lines are repeated/shared across all following ARI/AMI rows
+# until the next occurrence of that parameter updates the state.
+#
+# Usage: extract_clustering_results.sh <log_file> [<log_file> ...]
+
+set -euo pipefail
+
+if [[ $# -eq 0 ]]; then
+  echo "Usage: $0 <log_file> [<log_file> ...]" >&2
+  exit 1
+fi
+
+printf "kNNG k,Min Cluster Size,Found Clusters,MST clustering,Cluster coverage,ARI,AMI,file\n"
+
+for file in "$@"; do
+  awk -v fname="$file" '
+    # Return the last whitespace/colon separated token on the line, i.e. its value
+    function lastfield(s,    n, arr) {
+      n = split(s, arr, /[[:space:]:]+/)
+      return arr[n]
+    }
+
+    BEGIN {
+      k = "NA"; mcs = "NA"; fc = "NA"; mst_clustering = "NA"; coverage = "NA"; have_ari = 0; ari = "NA"
+    }
+
+    /kNNG k:/ || /^[[:space:]]*k:[[:space:]]*[0-9]+[[:space:]]*$/ { k = lastfield($0); next }
+    /Min cluster size/       { mcs = lastfield($0); next }
+    /MST-based cluster guess/ { mst_clustering = lastfield($0); next }
+    /#of final clusters/     { fc = lastfield($0); next }
+    /Cluster coverage \(%\)/ { coverage = lastfield($0); next }
+    /ARI/                    { ari = lastfield($0); have_ari = 1; next }
+    /AMI/ {
+      if (have_ari) {
+        ami = lastfield($0)
+        printf "%s,%s,%s,%s,%s,%s,%s,%s\n", k, mcs, fc, mst_clustering, coverage, ari, ami, fname
+        have_ari = 0
+      }
+      next
+    }
+  ' "$file"
+done

script/benchmark/extract_clams_clustering_time.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+# Copyright 2023-2026 Lawrence Livermore National Security, LLC and other ClaMS
+# Project Developers. See the top-level COPYRIGHT file for details.
+#
+# Extracts clustering execution times from ClaMS benchmark log files
+# (e.g., out.log). Each stage is delimited by a "<Stage>: <timestamp>" line
+# and a matching "Finished <Stage>: <timestamp>" line; the elapsed time
+# between the two is reported in seconds. Only the first occurrence of each
+# stage pair is used (e.g., "Running CLAMS-HDBSCAN" and
+# "Assign clusters to noise points" can repeat per min-cluster-size value).
+#
+# Usage: extract_clams_clustering_time.sh <log_file> [<log_file> ...]
+
+set -euo pipefail
+
+if [[ $# -eq 0 ]]; then
+  echo "Usage: $0 <log_file> [<log_file> ...]" >&2
+  exit 1
+fi
+
+printf "kNNG k,nodes,tasks/node,kNNG (s),MFC (s),AMST (s),CLAMS-HDBSCAN (s),Assigning noise points (s),file\n"
+
+for file in "$@"; do
+  awk -v fname="$file" '
+    # Convert a "YYYY/MM/DD HH:MM:SS" timestamp found at the end of the line to epoch seconds
+    function to_epoch(line,    ts, n, a) {
+      if (!match(line, /[0-9]{4}\/[0-9]{2}\/[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}$/)) return -1
+      ts = substr(line, RSTART, RLENGTH)
+      n = split(ts, a, /[\/ :]+/)
+      return mktime(a[1] " " a[2] " " a[3] " " a[4] " " a[5] " " a[6])
+    }
+
+    # Return the last whitespace/colon separated token on the line, i.e. its value
+    function lastfield(s,    n, arr) {
+      n = split(s, arr, /[[:space:]:]+/)
+      return arr[n]
+    }
+
+    function elapsed(start, finish) {
+      if (start == -1 || finish == -1) return "NA"
+      return finish - start
+    }
+
+    BEGIN {
+      k = "NA"; nodes = "NA"; tasks = "NA"
+      knng_start = -1; knng_end = -1
+      mfc_start = -1; mfc_end = -1
+      amst_start = -1; amst_end = -1
+      hdbscan_start = -1; hdbscan_end = -1
+      noise_start = -1; noise_end = -1
+    }
+
+    /^kNNG k:/ && k == "NA"           { k = lastfield($0) }
+    /^Compute nodes:/ && nodes == "NA" { nodes = lastfield($0) }
+    /^Tasks per node:/ && tasks == "NA" { tasks = lastfield($0) }
+
+    /^Building KNNG:/ && knng_start == -1                          { knng_start = to_epoch($0) }
+    /^Finished Building KNNG:/ && knng_start != -1 && knng_end == -1 { knng_end = to_epoch($0) }
+
+    /^Connecting the CCs using MFC:/ && mfc_start == -1                          { mfc_start = to_epoch($0) }
+    /^Finished Connecting the CCs using MFC:/ && mfc_start != -1 && mfc_end == -1 { mfc_end = to_epoch($0) }
+
+    /^Running AMST, approx bound/ && amst_start == -1                          { amst_start = to_epoch($0) }
+    /^Finished Running AMST, approx bound/ && amst_start != -1 && amst_end == -1 { amst_end = to_epoch($0) }
+
+    /^Running CLAMS-HDBSCAN:/ && hdbscan_start == -1                          { hdbscan_start = to_epoch($0) }
+    /^Finished Running CLAMS-HDBSCAN:/ && hdbscan_start != -1 && hdbscan_end == -1 { hdbscan_end = to_epoch($0) }
+
+    /^Assign clusters to noise points:/ && noise_start == -1                          { noise_start = to_epoch($0) }
+    /^Finished Assign clusters to noise points:/ && noise_start != -1 && noise_end == -1 { noise_end = to_epoch($0) }
+
+    END {
+      printf "%s,%s,%s,%s,%s,%s,%s,%s,%s\n", k, nodes, tasks, \
+        elapsed(knng_start, knng_end), elapsed(mfc_start, mfc_end), \
+        elapsed(amst_start, amst_end), elapsed(hdbscan_start, hdbscan_end), \
+        elapsed(noise_start, noise_end), fname
+    }
+  ' "$file"
+done