slisemap.metrics

Module that contains functions that can be used to evaluate SLISEMAP solutions.

The functions take a solution (plus other arguments) and returns a single float. This float should either be minimised or maximised for best results (see individual functions).

nanmean(x)

Compute the mean, ignoring any nan.

Source code in slisemap/metrics.py

def nanmean(x: np.ndarray) -> float:
    """Compute the mean, ignoring any nan."""
    mask = np.isfinite(x)
    if np.all(mask):
        return np.mean(x)
    elif not np.any(mask):
        return np.nan
    else:
        return np.mean(x[mask])

euclidean_nearest_neighbours(D, index, k=0.1, include_self=True)

Find the k nearest neighbours using euclidean distance.

Parameters:

Name	Type	Description	Default
D	Tensor	Distance matrix.	required
index	int	The item (row), for which to find neighbours.	required
k	Union[int, float]	The number of neighbours to find. Defaults to 0.1.	0.1
include_self	bool	include the item in its' neighbourhood. Defaults to True.	True

Returns:

Type	Description
LongTensor	Vector of indices for the neighbours.

Source code in slisemap/metrics.py

def euclidean_nearest_neighbours(
    D: torch.Tensor, index: int, k: Union[int, float] = 0.1, include_self: bool = True
) -> torch.LongTensor:
    """Find the k nearest neighbours using euclidean distance.

    Args:
        D: Distance matrix.
        index: The item (row), for which to find neighbours.
        k: The number of neighbours to find. Defaults to 0.1.
        include_self: include the item in its' neighbourhood. Defaults to True.

    Returns:
        Vector of indices for the neighbours.
    """
    if isinstance(k, float) and 0 < k <= 1:
        k = int(k * D.shape[0])
    if include_self:
        return torch.argsort(D[index])[:k]
    else:
        dist = D[index].clone()
        if not include_self:
            dist[index] = np.inf
            if k == D.shape[0]:
                k -= 1
        return torch.argsort(dist)[:k]

kernel_neighbours(D, index, epsilon=1.0, include_self=True)

Find the neighbours using a softmax kernel.

Parameters:

Name	Type	Description	Default
D	Tensor	Distance matrix.	required
index	int	The item for which we want to find neighbours.	required
epsilon	float	Treshold for selecting the neighbourhood (will be divided by n). Defaults to 1.0.	1.0
include_self	bool	include the item in its' neighbourhood. Defaults to True.	True

Returns:

Type	Description
LongTensor	Vector of indices for the neighbours.

Source code in slisemap/metrics.py

def kernel_neighbours(
    D: torch.Tensor, index: int, epsilon: float = 1.0, include_self: bool = True
) -> torch.LongTensor:
    """Find the neighbours using a softmax kernel.

    Args:
        D: Distance matrix.
        index: The item for which we want to find neighbours.
        epsilon: Treshold for selecting the neighbourhood (will be divided by `n`). Defaults to 1.0.
        include_self: include the item in its' neighbourhood. Defaults to True.

    Returns:
        Vector of indices for the neighbours.
    """
    K = torch.nn.functional.softmax(-D[index], 0)
    epsilon2 = epsilon / K.numel()
    if include_self:
        return torch.where(epsilon2 <= K)[0]
    else:
        mask = epsilon2 <= K
        mask[index] = False
        return torch.where(mask)[0]

cluster_neighbours(D, index, clusters, include_self=True)

Find the neighbours with given clusters.

Parameters:

Name	Type	Description	Default
D	Tensor	Distance matrix (ignored).	required
index	int	The item for which we want to find neighbours.	required
clusters	LongTensor	Cluster id:s for the data items.	required
include_self	bool	include the item in its' neighbourhood. Defaults to True.	True

Returns:

Type	Description
LongTensor	Vector of indices for the neighbours.

Source code in slisemap/metrics.py

def cluster_neighbours(
    D: torch.Tensor,
    index: int,
    clusters: torch.LongTensor,
    include_self: bool = True,
) -> torch.LongTensor:
    """Find the neighbours with given clusters.

    Args:
        D: Distance matrix (ignored).
        index: The item for which we want to find neighbours.
        clusters: Cluster id:s for the data items.
        include_self: include the item in its' neighbourhood. Defaults to True.

    Returns:
        Vector of indices for the neighbours.
    """
    if include_self:
        return torch.where(clusters == clusters[index])[0]
    else:
        mask = clusters == clusters[index]
        mask[index] = False
        return torch.where(mask)[0]

radius_neighbours(D, index, radius=None, quantile=0.2, include_self=True)

Find the neighbours within a radius.

Parameters:

Name	Type	Description	Default
D	Tensor	Distance matrix (ignored).	required
index	int	The item for which we want to find neighbours.	required
radius	Optional[float]	The radius of the neighbourhood. Defaults to None.	None
quantile	float	If radius is None then radius is set to the quantile of D. Defaults to 0.2.	0.2
include_self	bool	include the item in its' neighbourhood. Defaults to True.	True

Returns:

Type	Description
LongTensor	Vector of indices for the neighbours.

Source code in slisemap/metrics.py

def radius_neighbours(
    D: torch.Tensor,
    index: int,
    radius: Optional[float] = None,
    quantile: float = 0.2,
    include_self: bool = True,
) -> torch.LongTensor:
    """Find the neighbours within a radius.

    Args:
        D: Distance matrix (ignored).
        index: The item for which we want to find neighbours.
        radius: The radius of the neighbourhood. Defaults to None.
        quantile: If radius is None then radius is set to the quantile of D. Defaults to 0.2.
        include_self: include the item in its' neighbourhood. Defaults to True.

    Returns:
        Vector of indices for the neighbours.
    """
    if radius is None:
        radius = torch.quantile(D, quantile)
    if include_self:
        return torch.where(D[index] <= radius)[0]
    else:
        mask = D[index] <= radius
        mask[index] = False
        return torch.where(mask)[0]

Neighbours = Union[None, np.ndarray, torch.LongTensor, Callable[[torch.Tensor, int], torch.LongTensor]] module-attribute

Type annotation for specifying neighbouring items. Used in the get_neighbours function.

• If None, every item is or is not a neighbour.
• If a vector of cluster id:s, take neighbours from the same cluter.
• Or a function that gives neighbours (that takes a distance matrix and an index), primarily:
  • euclidean_nearest_neighbours
  • kernel_neighbours
  • cluster_neighbours
  • radius_neighbours

get_neighbours(sm, neighbours, full_if_none=False, **kwargs)

Create a function that takes the index of an item and returns the indices of its neighbours.

Parameters:

Name	Type	Description	Default
sm	Union[Slisemap, Tensor]	Trained Slisemap solution or an embedding vector (like Slisemap.Z).	required
neighbours	Neighbours	Either None (return self), a vector of cluster id:s (take neighbours from the same cluter), or a function that gives neighbours (that takes a distance matrix and an index).	required
full_if_none	bool	If neighbours is None, return the whole dataset. Defaults to False.	False

Other Parameters:

Name	Type	Description
**kwargs	Any	Arguments passed on to neighbours (if it is a function).

Returns:

Type	Description
Callable[[int], LongTensor]	Function that takes an index and returns neighbour indices.

Source code in slisemap/metrics.py

def get_neighbours(
    sm: Union[Slisemap, torch.Tensor],
    neighbours: Neighbours,
    full_if_none: bool = False,
    **kwargs: Any,
) -> Callable[[int], torch.LongTensor]:
    """Create a function that takes the index of an item and returns the indices of its neighbours.

    Args:
        sm: Trained Slisemap solution or an embedding vector (like Slisemap.Z).
        neighbours: Either None (return self), a vector of cluster id:s (take neighbours from the same cluter), or a function that gives neighbours (that takes a distance matrix and an index).
        full_if_none: If `neighbours` is None, return the whole dataset. Defaults to False.

    Keyword Args:
        **kwargs: Arguments passed on to `neighbours` (if it is a function).

    Returns:
        Function that takes an index and returns neighbour indices.
    """
    if neighbours is None:
        if full_if_none:
            try:
                n = sm.n
            except AttributeError:
                n = sm.shape[0]
            return lambda i: torch.arange(n)
        else:
            return lambda i: torch.LongTensor([i])
    if callable(neighbours):
        try:
            D = sm.get_D(numpy=False)
        except AttributeError:
            D = torch.cdist(sm, sm)
        return lambda i: neighbours(D, i, **kwargs)
    else:
        neighbours = torch.as_tensor(neighbours)
        return lambda i: cluster_neighbours(None, i, neighbours, **kwargs)

slisemap_loss(sm)

Evaluate a SLISEMAP solution by calculating the loss.

Smaller is better.

Parameters:

Name	Type	Description	Default
sm	Slisemap	Trained Slisemap solution.	required

Returns:

Type	Description
float	The loss value.

Source code in slisemap/metrics.py

def slisemap_loss(sm: Slisemap) -> float:
    """Evaluate a SLISEMAP solution by calculating the loss.

    Smaller is better.

    Args:
        sm: Trained Slisemap solution.

    Returns:
        The loss value.
    """
    return sm.value()

entropy(sm, aggregate=True, numpy=True)

Compute row-wise entropy of the W matrix induced by Z.

Parameters:

Name	Type	Description	Default
sm	Slisemap	Trained Slisemap solution.	required
aggregate	bool	Aggregate the row-wise entropies into one scalar. Defaults to True.	True
numpy	bool	Return a numpy.ndarray or float instead of a torch.Tensor. Defaults to True.	True

Returns:

Type	Description
Union[float, ndarray, Tensor]	The entropy.

Source code in slisemap/metrics.py

def entropy(
    sm: Slisemap, aggregate: bool = True, numpy: bool = True
) -> Union[float, np.ndarray, torch.Tensor]:
    """Compute row-wise entropy of the `W` matrix induced by `Z`.

    Args:
        sm: Trained Slisemap solution.
        aggregate: Aggregate the row-wise entropies into one scalar. Defaults to True.
        numpy: Return a `numpy.ndarray` or `float` instead of a `torch.Tensor`. Defaults to True.

    Returns:
        The entropy.
    """
    W = sm.get_W(numpy=False)
    entropy = -(W * W.log()).sum(dim=1)
    if aggregate:
        entropy = entropy.mean().exp() / sm.n
        return entropy.cpu().item() if numpy else entropy
    else:
        return tonp(entropy) if numpy else entropy

slisemap_entropy(sm)

Evaluate a SLISEMAP solution by calculating the entropy. DEPRECATED.

Parameters:

Name	Type	Description	Default
sm	Slisemap	Trained Slisemap solution.	required

Returns:

Type	Description
float	The embedding entropy.

Deprecated

1.4: Use entropy instead.

Source code in slisemap/metrics.py

def slisemap_entropy(sm: Slisemap) -> float:
    """Evaluate a SLISEMAP solution by calculating the entropy. **DEPRECATED**.

    Args:
        sm: Trained Slisemap solution.

    Returns:
        The embedding entropy.

    Deprecated:
        1.4: Use [entropy][slisemap.metrics.entropy] instead.
    """
    _deprecated(slisemap_entropy, entropy)
    return entropy(sm, aggregate=True, numpy=True)

fidelity(sm, neighbours=None, **kwargs)

Evaluate a SLISEMAP solution by calculating the fidelity (loss per item/neighbourhood).

Smaller is better.

Parameters:

Name	Type	Description	Default
sm	Slisemap	Trained Slisemap solution.	required
neighbours	Neighbours	Either None (only corresponding local model), a vector of cluster id:s, or a function that gives neighbours (see get_neighbours).	None

Other Parameters:

Name	Type	Description
**kwargs	Any	Arguments passed on to neighbours (if it is a function).

Returns:

Type	Description
float	The mean loss.

Source code in slisemap/metrics.py

def fidelity(sm: Slisemap, neighbours: Neighbours = None, **kwargs: Any) -> float:
    """Evaluate a SLISEMAP solution by calculating the fidelity (loss per item/neighbourhood).

    Smaller is better.

    Args:
        sm: Trained Slisemap solution.
        neighbours: Either None (only corresponding local model), a vector of cluster id:s, or a function that gives neighbours (see [get_neighbours][slisemap.metrics.get_neighbours]).

    Keyword Args:
        **kwargs: Arguments passed on to `neighbours` (if it is a function).

    Returns:
        The mean loss.
    """
    neighbours = get_neighbours(sm, neighbours, full_if_none=False, **kwargs)
    results = np.zeros(sm.n)
    L = sm.get_L(numpy=False)
    for i in range(len(results)):
        ni = neighbours(i)
        if ni.numel() == 0:
            results[i] = np.nan
        else:
            results[i] = torch.mean(L[i, ni]).cpu().detach().item()
    return nanmean(results)

coverage(sm, max_loss, neighbours=None, **kwargs)

Evaluate a SLISEMAP solution by calculating the coverage.

Larger is better.

Parameters:

Name	Type	Description	Default
sm	Slisemap	Trained Slisemap solution.	required
max_loss	float	Maximum tolerable loss.	required
neighbours	Neighbours	Either None (all), a vector of cluster id:s, or a function that gives neighbours (see get_neighbours).	None

Other Parameters:

Name	Type	Description
**kwargs	Any	Arguments passed on to neighbours (if it is a function).

Returns:

Type	Description
float	The mean fraction of items within the error bound.

Source code in slisemap/metrics.py

def coverage(
    sm: Slisemap, max_loss: float, neighbours: Neighbours = None, **kwargs: Any
) -> float:
    """Evaluate a SLISEMAP solution by calculating the coverage.

    Larger is better.

    Args:
        sm: Trained Slisemap solution.
        max_loss: Maximum tolerable loss.
        neighbours: Either None (all), a vector of cluster id:s, or a function that gives neighbours (see [get_neighbours][slisemap.metrics.get_neighbours]).

    Keyword Args:
        **kwargs: Arguments passed on to `neighbours` (if it is a function).

    Returns:
        The mean fraction of items within the error bound.
    """
    if torch.all(torch.isnan(sm.get_B(numpy=False).sum(1))).cpu().item():
        return np.nan
    neighbours = get_neighbours(sm, neighbours, full_if_none=True, **kwargs)
    results = np.zeros(sm.n)
    L = sm.get_L(numpy=False)
    for i in range(len(results)):
        ni = neighbours(i)
        if ni.numel() == 0:
            results[i] = np.nan
        else:
            results[i] = np.mean(tonp(L[i, ni] < max_loss))
    return nanmean(results)

median_loss(sm, neighbours=None, **kwargs)

Evaluate a SLISEMAP solution by calculating the median loss.

Smaller is better.

Parameters:

Name	Type	Description	Default
sm	Slisemap	Trained Slisemap solution.	required
neighbours	Neighbours	Either None (all), a vector of cluster id:s, or a function that gives neighbours (see get_neighbours).	None

Other Parameters:

Name	Type	Description
**kwargs	Any	Arguments passed on to neighbours (if it is a function).

Returns:

Type	Description
float	The mean median loss.

Source code in slisemap/metrics.py

def median_loss(sm: Slisemap, neighbours: Neighbours = None, **kwargs: Any) -> float:
    """Evaluate a SLISEMAP solution by calculating the median loss.

    Smaller is better.

    Args:
        sm: Trained Slisemap solution.
        neighbours: Either None (all), a vector of cluster id:s, or a function that gives neighbours (see [get_neighbours][slisemap.metrics.get_neighbours]).

    Keyword Args:
        **kwargs: Arguments passed on to `neighbours` (if it is a function).

    Returns:
        The mean median loss.
    """
    neighbours = get_neighbours(sm, neighbours, full_if_none=True, **kwargs)
    results = np.zeros(sm.n)
    L = sm.get_L(numpy=False)
    for i in range(len(results)):
        ni = neighbours(i)
        if ni.numel() == 0:
            results[i] = np.nan
        else:
            results[i] = _non_crashing_median(L[i, ni])
    return nanmean(results)

coherence(sm, neighbours=None, **kwargs)

Evaluate a SLISEMAP solution by calculating the coherence (max change in prediction divided by the change in variable values).

Smaller is better.

Parameters:

Name	Type	Description	Default
sm	Slisemap	Trained Slisemap solution.	required
neighbours	Neighbours	Either None (all), a vector of cluster id:s, or a function that gives neighbours (see get_neighbours).	None

Other Parameters:

Name	Type	Description
**kwargs	Any	Arguments passed on to neighbours (if it is a function).

Returns:

Type	Description
float	The mean coherence.

Source code in slisemap/metrics.py

def coherence(sm: Slisemap, neighbours: Neighbours = None, **kwargs: Any) -> float:
    """Evaluate a SLISEMAP solution by calculating the coherence (max change in prediction divided by the change in variable values).

    Smaller is better.

    Args:
        sm: Trained Slisemap solution.
        neighbours: Either None (all), a vector of cluster id:s, or a function that gives neighbours (see [get_neighbours][slisemap.metrics.get_neighbours]).

    Keyword Args:
        **kwargs: Arguments passed on to `neighbours` (if it is a function).

    Returns:
        The mean coherence.
    """
    neighbours = get_neighbours(
        sm, neighbours, full_if_none=True, include_self=False, **kwargs
    )
    results = np.zeros(sm.n)
    P = sm.local_model(sm._X, sm.get_B(numpy=False))
    for i in range(len(results)):
        ni = neighbours(i)
        if ni.numel() == 0:
            results[i] = np.nan
        else:
            dP = torch.sum((P[None, i, i] - P[ni, i] - P[i, ni] + P[ni, ni]) ** 2, 1)
            dX = torch.sum((sm._X[None, i, :] - sm._X[ni, :]) ** 2, 1) + 1e-8
            results[i] = torch.sqrt(torch.max(dP / dX))
    return nanmean(results)

stability(sm, neighbours=None, **kwargs)

Evaluate a SLISEMAP solution by calculating the stability (max change in the local model divided by the change in variable values).

Smaller is better.

Parameters:

Name	Type	Description	Default
sm	Slisemap	Trained Slisemap solution.	required
neighbours	Neighbours	Either None (all), a vector of cluster id:s, or a function that gives neighbours (see get_neighbours).	None

Other Parameters:

Name	Type	Description
**kwargs	Any	Arguments passed on to neighbours (if it is a function).

Returns:

Type	Description
float	The mean stability.

Source code in slisemap/metrics.py

def stability(sm: Slisemap, neighbours: Neighbours = None, **kwargs: Any) -> float:
    """Evaluate a SLISEMAP solution by calculating the stability (max change in the local model divided by the change in variable values).

    Smaller is better.

    Args:
        sm: Trained Slisemap solution.
        neighbours: Either None (all), a vector of cluster id:s, or a function that gives neighbours (see [get_neighbours][slisemap.metrics.get_neighbours]).

    Keyword Args:
        **kwargs: Arguments passed on to `neighbours` (if it is a function).

    Returns:
        The mean stability.
    """
    neighbours = get_neighbours(
        sm, neighbours, full_if_none=True, include_self=False, **kwargs
    )
    results = np.zeros(sm.n)
    B = sm.get_B(numpy=False)
    X = sm.get_X(numpy=False)
    for i in range(len(results)):
        ni = neighbours(i)
        if ni.numel() == 0:
            results[i] = np.nan
        else:
            dB = torch.sum((B[None, i, :] - B[ni, :]) ** 2, 1)
            dX = torch.sum((X[None, i, :] - X[ni, :]) ** 2, 1) + 1e-8
            results[i] = torch.sqrt(torch.max(dB / dX))
    return nanmean(results)

kmeans_matching(sm, clusters=range(2, 10), **kwargs)

Evaluate SLISE by measuring how well clusters in Z and B overlap (using kmeans to find the clusters).

The overlap is measured by finding the best matching clusters and dividing the size of intersect by the size of the union of each cluster pair. Larger is better.

Parameters:

Name	Type	Description	Default
sm	Slisemap	Trained Slisemap solution.	required
clusters	Union[int, Sequence[int]]	The number of clusters. Defaults to range(2, 10).	range(2, 10)

Other Parameters:

Name	Type	Description
**kwargs	Any	Additional arguments to sklearn.KMeans.

Returns:

Type	Description
float	The mean cluster matching.

Source code in slisemap/metrics.py

def kmeans_matching(
    sm: Slisemap, clusters: Union[int, Sequence[int]] = range(2, 10), **kwargs: Any
) -> float:
    """Evaluate SLISE by measuring how well clusters in Z and B overlap (using kmeans to find the clusters).

    The overlap is measured by finding the best matching clusters and dividing the size of intersect by the size of the union of each cluster pair.
    Larger is better.

    Args:
        sm: Trained Slisemap solution.
        clusters: The number of clusters. Defaults to range(2, 10).

    Keyword Args:
        **kwargs: Additional arguments to `sklearn.KMeans`.

    Returns:
        The mean cluster matching.
    """
    from scipy.optimize import linear_sum_assignment

    Z = sm.get_Z()
    B = sm.get_B()
    if np.all(np.var(Z, 0) < 1e-8) or np.all(np.var(B, 0) < 1e-8):
        return np.nan  # Do not compare singular clusters
    if not np.all(np.isfinite(Z)) or not np.all(np.isfinite(B)):
        return np.nan
    if isinstance(clusters, int):
        clusters = range(clusters, clusters + 1)
    results = []
    for k in clusters:
        cl_B = KMeans(n_clusters=k, **kwargs).fit(B)
        cl_Z = KMeans(n_clusters=k, **kwargs).fit(Z)
        sets_B = [set(np.where(cl_B.labels_ == i)[0]) for i in range(k)]
        sets_Z = [set(np.where(cl_Z.labels_ == i)[0]) for i in range(k)]
        mat = np.zeros((k, k))
        for i, sB in enumerate(sets_B):
            for j, sZ in enumerate(sets_Z):
                mat[i, j] = len(sB.intersection(sZ)) / (len(sB.union(sZ)) + 1e-8)
        # Hungarian algorithm to find the best match between the clusterings
        rows, cols = linear_sum_assignment(mat, maximize=True)
        results.append(mat[rows, cols].mean())
    return nanmean(results)

cluster_purity(sm, clusters)

Evaluate a SLISEMAP solution by calculating how many items in the same cluster are neighbours.

Larger is better.

Parameters:

Name	Type	Description	Default
sm	Union[Slisemap, ToTensor]	Trained Slisemap solution or embedding matrix.	required
clusters	Union[ndarray, LongTensor]	Cluster ids.	required

Returns:

Type	Description
float	The mean number of items sharing cluster that are neighbours.

Source code in slisemap/metrics.py

def cluster_purity(
    sm: Union[Slisemap, ToTensor], clusters: Union[np.ndarray, torch.LongTensor]
) -> float:
    """Evaluate a SLISEMAP solution by calculating how many items in the same cluster are neighbours.

    Larger is better.

    Args:
        sm: Trained Slisemap solution _or_ embedding matrix.
        clusters: Cluster ids.

    Returns:
        The mean number of items sharing cluster that are neighbours.
    """
    try:
        Z = sm.get_Z(numpy=False)
    except AttributeError:
        Z = to_tensor(sm)
    if isinstance(clusters, np.ndarray):
        clusters = torch.as_tensor(clusters, device=Z.device)
    res = np.zeros(Z.shape[0])
    D = torch.cdist(Z, Z)
    for i in range(len(res)):
        mask = clusters[i] == clusters
        knn = euclidean_nearest_neighbours(D, i, mask.sum())
        res[i] = torch.sum(mask[knn]) / knn.shape[0]
    return nanmean(res)

kernel_purity(sm, clusters, epsilon=1.0, losses=False)

Evaluate a SLISEMAP solution by calculating how many neighbours are in the same cluster.

Larger is better.

Parameters:

Name	Type	Description	Default
sm	Slisemap	Trained Slisemap solution.	required
clusters	Union[ndarray, LongTensor]	Cluster ids.	required
epsilon	float	Treshold for being a neighbour (softmax(D) < epsilon/n). Defaults to 1.0.	1.0
losses	bool	Use losses instead of embedding distances. Defaults to False.	False

Returns:

Type	Description
float	The mean number of neighbours that are in the same cluster.

Source code in slisemap/metrics.py

def kernel_purity(
    sm: Slisemap,
    clusters: Union[np.ndarray, torch.LongTensor],
    epsilon: float = 1.0,
    losses: bool = False,
) -> float:
    """Evaluate a SLISEMAP solution by calculating how many neighbours are in the same cluster.

    Larger is better.

    Args:
        sm: Trained Slisemap solution.
        clusters: Cluster ids.
        epsilon: Treshold for being a neighbour (`softmax(D) < epsilon/n`). Defaults to 1.0.
        losses: Use losses instead of embedding distances. Defaults to False.

    Returns:
        The mean number of neighbours that are in the same cluster.
    """
    if isinstance(clusters, np.ndarray):
        clusters = torch.tensor(clusters)
    res = np.zeros(sm.n)
    D = sm.get_L(numpy=False) if losses else sm.get_D(numpy=False)
    for i in range(len(res)):
        mask = clusters[i] == clusters
        neig = kernel_neighbours(D, i, epsilon)
        res[i] = torch.sum(mask[neig]) / neig.numel()
    return nanmean(res)

recall(sm, epsilon_D=1.0, epsilon_L=1.0)

Evaluate a SLISEMAP solution by calculating the recall.

We define recall as the intersection between the loss and embedding neighbourhoods divided by the loss neighbourhood.

Larger is better.

Parameters:

Name	Type	Description	Default
sm	Slisemap	Trained Slisemap solution.	required
epsilon_D	float	Treshold for being an embedding neighbour (softmax(D) < epsilon/n). Defaults to 1.0.	1.0
epsilon_L	float	Treshold for being a loss neighbour (softmax(L) < epsilon/n). Defaults to 1.0.	1.0

Returns:

Type	Description
float	The mean recall.

Source code in slisemap/metrics.py

def recall(sm: Slisemap, epsilon_D: float = 1.0, epsilon_L: float = 1.0) -> float:
    """Evaluate a SLISEMAP solution by calculating the recall.

    We define recall as the intersection between the loss and embedding neighbourhoods divided by the loss neighbourhood.

    Larger is better.

    Args:
        sm: Trained Slisemap solution.
        epsilon_D: Treshold for being an embedding neighbour (`softmax(D) < epsilon/n`). Defaults to 1.0.
        epsilon_L: Treshold for being a loss neighbour (`softmax(L) < epsilon/n`). Defaults to 1.0.

    Returns:
        The mean recall.
    """
    res = np.zeros(sm.n)
    D = sm.get_D(numpy=False)
    L = sm.get_L(numpy=False)
    for i in range(len(res)):
        nL = kernel_neighbours(L, i, epsilon_L)
        if nL.numel() == 0:
            res[i] = np.nan
        else:
            nD = kernel_neighbours(D, i, epsilon_D)
            inter = np.intersect1d(tonp(nD), tonp(nL), True)
            res[i] = len(inter) / nL.numel()
    return nanmean(res)

precision(sm, epsilon_D=1.0, epsilon_L=1.0)

Evaluate a SLISEMAP solution by calculating the recall.

We define recall as the intersection between the loss and embedding neighbourhoods divided by the embedding neighbourhood.

Larger is better.

Parameters:

Name	Type	Description	Default
sm	Slisemap	Trained Slisemap solution.	required
epsilon_D	float	Treshold for being an embedding neighbour (softmax(D) < epsilon/n). Defaults to 1.0.	1.0
epsilon_L	float	Treshold for being a loss neighbour (softmax(L) < epsilon/n). Defaults to 1.0.	1.0

Returns:

Type	Description
float	The mean precision.

Source code in slisemap/metrics.py

def precision(sm: Slisemap, epsilon_D: float = 1.0, epsilon_L: float = 1.0) -> float:
    """Evaluate a SLISEMAP solution by calculating the recall.

    We define recall as the intersection between the loss and embedding neighbourhoods divided by the embedding neighbourhood.

    Larger is better.

    Args:
        sm: Trained Slisemap solution.
        epsilon_D: Treshold for being an embedding neighbour (`softmax(D) < epsilon/n`). Defaults to 1.0.
        epsilon_L: Treshold for being a loss neighbour (`softmax(L) < epsilon/n`). Defaults to 1.0.

    Returns:
        The mean precision.
    """
    res = np.zeros(sm.n)
    D = sm.get_D(numpy=False)
    L = sm.get_L(numpy=False)
    for i in range(len(res)):
        nD = kernel_neighbours(D, i, epsilon_D)
        if nD.numel() == 0:
            res[i] = np.nan
        else:
            nL = kernel_neighbours(L, i, epsilon_L)
            inter = np.intersect1d(tonp(nD), tonp(nL), True)
            res[i] = len(inter) / nD.numel()
    return nanmean(res)

relevance(sm, pred_fn, change)

Evaluate a SLISEMAP solution by calculating the relevance.

Smaller is better.

TODO: This does not (currently) work for multi-class predictions

Parameters:

Name	Type	Description	Default
sm	Slisemap	Trained Slisemap solution.	required
pred_fn	Callable	Function that gives y:s for new x:s (the "black box model").	required
change	float	How much should the prediction change?	required

Returns:

Type	Description
float	The mean number of mutated variables required to cause a large enough change in the prediction.

Source code in slisemap/metrics.py

def relevance(sm: Slisemap, pred_fn: Callable, change: float) -> float:
    """Evaluate a SLISEMAP solution by calculating the relevance.

    Smaller is better.

    TODO: This does not (currently) work for multi-class predictions

    Args:
        sm: Trained Slisemap solution.
        pred_fn: Function that gives y:s for new x:s (the "black box model").
        change: How much should the prediction change?

    Returns:
        The mean number of mutated variables required to cause a large enough change in the prediction.
    """
    rel = np.ones(sm.n) * sm.m
    B = sm.get_B(numpy=False)
    for i in range(len(rel)):
        b = B[i, :]
        x = sm._X[i, :]
        y = sm._Y[i, 0]
        xmax = torch.max(sm._X, 0)[0]
        xmin = torch.min(sm._X, 0)[0]
        xinc = torch.where(b > 0, xmax, xmin)
        xdec = torch.where(b < 0, xmax, xmin)
        babs = torch.abs(b)
        for i, bs in enumerate(torch.sort(babs)[0]):
            yinc = pred_fn(torch.where(babs >= bs, xinc, x))
            ydec = pred_fn(torch.where(babs >= bs, xdec, x))
            if yinc - y > change or y - ydec > change:
                rel[i] = i
                break
    return nanmean(rel)

accuracy(sm, X=None, Y=None, fidelity=True, optimise=False, fit_new=False, **kwargs)

Evaluate a SLISEMAP solution by checking how well the fitted models work on new points.

Parameters:

Name	Type	Description	Default
sm	Slisemap	Trained Slisemap solution.	required
X	Optional[ToTensor]	New data matrix (uses the training data if None). Defaults to None.	None
Y	Optional[ToTensor]	New target matrix (uses the training data if None). Defaults to None.	None
fidelity	bool	Return the mean local loss (fidelity) instead of the mean embedding weighted loss. Defaults to True.	True
optimise	bool	If fit_new, optimise the new points. Defaults to False.	False
fit_new	bool	Use [Slisemap.fit_new][slisemap.slisemap.Slisemap.fit_new] instead of [Slisemap.predict][slisemap.slisemap.Slisemap.predict] (if fidelity=True). Defaults to True.	False

Other Parameters:

Name	Type	Description
**kwargs	Any	Optional keyword arguments to Slisemap.predict.

Returns:

Type	Description
float	Mean loss for the new points.

Deprecated

1.6: fit_new, fidelity, optimise.

Source code in slisemap/metrics.py

def accuracy(
    sm: Slisemap,
    X: Optional[ToTensor] = None,
    Y: Optional[ToTensor] = None,
    fidelity: bool = True,
    optimise: bool = False,
    fit_new: bool = False,
    **kwargs: Any,
) -> float:
    """Evaluate a SLISEMAP solution by checking how well the fitted models work on new points.

    Args:
        sm: Trained Slisemap solution.
        X: New data matrix (uses the training data if None). Defaults to None.
        Y: New target matrix (uses the training data if None). Defaults to None.
        fidelity: Return the mean local loss (fidelity) instead of the mean embedding weighted loss. Defaults to True.
        optimise: If `fit_new`, optimise the new points. Defaults to False.
        fit_new: Use `[Slisemap.fit_new][slisemap.slisemap.Slisemap.fit_new]` instead of `[Slisemap.predict][slisemap.slisemap.Slisemap.predict]` (if `fidelity=True`). Defaults to True.

    Keyword Args:
        **kwargs: Optional keyword arguments to [Slisemap.predict][slisemap.slisemap.Slisemap.predict].

    Returns:
        Mean loss for the new points.

    Deprecated:
        1.6: `fit_new`, `fidelity`, `optimise`.
    """
    if X is None or Y is None:
        X = sm.get_X(intercept=False, numpy=False)
        Y = sm.get_Y(numpy=False)
    if fit_new:
        _deprecated("accuracy(..., fit_new=True)")
    if not fidelity:
        _deprecated("accuracy(..., fidelity=False)")
    if optimise:
        _deprecated("accuracy(..., optimise=True)")
    if not fidelity:
        loss = sm.fit_new(X, Y, loss=True, optimise=optimise, numpy=False, **kwargs)[2]
        return loss.mean().cpu().item()
    else:
        X = sm._as_new_X(X)
        Y = sm._as_new_Y(Y, X.shape[0])
        if fit_new:
            B, _ = sm.fit_new(
                X, Y, loss=False, optimise=optimise, numpy=False, **kwargs
            )
            return sm.local_loss(sm.predict(X, B, numpy=False), Y).mean().cpu().item()
        else:
            loss = sm.local_loss(sm.predict(X, **kwargs, numpy=False), Y)
            assert loss.shape == X.shape[:1]
            return loss.mean().cpu().item()