Source code for compiam.melody.pattern.sancara_search.extraction.self_sim
import os
import shutil
import skimage
import matplotlib.pyplot as plt
import numpy as np
from scipy.spatial.distance import pdist, squareform
from scipy.signal import convolve2d
from compiam.melody.pattern.sancara_search.extraction.img import (
remove_diagonal,
convolve_array,
binarize,
diagonal_gaussian,
apply_bin_op,
make_symmetric,
edges_to_contours,
)
from compiam.utils import create_if_not_exists, run_or_cache
from compiam.melody.pattern.sancara_search.extraction.sequence import (
convert_seqs_to_timestep,
remove_below_length,
add_center_to_mask,
)
from compiam.melody.pattern.sancara_search.extraction.evaluation import (
evaluate,
get_coverage,
get_grouping_accuracy,
)
from compiam.melody.pattern.sancara_search.extraction.visualisation import (
plot_all_sequences,
plot_pitch,
flush_matplotlib,
)
from compiam.melody.pattern.sancara_search.extraction.io import (
load_sim_matrix,
write_all_sequence_audio,
load_yaml,
load_pkl,
write_pkl,
)
from compiam.melody.pattern.sancara_search.extraction.pitch import (
cents_to_pitch,
pitch_seq_to_cents,
pitch_to_cents,
get_timeseries,
interpolate_below_length,
)
from compiam.melody.pattern.sancara_search.extraction.segments import (
line_through_points,
trim_silence,
break_all_segments,
remove_short,
extend_segments,
join_all_segments,
extend_groups_to_mask,
group_segments,
group_overlapping,
group_by_distance,
trim_silence,
)
from compiam.utils import get_logger
logger = get_logger(__name__)
[docs]def self_similarity(
features, exclusion_mask=None, timestep=None, hop_length=None, sr=44100
):
"""
Compute self similarity matrix between features in <features>. If an <exclusion_mask>
is passed. Regions corresponding to that mask will be excluded from the computation and
the returned matrix will correspond only to those regions marked as 0 in the mask.
:param features: array of features extracted from audio
:type features: np.ndarray
:param exclusion_mask: array of 0 and 1, should be masked or not? [Optional]
:type exclusion_mask: np.ndarray or None
:param timestep: time in seconds between elements of <exclusion_mask>
Only required if <exclusion_mask> is passed
:type timestep: float or None
:param hop_length: number of audio frames corresponding to one element in <features>
Only required if <exclusion_mask> is passed
:type hop_length: int or None
:param sr: sampling rate of audio corresponding to <features>
Only required if <exclusion_mask> is passed
:type sr: int or None
:returns:
if exclusion mask is passed return...
matrix - self similarity matrix
orig_sparse_lookup - dict of {index in orig array: index of same element in sparse array}
sparse_orig_lookup - dict of {index in sparse array: index of same element in orig array}
boundaries_orig - list of boundaries between wanted and unwanted regions in orig array
boundaries_sparse - list of boundaries between formally separated wanted regions in sparse array
else return
matrix - self similarity matrix
:rtype: (np.ndarray, dict, dict, list, list) or np.ndarray
"""
em = not (exclusion_mask is None)
if em:
assert all(
[not timestep is None, not hop_length is None, not sr is None]
), "To use exclusion mask, <timestep>, <hop_length> and <sr> must also be passed"
# Deal with masking if any
if em:
features_mask = convert_mask(features, exclusion_mask, timestep, hop_length, sr)
(
orig_sparse_lookup,
sparse_orig_lookup,
boundaries_orig,
boundaries_sparse,
) = get_conversion_mappings(features_mask)
else:
orig_sparse_lookup = None
sparse_orig_lookup = None
boundaries_orig = None
boundaries_sparse = None
# Indices we want to keep
good_ix = np.where(features_mask == 0)[0]
# Compute self similarity
sparse_features = features[good_ix]
matrix = create_ss_matrix(sparse_features)
# Normalise self similarity matrix
matrix_norm = normalise_self_sim(matrix)
if em:
return (
matrix_norm,
orig_sparse_lookup,
sparse_orig_lookup,
boundaries_orig,
boundaries_sparse,
)
else:
return matrix_norm
[docs]def convert_mask(arr, mask, timestep, hop_length, sr):
"""
Get mask of excluded regions in the same dimension as array, <arr>
:param arr: array corresponding to features extracted from audio
:type arr: np.ndarray
:param mask: Mask indicating whether element should be excluded (different dimensions to <arr>)
:type mask: np.ndarray
:param timestep: time in seconds between each element in <mask>
:type timestep: float
:param hop_length: how many frames of audio correspond to each element in <arr>
:type hop_length: int
:param sr: sampling rate of audio from which <arr> was computed
:type sr: int
:returns: array of mask values equal in length to one dimension of <arr> - 0/1 is masked?
:rtype: np.ndarray
"""
# get mask of silent and stable regions
new_mask = []
for i in range(arr.shape[0]):
# what is the time at this element of arr?
t = (i + 1) * hop_length / sr
# find index in mask
ix = round(t / timestep)
# append mask value for this point
new_mask.append(mask[ix])
return np.array(new_mask)
[docs]def get_conversion_mappings(mask):
"""
Before reducing an array to only include elements that do not correspond
to <mask>. We want to record the relationship between the new (sparse) array
index and the old (orig) array.
:param mask: mask of 0/1 - is element to be excluded
:param type: np.ndarray
:returns:
orig_sparse_lookup - dict of {index in orig array: index of same element in sparse array}
sparse_orig_lookup - dict of {index in sparse array: index of same element in orig array}
boundaries_orig - list of boundaries between wanted and unwanted regions in orig array
boundaries_sparse - list of boundaries between formally separated wanted regions in sparse array
:rtype: (dict, dict, list, list)
"""
# Indices we want to keep
good_ix = np.where(mask == 0)[0]
# Mapping between old and new indices
orig_sparse_lookup = {g: s for s, g in enumerate(good_ix)}
sparse_orig_lookup = {s: g for g, s in orig_sparse_lookup.items()}
# Indices corresponding to boundaries
# between wanted and unwanted regions
# in original array
boundaries_orig = []
for i in range(1, len(mask)):
curr = mask[i]
prev = mask[i - 1]
if curr == 0 and prev == 1:
boundaries_orig.append(i)
elif curr == 1 and prev == 0:
boundaries_orig.append(i - 1)
# Boundaries corresponding to newly joined
# regions in sparse array
boundaries_sparse = np.array([orig_sparse_lookup[i] for i in boundaries_orig])
# Boundaries contain two consecutive boundaries for each gap
# but not if the excluded region leads to the end of the track
red_boundaries_sparse = []
boundaries_mask = [0] * len(boundaries_sparse)
for i in range(len(boundaries_sparse)):
if i == 0:
red_boundaries_sparse.append(boundaries_sparse[i])
boundaries_mask[i] = 1
if boundaries_mask[i] == 1:
continue
curr = boundaries_sparse[i]
prev = boundaries_sparse[i - 1]
if curr - prev == 1:
red_boundaries_sparse.append(prev)
boundaries_mask[i] = 1
boundaries_mask[i - 1] = 1
else:
red_boundaries_sparse.append(curr)
boundaries_mask[i] = 1
boundaries_sparse = np.array(sorted(list(set(red_boundaries_sparse))))
return orig_sparse_lookup, sparse_orig_lookup, boundaries_orig, boundaries_sparse
[docs]def create_ss_matrix(feats, mode="cosine"):
"""
Compute self similarity matrix between features in <feats>
using distance measure, <mode>
:param feats: array of features
:type feats: np.ndarray
:param mode: name of distance measure (recognised by scipy.spatial.distance)
:type mode: str
:returns: self similarity matrix
:rtype: np.ndarray
"""
matrix = squareform(pdist(np.vstack(feats.detach().numpy()), metric=mode))
return matrix
[docs]def normalise_self_sim(matrix):
"""
Normalise self similarity matrix:
invert and convolve
:param matrix: self similarity matrix
:type matrix: np.ndarray
:returns: matrix normalized, same dimensions
:rtype: np.ndarray
"""
matrix = 1 / (matrix + 1e-6)
for k in range(-8, 9):
eye = 1 - np.eye(*matrix.shape, k=k)
matrix = matrix * eye
flength = 10
ey = np.eye(flength) + np.eye(flength, k=1) + np.eye(flength, k=-1)
matrix = convolve2d(matrix, ey, mode="same")
diag_mask = np.ones(matrix.shape)
diag_mask = (diag_mask - np.diag(np.ones(matrix.shape[0]))).astype(np.bool)
mat_min = np.min(matrix[diag_mask])
mat_max = np.max(matrix[diag_mask])
matrix[~diag_mask] = 0
matrix = zero_normalise(matrix)
return matrix
def zero_normalise(matrix):
matrix = matrix - matrix.min()
matrix = matrix / (matrix.max() + 1e-8)
return matrix
[docs]def get_report_paths(out_dir):
"""
Get dictionary of fielpaths relevant to progress plots
in extract_segments()
:params out_dir: directory path to save plots in
:type out_dir: str
:returns: dict of filepaths
:rtype: dict
"""
sim = os.path.join(out_dir, "1_simsave.png") if out_dir else None
conv = os.path.join(out_dir, "2_conv.png") if out_dir else None
binar = os.path.join(out_dir, "3_binary.png") if out_dir else None
diag = os.path.join(out_dir, "4_diag.png") if out_dir else None
gauss = os.path.join(out_dir, "5_gauss.png") if out_dir else None
cont = os.path.join(out_dir, "6_cont.png") if out_dir else None
close = os.path.join(out_dir, "6_close.png") if out_dir else None
binop = os.path.join(out_dir, "7_binop.png") if out_dir else None
return {
"sim": sim,
"conv": conv,
"binar": binar,
"diag": diag,
"gauss": gauss,
"cont": cont,
"close": close,
"binop": binop,
}
[docs]def save_matrix(X, filepath):
"""
if <filepath>, save <X> at <filepath>
:param X: matrix to save
:type X: np.ndarray
:param filepath: filepath
:type filepath: str or None
"""
if filepath:
create_if_not_exists(filepath)
skimage.io.imsave(filepath, X)
[docs]def get_param_hash_filepath(out_dir, *params):
"""
Build filepath by creating string of input <params>
in <out_dir>
:params out_dir: directory path
:type out_dir: str
:params params: arguments, any type
:type params: arguments, any type
:returns: filepath unique to input params in <out_dir>
:rtype: str
"""
if out_dir is None:
return None
param_hash = str(params)
return os.path.join(out_dir, f"{param_hash}.pkl")
[docs]def sparse_to_original(all_segments, boundaries_sparse, lookup):
"""
Convert indices corresponding to segments in <all_segments>
to their non-sparse form using mapping in <lookup>
:param all_segments: list of segments, [(x0,y0),(x1,y1),...]
:type all_segments: list
:param boundaries_sparse: list indices in sparse array corresponding to splits in original array
:type boundaries_sparse: list
:param lookup: dict of sparse_index:non-sparse index
:type lookup: dict
:returns: <all_segments> with indices replaced according to lookup
:rtype: list
"""
boundaries_sparse = [x for x in boundaries_sparse if x != 0]
all_segments_scaled_x = []
for seg in all_segments:
((x0, y0), (x1, y1)) = seg
get_x, get_y = line_through_points(x0, y0, x1, y1)
boundaries_in_x = sorted([i for i in boundaries_sparse if i >= x0 and i <= x1])
current_x0 = x0
if boundaries_in_x:
for b in boundaries_in_x:
x0_ = current_x0
x1_ = b - 1
y0_ = int(get_y(x0_))
y1_ = int(get_y(x1_))
all_segments_scaled_x.append(((x0_, y0_), (x1_, y1_)))
current_x0 = b + 1
if current_x0 > x1:
x0_ = current_x0
x1_ = x1
y0_ = int(get_y(x0_))
y1_ = int(get_y(x1_))
all_segments_scaled_x.append(((x0_, y0_), (x1_, y1_)))
else:
all_segments_scaled_x.append(((x0, y0), (x1, y1)))
all_segments_scaled_x_reduced = remove_short(all_segments_scaled_x, 1)
all_segments_scaled = []
for seg in all_segments_scaled_x_reduced:
((x0, y0), (x1, y1)) = seg
get_x, get_y = line_through_points(x0, y0, x1, y1)
boundaries_in_y = sorted([i for i in boundaries_sparse if i >= y0 and i <= y1])
current_y0 = y0
if boundaries_in_y:
for b in boundaries_in_y:
y0_ = current_y0
y1_ = b - 1
x0_ = int(get_x(y0_))
x1_ = int(get_x(y1_))
all_segments_scaled.append(((x0_, y0_), (x1_, y1_)))
current_y0 = b + 1
if current_y0 < y1:
y0_ = current_y0
y1_ = y1
x0_ = int(get_x(y0_))
x1_ = int(get_x(y1_))
all_segments_scaled.append(((x0_, y0_), (x1_, y1_)))
else:
all_segments_scaled.append(((x0, y0), (x1, y1)))
all_segments_scaled_reduced = remove_short(all_segments_scaled, 1)
all_segments_converted = []
de = 0
for i, seg in enumerate(all_segments_scaled_reduced):
((x0, y0), (x1, y1)) = seg
while (
(x0 in boundaries_sparse)
or (x1 in boundaries_sparse)
or (y0 in boundaries_sparse)
or (y1 in boundaries_sparse)
):
if x0 in boundaries_sparse:
get_x, get_y = line_through_points(x0, y0, x1, y1)
x0 = x0 + 1
x1 = x1
y0 = round(get_y(x0))
y1 = round(get_y(x1))
if x1 in boundaries_sparse:
get_x, get_y = line_through_points(x0, y0, x1, y1)
x0 = x0
x1 = x1 - 1
y0 = round(get_y(x0))
y1 = round(get_y(x1))
if y0 in boundaries_sparse:
get_x, get_y = line_through_points(x0, y0, x1, y1)
y0 = y0 + 1
y1 = y1
x0 = round(get_x(y0))
x1 = round(get_x(y1))
if y1 in boundaries_sparse:
get_x, get_y = line_through_points(x0, y0, x1, y1)
y0 = y0
y1 = y1 - 1
x0 = round(get_x(y0))
x1 = round(get_x(y1))
x0_ = lookup[x0 + de]
y0_ = lookup[y0 + de]
x1_ = lookup[x1 + de]
y1_ = lookup[y1 + de]
all_segments_converted.append(((x0_, y0_), (x1_, y1_)))
return all_segments_converted
def zero_norm_matrix(X):
X = X - X.min()
X /= X.max() + 1e-8
return X
[docs]class segmentExtractor:
"""
Manipulate and extract segments from self similarity matrix
"""
def __init__(self, X, window_size, sr=44100, cache_dir=None):
self.X = X
self.shape = X.shape
self.window_size = window_size
self.sr = sr
self.cache_dir = cache_dir
# initialise arrays
self.X_conv = None
self.X_bin = None
self.X_diag = None
self.X_gauss = None
self.X_cont = None
self.X_sym = None
self.X_fill = None
self.X_binop = None
self.X_proc = None
# Initialize status
self.emphasized = False
self.extracted = False
# cache paths
self._cache_base = os.path.join(cache_dir, "{0}", "") if cache_dir else None
self._segment_convolve_cache = (
self._cache_base.format("convolve") if cache_dir else None
)
self._segment_cache = self._cache_base.format("segments") if cache_dir else None
self._segment_ext_cache = (
self._cache_base.format("segments_extended") if cache_dir else None
)
self._segment_join_cache = (
self._cache_base.format("segments_joined") if cache_dir else None
)
self._segment_group_cache = (
self._cache_base.format("segments_groups") if cache_dir else None
)
self._segment_group_overlap_cache = (
self._cache_base.format("segment_overlap") if cache_dir else None
)
[docs] def emphasize_diagonals(
self,
bin_thresh=0.025,
gauss_sigma=None,
cont_thresh=None,
etc_kernel_size=10,
binop_dim=3,
image_report=False,
verbose=False,
):
"""
From self similarity matrix, self.X. Emphasize diagonals using a series
of image processing steps.
:param bin_thresh: Threshold for binarization of self similarity array.
Values below this threshold are set to 0 (not significant), those
above or equal too are set to 1. Very important parameter
:type bin_thresh: float
:param gauss_sigma: If not None, sigma for diagonal gaussian blur to
apply to matrix
:type gauss_sigma: float or None
:param cont_thresh: Only applicable if <gauss_sigma>. This binary
threshold isreapplied after gaussian blur to ensure matrix of
0 and 1. if None, equal to <bin_thresh>
:type cont_thresh: float or None
:param etc_kernel_size: Kernel size for morphological closing
:type etc_kernel_size: int
:param binop_dim: square dimension of binary opening structure
(square matrix of zeros with 1 across the diagonal)
:type binop_dim: int
:param image_report: str corresponding to folder to save progress images in.
:type image_report: None
:param verbose: Display progress
:type verbose: bool
:returns: list of segments in the form [((x0,y0),(x1,y1)),..]
:rtype: list
"""
self.bin_thresh = bin_thresh
self.gauss_sigma = gauss_sigma
self.etc_kernel_size = etc_kernel_size
self.binop_dim = binop_dim
self.image_report = image_report
self.report_fns = get_report_paths(image_report)
# Save original self similarity matrix
save_matrix(self.X, self.report_fns["sim"])
####################
## Convert params ##
####################
self.cont_thresh = self.bin_thresh if not cont_thresh else cont_thresh
#########################
## Emphasize Diagonals ##
#########################
if verbose:
logger.info("Convolving similarity matrix")
self.conv_path = get_param_hash_filepath(
self._segment_convolve_cache,
)
self.X_conv = run_or_cache(convolve_array, [self.X], self.conv_path)
self.X_conv = zero_norm_matrix(self.X_conv)
save_matrix(self.X_conv, self.report_fns["conv"])
if verbose:
logger.info("Binarizing convolved array")
self.X_bin = binarize(self.X_conv, self.bin_thresh)
save_matrix(self.X_bin, self.report_fns["binar"])
if verbose:
logger.info("Removing diagonal")
self.X_diag = remove_diagonal(self.X_bin)
save_matrix(self.X_diag, self.report_fns["diag"])
if self.gauss_sigma:
if verbose:
logger.info("Applying diagonal gaussian filter")
self.X_gauss = diagonal_gaussian(self.X_diag, self.gauss_sigma)
save_matrix(self.X_gauss, self.report_fns["gauss"])
if verbose:
logger.info("Binarize gaussian blurred similarity matrix")
self.X_cont = binarize(self.X_gauss, self.cont_thresh)
save_matrix(self.X_cont, self.report_fns["cont"])
else:
self.X_gauss = self.X_diag
self.X_cont = self.X_gauss
if verbose:
logger.info("Ensuring symmetry between upper and lower triangle in array")
self.X_sym = make_symmetric(self.X_cont)
if verbose:
logger.info("Identifying and isolating regions between edges")
self.X_fill = edges_to_contours(self.X_sym, self.etc_kernel_size)
save_matrix(self.X_fill, self.report_fns["close"])
if verbose:
logger.info(
"Cleaning isolated non-directional regions using morphological opening"
)
self.X_binop = apply_bin_op(self.X_fill, self.binop_dim)
if verbose:
logger.info("Ensuring symmetry between upper and lower triangle in array")
self.X_proc = make_symmetric(self.X_binop)
save_matrix(self.X_proc, self.report_fns["binop"])
self.emphasized = True
return self.X_proc
[docs] def extract_segments(
self,
etc_kernel_size=10,
binop_dim=3,
perc_tail=0.5,
bin_thresh_segment=None,
min_diff_trav=0.5,
min_pattern_length_seconds=2,
boundaries=None,
lookup=None,
break_mask=None,
timestep=None,
verbose=False,
):
"""
From self similarity matrix, <self.X_proc>. Return list of segments,
each corresponding to two regions of the input axis.
:param etc_kernel_size: Kernel size for morphological closing
:type etc_kernel_size: int
:param binop_dim: square dimension of binary opening structure
(square matrix of zeros with 1 across the diagonal)
:type binop_dim: int
:param perc_tail: Percentage either size of a segment along its
trajectory considered for lower threshold for significance
:type perc_tail: int
:param bin_thresh_segment: Reduced <bin_thresh> threshold for
areas neighbouring identified segments. If None, use 0.5*<bin_thresh>
:type bin_thresh_segment: float
:param min_diff_trav: Min time difference in seconds between
two segments for them to be joined to one.
:type min_diff_trav: float
:param min_pattern_length_seconds: Minimum length of any
returned pattern in seconds
:type min_pattern_length_seconds: float
:param boundaries: list of boundaries in <X> corresponding
to breaks due to sparsity
:type boundaries: list or None
:param lookup: Lookup of sparse index (in X): non-sparse index
:type lookup: dict
:param break_mask: any segment that traverses a non-zero element
in <break_mask> is broken into two according to this non-zero value
:type break_mask: array
:param timestep: Time in seconds between each element in <break_mask>
:type timestep: float or None
:param verbose: Display progress
:type verbose: bool
:returns: list of segments in the form [((x0,y0),(x1,y1)),..]
:rtype: list
"""
############
## Checks ##
############
if not self.emphasized:
raise Exception(
"Please run self.emphasize_diagonals before attempting to extract segments."
)
if break_mask is not None:
assert (
timestep is not None
), "If <break_mask> is passed, timestep too must be specified"
if boundaries is not None:
assert (
lookup is not None
), "If <boundaries> is passed, lookup too must be specified"
############
## Params ##
############
self.min_diff_trav = min_diff_trav
# in terms of elements matrix elements
self.min_length_cqt = min_pattern_length_seconds * self.sr / self.window_size
# translate min_diff_trav to corresponding diagonal distance
self.min_diff_trav_hyp = (2 * min_diff_trav**2) ** 0.5
self.min_diff_trav_seq = self.min_diff_trav_hyp * self.sr / self.window_size
self.bin_thresh_segment = (
self.bin_thresh * 0.5 if not bin_thresh_segment else bin_thresh_segment
)
self.perc_tail = perc_tail
self.min_pattern_length_seconds = min_pattern_length_seconds
self.boundaries = boundaries
self.lookup = lookup
self.break_mask = break_mask
self.timestep = timestep
######################
## Extract segments ##
######################
if verbose:
logger.info("Extracting segments")
self.seg_path = get_param_hash_filepath(
self._segment_cache,
self.bin_thresh,
self.gauss_sigma,
self.cont_thresh,
self.etc_kernel_size,
self.binop_dim,
)
self.all_segments = run_or_cache(
segments_from_matrix, [self.X_bin], self.seg_path
)
if verbose:
logger.info("Extending Segments")
self.seg_ext_path = get_param_hash_filepath(
self._segment_ext_cache,
self.bin_thresh,
self.gauss_sigma,
self.cont_thresh,
self.etc_kernel_size,
self.binop_dim,
self.perc_tail,
self.bin_thresh_segment,
)
args = [
self.all_segments,
self.X_sym,
self.X_conv,
self.perc_tail,
self.bin_thresh_segment,
]
self.all_segments_extended = run_or_cache(
extend_segments, args, self.seg_ext_path
)
if verbose:
logger.info(f" {len(self.all_segments_extended)} extended segments...")
self.all_segments_extended_reduced = remove_short(self.all_segments_extended, 1)
if verbose:
logger.info("Converting sparse segment indices to original")
if not self.boundaries is None:
self.all_segments_converted = sparse_to_original(
self.all_segments_extended_reduced, self.boundaries, self.lookup
)
else:
self.all_segments_converted = self.all_segments_extended_reduced
if verbose:
logger.info("Joining segments that are sufficiently close")
self.seg_join_path = get_param_hash_filepath(
self._segment_join_cache,
self.bin_thresh,
self.gauss_sigma,
self.cont_thresh,
self.etc_kernel_size,
self.binop_dim,
self.perc_tail,
self.bin_thresh_segment,
self.min_diff_trav_seq,
)
args = [self.all_segments_converted, self.min_diff_trav_seq]
self.all_segments_joined = run_or_cache(
join_all_segments,
[self.all_segments_converted, self.min_diff_trav_seq],
self.seg_join_path,
)
if verbose:
logger.info(f" {len(self.all_segments_joined)} joined segments...")
if verbose:
logger.info("Breaking segments with silent/stable regions")
if not self.break_mask is None:
self.all_broken_segments = break_all_segments(
self.all_segments_joined,
self.break_mask,
self.window_size,
self.sr,
self.timestep,
)
else:
self.all_broken_segments = self.all_segments_joined
if verbose:
logger.info(f" {len(self.all_broken_segments)} broken segments...")
if verbose:
logger.info("Reducing Segments")
self.all_segments_reduced = remove_short(
self.all_broken_segments, self.min_length_cqt
)
if verbose:
logger.info(
f" {len(self.all_segments_reduced)} segments above minimum length of {self.min_pattern_length_seconds}s..."
)
self.extracted = True
return self.all_segments_reduced
def group_segments(
self,
all_segments,
break_mask,
pitch,
ext_mask_tol=0.5,
match_tol=1,
dupl_perc_overlap_inter=0.9,
dupl_perc_overlap_intra=0.55,
group_len_var=1.0,
n_dtw=10,
thresh_dtw=10,
thresh_cos=None,
min_pattern_length_seconds=2,
min_in_group=2,
verbose=False,
):
############
## Params ##
############
self.pitch = pitch
break_mask = break_mask
self.ext_mask_tol = ext_mask_tol
self.match_tol = match_tol
self.dupl_perc_overlap_inter = dupl_perc_overlap_inter
self.dupl_perc_overlap_intra = dupl_perc_overlap_intra
self.group_len_var = group_len_var
self.n_dtw = n_dtw
self.thresh_dtw = thresh_dtw
self.thresh_cos = thresh_cos
self.min_pattern_length_seconds = min_pattern_length_seconds
self.min_in_group = min_in_group
if verbose:
logger.info("Identifying Segment Groups")
self.group_path = get_param_hash_filepath(
self._segment_group_cache,
self.bin_thresh,
self.gauss_sigma,
self.cont_thresh,
self.etc_kernel_size,
self.binop_dim,
self.perc_tail,
self.bin_thresh_segment,
self.min_diff_trav_seq,
self.min_length_cqt,
self.match_tol,
)
args = [
all_segments,
self.min_length_cqt,
self.match_tol,
break_mask,
self.window_size,
self.timestep,
self.sr,
self.pitch,
]
all_groups = run_or_cache(group_segments, args, self.group_path)
if verbose:
logger.info("Extending segments to silence/stability")
all_groups_ext = extend_groups_to_mask(
all_groups,
break_mask,
self.window_size,
self.sr,
self.timestep,
toler=self.ext_mask_tol,
)
if verbose:
logger.info("Trimming Silence")
all_groups_sil = trim_silence(
all_groups_ext, self.pitch, self.window_size, self.sr, self.timestep
)
all_groups_sil = [[(i, j) for i, j in x if j > i] for x in all_groups_sil]
all_groups_sil = [
remove_group_duplicates(g, self.dupl_perc_overlap_intra)
for g in all_groups_sil
]
if verbose:
logger.info("Identifying Segment Groups")
self.segment_overlap_path = get_param_hash_filepath(
self._segment_group_overlap_cache,
self.bin_thresh,
self.gauss_sigma,
self.cont_thresh,
self.etc_kernel_size,
self.binop_dim,
self.perc_tail,
self.bin_thresh_segment,
self.min_diff_trav_seq,
self.min_length_cqt,
self.match_tol,
self.dupl_perc_overlap_inter,
self.group_len_var,
)
all_groups = run_or_cache(
group_overlapping,
[all_groups_sil, self.dupl_perc_overlap_inter, self.group_len_var],
self.segment_overlap_path,
)
if self.thresh_dtw:
if verbose:
logger.info("Joining geometrically close groups using pitch tracks")
all_groups_dtw = group_by_distance(
all_groups,
self.pitch,
self.n_dtw,
self.thresh_dtw,
self.thresh_cos,
self.group_len_var,
self.window_size,
self.sr,
self.timestep,
)
if verbose:
logger.info(f" {len(all_groups_dtw)} groups after join...")
else:
all_groups_dtw = all_groups
# all_groups_over = group_overlapping(all_groups_dtw, 0.1, group_len_var)
all_groups_rgd = [
remove_group_duplicates(g, self.dupl_perc_overlap_intra)
for g in all_groups_dtw
]
if verbose:
logger.info("Grouping overlapping")
all_groups_dov = group_overlapping(
all_groups_rgd, self.dupl_perc_overlap_inter, self.group_len_var
)
if verbose:
logger.info(f" {len(all_groups_dov)} groups after join...")
if verbose:
logger.info("Extending to mask")
all_groups_extdov = extend_groups_to_mask(
all_groups_dov,
break_mask,
self.window_size,
self.sr,
self.timestep,
toler=self.ext_mask_tol,
)
if verbose:
logger.info("Trimming Silence")
all_groups_ts = trim_silence(
all_groups_extdov, self.pitch, self.window_size, self.sr, self.timestep
)
all_groups_final = [
remove_group_duplicates(g, self.dupl_perc_overlap_intra)
for g in all_groups_ts
]
if verbose:
logger.info("Convert sequences to pitch track timesteps")
starts_seq, lengths_seq = convert_seqs_to_timestep(
all_groups_final, self.window_size, self.sr, self.timestep
)
if verbose:
logger.info("Applying exclusion functions")
starts_seq_exc, lengths_seq_exc = remove_below_length(
starts_seq, lengths_seq, self.timestep, self.min_pattern_length_seconds
)
starts = [p for p in starts_seq_exc if len(p) >= self.min_in_group]
lengths = [p for p in lengths_seq_exc if len(p) >= self.min_in_group]
starts_sec = [[x * self.timestep for x in p] for p in starts]
lengths_sec = [[x * self.timestep for x in l] for l in lengths]
return starts, lengths
def display_matrix(self, X, title=None, title_size=9, figsize=(3, 3)):
if not isinstance(X, np.ndarray):
raise ValueError("X must be a 2d numpy array")
fig, ax = plt.subplots(figsize=figsize)
if title:
plt.title(title, fontsize=title_size)
ax.imshow(X, interpolation="nearest")
plt.axis("off")
plt.tight_layout()
plt.show()
def display_all_matrices(self, title_size=9, figsize=(3, 3)):
if not self.emphasized:
raise Exception(
"Please run self.emphasize_diagonals before attempting to extract segments."
)
self.display_matrix(
self.X, "Self Similarity", title_size=title_size, figsize=figsize
)
self.display_matrix(
self.X_conv, "Convolved", title_size=title_size, figsize=figsize
)
self.display_matrix(
self.X_diag,
f"Binarized (threshold={self.bin_thresh})",
title_size=title_size,
figsize=figsize,
)
if self.gauss_sigma:
self.display_matrix(
self.X_gauss,
f"Diagonal gaussian blur (sigma={self.gauss_sigma})",
title_size=title_size,
figsize=figsize,
)
self.display_matrix(
self.X_cont,
f"Gaussian binarized (threshold={self.bin_thresh_segment})",
title_size=title_size,
figsize=figsize,
)
self.display_matrix(
self.X_fill,
f"Morphological opening (kernel size={self.etc_kernel_size})",
title_size=title_size,
figsize=figsize,
)
self.display_matrix(
self.X_binop,
f"Morphological closing (square dimension={self.binop_dim}",
title_size=title_size,
figsize=figsize,
)
self.display_matrix(
self.X_proc, "Final Matrix", title_size=title_size, figsize=figsize
)
def print_steps(self):
logger.info("Current Status")
logger.info("--------------")
logger.info(f"Input matrix of shape: {self.shape}")
logger.info(f"Windows size: {self.window_size}")
logger.info(f"Sampling rate of original audio: {self.sr}\n")
logger.info("Segment extraction")
logger.info("------------------")
if self.emphasized:
logger.info("Convolved matrix available at self.X_conv")
logger.info(
f"Binarized matrix available at self.X_bin (threshold={self.bin_thresh})"
)
if self.gauss_sigma:
logger.info(
f"Gaussian smoothed matrix available at self.X_gauss (sigma={self.gauss_sigma})"
)
else:
logger.info(f"No gaussian smoothing was applied")
logger.info(
f"Morphologically closed matrix available at self.X_fill (kernel size={self.etc_kernel_size})"
)
logger.info(
f"Morphologically opened matrix available at self.X_binop (square dimension of binary opening structure={self.binop_dim})"
)
logger.info(
"Final matrix after all steps applied, available at self.X_proc"
)
else:
logger.info(
"No processes have been applied to the input matrix (see self.emphasize_diagonals)"
)
if self.extracted:
logger.info("")
else:
logger.info("No segments have been extracted (see self.extract_segments)")
def cache_paths(self):
return {
"convolved": self.conv_path,
"extracted_segments": self.seg_path,
"extended_segments": self.seg_ext_path,
"joined_segments": self.seg_join_path,
}
def clear_cache(self):
shutil.rmtree(self.cache_dir)
def __repr__(self):
return f"segmentExtractor(X={self.shape}, window_size={self.window_size}, sr={self.sr}, cache_dir={self.cache_dir})"