From ef9b6a41bc2f64981b8adba028a277c97810a067 Mon Sep 17 00:00:00 2001 From: flixha Date: Fri, 2 Dec 2022 16:46:43 +0100 Subject: [PATCH 1/9] fix merge conflict --- eqcorrscan/core/match_filter/template.py | 32 ++++++++++++++++++++++++ eqcorrscan/core/match_filter/tribe.py | 6 +++-- eqcorrscan/tests/match_filter_test.py | 13 ++++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/eqcorrscan/core/match_filter/template.py b/eqcorrscan/core/match_filter/template.py index 8b792de90..0cd78607c 100644 --- a/eqcorrscan/core/match_filter/template.py +++ b/eqcorrscan/core/match_filter/template.py @@ -707,6 +707,38 @@ def group_templates(templates): return template_groups +def quick_group_templates(templates): + """ + Group templates into sets of similarly processed templates. + + :type templates: List of Tribe of Templates + :return: List of Lists of Templates. + """ + # Get the template's processing parameters and a unique list of + # parameter-combinations + processing_tuples = [ + tuple( + [value for key, value in template.__dict__.items() + if key not in ['name', 'st', 'prepick', 'event', 'template_info']] + ) + for template in templates] + # Convert to numpy array to efficiently search for rows with same values + processing_array = np.array(processing_tuples) + uniq_processing_parameters = sorted(list(set(processing_tuples))) + # sort templates into groups + template_groups = [] + for parameter_combination in uniq_processing_parameters: + # find indices of rows with same parameters + template_indices_for_group = np.where( + (processing_array == np.array(parameter_combination)).all(axis=1)) + new_group = list() + for template_index in template_indices_for_group[0]: + # use indices to sort templates into groups + new_group.append(templates[int(template_index)]) + template_groups.append(new_group) + return template_groups + + if __name__ == "__main__": import doctest diff --git a/eqcorrscan/core/match_filter/tribe.py b/eqcorrscan/core/match_filter/tribe.py index f03bf77a4..dfdff712f 100644 --- a/eqcorrscan/core/match_filter/tribe.py +++ b/eqcorrscan/core/match_filter/tribe.py @@ -24,7 +24,8 @@ from obspy import Catalog, Stream, read, read_events from obspy.core.event import Comment, CreationInfo -from eqcorrscan.core.match_filter.template import Template, group_templates +from eqcorrscan.core.match_filter.template import ( + Template, group_templates, quick_group_templates) from eqcorrscan.core.match_filter.party import Party from eqcorrscan.core.match_filter.helpers import ( _safemembers, _par_read, get_waveform_client) @@ -584,7 +585,8 @@ def detect(self, stream, threshold, threshold_type, trig_int, plot=False, length is the number of channels within this template. """ party = Party() - template_groups = group_templates(self.templates) + # template_groups = group_templates(self.templates) + template_groups = quick_group_templates(self.templates) if len(template_groups) > 1 and pre_processed: raise NotImplementedError( "Inconsistent template processing and pre-processed data - " diff --git a/eqcorrscan/tests/match_filter_test.py b/eqcorrscan/tests/match_filter_test.py index 95a8f8a08..7129919a7 100644 --- a/eqcorrscan/tests/match_filter_test.py +++ b/eqcorrscan/tests/match_filter_test.py @@ -586,6 +586,19 @@ def test_tribe_copy(self): self.assertEqual(t, copy_t) self.assertEqual(tribe, copied) + def test_tribe_copy_quick(self): + from robustraqn.templates_creation import _quick_tribe_copy + party = Party().read( + filename=os.path.join( + os.path.abspath(os.path.dirname(__file__)), + 'test_data', 'test_party.tgz')) + tribe = Tribe(f.template for f in party.families) + copied = _quick_tribe_copy(tribe) + self.assertEqual(len(tribe), len(copied)) + for t, copy_t in zip(tribe.templates, copied.templates): + self.assertEqual(t, copy_t) + self.assertEqual(tribe, copied) + @pytest.mark.network class TestTribeConstruction(unittest.TestCase): From 32c20dd4a72f3f3d4da413b564a0893ce9ffbb7e Mon Sep 17 00:00:00 2001 From: flixha Date: Mon, 5 Dec 2022 16:04:32 +0100 Subject: [PATCH 2/9] use list of tuples for making unique template groups --- eqcorrscan/core/match_filter/template.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/eqcorrscan/core/match_filter/template.py b/eqcorrscan/core/match_filter/template.py index 0cd78607c..67380d1c6 100644 --- a/eqcorrscan/core/match_filter/template.py +++ b/eqcorrscan/core/match_filter/template.py @@ -714,25 +714,26 @@ def quick_group_templates(templates): :type templates: List of Tribe of Templates :return: List of Lists of Templates. """ - # Get the template's processing parameters and a unique list of - # parameter-combinations + # Get the template's processing parameters processing_tuples = [ tuple( [value for key, value in template.__dict__.items() if key not in ['name', 'st', 'prepick', 'event', 'template_info']] ) for template in templates] - # Convert to numpy array to efficiently search for rows with same values - processing_array = np.array(processing_tuples) + # Get list of unique parameter-tuples. Sort it so that the order in which + # the groups are processed is consistent across different runs. uniq_processing_parameters = sorted(list(set(processing_tuples))) # sort templates into groups template_groups = [] for parameter_combination in uniq_processing_parameters: - # find indices of rows with same parameters - template_indices_for_group = np.where( - (processing_array == np.array(parameter_combination)).all(axis=1)) + # find indices of tuples in list with same parameters + template_indices_for_group = [ + j for j, param_tuple in enumerate(processing_tuples) + if param_tuple == parameter_combination] + new_group = list() - for template_index in template_indices_for_group[0]: + for template_index in template_indices_for_group: #[0]: # use indices to sort templates into groups new_group.append(templates[int(template_index)]) template_groups.append(new_group) From 0e69bd859bd55ef3285aeb0fa8ff378adf55369f Mon Sep 17 00:00:00 2001 From: flixha Date: Mon, 12 Dec 2022 09:46:14 +0100 Subject: [PATCH 3/9] add changelog entry --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 024c576b4..aca9d6b0c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,6 @@ ## Current +* core.match_filter.template + - new quick_group_templates function for 50x quicker template grouping. ## 0.4.4 * core.match_filter From e14eb8f28b5a703596dc8293a5145c51775e9d66 Mon Sep 17 00:00:00 2001 From: Calum Chamberlain Date: Fri, 23 Dec 2022 08:49:06 +1300 Subject: [PATCH 4/9] Stickler --- eqcorrscan/core/match_filter/tribe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eqcorrscan/core/match_filter/tribe.py b/eqcorrscan/core/match_filter/tribe.py index dfdff712f..22329c484 100644 --- a/eqcorrscan/core/match_filter/tribe.py +++ b/eqcorrscan/core/match_filter/tribe.py @@ -25,7 +25,7 @@ from obspy.core.event import Comment, CreationInfo from eqcorrscan.core.match_filter.template import ( - Template, group_templates, quick_group_templates) + Template, quick_group_templates) from eqcorrscan.core.match_filter.party import Party from eqcorrscan.core.match_filter.helpers import ( _safemembers, _par_read, get_waveform_client) From 594792e97ab91ef5097e0ec4f6fbe3a5bb51391e Mon Sep 17 00:00:00 2001 From: Calum Chamberlain Date: Fri, 23 Dec 2022 08:49:16 +1300 Subject: [PATCH 5/9] Stickler --- eqcorrscan/core/match_filter/template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eqcorrscan/core/match_filter/template.py b/eqcorrscan/core/match_filter/template.py index 67380d1c6..0e00d915b 100644 --- a/eqcorrscan/core/match_filter/template.py +++ b/eqcorrscan/core/match_filter/template.py @@ -733,7 +733,7 @@ def quick_group_templates(templates): if param_tuple == parameter_combination] new_group = list() - for template_index in template_indices_for_group: #[0]: + for template_index in template_indices_for_group: # use indices to sort templates into groups new_group.append(templates[int(template_index)]) template_groups.append(new_group) From bce2f46c556732a393cb48e57576010b6c7ed92d Mon Sep 17 00:00:00 2001 From: Calum Chamberlain Date: Fri, 23 Dec 2022 09:07:04 +1300 Subject: [PATCH 6/9] Use eqcorscan func rather than robustraqn. --- eqcorrscan/tests/match_filter_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eqcorrscan/tests/match_filter_test.py b/eqcorrscan/tests/match_filter_test.py index 7129919a7..19a40b620 100644 --- a/eqcorrscan/tests/match_filter_test.py +++ b/eqcorrscan/tests/match_filter_test.py @@ -587,7 +587,7 @@ def test_tribe_copy(self): self.assertEqual(tribe, copied) def test_tribe_copy_quick(self): - from robustraqn.templates_creation import _quick_tribe_copy + from eqcorrscan.core.match_filter.template import quick_group_templates party = Party().read( filename=os.path.join( os.path.abspath(os.path.dirname(__file__)), From b15017d1e172c37bfe1399aebe4b5d16c84f437b Mon Sep 17 00:00:00 2001 From: Calum Chamberlain Date: Fri, 23 Dec 2022 09:27:38 +1300 Subject: [PATCH 7/9] Add tests for grouping and remove copy test --- eqcorrscan/tests/match_filter_test.py | 31 ++++++++++++++++----------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/eqcorrscan/tests/match_filter_test.py b/eqcorrscan/tests/match_filter_test.py index 19a40b620..dd642dbbb 100644 --- a/eqcorrscan/tests/match_filter_test.py +++ b/eqcorrscan/tests/match_filter_test.py @@ -21,6 +21,8 @@ from eqcorrscan.core.match_filter.matched_filter import ( match_filter, MatchFilterError) from eqcorrscan.core.match_filter.helpers import get_waveform_client +from eqcorrscan.core.match_filter.template import quick_group_templates + from eqcorrscan.utils import pre_processing, catalog_utils from eqcorrscan.utils.correlate import fftw_normxcorr, numpy_normxcorr from eqcorrscan.utils.catalog_utils import filter_picks @@ -586,19 +588,6 @@ def test_tribe_copy(self): self.assertEqual(t, copy_t) self.assertEqual(tribe, copied) - def test_tribe_copy_quick(self): - from eqcorrscan.core.match_filter.template import quick_group_templates - party = Party().read( - filename=os.path.join( - os.path.abspath(os.path.dirname(__file__)), - 'test_data', 'test_party.tgz')) - tribe = Tribe(f.template for f in party.families) - copied = _quick_tribe_copy(tribe) - self.assertEqual(len(tribe), len(copied)) - for t, copy_t in zip(tribe.templates, copied.templates): - self.assertEqual(t, copy_t) - self.assertEqual(tribe, copied) - @pytest.mark.network class TestTribeConstruction(unittest.TestCase): @@ -1355,6 +1344,22 @@ def test_template_io(self): if os.path.isfile('test_template.tgz'): os.remove('test_template.tgz') + def test_template_grouping(self): + # PR #524 + # Test that this works directly on the tribe - it should + tribe_len = len(self.tribe) + groups = quick_group_templates(self.tribe) + self.assertEqual(len(groups), 1) + # Add one copy of a template with a different processing length + t2 = self.tribe[0].copy() + t2.process_length -= 100 + templates = [t2] + templates.extend(self.tribe.templates) + # Quick check that we haven't changed the tribe + self.assertEqual(len(self.tribe), tribe_len) + groups2 = quick_group_templates(templates) + self.assertEqual(len(groups2), 2) + def test_party_io(self): """Test reading and writing party objects.""" if os.path.isfile('test_party_out.tgz'): From 75915af7f8c0039fa5d4aa01b810a298f8962492 Mon Sep 17 00:00:00 2001 From: flixha Date: Tue, 3 Jan 2023 14:11:56 +0100 Subject: [PATCH 8/9] add _processing_parameters as function / attribute for Template --- eqcorrscan/core/match_filter/template.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/eqcorrscan/core/match_filter/template.py b/eqcorrscan/core/match_filter/template.py index 0e00d915b..58f61ecae 100644 --- a/eqcorrscan/core/match_filter/template.py +++ b/eqcorrscan/core/match_filter/template.py @@ -105,6 +105,15 @@ def __init__(self, name=None, st=None, lowcut=None, highcut=None, author=getpass.getuser()))) self.event = event + @property + def _processing_parameters(self): + """ + Internal function / attribute to return all processing parameters for + quick grouping of templates as tuple. + """ + return (self.lowcut, self.highcut, self.samp_rate, self.filt_order, + self.process_length, self.prepick) + def __repr__(self): """ Print the template. @@ -715,12 +724,8 @@ def quick_group_templates(templates): :return: List of Lists of Templates. """ # Get the template's processing parameters - processing_tuples = [ - tuple( - [value for key, value in template.__dict__.items() - if key not in ['name', 'st', 'prepick', 'event', 'template_info']] - ) - for template in templates] + processing_tuples = [template._processing_parameters + for template in templates] # Get list of unique parameter-tuples. Sort it so that the order in which # the groups are processed is consistent across different runs. uniq_processing_parameters = sorted(list(set(processing_tuples))) From d9fd1a16a9d5bbef0667b64e978413c12059c2fc Mon Sep 17 00:00:00 2001 From: flixha Date: Tue, 3 Jan 2023 14:16:23 +0100 Subject: [PATCH 9/9] simplify same_processing function, do not include pre_pick in _processing_parameters --- eqcorrscan/core/match_filter/template.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/eqcorrscan/core/match_filter/template.py b/eqcorrscan/core/match_filter/template.py index 58f61ecae..ba8955c3b 100644 --- a/eqcorrscan/core/match_filter/template.py +++ b/eqcorrscan/core/match_filter/template.py @@ -112,7 +112,7 @@ def _processing_parameters(self): quick grouping of templates as tuple. """ return (self.lowcut, self.highcut, self.samp_rate, self.filt_order, - self.process_length, self.prepick) + self.process_length) def __repr__(self): """ @@ -293,12 +293,9 @@ def same_processing(self, other): >>> template_a.same_processing(template_b) False """ - for key in self.__dict__.keys(): - if key in ['name', 'st', 'prepick', 'event', 'template_info']: - continue - if not self.__dict__[key] == other.__dict__[key]: - return False - return True + if self._processing_parameters == other._processing_parameters: + return True + return False def write(self, filename, format='tar'): """