From 6de4f9b1da071c35fce8ca8d51c541bcde67d11e Mon Sep 17 00:00:00 2001 From: andrewqian2001datadog Date: Mon, 9 Sep 2024 10:21:17 -0400 Subject: [PATCH 1/6] WIP --- datadog/dogstatsd/aggregator.py | 24 +++++++++ datadog/dogstatsd/buffered_metrics.py | 70 +++++++++++++++++++++++++++ datadog/dogstatsd/metric_types.py | 3 ++ 3 files changed, 97 insertions(+) create mode 100644 datadog/dogstatsd/buffered_metrics.py diff --git a/datadog/dogstatsd/aggregator.py b/datadog/dogstatsd/aggregator.py index 4a805b75e..e2896af3d 100644 --- a/datadog/dogstatsd/aggregator.py +++ b/datadog/dogstatsd/aggregator.py @@ -4,6 +4,11 @@ GaugeMetric, SetMetric, ) +from datadog.dogstatsd.buffered_metrics import ( + HistogramMetric, + DistributionMetric, + TimingMetric +) from datadog.dogstatsd.metric_types import MetricType @@ -14,10 +19,18 @@ def __init__(self): MetricType.GAUGE: {}, MetricType.SET: {}, } + self.buffered_metrics_map = { + MetricType.HISTOGRAM: {}, + MetricType.DISTRIBUTION: {}, + MetricType.TIMING: {} + } self._locks = { MetricType.COUNT: threading.RLock(), MetricType.GAUGE: threading.RLock(), MetricType.SET: threading.RLock(), + MetricType.HISTOGRAM: threading.RLock(), + MetricType.DISTRIBUTION: threading.RLock(), + MetricType.TIMING: threading.RLock() } def flush_aggregated_metrics(self): @@ -30,6 +43,16 @@ def flush_aggregated_metrics(self): metrics.extend(metric.get_data() if isinstance(metric, SetMetric) else [metric]) return metrics + def flush_aggregated_buffered_metrics(self): + metrics = [] + for metric_type in self.buffered_metrics_map.keys(): + with self._locks[metric_type]: + current_metrics = self.buffered_metrics_map[metric_type] + self.buffered_metrics_map[metric_type] = {} + for metric in current_metrics.values(): + metrics.append(metric) + return metrics + def get_context(self, name, tags): tags_str = ",".join(tags) if tags is not None else "" return "{}:{}".format(name, tags_str) @@ -60,3 +83,4 @@ def add_metric( self.metrics_map[metric_type][context] = metric_class( name, value, tags, rate, timestamp ) + diff --git a/datadog/dogstatsd/buffered_metrics.py b/datadog/dogstatsd/buffered_metrics.py new file mode 100644 index 000000000..d94c00e51 --- /dev/null +++ b/datadog/dogstatsd/buffered_metrics.py @@ -0,0 +1,70 @@ +import random +from datadog.dogstatsd.metric_types import MetricType + + +class BufferedMetric(object): + def __init__(self, name, tags, metric_type, max_metrics=0, specified_rate=1.0): + self.name = name + self.tags = tags + self.metric_type = metric_type + self.max_metrics = max_metrics + self.specified_rate = specified_rate + self.data = [] + self.stored_metrics = 0 + self.total_metrics = 0 + + def aggregate(self, value): + self.data.append(value) + self.stored_metrics += 1 + self.total_metrics += 1 + + def maybe_add_metric(self, value): + if self.max_metrics > 0: + if self.stored_metrics >= self.max_metrics: + i = random.randint(0, self.total_metrics - 1) + if i < self.max_metrics: + self.data[i] = value + else: + self.data.append(value) + self.stored_metrics += 1 + self.total_metrics += 1 + else: + self.aggregate(value) + + def skip_metric(self): + self.total_metrics += 1 + + def flush(self): + total_metrics = self.total_metrics + if self.specified_rate != 1.0: + rate = self.specified_rate + else: + if total_metrics != 0: + rate = self.stored_metrics / total_metrics + else: + rate = 1.0 + return { + 'name': self.name, + 'tags': self.tags, + 'metric_type': self.metric_type, + 'rate': rate, + 'values': self.data[:] + } + + +class HistogramMetric(BufferedMetric): + def __init__(self, name, value, tags, max_metrics=0, rate=1.0): + super(HistogramMetric, self).__init__(name, tags, MetricType.HISTOGRAM, max_metrics, rate) + self.aggregate(value) + + +class DistributionMetric(BufferedMetric): + def __init__(self, name, value, tags, max_metrics=0, rate=1.0): + super(DistributionMetric, self).__init__(name, tags, MetricType.DISTRIBUTION, max_metrics, rate) + self.aggregate(value) + + +class TimingMetric(BufferedMetric): + def __init__(self, name, value, tags, max_metrics=0, rate=1.0): + super(TimingMetric, self).__init__(name, tags, MetricType.TIMING, max_metrics, rate) + self.aggregate(value) diff --git a/datadog/dogstatsd/metric_types.py b/datadog/dogstatsd/metric_types.py index 8eee29849..584b1a3ca 100644 --- a/datadog/dogstatsd/metric_types.py +++ b/datadog/dogstatsd/metric_types.py @@ -2,3 +2,6 @@ class MetricType: COUNT = "c" GAUGE = "g" SET = "s" + HISTOGRAM = "h" + DISTRIBUTION = "d" + TIMING = "ms" From c171911f42e1a548c5cb34dc688d573a17a4491b Mon Sep 17 00:00:00 2001 From: andrewqian2001datadog Date: Tue, 10 Sep 2024 14:10:24 -0400 Subject: [PATCH 2/6] add buffered_metrics object type (#853) * add buffered_metrics object type * update metric_types to include histogram, distribution, timing * Run tests on any branch --- .github/workflows/test.yml | 2 +- datadog/dogstatsd/buffered_metrics.py | 65 +++++++++++ datadog/dogstatsd/metric_types.py | 3 + tests/unit/dogstatsd/test_buffered_metrics.py | 104 ++++++++++++++++++ 4 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 datadog/dogstatsd/buffered_metrics.py create mode 100644 tests/unit/dogstatsd/test_buffered_metrics.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 36572d945..a81b1ed51 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,7 +6,7 @@ on: - master pull_request: branches: - - master + - '*' # TODO: Revert when merged to master concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} diff --git a/datadog/dogstatsd/buffered_metrics.py b/datadog/dogstatsd/buffered_metrics.py new file mode 100644 index 000000000..fa092e998 --- /dev/null +++ b/datadog/dogstatsd/buffered_metrics.py @@ -0,0 +1,65 @@ +import random +from datadog.dogstatsd.metric_types import MetricType + + +class BufferedMetric(object): + def __init__(self, name, value, tags, metric_type, max_metric_samples=0, specified_rate=1.0): + self.name = name + self.tags = tags + self.metric_type = metric_type + self.max_metric_samples = max_metric_samples + self.specified_rate = specified_rate + self.data = [value] + self.stored_metric_samples = 1 + self.total_metric_samples = 1 + + def aggregate(self, value): + self.data.append(value) + self.stored_metric_samples += 1 + self.total_metric_samples += 1 + + def maybe_add_metric(self, value): + if self.max_metric_samples > 0: + if self.stored_metric_samples >= self.max_metric_samples: + i = random.randint(0, self.total_metric_samples - 1) + if i < self.max_metric_samples: + self.data[i] = value + else: + self.data.append(value) + self.stored_metric_samples += 1 + self.total_metric_samples += 1 + else: + self.aggregate(value) + + def skip_metric(self): + self.total_metric_samples += 1 + + def flush(self): + total_metric_samples = self.total_metric_samples + if self.specified_rate != 1.0: + rate = self.specified_rate + else: + rate = self.stored_metric_samples / total_metric_samples + + return { + 'name': self.name, + 'tags': self.tags, + 'metric_type': self.metric_type, + 'rate': rate, + 'values': self.data[:] + } + + +class HistogramMetric(BufferedMetric): + def __init__(self, name, value, tags, max_metric_samples=0, rate=1.0): + super(HistogramMetric, self).__init__(name, value, tags, MetricType.HISTOGRAM, max_metric_samples, rate) + + +class DistributionMetric(BufferedMetric): + def __init__(self, name, value, tags, max_metric_samples=0, rate=1.0): + super(DistributionMetric, self).__init__(name, value, tags, MetricType.DISTRIBUTION, max_metric_samples, rate) + + +class TimingMetric(BufferedMetric): + def __init__(self, name, value, tags, max_metric_samples=0, rate=1.0): + super(TimingMetric, self).__init__(name, value, tags, MetricType.TIMING, max_metric_samples, rate) diff --git a/datadog/dogstatsd/metric_types.py b/datadog/dogstatsd/metric_types.py index 8eee29849..584b1a3ca 100644 --- a/datadog/dogstatsd/metric_types.py +++ b/datadog/dogstatsd/metric_types.py @@ -2,3 +2,6 @@ class MetricType: COUNT = "c" GAUGE = "g" SET = "s" + HISTOGRAM = "h" + DISTRIBUTION = "d" + TIMING = "ms" diff --git a/tests/unit/dogstatsd/test_buffered_metrics.py b/tests/unit/dogstatsd/test_buffered_metrics.py new file mode 100644 index 000000000..b4ac90ab7 --- /dev/null +++ b/tests/unit/dogstatsd/test_buffered_metrics.py @@ -0,0 +1,104 @@ +import unittest +from datadog.dogstatsd.buffered_metrics import HistogramMetric, DistributionMetric, TimingMetric +from datadog.dogstatsd.metric_types import MetricType + +class TestBufferedMetric(unittest.TestCase): + + def test_new_histogram_metric(self): + s = HistogramMetric(name="test", value=1.0, tags="tag1,tag2", max_metric_samples=0, rate=1.0) + self.assertEqual(s.data, [1.0]) + self.assertEqual(s.name, "test") + self.assertEqual(s.tags, "tag1,tag2") + self.assertEqual(s.specified_rate, 1.0) + self.assertEqual(s.metric_type, MetricType.HISTOGRAM) + + def test_histogram_metric_aggregate(self): + s = HistogramMetric(name="test", value=1.0, tags="tag1,tag2", max_metric_samples=0, rate=1.0) + s.aggregate(123.45) + self.assertEqual(s.data, [1.0, 123.45]) + self.assertEqual(s.name, "test") + self.assertEqual(s.tags, "tag1,tag2") + self.assertEqual(s.specified_rate, 1.0) + self.assertEqual(s.metric_type, MetricType.HISTOGRAM) + + def test_flush_histogram_metric_aggregate(self): + s = HistogramMetric(name="test", value=1.0, tags="tag1,tag2", max_metric_samples=0, rate=1.0) + m = s.flush() + self.assertEqual(m['metric_type'], MetricType.HISTOGRAM) + self.assertEqual(m['values'], [1.0]) + self.assertEqual(m['name'], "test") + self.assertEqual(m['tags'], "tag1,tag2") + + s.aggregate(21) + s.aggregate(123.45) + m = s.flush() + self.assertEqual(m['metric_type'], MetricType.HISTOGRAM) + self.assertEqual(m['values'], [1.0, 21.0, 123.45]) + self.assertEqual(m['name'], "test") + self.assertEqual(m['rate'], 1.0) + self.assertEqual(m['tags'], "tag1,tag2") + + def test_new_distribution_metric(self): + s = DistributionMetric(name="test", value=1.0, tags="tag1,tag2", max_metric_samples=0, rate=1.0) + self.assertEqual(s.data, [1.0]) + self.assertEqual(s.name, "test") + self.assertEqual(s.tags, "tag1,tag2") + self.assertEqual(s.metric_type, MetricType.DISTRIBUTION) + + def test_distribution_metric_aggregate(self): + s = DistributionMetric(name="test", value=1.0, tags="tag1,tag2", max_metric_samples=0, rate=1.0) + s.aggregate(123.45) + self.assertEqual(s.data, [1.0, 123.45]) + self.assertEqual(s.name, "test") + self.assertEqual(s.tags, "tag1,tag2") + self.assertEqual(s.metric_type, MetricType.DISTRIBUTION) + + def test_flush_distribution_metric_aggregate(self): + s = DistributionMetric(name="test", value=1.0, tags="tag1,tag2", max_metric_samples=0, rate=1.0) + m = s.flush() + self.assertEqual(m['metric_type'], MetricType.DISTRIBUTION) + self.assertEqual(m['values'], [1.0]) + self.assertEqual(m['name'], "test") + self.assertEqual(m['tags'], "tag1,tag2") + + s.aggregate(21) + s.aggregate(123.45) + m = s.flush() + self.assertEqual(m['metric_type'], MetricType.DISTRIBUTION) + self.assertEqual(m['values'], [1.0, 21.0, 123.45]) + self.assertEqual(m['name'], "test") + self.assertEqual(m['tags'], "tag1,tag2") + + def test_new_timing_metric(self): + s = TimingMetric(name="test", value=1.0, tags="tag1,tag2", max_metric_samples=0, rate=1.0) + self.assertEqual(s.data, [1.0]) + self.assertEqual(s.name, "test") + self.assertEqual(s.tags, "tag1,tag2") + self.assertEqual(s.metric_type, MetricType.TIMING) + + def test_timing_metric_aggregate(self): + s = TimingMetric(name="test", value=1.0, tags="tag1,tag2", max_metric_samples=0, rate=1.0) + s.aggregate(123.45) + self.assertEqual(s.data, [1.0, 123.45]) + self.assertEqual(s.name, "test") + self.assertEqual(s.tags, "tag1,tag2") + self.assertEqual(s.metric_type, MetricType.TIMING) + + def test_flush_timing_metric_aggregate(self): + s = TimingMetric(name="test", value=1.0, tags="tag1,tag2", max_metric_samples=0, rate=1.0) + m = s.flush() + self.assertEqual(m['metric_type'], MetricType.TIMING) + self.assertEqual(m['values'], [1.0]) + self.assertEqual(m['name'], "test") + self.assertEqual(m['tags'], "tag1,tag2") + + s.aggregate(21) + s.aggregate(123.45) + m = s.flush() + self.assertEqual(m['metric_type'], MetricType.TIMING) + self.assertEqual(m['values'], [1.0, 21.0, 123.45]) + self.assertEqual(m['name'], "test") + self.assertEqual(m['tags'], "tag1,tag2") + +if __name__ == '__main__': + unittest.main() From f3a8cc1ec80fe16173a4938b3bc230da6eb421e4 Mon Sep 17 00:00:00 2001 From: jack-edmonds-dd Date: Thu, 12 Sep 2024 12:06:33 -0400 Subject: [PATCH 3/6] Set more specific GitHub token permissions on workflows. (#854) --- .github/workflows/changelog.yaml | 4 ++++ .github/workflows/codeql-analysis.yml | 4 ++++ .github/workflows/labeler.yml | 5 +++++ .github/workflows/release.yaml | 4 ++++ .github/workflows/stale.yml | 6 ++++++ .github/workflows/test.yml | 3 +++ .github/workflows/test_integration.yml | 3 +++ 7 files changed, 29 insertions(+) diff --git a/.github/workflows/changelog.yaml b/.github/workflows/changelog.yaml index 6dd28dcc4..c3454b79c 100644 --- a/.github/workflows/changelog.yaml +++ b/.github/workflows/changelog.yaml @@ -1,4 +1,8 @@ name: "Ensure labels" + +permissions: + pull-requests: read + on: # yamllint disable-line rule:truthy pull_request: types: diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 330c87a2f..387ae5dd1 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -1,5 +1,9 @@ name: "CodeQL" +permissions: + contents: read + checks: write + on: push: branches: [ master ] diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 0bd3565ac..efe88ab83 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -1,4 +1,9 @@ name: "Pull Request Labeler" + +permissions: + contents: read + pull-requests: write + on: - pull_request diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 8008ff16f..1898f83c6 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,5 +1,9 @@ name: Build +permissions: + contents: write + pull-requests: write + on: pull_request: release: diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index c5291bfc5..51f9ad156 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -1,6 +1,12 @@ # Configuration for https://github.com/actions/stale name: "Stale issues and pull requests" + +permissions: + contents: write + issues: write + pull-requests: write + on: schedule: - cron: "0 5 * * *" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 36572d945..f86f06175 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,5 +1,8 @@ name: test +permissions: + contents: read + on: push: branches: diff --git a/.github/workflows/test_integration.yml b/.github/workflows/test_integration.yml index cd8d494ad..152c0b500 100644 --- a/.github/workflows/test_integration.yml +++ b/.github/workflows/test_integration.yml @@ -1,5 +1,8 @@ name: Run Integration Tests +permissions: + contents: read + on: # yamllint disable-line rule:truthy pull_request: types: From df5959c05c87b37f539d844f26744a0acd3cd53b Mon Sep 17 00:00:00 2001 From: Bryce Thuilot Date: Fri, 13 Sep 2024 15:16:48 -0400 Subject: [PATCH 4/6] docs: update datadog_checks_dev installation guide (#855) Signed-off-by: Bryce Thuilot --- RELEASING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASING.md b/RELEASING.md index 2d8c3e9fb..f3e6e16b1 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -21,7 +21,7 @@ This project does not have a strict release schedule. However, we would make a r Our team will trigger the release pipeline. ### Prerequisite -- Install [datadog_checks_dev](https://datadog-checks-base.readthedocs.io/en/latest/datadog_checks_dev.cli.html#installation) using Python 3. +- Install [datadog_checks_dev](https://datadoghq.dev/integrations-core/setup/#ddev) using Python 3. - Setup PyPI, see the internal documentation for more details ### Update Changelog and version From 7897910dadbfe4efb288f32383002982766f9ac2 Mon Sep 17 00:00:00 2001 From: Carlos Date: Wed, 18 Sep 2024 21:33:25 +0100 Subject: [PATCH 5/6] Release 0.50.1 (#856) * Release 0.50.1 * Update CHANGELOG.md Co-authored-by: Bryce Eadie --------- Co-authored-by: andrewqian2001datadog Co-authored-by: Bryce Eadie --- CHANGELOG.md | 4 ++++ datadog/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e46391318..c9e32dce2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.50.1 / 2024-09-18 + +* [Added] Add the ability for buffering and aggregation to work at the same time. See [#851](https://github.com/DataDog/datadogpy/pull/851). + ## v0.50.0 / 2024-08-20 * [Added] Add client side aggregation. See [#844](https://github.com/DataDog/datadogpy/pull/844). diff --git a/datadog/version.py b/datadog/version.py index 34f41b46c..834178665 100644 --- a/datadog/version.py +++ b/datadog/version.py @@ -1 +1 @@ -__version__ = "0.50.1.dev" +__version__ = "0.50.1" From ab20e2950221b8c35b52d5f308582681755ec682 Mon Sep 17 00:00:00 2001 From: Carlos Date: Thu, 19 Sep 2024 10:36:59 -0400 Subject: [PATCH 6/6] start dev cycle (#857) --- datadog/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datadog/version.py b/datadog/version.py index 834178665..0b5aa08f1 100644 --- a/datadog/version.py +++ b/datadog/version.py @@ -1 +1 @@ -__version__ = "0.50.1" +__version__ = "0.50.2.dev"