Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Ignore services with 1 replica if needed #542

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Ignore services with 1 replica if needed #542

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions chaoskube/chaoskube.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,9 @@ type Chaoskube struct {
Notifier notifier.Notifier
// namespace scope for the Kubernetes client
ClientNamespaceScope string

// ignoreSingleReplicas will make chaoskube ignore services with one replica when needed.
IgnoreSingleReplicas bool
}

var (
Expand All @@ -97,7 +100,7 @@ var (
// * a logger implementing logrus.FieldLogger to send log output to
// * what specific terminator to use to imbue chaos on victim pods
// * whether to enable/disable dry-run mode
func New(client kubernetes.Interface, labels, annotations, kinds, namespaces, namespaceLabels labels.Selector, includedPodNames, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool, terminator terminator.Terminator, maxKill int, notifier notifier.Notifier, clientNamespaceScope string) *Chaoskube {
func New(client kubernetes.Interface, labels, annotations, kinds, namespaces, namespaceLabels labels.Selector, includedPodNames, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool, terminator terminator.Terminator, maxKill int, notifier notifier.Notifier, clientNamespaceScope string, ignoreSingleReplicas bool) *Chaoskube {
broadcaster := record.NewBroadcaster()
broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: client.CoreV1().Events(clientNamespaceScope)})
recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "chaoskube"})
Expand All @@ -124,6 +127,7 @@ func New(client kubernetes.Interface, labels, annotations, kinds, namespaces, na
MaxKill: maxKill,
Notifier: notifier,
ClientNamespaceScope: clientNamespaceScope,
IgnoreSingleReplicas: ignoreSingleReplicas,
}
}

Expand Down Expand Up @@ -239,7 +243,7 @@ func (c *Chaoskube) Candidates(ctx context.Context) ([]v1.Pod, error) {
pods = filterTerminatingPods(pods)
pods = filterByMinimumAge(pods, c.MinimumAge, c.Now())
pods = filterByPodName(pods, c.IncludedPodNames, c.ExcludedPodNames)
pods = filterByOwnerReference(pods)
pods = filterByOwnerReference(pods, c.IgnoreSingleReplicas)

return pods, nil
}
Expand Down Expand Up @@ -513,7 +517,7 @@ func filterByPodName(pods []v1.Pod, includedPodNames, excludedPodNames *regexp.R
return filteredList
}

func filterByOwnerReference(pods []v1.Pod) []v1.Pod {
func filterByOwnerReference(pods []v1.Pod, ignoreSingleReplicas bool) []v1.Pod {
owners := make(map[types.UID][]v1.Pod)
filteredList := []v1.Pod{}
for _, pod := range pods {
Expand All @@ -531,6 +535,9 @@ func filterByOwnerReference(pods []v1.Pod) []v1.Pod {

// For each owner reference select a random pod from its group
for _, pods := range owners {
if ignoreSingleReplicas && len(pods) <= 1 {
continue
}
filteredList = append(filteredList, util.RandomPodSubSlice(pods, 1)...)
}

Expand Down
87 changes: 64 additions & 23 deletions chaoskube/chaoskube_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,22 +48,23 @@ func (suite *Suite) SetupTest() {
// TestNew tests that arguments are passed to the new instance correctly
func (suite *Suite) TestNew() {
var (
client = fake.NewSimpleClientset()
labelSelector, _ = labels.Parse("foo=bar")
annotations, _ = labels.Parse("baz=waldo")
kinds, _ = labels.Parse("job")
namespaces, _ = labels.Parse("qux")
namespaceLabels, _ = labels.Parse("taz=wubble")
includedPodNames = regexp.MustCompile("foo")
excludedPodNames = regexp.MustCompile("bar")
excludedWeekdays = []time.Weekday{time.Friday}
excludedTimesOfDay = []util.TimePeriod{util.TimePeriod{}}
excludedDaysOfYear = []time.Time{time.Now()}
minimumAge = time.Duration(42)
dryRun = true
terminator = terminator.NewDeletePodTerminator(client, logger, 10*time.Second)
maxKill = 1
notifier = testNotifier
client = fake.NewSimpleClientset()
labelSelector, _ = labels.Parse("foo=bar")
annotations, _ = labels.Parse("baz=waldo")
kinds, _ = labels.Parse("job")
namespaces, _ = labels.Parse("qux")
namespaceLabels, _ = labels.Parse("taz=wubble")
includedPodNames = regexp.MustCompile("foo")
excludedPodNames = regexp.MustCompile("bar")
excludedWeekdays = []time.Weekday{time.Friday}
excludedTimesOfDay = []util.TimePeriod{{}}
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Irrelevant to PR, but seemed redundunt.

excludedDaysOfYear = []time.Time{time.Now()}
minimumAge = time.Duration(42)
dryRun = true
terminator = terminator.NewDeletePodTerminator(client, logger, 10*time.Second)
maxKill = 1
notifier = testNotifier
ignoreSingleReplicas = false
)

chaoskube := New(
Expand All @@ -86,6 +87,7 @@ func (suite *Suite) TestNew() {
maxKill,
notifier,
v1.NamespaceAll,
ignoreSingleReplicas,
)
suite.Require().NotNil(chaoskube)

Expand All @@ -105,6 +107,7 @@ func (suite *Suite) TestNew() {
suite.Equal(logger, chaoskube.Logger)
suite.Equal(dryRun, chaoskube.DryRun)
suite.Equal(terminator, chaoskube.Terminator)
suite.Equal(ignoreSingleReplicas, chaoskube.IgnoreSingleReplicas)
}

// TestRunContextCanceled tests that a canceled context will exit the Run function.
Expand All @@ -126,6 +129,7 @@ func (suite *Suite) TestRunContextCanceled() {
10,
1,
v1.NamespaceAll,
false,
)

ctx, cancel := context.WithCancel(context.Background())
Expand Down Expand Up @@ -182,6 +186,7 @@ func (suite *Suite) TestCandidates() {
false,
10,
v1.NamespaceAll,
false,
)

suite.assertCandidates(chaoskube, tt.pods)
Expand Down Expand Up @@ -228,6 +233,7 @@ func (suite *Suite) TestCandidatesNamespaceLabels() {
false,
10,
v1.NamespaceAll,
false,
)

suite.assertCandidates(chaoskube, tt.pods)
Expand Down Expand Up @@ -262,6 +268,7 @@ func (suite *Suite) TestCandidatesClientNamespaceScope() {
false,
10,
tt.clientNamespaceScope,
false,
)

suite.assertCandidates(chaoskube, tt.pods)
Expand Down Expand Up @@ -306,6 +313,7 @@ func (suite *Suite) TestCandidatesPodNameRegexp() {
false,
10,
v1.NamespaceAll,
false,
)

suite.assertCandidates(chaoskube, tt.pods)
Expand Down Expand Up @@ -347,6 +355,7 @@ func (suite *Suite) TestVictim() {
false,
10,
v1.NamespaceAll,
false,
)

suite.assertVictim(chaoskube, tt.victim)
Expand Down Expand Up @@ -402,6 +411,7 @@ func (suite *Suite) TestVictims() {
10,
tt.maxKill,
v1.NamespaceAll,
false,
)
suite.createPods(chaoskube.Client, podsInfo)

Expand All @@ -428,6 +438,7 @@ func (suite *Suite) TestNoVictimReturnsError() {
10,
1,
v1.NamespaceAll,
false,
)

_, err := chaoskube.Victims(context.Background())
Expand Down Expand Up @@ -463,6 +474,7 @@ func (suite *Suite) TestDeletePod() {
tt.dryRun,
10,
v1.NamespaceAll,
false,
)

victim := util.NewPod("default", "foo", v1.PodRunning)
Expand Down Expand Up @@ -494,6 +506,7 @@ func (suite *Suite) TestDeletePodNotFound() {
10,
1,
v1.NamespaceAll,
false,
)

victim := util.NewPod("default", "foo", v1.PodRunning)
Expand Down Expand Up @@ -726,6 +739,7 @@ func (suite *Suite) TestTerminateVictim() {
false,
10,
v1.NamespaceAll,
false,
)
chaoskube.Now = tt.now

Expand Down Expand Up @@ -758,6 +772,7 @@ func (suite *Suite) TestTerminateNoVictimLogsInfo() {
10,
1,
v1.NamespaceAll,
false,
)

err := chaoskube.TerminateVictims(context.Background())
Expand Down Expand Up @@ -792,7 +807,7 @@ func (suite *Suite) assertNotified(notifier *notifier.Noop) {
suite.Assert().Greater(notifier.Calls, 0)
}

func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration, clientNamespaceScope string) *Chaoskube {
func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration, clientNamespaceScope string, ignoreSingleReplicas bool) *Chaoskube {
chaoskube := suite.setup(
labelSelector,
annotations,
Expand All @@ -810,6 +825,7 @@ func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations lab
gracePeriod,
1,
clientNamespaceScope,
ignoreSingleReplicas,
)

for _, namespace := range []v1.Namespace{
Expand Down Expand Up @@ -845,7 +861,7 @@ func (suite *Suite) createPods(client kubernetes.Interface, podsInfo []podInfo)
}
}

func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration, maxKill int, clientNamespaceScope string) *Chaoskube {
func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration, maxKill int, clientNamespaceScope string, ignoreSingleReplicas bool) *Chaoskube {
logOutput.Reset()

client := fake.NewSimpleClientset()
Expand All @@ -871,6 +887,7 @@ func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Sele
maxKill,
testNotifier,
clientNamespaceScope,
ignoreSingleReplicas,
)
}

Expand Down Expand Up @@ -977,6 +994,7 @@ func (suite *Suite) TestMinimumAge() {
10,
1,
v1.NamespaceAll,
false,
)
chaoskube.Now = tt.now

Expand Down Expand Up @@ -1091,10 +1109,11 @@ func (suite *Suite) TestFilterByOwnerReference() {
baz1 := util.NewPod("default", "baz-1", v1.PodRunning)

for _, tt := range []struct {
seed int64
name string
pods []v1.Pod
expected []v1.Pod
seed int64
name string
pods []v1.Pod
expected []v1.Pod
ignoreSingleReplicas bool
}{
{
seed: 1000,
Expand Down Expand Up @@ -1126,10 +1145,31 @@ func (suite *Suite) TestFilterByOwnerReference() {
pods: []v1.Pod{baz, baz1},
expected: []v1.Pod{baz, baz1},
},
{
seed: 1000,
name: "1 pod, with parent, don't pick",
pods: []v1.Pod{foo},
expected: []v1.Pod{},
ignoreSingleReplicas: true,
},
{
seed: 1000,
name: "3 pods, 2 with same parent one without, pick first from first parent and ignore second",
pods: []v1.Pod{foo, foo1, bar},
expected: []v1.Pod{foo},
ignoreSingleReplicas: true,
},
{
seed: 1000,
name: "2 pods, different parents, don't pick",
pods: []v1.Pod{foo, bar},
expected: []v1.Pod{},
ignoreSingleReplicas: true,
},
} {
rand.Seed(tt.seed)

results := filterByOwnerReference(tt.pods)
results := filterByOwnerReference(tt.pods, tt.ignoreSingleReplicas)
suite.Require().Len(results, len(tt.expected))

// ensure returned pods are ordered by name
Expand Down Expand Up @@ -1160,6 +1200,7 @@ func (suite *Suite) TestNotifierCall() {
false,
10,
v1.NamespaceAll,
false,
)

victim := util.NewPod("default", "foo", v1.PodRunning)
Expand Down
4 changes: 4 additions & 0 deletions main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ var (
logCaller bool
slackWebhook string
clientNamespaceScope string
ignoreSingleReplicas bool
)

func cliEnvVar(name string) string {
Expand Down Expand Up @@ -97,6 +98,7 @@ func init() {
kingpin.Flag("log-caller", "Include the calling function name and location in the log messages.").Envar(cliEnvVar("LOG_CALLER")).BoolVar(&logCaller)
kingpin.Flag("slack-webhook", "The address of the slack webhook for notifications").Envar(cliEnvVar("SLACK_WEBHOOK")).StringVar(&slackWebhook)
kingpin.Flag("client-namespace-scope", "Scope Kubernetes API calls to the given namespace. Defaults to v1.NamespaceAll which requires global read permission.").Envar(cliEnvVar("CLIENT_NAMESPACE_SCOPE")).Default(v1.NamespaceAll).StringVar(&clientNamespaceScope)
kingpin.Flag("ignore-single-replicas", "Ignore services with one replica when needed.").Envar(cliEnvVar("IGNORE_SINGLE_REPLICAS")).BoolVar(&ignoreSingleReplicas)
}

func main() {
Expand Down Expand Up @@ -141,6 +143,7 @@ func main() {
"logFormat": logFormat,
"slackWebhook": slackWebhook,
"clientNamespaceScope": clientNamespaceScope,
"ignoreSingleReplicas": ignoreSingleReplicas,
}).Debug("reading config")

log.WithFields(log.Fields{
Expand Down Expand Up @@ -234,6 +237,7 @@ func main() {
maxKill,
notifiers,
clientNamespaceScope,
ignoreSingleReplicas,
)

if metricsAddress != "" {
Expand Down