From ae2fd513852ff3e3655a45fb88571d8d12eb684b Mon Sep 17 00:00:00 2001 From: Alberto Jesus Gutierrez Juanes Date: Mon, 29 May 2017 20:44:18 +0200 Subject: [PATCH] scheduler --- README.md | 4 +- lib/fog/hadoop/compute.rb | 1 + .../hadoop/requests/compute/get_scheduler.rb | 279 ++++++++++++++++++ .../requests/compute/scheduler_tests.rb | 276 +++++++++++++++++ 4 files changed, 558 insertions(+), 2 deletions(-) create mode 100644 lib/fog/hadoop/requests/compute/get_scheduler.rb create mode 100644 tests/hadoop/requests/compute/scheduler_tests.rb diff --git a/README.md b/README.md index 8b54709..798057d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # fog-hadoop -[![Build Status](https://travis-ci.org/fog/fog-hadoop.svg?branch=master)](https://travis-ci.org/fog/fog-hadoop) [![Code Climate](https://lima.codeclimate.com/github/fog/fog-hadoop/badges/gpa.svg)](https://lima.codeclimate.com/github/fog/fog-hadoop) [![Test Coverage](https://lima.codeclimate.com/github/fog/fog-hadoop/badges/coverage.svg)](https://lima.codeclimate.com/github/fog/fog-hadoop/coverage) +[![Gem Version](https://badge.fury.io/rb/fog-hadoop.svg)](https://badge.fury.io/rb/fog-hadoop)[![Build Status](https://travis-ci.org/fog/fog-hadoop.svg?branch=master)](https://travis-ci.org/fog/fog-hadoop) [![Code Climate](https://lima.codeclimate.com/github/fog/fog-hadoop/badges/gpa.svg)](https://lima.codeclimate.com/github/fog/fog-hadoop) [![Test Coverage](https://lima.codeclimate.com/github/fog/fog-hadoop/badges/coverage.svg)](https://lima.codeclimate.com/github/fog/fog-hadoop/coverage) ## Fog connector for Hadoop @@ -44,7 +44,7 @@ Connection parameters: Check in this [link](supported.md) # Contribute -Read the [contribute](docs/CONTRIBUTING.md) documentation +Read the [contribute](CONTRIBUTING.md) documentation # Development diff --git a/lib/fog/hadoop/compute.rb b/lib/fog/hadoop/compute.rb index c5930df..9b60313 100644 --- a/lib/fog/hadoop/compute.rb +++ b/lib/fog/hadoop/compute.rb @@ -8,6 +8,7 @@ class Hadoop < Fog::Service request_path 'fog/hadoop/requests/compute' request :get_info + request :get_scheduler request :get_app_stats request :get_app_stats_detail request :get_metrics diff --git a/lib/fog/hadoop/requests/compute/get_scheduler.rb b/lib/fog/hadoop/requests/compute/get_scheduler.rb new file mode 100644 index 0000000..f5a604f --- /dev/null +++ b/lib/fog/hadoop/requests/compute/get_scheduler.rb @@ -0,0 +1,279 @@ +module Fog + module Compute + class Hadoop + class Real + def get_scheduler + request( + :expects => 200, + :method => 'GET', + :path => "#{Fog::Hadoop.yarn_endpoint}/scheduler" + ).body["scheduler"] + end + end + class Mock + def get_scheduler + response = Excon::Response.new + response.status = 200 + response.body = + { + "schedulerInfo" => { + "capacity" => 100.0, + "maxCapacity" => 100.0, + "queueName" => "root", + "queues" => { + "queue" => [ + { + "absoluteCapacity" => 10.5, + "absoluteMaxCapacity" => 50.0, + "absoluteUsedCapacity" => 0.0, + "capacity" => 10.5, + "maxCapacity" => 50.0, + "numApplications" => 0, + "queueName" => "a", + "queues" => { + "queue" => [ + { + "absoluteCapacity" => 3.15, + "absoluteMaxCapacity" => 25.0, + "absoluteUsedCapacity" => 0.0, + "capacity" => 30.000002, + "maxCapacity" => 50.0, + "numApplications" => 0, + "queueName" => "a1", + "queues" => { + "queue" => [ + { + "absoluteCapacity" => 2.6775, + "absoluteMaxCapacity" => 25.0, + "absoluteUsedCapacity" => 0.0, + "capacity" => 85.0, + "maxActiveApplications" => 1, + "maxActiveApplicationsPerUser" => 1, + "maxApplications" => 267, + "maxApplicationsPerUser" => 267, + "maxCapacity" => 100.0, + "numActiveApplications" => 0, + "numApplications" => 0, + "numContainers" => 0, + "numPendingApplications" => 0, + "queueName" => "a1a", + "resourcesUsed" => { + "memory" => 0, + "vCores" => 0 + }, + "state" => "RUNNING", + "type" => "capacitySchedulerLeafQueueInfo", + "usedCapacity" => 0.0, + "usedResources" => "", + "userLimit" => 100, + "userLimitFactor" => 1.0, + "users" => null + }, + { + "absoluteCapacity" => 0.47250003, + "absoluteMaxCapacity" => 25.0, + "absoluteUsedCapacity" => 0.0, + "capacity" => 15.000001, + "maxActiveApplications" => 1, + "maxActiveApplicationsPerUser" => 1, + "maxApplications" => 47, + "maxApplicationsPerUser" => 47, + "maxCapacity" => 100.0, + "numActiveApplications" => 0, + "numApplications" => 0, + "numContainers" => 0, + "numPendingApplications" => 0, + "queueName" => "a1b", + "resourcesUsed" => { + "memory" => 0, + "vCores" => 0 + }, + "state" => "RUNNING", + "type" => "capacitySchedulerLeafQueueInfo", + "usedCapacity" => 0.0, + "usedResources" => "", + "userLimit" => 100, + "userLimitFactor" => 1.0, + "users" => null + } + ] + }, + "resourcesUsed" => { + "memory" => 0, + "vCores" => 0 + }, + "state" => "RUNNING", + "usedCapacity" => 0.0, + "usedResources" => "" + }, + { + "absoluteCapacity" => 7.35, + "absoluteMaxCapacity" => 50.0, + "absoluteUsedCapacity" => 0.0, + "capacity" => 70.0, + "maxActiveApplications" => 1, + "maxActiveApplicationsPerUser" => 100, + "maxApplications" => 735, + "maxApplicationsPerUser" => 73500, + "maxCapacity" => 100.0, + "numActiveApplications" => 0, + "numApplications" => 0, + "numContainers" => 0, + "numPendingApplications" => 0, + "queueName" => "a2", + "resourcesUsed" => { + "memory" => 0, + "vCores" => 0 + }, + "state" => "RUNNING", + "type" => "capacitySchedulerLeafQueueInfo", + "usedCapacity" => 0.0, + "usedResources" => "", + "userLimit" => 100, + "userLimitFactor" => 100.0, + "users" => null + } + ] + }, + "resourcesUsed" => { + "memory" => 0, + "vCores" => 0 + }, + "state" => "RUNNING", + "usedCapacity" => 0.0, + "usedResources" => "" + }, + { + "absoluteCapacity" => 89.5, + "absoluteMaxCapacity" => 100.0, + "absoluteUsedCapacity" => 0.0, + "capacity" => 89.5, + "maxCapacity" => 100.0, + "numApplications" => 2, + "queueName" => "b", + "queues" => { + "queue" => [ + { + "absoluteCapacity" => 53.7, + "absoluteMaxCapacity" => 100.0, + "absoluteUsedCapacity" => 0.0, + "capacity" => 60.000004, + "maxActiveApplications" => 1, + "maxActiveApplicationsPerUser" => 100, + "maxApplications" => 5370, + "maxApplicationsPerUser" => 537000, + "maxCapacity" => 100.0, + "numActiveApplications" => 1, + "numApplications" => 2, + "numContainers" => 0, + "numPendingApplications" => 1, + "queueName" => "b1", + "resourcesUsed" => { + "memory" => 0, + "vCores" => 0 + }, + "state" => "RUNNING", + "type" => "capacitySchedulerLeafQueueInfo", + "usedCapacity" => 0.0, + "usedResources" => "", + "userLimit" => 100, + "userLimitFactor" => 100.0, + "users" => { + "user" => [ + { + "numActiveApplications" => 0, + "numPendingApplications" => 1, + "resourcesUsed" => { + "memory" => 0, + "vCores" => 0 + }, + "username" => "user2" + }, + { + "numActiveApplications" => 1, + "numPendingApplications" => 0, + "resourcesUsed" => { + "memory" => 0, + "vCores" => 0 + }, + "username" => "user1" + } + ] + } + }, + { + "absoluteCapacity" => 35.3525, + "absoluteMaxCapacity" => 100.0, + "absoluteUsedCapacity" => 0.0, + "capacity" => 39.5, + "maxActiveApplications" => 1, + "maxActiveApplicationsPerUser" => 100, + "maxApplications" => 3535, + "maxApplicationsPerUser" => 353500, + "maxCapacity" => 100.0, + "numActiveApplications" => 0, + "numApplications" => 0, + "numContainers" => 0, + "numPendingApplications" => 0, + "queueName" => "b2", + "resourcesUsed" => { + "memory" => 0, + "vCores" => 0 + }, + "state" => "RUNNING", + "type" => "capacitySchedulerLeafQueueInfo", + "usedCapacity" => 0.0, + "usedResources" => "", + "userLimit" => 100, + "userLimitFactor" => 100.0, + "users" => null + }, + { + "absoluteCapacity" => 0.4475, + "absoluteMaxCapacity" => 100.0, + "absoluteUsedCapacity" => 0.0, + "capacity" => 0.5, + "maxActiveApplications" => 1, + "maxActiveApplicationsPerUser" => 100, + "maxApplications" => 44, + "maxApplicationsPerUser" => 4400, + "maxCapacity" => 100.0, + "numActiveApplications" => 0, + "numApplications" => 0, + "numContainers" => 0, + "numPendingApplications" => 0, + "queueName" => "b3", + "resourcesUsed" => { + "memory" => 0, + "vCores" => 0 + }, + "state" => "RUNNING", + "type" => "capacitySchedulerLeafQueueInfo", + "usedCapacity" => 0.0, + "usedResources" => "", + "userLimit" => 100, + "userLimitFactor" => 100.0, + "users" => null + } + ] + }, + "resourcesUsed" => { + "memory" => 0, + "vCores" => 0 + }, + "state" => "RUNNING", + "usedCapacity" => 0.0, + "usedResources" => "" + } + ] + }, + "type" => "capacityScheduler", + "usedCapacity" => 0.0 + } + } + response + end + end + end + end +end \ No newline at end of file diff --git a/tests/hadoop/requests/compute/scheduler_tests.rb b/tests/hadoop/requests/compute/scheduler_tests.rb new file mode 100644 index 0000000..8e6d71f --- /dev/null +++ b/tests/hadoop/requests/compute/scheduler_tests.rb @@ -0,0 +1,276 @@ +Shindo.tests('Fog::Compute[:hadoop] | scheduler request', ['hadoop', 'compute']) do + + @scheduler_format = { + "schedulerInfo" => { + "capacity" => Integer, + "maxCapacity" => Integer, + "queueName" => String, + "queues" => { + "queue" => [ + { + "absoluteCapacity" => Integer, + "absoluteMaxCapacity" => Integer, + "absoluteUsedCapacity" => Integer, + "capacity" => Integer, + "maxCapacity" => Integer, + "numApplications" => Integer, + "queueName" => String, + "queues" => { + "queue" => [ + { + "absoluteCapacity" => Integer, + "absoluteMaxCapacity" => Integer, + "absoluteUsedCapacity" => Integer, + "capacity" => Integer, + "maxCapacity" => Integer, + "numApplications" => Integer, + "queueName" => String, + "queues" => { + "queue" => [ + { + "absoluteCapacity" => Integer, + "absoluteMaxCapacity" => Integer, + "absoluteUsedCapacity" => Integer, + "capacity" => Integer, + "maxActiveApplications" => Integer, + "maxActiveApplicationsPerUser" => Integer, + "maxApplications" => Integer, + "maxApplicationsPerUser" => Integer, + "maxCapacity" => Integer, + "numActiveApplications" => Integer, + "numApplications" => Integer, + "numContainers" => Integer, + "numPendingApplications" => Integer, + "queueName" => String, + "resourcesUsed" => { + "memory" => Integer, + "vCores" => Integer + }, + "state" => String, + "type" => String, + "usedCapacity" => Integer, + "usedResources" => String, + "userLimit" => Integer, + "userLimitFactor" => Integer, + "users" => nil + }, + { + "absoluteCapacity" => Integer, + "absoluteMaxCapacity" => Integer, + "absoluteUsedCapacity" => Integer, + "capacity" => Integer, + "maxActiveApplications" => Integer, + "maxActiveApplicationsPerUser" => Integer, + "maxApplications" => Integer, + "maxApplicationsPerUser" => Integer, + "maxCapacity" => Integer, + "numActiveApplications" => Integer, + "numApplications" => Integer, + "numContainers" => Integer, + "numPendingApplications" => Integer, + "queueName" => String, + "resourcesUsed" => { + "memory" => Integer, + "vCores" => Integer + }, + "state" => String, + "type" => String, + "usedCapacity" => Integer, + "usedResources" => String, + "userLimit" => Integer, + "userLimitFactor" => Integer, + "users" => nil + } + ] + }, + "resourcesUsed" => { + "memory" => Integer, + "vCores" => Integer + }, + "state" => String, + "usedCapacity" => Integer, + "usedResources" => String + }, + { + "absoluteCapacity" => Integer, + "absoluteMaxCapacity" => Integer, + "absoluteUsedCapacity" => Integer, + "capacity" => Integer, + "maxActiveApplications" => Integer, + "maxActiveApplicationsPerUser" => Integer, + "maxApplications" => Integer, + "maxApplicationsPerUser" => Integer, + "maxCapacity" => Integer, + "numActiveApplications" => Integer, + "numApplications" => Integer, + "numContainers" => Integer, + "numPendingApplications" => Integer, + "queueName" => String, + "resourcesUsed" => { + "memory" => Integer, + "vCores" => Integer + }, + "state" => String, + "type" => String, + "usedCapacity" => Integer, + "usedResources" => String, + "userLimit" => Integer, + "userLimitFactor" => Integer, + "users" => nil + } + ] + }, + "resourcesUsed" => { + "memory" => Integer, + "vCores" => Integer + }, + "state" => String, + "usedCapacity" => Integer, + "usedResources" => String + }, + { + "absoluteCapacity" => Integer, + "absoluteMaxCapacity" => Integer, + "absoluteUsedCapacity" => Integer, + "capacity" => Integer, + "maxCapacity" => Integer, + "numApplications" => Integer, + "queueName" => String, + "queues" => { + "queue" => [ + { + "absoluteCapacity" => Integer, + "absoluteMaxCapacity" => Integer, + "absoluteUsedCapacity" => Integer, + "capacity" => Integer, + "maxActiveApplications" => Integer, + "maxActiveApplicationsPerUser" => Integer, + "maxApplications" => Integer, + "maxApplicationsPerUser" => Integer, + "maxCapacity" => Integer, + "numActiveApplications" => Integer, + "numApplications" => Integer, + "numContainers" => Integer, + "numPendingApplications" => Integer, + "queueName" => String, + "resourcesUsed" => { + "memory" => Integer, + "vCores" => Integer + }, + "state" => String, + "type" => String, + "usedCapacity" => Integer, + "usedResources" => String, + "userLimit" => Integer, + "userLimitFactor" => Integer, + "users" => { + "user" => [ + { + "numActiveApplications" => Integer, + "numPendingApplications" => Integer, + "resourcesUsed" => { + "memory" => Integer, + "vCores" => Integer + }, + "username" => String + }, + { + "numActiveApplications" => Integer, + "numPendingApplications" => Integer, + "resourcesUsed" => { + "memory" => Integer, + "vCores" => Integer + }, + "username" => String + } + ] + } + }, + { + "absoluteCapacity" => Integer, + "absoluteMaxCapacity" => Integer, + "absoluteUsedCapacity" => Integer, + "capacity" => Integer, + "maxActiveApplications" => Integer, + "maxActiveApplicationsPerUser" => Integer, + "maxApplications" => Integer, + "maxApplicationsPerUser" => Integer, + "maxCapacity" => Integer, + "numActiveApplications" => Integer, + "numApplications" => Integer, + "numContainers" => Integer, + "numPendingApplications" => Integer, + "queueName" => String, + "resourcesUsed" => { + "memory" => Integer, + "vCores" => Integer + }, + "state" => String, + "type" => String, + "usedCapacity" => Integer, + "usedResources" => String, + "userLimit" => Integer, + "userLimitFactor" => Integer, + "users" => nil + }, + { + "absoluteCapacity" => Integer, + "absoluteMaxCapacity" => Integer, + "absoluteUsedCapacity" => Integer, + "capacity" => Integer, + "maxActiveApplications" => Integer, + "maxActiveApplicationsPerUser" => Integer, + "maxApplications" => Integer, + "maxApplicationsPerUser" => Integer, + "maxCapacity" => Integer, + "numActiveApplications" => Integer, + "numApplications" => Integer, + "numContainers" => Integer, + "numPendingApplications" => Integer, + "queueName" => String, + "resourcesUsed" => { + "memory" => Integer, + "vCores" => Integer + }, + "state" => String, + "type" => String, + "usedCapacity" => Integer, + "usedResources" => String, + "userLimit" => Integer, + "userLimitFactor" => Integer, + "users" => nil + } + ] + }, + "resourcesUsed" => { + "memory" => Integer, + "vCores" => Integer + }, + "state" => String, + "usedCapacity" => Integer, + "usedResources" => String + } + ] + }, + "type" => String, + "usedCapacity" => Integer + } + } + + tests('success') do + tests('#get_metrics') do + schedulers = hadoop_compute_service.get_scheduler + test 'returns a Hash' do + schedulers.body.is_a? Hash + end + if Fog.mock? + tests('Scheduler').formats(@scheduler_format, false) do + schedulers.body + end + else + returns(200) { schedulers.status } + returns(true) { schedulers.body.is_a? Hash } + end + end + end +end