Divide multi-valued seriesList by another multi-valued seriesList

[tlktoash]

Hi i am getting build data from jenkins and dumping into elastic to generate code coverage using awesome timlion plugin for kibana. my document has following data structure . Where product has different module with name. Now want to get code coverage for each module and sum it up for each product. I am using codecoverageline and total line to get right code coverage %. I am using fit(carry) function to carry previous value if build data is not available for module at particular day. I was able to achieve right code coverage % my query look like this . But i have to do this for each modules to get right code coverage for product. Is there any way i can use field: name in query so it group by module name so i don't have to do for each module. Any help or pointer will be really appropriated. I love timlion it's awesome!!

(.es(metric='sum:codecoverageline' q='XXX.Web.Build.R2016').fit(carry).divide(.es(metric='sum:totalline' q='XXX.Web.Build.R2016').fit(carry))).multiply(100)

{
  "_index": "apd",
  "_type": "codecoverage",
  "_id": "AVWiVxrVLi1D2E554p1l",
  "_score": 4.0385523,
  "_source": {
    "company": "XXXX",
    "department": "XXXX",
    "product": "XXX",
    "name": "XXWeb.Build.R2016",
    "buildnumber": 36,
    "buildtimestamp": "2016-05-12T10:38:00.000Z",
    "result": "SUCCESS",
    "buiildtriggerby": "XXX",
    "checkinfiles": [
      "XX"
    ],
    "checkinauthors": "XXX",
    "checkincomments": "Modified implementation for Req Checkbox",
    "totalline": 622,
    "codecoverageline": 502,
    "codecoverageper": 80.7074
  }
}

[rashidkpc]

You can use .es(split=name:10) to get a line for each of the top 10 products.

The issue is with the division later on, timelion isn't equipped to do the sort of matrix math in which you get a value for each name, then divide it by a different value for the same name.

[tlktoash]

@rashidkpc thanks for response. In kibana with line chart it also not showing correct result. any pointer ?

{ "size": 0, "query": { "query_string": { "query": "*", "analyze_wildcard": true } }, "aggs": { "2": { "date_histogram": { "field": "buildtimestamp", "interval": "1d", "time_zone": "America/Los_Angeles", "min_doc_count": 1 }, "aggs": { "3": { "filters": { "filters": { "product : \"XXX\"": { "query": { "query_string": { "query": "product : \"XXX\"", "analyze_wildcard": true } } } } }, "aggs": { "1": { "percentiles": { "field": "codecoverageline", "percents": [ 100 ], "script": "_value ==0? 0: ((_value)/doc['totalline'].value)*100" } } } } } } } }

[leeway23]

+1 unfortunately I can not use timelion for my usecases without this functionality e.g.: .es(split=id:0).divide(.es(split=id:0, metric=sum:fieldname)) . Thanks for having a serious look guys ;)

[tlktoash]

@rashidkpc , i have implemented divideseries function added in timelion series_function folder in local . I didn't spend much time. if you can review it that would be great. This code might not up to standard but we have tight deadline to generate report so.

var alter = require('../lib/alter.js');
var _ = require('lodash');
var Chainable = require('../lib/classes/chainable');
var reduce = require('../lib/reduce.js');

module.exports = new Chainable('divideseries', {
  args: [
    {
      name: 'inputSeries',
      types: ['seriesList']
    },
    {
      name: 'divisorSeries',
      types: ['seriesList'],
      help: 'seriesList to divide by'
    }
  ],
  help: 'Divides the values of one or more series in a seriesList to each position, in each series, of the input seriesList',
  fn: function divideseriesFn(args) {
      var seriesList = args[0];
      var divisorseriesList = args[1];

      if(seriesList.type !== 'seriesList') {
        throw new Error ('input must be a seriesList');
      }

       if(divisorseriesList.type !== 'seriesList') {
        throw new Error ('divisorSeries must be a seriesList');
      }
      var serieslistdata = _.chain(seriesList.list).map(function (series) {
            return series.data;
        }).flatten().value();

      var divisorserieslistdata = _.chain(divisorseriesList.list).map(function (series) {
            return series.data;
        }).flatten().value();

    var ziparray=_.zipWith(serieslistdata,divisorserieslistdata,function (a,b) {
            if(a[1] > 0 )
                return [a[0],a[1] / b[1]];
            else 
            return [a[0],0];
    });
    return alter(args,function (inputSeries) {
        var pairs = inputSeries.data;
        inputSeries.data=ziparray;
        return inputSeries;
    })

  }
});

[rashidkpc]

I'd probably suggest implementing this as an option to the divide() function instead of as its own thing. Though it would probably make sense to implement for everything that uses the reduce() helper

[tlktoash]

That is good point. I would add to divide() function so it can handle series. any comments on code?

[rashidkpc]

You might look at how reduce is implemented, that should give you some hints on the path to go down to make it more generalized. Alternatively, you could publish it as a plugin based on the example plugin here: https://github.com/rashidkpc/timelion-random

[tlktoash]

yes i changed reduce.js file in my local. i will look into plugin how can be made. Thanks !!

[ddavidebor]

@tlktoash @leeway23 could this solve your problem? https://github.com/fermiumlabs/mathlion