Refactor inventory gathering in preparation for new source of applica…

…tion inventory

agent/plugins/inventory/datauploader/uploader_test.go

@@ -15,6 +15,7 @@
 package datauploader
 
 import (
+	"encoding/json"
 	"testing"
 
 	"github.com/aws/amazon-ssm-agent/agent/context"
@@ -51,6 +52,26 @@ func FakeInventoryItems(count int) (items []model.Item) {
 	return
 }
 
+func ApplicationInventoryItem() (items []model.Item) {
+	// Omit Version which is not omitempty
+	// Omit InstalledTime and URL which are omitempty
+	// Include CompType which should be omitted in all cases
+	items = append(items, model.Item{
+		Name: "RandomInventoryItem",
+		Content: model.ApplicationData{
+			Name:            "Test1",
+			Publisher:       "Pub1",
+			ApplicationType: "Foo",
+			Architecture:    "Brutalism",
+			CompType:        model.AWSComponent,
+		},
+		SchemaVersion: "1.0",
+		CaptureTime:   "time",
+	})
+
+	return
+}
+
 //TODO: add unit tests for ShouldUpdate scenario once content hash is implemented
 
 func TestConvertToSsmInventoryItems(t *testing.T) {
@@ -81,3 +102,27 @@ func TestConvertToSsmInventoryItems(t *testing.T) {
 	inventoryItems, _, err = u.ConvertToSsmInventoryItems(c, items)
 	assert.NotNil(t, err, "Error should be thrown for unsupported Item.Content")
 }
+
+func TestConvertExcludedAndEmptyToSsmInventoryItems(t *testing.T) {
+
+	var items []model.Item
+	var inventoryItems []*ssm.InventoryItem
+	var err error
+
+	c := context.NewMockDefault()
+	u := MockInventoryUploader()
+
+	//testing positive scenario
+
+	//setting up inventory.Item
+	items = append(items, ApplicationInventoryItem()...)
+	inventoryItems, _, err = u.ConvertToSsmInventoryItems(c, items)
+
+	assert.Nil(t, err, "Error shouldn't be thrown for application inventory item")
+	assert.Equal(t, len(items), len(inventoryItems), "Count of inventory items should be equal to input")
+
+	bytes, err := json.Marshal(items[0].Content)
+	assert.Nil(t, err, "Error shouldn't be thrown when marshalling content")
+	// CompType not present even though it has value.  Version should be present even though it doesn't.  InstallTime and Url should not be present because they have no value.
+	assert.Equal(t, "{\"Name\":\"Test1\",\"Publisher\":\"Pub1\",\"Version\":\"\",\"ApplicationType\":\"Foo\",\"Architecture\":\"Brutalism\"}", string(bytes[:]))
+}

agent/plugins/inventory/gatherers/application/dataProvider.go

@@ -0,0 +1,54 @@
+package application
+
+import (
+	"strings"
+
+	"github.com/aws/amazon-ssm-agent/agent/context"
+	"github.com/aws/amazon-ssm-agent/agent/plugins/inventory/model"
+)
+
+const (
+	amazonPublisherName = "amazon"
+	amazonSsmAgentLinux = "amazon-ssm-agent"
+	amazonSsmAgentWin   = "amazon ssm agent"
+	awsToolsWindows     = "aws tools for windows"
+	ec2ConfigService    = "ec2configservice"
+	awsCfnBootstrap     = "aws-cfn-bootstrap"
+	awsPVDrivers        = "aws pv drivers"
+	awsAPIToolsPrefix   = "aws-apitools-"
+	awsAMIToolsPrefix   = "aws-amitools-"
+)
+
+var selectAwsApps map[string]string
+
+func init() {
+	//NOTE:
+	// For V1 - to filter out aws components from aws applications - we are using a list of all aws components that
+	// have been identified in various OS - amazon linux, ubuntu, windows etc.
+	// This is also useful for amazon linux ami - where all packages have Amazon.com as publisher.
+	selectAwsApps = make(map[string]string)
+	selectAwsApps[amazonSsmAgentLinux] = amazonPublisherName
+	selectAwsApps[amazonSsmAgentWin] = amazonPublisherName
+	selectAwsApps[awsToolsWindows] = amazonPublisherName
+	selectAwsApps[ec2ConfigService] = amazonPublisherName
+	selectAwsApps[awsCfnBootstrap] = amazonPublisherName
+	selectAwsApps[awsPVDrivers] = amazonPublisherName
+}
+
+func componentType(applicationName string) model.ComponentType {
+	formattedName := strings.TrimSpace(applicationName)
+	formattedName = strings.ToLower(formattedName)
+
+	var compType model.ComponentType
+
+	//check if application is a known aws component or part of aws-apitool- or aws-amitools- tool set.
+	if _, found := selectAwsApps[formattedName]; found || strings.Contains(formattedName, awsAPIToolsPrefix) || strings.Contains(formattedName, awsAMIToolsPrefix) {
+		compType |= model.AWSComponent
+	}
+
+	return compType
+}
+
+func CollectApplicationData(context context.T) (appData []model.ApplicationData) {
+	return collectPlatformDependentApplicationData(context)
+}

agent/plugins/inventory/gatherers/application/dataProvider_test.go

@@ -0,0 +1,35 @@
+// Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may not
+// use this file except in compliance with the License. A copy of the
+// License is located at
+//
+// http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+// either express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+// Package application contains a application gatherer.
+package application
+
+import (
+	"testing"
+
+	"github.com/aws/amazon-ssm-agent/agent/plugins/inventory/model"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestComponentType(t *testing.T) {
+	awsComponents := []string{"amazon-ssm-agent", "aws-apitools-mon", "aws-amitools-ec2", "AWS Tools for Windows", "AWS PV Drivers"}
+	nonawsComponents := []string{"Notepad++", "Google Update Helper", "accountsservice", "pcre", "kbd-misc"}
+
+	for _, name := range awsComponents {
+		assert.Equal(t, model.AWSComponent, componentType(name))
+	}
+
+	for _, name := range nonawsComponents {
+		assert.Equal(t, model.ComponentType(0), componentType(name))
+	}
+}

agent/plugins/inventory/gatherers/application/dataProvider_unix.go

@@ -61,8 +61,8 @@ func platformInfoProvider(log log.T) (name string, err error) {
 	return platform.PlatformName(log)
 }
 
-// CollectApplicationData collects all application data from the system using rpm or dpkg query.
-func CollectApplicationData(context context.T) (appData []model.ApplicationData) {
+// collectPlatformDependentApplicationData collects all application data from the system using rpm or dpkg query.
+func collectPlatformDependentApplicationData(context context.T) (appData []model.ApplicationData) {
 
 	var err error
 	log := context.Log()
@@ -71,11 +71,11 @@ func CollectApplicationData(context context.T) (appData []model.ApplicationData)
 	cmd := dpkgCmd
 
 	// try dpkg first, if any error occurs, use rpm
-	if appData, err = GetApplicationData(context, cmd, args); err != nil {
+	if appData, err = getApplicationData(context, cmd, args); err != nil {
 		log.Info("Getting applications information using dpkg failed, trying rpm now")
 		cmd = rpmCmd
 		args = []string{rpmCmdArgToGetAllApplications, rpmQueryFormat, rpmQueryFormatArgs}
-		if appData, err = GetApplicationData(context, cmd, args); err != nil {
+		if appData, err = getApplicationData(context, cmd, args); err != nil {
 			log.Errorf("Unable to detect package manager - hence no inventory data for %v", GathererName)
 		}
 	}
@@ -86,8 +86,8 @@ func CollectApplicationData(context context.T) (appData []model.ApplicationData)
 	return
 }
 
-// GetApplicationData runs a shell command and gets information about all packages/applications
-func GetApplicationData(context context.T, command string, args []string) (data []model.ApplicationData, err error) {
+// getApplicationData runs a shell command and gets information about all packages/applications
+func getApplicationData(context context.T, command string, args []string) (data []model.ApplicationData, err error) {
 
 	/*
 		Note: Following are samples of how rpm & dpkg stores package information.
@@ -190,7 +190,7 @@ func GetApplicationData(context context.T, command string, args []string) (data
 		cmdOutput := string(output)
 		log.Debugf("Command output: %v", cmdOutput)
 
-		if data, err = ConvertToApplicationData(cmdOutput); err != nil {
+		if data, err = convertToApplicationData(cmdOutput); err != nil {
 			err = fmt.Errorf("Unable to convert query output to ApplicationData - %v", err.Error())
 		} else {
 			log.Infof("Number of applications detected - %v", len(data))
@@ -200,8 +200,8 @@ func GetApplicationData(context context.T, command string, args []string) (data
 	return
 }
 
-// ConvertToApplicationData converts query output into json string so that it can be deserialized easily
-func ConvertToApplicationData(input string) (data []model.ApplicationData, err error) {
+// convertToApplicationData converts query output into json string so that it can be deserialized easily
+func convertToApplicationData(input string) (data []model.ApplicationData, err error) {
 
 	//This implementation is closely tied to the kind of rpm/dpkg query. A change in query MUST be accompanied
 	//with a change in transform logic or else json formatting will be impacted.
@@ -236,15 +236,17 @@ func ConvertToApplicationData(input string) (data []model.ApplicationData, err e
 	if err = json.Unmarshal([]byte(str), &data); err == nil {
 
 		//transform the date - by iterating over all elements
-		for j, item := range data {
+		for i, item := range data {
 			if item.InstalledTime != "" {
-				if i, err := strconv.ParseInt(item.InstalledTime, 10, 64); err == nil {
+				if sec, err := strconv.ParseInt(item.InstalledTime, 10, 64); err == nil {
 					//InstalledTime must comply with format: 2016-07-30T18:15:37Z to provide better search experience for customers
-					tm := time.Unix(i, 0).UTC()
-					data[j].InstalledTime = tm.Format(time.RFC3339)
+					tm := time.Unix(sec, 0).UTC()
+					item.InstalledTime = tm.Format(time.RFC3339)
 				}
 				//ignore the date transformation if error is encountered
 			}
+			item.CompType = componentType(item.Name)
+			data[i] = item
 		}
 	}
 

agent/plugins/inventory/gatherers/application/dataProvider_unix_test.go

@@ -49,7 +49,7 @@ func MockTestExecutorWithAndWithoutError(command string, args ...string) ([]byte
 }
 
 func TestConvertToApplicationData(t *testing.T) {
-	data, err := ConvertToApplicationData(sampleData)
+	data, err := convertToApplicationData(sampleData)
 
 	assert.Nil(t, err, "Check conversion logic - since sample data in unit test is tied to implementation")
 	assert.Equal(t, 2, len(data), "Given sample data must return 2 entries of application data")
@@ -71,15 +71,15 @@ func TestGetApplicationData(t *testing.T) {
 	//testing with error
 	cmdExecutor = MockTestExecutorWithError
 
-	data, err = GetApplicationData(mockContext, mockCommand, mockArgs)
+	data, err = getApplicationData(mockContext, mockCommand, mockArgs)
 
 	assert.NotNil(t, err, "Error must be thrown when command execution fails")
 	assert.Equal(t, 0, len(data), "When command execution fails - application dataset must be empty")
 
 	//testing without error
 	cmdExecutor = MockTestExecutorWithoutError
 
-	data, err = GetApplicationData(mockContext, mockCommand, mockArgs)
+	data, err = getApplicationData(mockContext, mockCommand, mockArgs)
 
 	assert.Nil(t, err, "Error must not be thrown with MockTestExecutorWithoutError")
 	assert.Equal(t, 2, len(data), "Given sample data must return 2 entries of application data")
@@ -90,16 +90,16 @@ func TestCollectApplicationData(t *testing.T) {
 
 	// both dpkg and rpm return result without error
 	cmdExecutor = MockTestExecutorWithoutError
-	data := CollectApplicationData(mockContext)
+	data := collectPlatformDependentApplicationData(mockContext)
 	assert.Equal(t, 2, len(data), "Given sample data must return 2 entries of application data")
 
 	// both dpkg and rpm return errors
 	cmdExecutor = MockTestExecutorWithError
-	data = CollectApplicationData(mockContext)
+	data = collectPlatformDependentApplicationData(mockContext)
 	assert.Equal(t, 0, len(data), "When command execution fails - application dataset must be empty")
 
 	// dpkg returns error and rpm return some result
 	cmdExecutor = MockTestExecutorWithAndWithoutError
-	data = CollectApplicationData(mockContext)
+	data = collectPlatformDependentApplicationData(mockContext)
 	assert.Equal(t, 2, len(data), "Given sample data must return 2 entries of application data")
 }

agent/plugins/inventory/gatherers/application/dataProvider_windows.go

@@ -44,8 +44,8 @@ func executeCommand(command string, args ...string) ([]byte, error) {
 	return exec.Command(command, args...).CombinedOutput()
 }
 
-// CollectApplicationData collects application data for windows platform
-func CollectApplicationData(context context.T) []model.ApplicationData {
+// collectPlatformDependentApplicationData collects application data for windows platform
+func collectPlatformDependentApplicationData(context context.T) []model.ApplicationData {
 	/*
 		Note:
 
@@ -69,11 +69,11 @@ func CollectApplicationData(context context.T) []model.ApplicationData {
 	var data, apps []model.ApplicationData
 
 	//getting all 64 bit applications
-	apps = ExecutePowershellCommands(context, PowershellCmd, ArgsFor64BitApplications, Arch64Bit)
+	apps = executePowershellCommands(context, PowershellCmd, ArgsFor64BitApplications, Arch64Bit)
 	data = append(data, apps...)
 
 	//getting all 32 bit applications
-	apps = ExecutePowershellCommands(context, PowershellCmd, ArgsFor32BitApplications, Arch32Bit)
+	apps = executePowershellCommands(context, PowershellCmd, ArgsFor32BitApplications, Arch32Bit)
 	data = append(data, apps...)
 
 	//sorts the data based on application-name
@@ -82,8 +82,8 @@ func CollectApplicationData(context context.T) []model.ApplicationData {
 	return data
 }
 
-// ExecutePowershellCommands executes commands in powershell to get all windows applications installed.
-func ExecutePowershellCommands(context context.T, command, args, arch string) (data []model.ApplicationData) {
+// executePowershellCommands executes commands in powershell to get all windows applications installed.
+func executePowershellCommands(context context.T, command, args, arch string) (data []model.ApplicationData) {
 
 	var output []byte
 	var err error
@@ -105,7 +105,7 @@ func ExecutePowershellCommands(context context.T, command, args, arch string) (d
 		cmdOutput := string(output)
 		log.Debugf("Command output: %v", cmdOutput)
 
-		if data, err = ConvertToApplicationData(cmdOutput, arch); err != nil {
+		if data, err = convertToApplicationData(cmdOutput, arch); err != nil {
 			err = fmt.Errorf("Unable to convert query output to ApplicationData - %v", err.Error())
 			log.Error(err.Error())
 			log.Infof("No application data to return")
@@ -120,8 +120,8 @@ func ExecutePowershellCommands(context context.T, command, args, arch string) (d
 	return
 }
 
-// ConvertToApplicationData converts powershell command output to an array of model.ApplicationData
-func ConvertToApplicationData(cmdOutput, architecture string) (data []model.ApplicationData, err error) {
+// convertToApplicationData converts powershell command output to an array of model.ApplicationData
+func convertToApplicationData(cmdOutput, architecture string) (data []model.ApplicationData, err error) {
 	//This implementation is closely tied to the kind of powershell command we run in windows. A change in command
 	//MUST be accompanied with a change in json conversion logic as well.
 
@@ -163,10 +163,11 @@ func ConvertToApplicationData(cmdOutput, architecture string) (data []model.Appl
 	if err = json.Unmarshal([]byte(str), &data); err == nil {
 
 		//iterate over all entries and add default value of architecture as given input
-		for i, v := range data {
+		for i, item := range data {
 			//set architecture to given input
-			v.Architecture = architecture
-			data[i] = v
+			item.Architecture = architecture
+			item.CompType = componentType(item.Name)
+			data[i] = item
 		}
 	}
 

agent/plugins/inventory/gatherers/application/dataProvider_windows_test.go

@@ -50,7 +50,7 @@ func TestConvertToApplicationData(t *testing.T) {
 	var data []model.ApplicationData
 	var err error
 
-	data, err = ConvertToApplicationData(sampleData, mockArch)
+	data, err = convertToApplicationData(sampleData, mockArch)
 
 	assert.Nil(t, err, "Error is not expected for processing sample data - %v", sampleData)
 	assert.Equal(t, 3, len(data))
@@ -66,19 +66,19 @@ func TestExecutePowershellCommands(t *testing.T) {
 
 	//testing command executor without errors
 	cmdExecutor = MockTestExecutorWithoutError
-	data = ExecutePowershellCommands(c, mockCmd, mockArgs, mockArch)
+	data = executePowershellCommands(c, mockCmd, mockArgs, mockArch)
 
 	assert.Equal(t, 3, len(data), "There must be 3 applications for given sample data - %v", sampleData)
 
 	//testing command executor with errors
 	cmdExecutor = MockTestExecutorWithError
-	data = ExecutePowershellCommands(c, mockCmd, mockArgs, mockArch)
+	data = executePowershellCommands(c, mockCmd, mockArgs, mockArch)
 
 	assert.Equal(t, 0, len(data), "On encountering error - application dataset must be empty")
 
 	//testing command executor with ConvertToApplicationData throwing errors
 	cmdExecutor = MockTestExecutorWithConvertToApplicationDataReturningRandomString
-	data = ExecutePowershellCommands(c, mockCmd, mockArgs, mockArch)
+	data = executePowershellCommands(c, mockCmd, mockArgs, mockArch)
 
 	assert.Equal(t, 0, len(data), "On encountering error during json conversion - application dataset must be empty")
 }
@@ -90,13 +90,13 @@ func TestCollectApplicationData(t *testing.T) {
 
 	//testing command executor without errors
 	cmdExecutor = MockTestExecutorWithoutError
-	data = CollectApplicationData(c)
+	data = collectPlatformDependentApplicationData(c)
 
 	assert.Equal(t, 6, data, "MockExecutor will be called 2 times hence total entries must be 6")
 
 	//testing command executor with errors
 	cmdExecutor = MockTestExecutorWithError
-	data = CollectApplicationData(c)
+	data = collectPlatformDependentApplicationData(c)
 
 	assert.Equal(t, 0, data, "If MockExecutor throws error, application dataset must be empty")
 }