Fixes for analyser, vis + deltas; finalised correct scout messages; m…

…inor changes; correct reset config behaviour deltas + createCategories

Details:
* Closes #14 (scout)
* Fixed some typos
* Corrected createCategories behaviour; fixed some linter errors;
* Added delta docstrings; some refactoring
* Closes #46 (reset config behaviour deltas)
* Some clean-ups for teamScale reports (#23)
* Unified path generation for all reports
* Fixed bug in visualiser
* Modified functional tests for teamScale output (now in correct path) -> #23

.pylintrc

@@ -148,7 +148,7 @@ enable=c-extension-no-member
 # Python expression which should return a note less than 10 (10 is the highest
 # note). You have access to the variables errors warning, statement which
 # respectively contain the number of errors / warnings messages and the total
-# number of statements analyzed. This is used by the global evaluation report
+# number of statements analysed. This is used by the global evaluation report
 # (RP0004).
 evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
 

Emma/emma.py

@@ -48,7 +48,10 @@ def main(arguments):
     memoryManager = Emma.emma_libs.memoryManager.MemoryManager(*processArguments(arguments))
     memoryManager.readConfiguration()
     memoryManager.processMapfiles()
-    memoryManager.createReports()
+    if memoryManager.settings.createCategories:
+        sc().info("No results were generated since categorisation option is active.")
+    else:
+        memoryManager.createReports()
 
     # Stop and display time measurement
     TIME_END = timeit.default_timer()

Emma/emma_delta_libs/FilePresenter.py

@@ -41,42 +41,50 @@ def __init__(self, fileSelector: Emma.emma_delta_libs.FileSelector):
         self.__fileSelector: Emma.emma_delta_libs.FileSelector = fileSelector
 
     def chooseCandidates(self) -> typing.List[str]:
+        """
+        Select two files for the further analysis
+        :return: list with chosen filenames
+        """
         # TODO: Validate all inputs (FM)
         self.__printFileType()
         while True:
-            selectedNumber = int((input("Choose File type >\n")))
-            if selectedNumber in self.__filetypes:
-                filetype: str = self.__filetypes[selectedNumber]
-                break
+            selectedNumber = (input("Choose File type >\n"))
+            if len(str(selectedNumber)) == 1:
+                if int(selectedNumber) in self.__filetypes:                      # pylint: disable=no-else-break
+                    filetype: str = self.__filetypes[int(selectedNumber)]
+                    break
+                else:
+                    sc().warning("Select valid number.\n")
             else:
-                print("Select valid number.\n")
-
+                sc().warning("Select valid number.\n")
         candidates: typing.Dict = self.__fileSelector.getCandidates(filetype)
         self.__printCandidates(candidates)
 
         while True:
             indices: str = input("Select two indices separated by one space >")
             indices: typing.List[str] = indices.split(" ")
             if len(indices) == 2:
-                if int(indices[0])in candidates and int(indices[1]) in candidates:
+                if int(indices[0]) in candidates and int(indices[1]) in candidates:      # pylint: disable=no-else-break
                     break
                 else:
-                    print("select 2 of the numbers presented above")
+                    sc().warning("select two of the numbers presented above")
             else:
-                print("Select exactly 2 files.")
+                sc().warning("Select exactly two files.")
 
         selectedFiles: typing.List[str] = self.__fileSelector.selectFiles(indices)
         self.__printSelectedFiles(selectedFiles)
         return selectedFiles
 
-    def __printCandidates(self, candidates) -> None:
-        for i, candidate in candidates.items():
+    @staticmethod
+    def __printCandidates(candidates) -> None:
+        for i in candidates:
             string = "    " + str(i) + ": " + candidates[i]
             print(string)
 
-    def __printSelectedFiles(self, paths: typing.List[str]) -> None:
+    @staticmethod
+    def __printSelectedFiles( paths: typing.List[str]) -> None:
         sc().info("Selected files:")
-        for i, path in enumerate(paths):
+        for path in paths:
             pathSplit: typing.List[str] = os.path.split(path)
             version: str = os.path.split(os.path.split(pathSplit[0])[0])[1]
             file: str = pathSplit[1]

Emma/emma_delta_libs/FileSelector.py

@@ -17,15 +17,13 @@
 """
 
 
-import sys
 import os
 import typing
 
-from pypiscout.SCout_Logger import Logger as sc
 
-from Emma.shared_libs.stringConstants import *                           # pylint: disable=unused-wildcard-import,wildcard-import
+from Emma.shared_libs.stringConstants import *                 # pylint: disable=unused-wildcard-import,wildcard-import
 import Emma.shared_libs.emma_helper
-
+from pypiscout.SCout_Logger import Logger as sc
 
 class FileSelector:
     """
@@ -37,17 +35,28 @@ def __init__(self, projectDir: str):
         self.__versionCandidates = {}
 
     def getCandidates(self, filetype: str):
+        """
+        Find files according to the chosen filetype
+        :param filetype: chosen filetype
+        :return: list of files that can be processed
+        """
         index = 0
         for file in os.listdir(self.__path):
             if filetype in file:
                 index += 1
                 self.__versionCandidates[index] = file
-
+        if len(self.__versionCandidates) == 0:
+            sc().error("No matching files in " + self.__path)
         return self.__versionCandidates
 
     def selectFiles(self, indices: typing.List[int]) -> typing.List[str]:
+        """
+        Find and return files by selected indices
+        :param indices: list with indices of 2 chosen files
+        :return: list of chosen filenames
+        """
         memStatsCandidates: typing.List[str] = []
         for i in indices:
             candidate = Emma.shared_libs.emma_helper.joinPath(self.__path, self.__versionCandidates[int(i)])
-            memStatsCandidates.append(candidate)            
+            memStatsCandidates.append(candidate)
         return memStatsCandidates

Emma/emma_deltas.py

@@ -21,7 +21,7 @@
 import timeit
 import datetime
 import argparse
-
+import os
 from pypiscout.SCout_Logger import Logger as sc
 
 import Emma
@@ -103,17 +103,27 @@ def main(arguments):
     :param arguments: parsed arguments
     :return: None
     """
-    sc(invVerbosity=arguments.verbosity, actionWarning=(lambda : sys.exit(-10) if arguments.Werror is not None else None), actionError=lambda: sys.exit(-10))
+    sc(invVerbosity=arguments.verbosity, actionWarning=(lambda: sys.exit(-10) if arguments.Werror is not None else None), actionError=lambda: sys.exit(-10))
 
     sc().header("Emma Memory and Mapfile Analyser - Deltas", symbol="/")
 
     # Start and display time measurement
     TIME_START = timeit.default_timer()
     sc().info("Started processing at", datetime.datetime.now().strftime("%H:%M:%S"))
+    if arguments.r:
+        sc().info("The configuration file will be deleted")
+        os.remove("./" + DELTA_CONFIG)
+        if arguments.project is None:
+            sys.exit()
+        else:
+            # Do nothing -> normal first run
+            pass
 
     if arguments.infiles and arguments.outfile is not None:
         candidates = arguments.infiles
     elif arguments.project:
+        rootpath = Emma.emma_delta_libs.RootSelector.selectRoot()           # TODO: rewrite the config file (Daria)
+        Emma.emma_delta_libs.RootSelector.saveNewRootpath(rootpath)
         Emma.shared_libs.emma_helper.checkIfFolderExists(arguments.project)
         fileSelector = Emma.emma_delta_libs.FileSelector.FileSelector(projectDir=arguments.project)
         filePresenter = Emma.emma_delta_libs.FilePresenter.FilePresenter(fileSelector=fileSelector)

Emma/emma_libs/memoryManager.py

@@ -90,7 +90,6 @@ def processMapfiles(self):
         """
         # Check if the configuration loaded
         if self.configuration is not None:
-
             # We will create an empty memory content that will be filled now
             self.memoryContent = {}
 
@@ -114,8 +113,9 @@ def processMapfiles(self):
                 # Updating the categorisation files from the categorisation keywords and remove the unmatched one based on the settings
                 self.categorisation.manageCategoriesFiles(self.settings.createCategories, self.settings.removeUnmatched, sectionCollection, objectCollection)
 
-                # Resolving the duplicate, containment and Overlap in the consumerCollections
+                # Do not resolve duplicate, containment and overlap when createCategories is active
                 if not self.settings.createCategories:
+                    # Resolving the duplicate, containment and overlap in the consumerCollections
                     if not self.settings.noResolveOverlap:
                         sc().info("Resolving section overlaps. This may take some time...")
                         Emma.emma_libs.memoryMap.resolveDuplicateContainmentOverlap(sectionCollection, Emma.emma_libs.memoryEntry.SectionEntry)
@@ -130,7 +130,7 @@ def processMapfiles(self):
                     sc().info("Calculating objects in sections. This may take some time...")
                     self.memoryContent[configId][FILE_IDENTIFIER_OBJECTS_IN_SECTIONS] = Emma.emma_libs.memoryMap.calculateObjectsInSections(self.memoryContent[configId][FILE_IDENTIFIER_SECTION_SUMMARY], self.memoryContent[configId][FILE_IDENTIFIER_OBJECT_SUMMARY])
                 else:
-                    sc().info("No results were generated since categorisation option is active.")
+                    pass
         else:
             sc().error("The configuration needs to be loaded before processing the mapfiles!")
 
@@ -155,7 +155,6 @@ def consumerCollections2GlobalList():
                     consumerCollections[collectionType].extend(self.memoryContent[configId][collectionType])
             return consumerCollections
 
-
         def createStandardReports():
             """
             Create Section, Object and ObjectsInSections reports
@@ -165,7 +164,7 @@ def createStandardReports():
 
             # Creating reports from the consumer collections
             for collectionType in consumerCollections:
-                reportPath = Emma.emma_libs.memoryMap.createReportPath(self.settings.outputPath, self.settings.projectName, collectionType)
+                reportPath = Emma.emma_libs.memoryMap.createReportPath(self.settings.outputPath, self.settings.projectName, collectionType, "csv")
                 Emma.emma_libs.memoryMap.writeReportToDisk(reportPath, consumerCollections[collectionType])
                 sc().info("A report was stored:", os.path.abspath(reportPath))
 
@@ -197,16 +196,12 @@ def createTeamScaleReports():
             # Creating reports from the consumer collections
             for memEntryRow in consumerCollections["Section_Summary"]:
                 fqn = memEntryRow.getFQN(sep="/")
-                size = Emma.shared_libs.emma_helper.toHumanReadable(memEntryRow.addressLength) if (memEntryRow.objectName != OBJECTS_IN_SECTIONS_SECTION_ENTRY) else ""
-                # resultsLst.append({"path": fqn, "count": size})
                 resultsLst.append({"path": fqn, "count": memEntryRow.addressLength})
             for memEntryRow in consumerCollections["Object_Summary"]:
                 fqn = memEntryRow.getFQN(sep="/")
-                size = Emma.shared_libs.emma_helper.toHumanReadable(memEntryRow.addressLength) if (memEntryRow.objectName != OBJECTS_IN_SECTIONS_SECTION_ENTRY) else ""
-                # resultsLst.append({"path": fqn, "count": size})
                 resultsLst.append({"path": fqn, "count": memEntryRow.addressLength})
-
-            Emma.shared_libs.emma_helper.writeJson("testTeamScaleJSON.json", resultsLst)
+            reportPath = Emma.emma_libs.memoryMap.createReportPath(self.settings.outputPath, self.settings.projectName, TEAMSCALE_PREFIX, "json")
+            Emma.shared_libs.emma_helper.writeJson(reportPath, resultsLst)
 
         if self.memoryContent is not None:
             # TODO: Implement handling and choosing of which reports to create (via cmd line argument (like a comma separted string) (MSc)

Emma/emma_libs/memoryMap.py

@@ -261,16 +261,17 @@ def cutOffTheBeginningOfTheSection(sectionToCut, newAddressStart):
     return objectsInSections
 
 
-def createReportPath(outputPath, projectName, reportName):
+def createReportPath(outputPath, projectName, reportName, fileExtension):
     """
     Function to create a string representing the path of a report.
     :param outputPath: The folder where the report will be.
     :param projectName: The name of the project.
     :param reportName: The name of the report.
+    :param fileExtension: the name of file extension
     :return: The created path string.
     """
     Emma.shared_libs.emma_helper.mkDirIfNeeded(outputPath)
-    memStatsFileName = projectName + "_" + reportName + "_" + TIMESTAMP + ".csv"
+    memStatsFileName = projectName + "_" + reportName + "_" + TIMESTAMP + "." + fileExtension
     return Emma.shared_libs.emma_helper.joinPath(outputPath, memStatsFileName)
 
 

Emma/emma_vis.py

@@ -169,7 +169,7 @@ def processArguments(arguments):
     del arguments.subDir
     del arguments.inOutDir
 
-    return arguments.verbosity, arguments.inOutPath, arguments.quiet, arguments.append, arguments.noprompt, arguments.projectDir, arguments.categorised_image_csv, arguments.overview  
+    return arguments.verbosity, arguments.inOutPath, arguments.quiet, arguments.append, arguments.noprompt, arguments.projectDir, arguments.categorised_image_csv, arguments.overview
 
 
 def main(arguments):
@@ -181,7 +181,7 @@ def main(arguments):
     verbosity, inOutPath, quiet, append, noprompt, projectDir, categorised_image_csv, overview = processArguments(arguments)
 
     # Setup SCout
-    sc(invVerbosity=verbosity, actionWarning=(lambda: sys.exit(-10) if Werror is not None else None), actionError=lambda: sys.exit(-10))
+    sc(invVerbosity=verbosity, actionWarning=(lambda: sys.exit(-10) if arguments.Werror is not None else None), actionError=lambda: sys.exit(-10))
 
     sc().header("Emma Memory and Mapfile Analyser - Visualiser", symbol="/")
 

Emma/shared_libs/stringConstants.py

@@ -76,6 +76,7 @@
 MEM_TYPE_TAG = "tag"
 TIMESTAMP = "timestamp"
 TOTAL_USED_PERCENT = "Total used [%]"
+TEAMSCALE_PREFIX = "TeamScaleJSON"
 UNIQUE_PATTERN_SECTIONS = "UniquePatternSections"
 UNIQUE_PATTERN_OBJECTS = "UniquePatternObjects"
 USED_BYTE = "Used [Byte]"

doc/readme-emma.md

@@ -34,7 +34,7 @@ The devices must have a single linear physical address space:
 
 * Devices that use for example a Intel 8051 architecture have separate code and data address spaces that both start at address `0x0000`. Devices based on architectures like this can not entirely analysed with Emma.
 * Devices that use for example the ARMv6M architecture have a single linear address space.
-    Devices based on architectures like this can be analyzed with Emma.
+    Devices based on architectures like this can be analysed with Emma.
 
 
 ## Arguments
@@ -70,7 +70,7 @@ Based on this description the user will have to create his/her own configuration
 Creating a configuration is done by writing several JSON files (if you are not familiar with JSON, please visit [https://www.json.org](https://www.json.org)).
 This chapter will go trough the topic by formally defining the format, rules and the functionality of the configuration files.
 There are practical example projects available in the **`doc`** folder. These projects will lead you step by step trough the process of
-creating a configuration and they also contain map files that can be analyzed.
+creating a configuration and they also contain map files that can be analysed.
 
 Currently the following example projects are available:
 
@@ -457,7 +457,7 @@ The following rules apply:
     * `<SECTION_NAME>` is a string
 * The `<VAS_NAME>` keys are the ones that were defined in the `patterns*.json`
 * Every `<VAS_NAME>` key has an array as value that lists the sections that belong to the virtual address space 
-* There are no rules for the assignment, this needs to be done intuitively based on the project being analyzed
+* There are no rules for the assignment, this needs to be done intuitively based on the project being analysed
 
 ## Output Files
 The output Files will be saved to the memStats folder of the respective project. The filename will have this form: 

doc/test_project/readme.md

@@ -3,7 +3,7 @@ This is an imaginary system, the mapfiles were created by hand and they are foll
 The goal of the project is to present a simple system without any complicated parts in order
 to introduce the new users step-by-step to creating an Emma configuration.
 
-The project can be analyzed by running the following commands from the Emma top-level folder:
+The project can be analysed by running the following commands from the Emma top-level folder:
 
 Emma:
 

tests/functional_tests/test__test_project.py

@@ -71,16 +71,19 @@ def setUp(self):
         for _, directories, files in os.walk(self.memStatsFolder):
             # The result folder shall have 0 subdirectories and three summary files
             self.assertEqual(len(directories), 0)
-            self.assertEqual(len(files), 3)
+            self.assertEqual(len(files), 4)
 
         # Setting up the file name related variables
         projectName = "test_project"
         timeStampLength = len(datetime.datetime.now().strftime("%Y-%m-%d-%Hh%Ms%S"))
         reportFileExtension = ".csv"
+        teamScaleFileExtension = ".json"
         reportFileExtensionLength = len(reportFileExtension)
+        teamScaletFileExtensionLength = len(teamScaleFileExtension)
         imageSummaryFileNameFixPart = projectName + "_" + FILE_IDENTIFIER_SECTION_SUMMARY + "_"
         moduleSummaryFileNameFixPart = projectName + "_" + FILE_IDENTIFIER_OBJECT_SUMMARY + "_"
         objectsInSectionsFileNameFixPart = projectName + "_" + FILE_IDENTIFIER_OBJECTS_IN_SECTIONS + "_"
+        teamScaleFileNameFixPart = projectName + "_" + TEAMSCALE_PREFIX + "_"
 
         # Checking whether the expected report names are there and setting up the variables with their paths
         for file in os.listdir(self.memStatsFolder):
@@ -90,6 +93,8 @@ def setUp(self):
                 self.moduleSummaryPath = os.path.join(self.memStatsFolder, file)
             elif objectsInSectionsFileNameFixPart == file[:-(timeStampLength + reportFileExtensionLength)]:
                 self.objectsInSectionsPath = os.path.join(self.memStatsFolder, file)
+            elif teamScaleFileNameFixPart == file[:-(timeStampLength + teamScaletFileExtensionLength)]:
+                self.teamScaleReportPath = os.path.join(self.memStatsFolder, file)
             else:
                 raise EnvironmentError("Unexpected file: " + os.path.join(self.memStatsFolder, file))
 
@@ -262,6 +267,7 @@ def test_objectsInSectionsReport(self):
         expectedMcuIntRamData = self.ExpectedDataTypeData("INT_RAM", 8, 134656)
         expectedMcuData = self.ExpectedConfigIdData("MCU", 40, [expectedMcuIntFlashData, expectedMcuIntRamData])
 
+
         # Define the expected data of the SOC configId
         expectedSocExtRamData = self.ExpectedDataTypeData("EXT_RAM", 59, 6164824)
         expectedSocData = self.ExpectedConfigIdData("SOC", 59, [expectedSocExtRamData])