Merge branch 'main' of github.com:OpenPathfinder/visionBoard into owa…

…sp-table

.github/ISSUE_TEMPLATE/add-a-new-compliance-check.md

@@ -35,6 +35,6 @@ You can find more details in [the contributing guide](/CONTRIBUTING.md#current-i
   - [ ] Add new integration test cases for this check.
   - [ ] Verify that all tests are passing.
   - [ ] Run the command `check run --name {check_code_name}` and verify the changes in the database. Update the seed script if needed (`npm run db:seed`)
-- [ ] **5. Update the website [Example](https://github.com/OpenPathfinder/website/pull/9)**
-  - [ ] Review the current content it in `https://openjs-security-program-standards.netlify.app/details/{check_code_name}`
-  - [ ] Create a PR in https://github.com/OpenPathfinder/website to include how we calculate this check and include additional information on the mitigation if needed.
+- [ ] **5. Update the website [Example](https://github.com/OpenPathfinder/openpathfinder.com/pull/20)**
+  - [ ] Review the current content it in `https://openpathfinder.com/docs/checks/{check_code_name}`
+  - [ ] Create a PR in https://github.com/OpenPathfinder/openpathfinder.com to include how we calculate this check and include additional information on the mitigation if needed.

.github/workflows/review-compliance-checks.yml

@@ -48,7 +48,7 @@ jobs:
                 "- [ ] Have you included severity validation (`getSeverityFromPriorityGroup`)?\n" +
                 "- [ ] Have you included the tasks, alerts, and results in the database tables?\n" +
                 "- [ ] Have you tested the check with `check run --name {check_code_name}` using the seeded database (`npm run db:seed`)?\n" +
-                "- [ ] Have you created a PR in [the website](https://github.com/OpenPathfinder/website) with the calculation details?\n" +
+                "- [ ] Have you created a PR in [the website](https://github.com/OpenPathfinder/openpathfinder.com) with the calculation details?\n" +
                 "\n" +
                 "You can find more information in [the contributing guide](/CONTRIBUTING.md#add-compliance-checks).\n"
               });

__fixtures__/index.js

@@ -909,9 +909,17 @@ const sampleOSSFScorecardResult = {
   metadata: null
 }
 
+const sampleBulkImportFileContent = [{
+  type: 'softwareDesignTraining',
+  description: 'Course on software design principles done by the team',
+  project_id: 1,
+  implementation_status: 'completed'
+}]
+
 module.exports = {
   sampleGithubOrg,
   sampleGithubListOrgRepos,
   sampleGithubRepository,
-  sampleOSSFScorecardResult
+  sampleOSSFScorecardResult,
+  sampleBulkImportFileContent
 }

__tests__/cli/__snapshots__/workflows.test.js.snap

@@ -32,5 +32,10 @@ exports[`list - Non-Interactive Mode Should provide a list of available workflow
     "name": "generate-reports",
     "workflow": [Function],
   },
+  {
+    "description": "Bulk import data from a CSV file.",
+    "name": "bulk-import",
+    "workflow": [Function],
+  },
 ]
 `;

__tests__/cli/workflows.test.js

@@ -157,3 +157,18 @@ describe('run run-all-checks', () => {
 describe('run upsert-ossf-scorecard', () => {
   test.todo('Should upsert the OSSF Scorecard scoring by running and checking every repository in the database')
 })
+
+describe('run bulk-import', () => {
+  test.todo('Should bulk import data from a JSON file')
+  test.todo('Should throw an error when the file path is not provided')
+  test.todo('Should throw an error when the JSON file can\'t be parsed')
+  test.todo('Should throw an error when the JSON file is not valid due schema validation')
+})
+
+describe('run show-reports', () => {
+  test.todo('Should start a http server that shows all the files and folders in the output directory')
+})
+
+describe('run generate-reports', () => {
+  test.todo('Should generate the reports for the stored data')
+})

__tests__/schemas.test.js

@@ -1,5 +1,5 @@
-const { sampleGithubOrg, sampleGithubListOrgRepos, sampleGithubRepository, sampleOSSFScorecardResult } = require('../__fixtures__')
-const { validateGithubOrg, validateGithubListOrgRepos, validateGithubRepository, validateOSSFResult } = require('../src/schemas')
+const { sampleGithubOrg, sampleGithubListOrgRepos, sampleGithubRepository, sampleOSSFScorecardResult, sampleBulkImportFileContent } = require('../__fixtures__')
+const { validateGithubOrg, validateGithubListOrgRepos, validateGithubRepository, validateOSSFResult, validateBulkImport } = require('../src/schemas')
 
 describe('schemas', () => {
   describe('validateGithubOrg', () => {
@@ -68,4 +68,17 @@ describe('schemas', () => {
       expect(() => validateOSSFResult(invalidData)).toThrow()
     })
   })
+  describe('validateBulkImport', () => {
+    test('Should not throw an error with valid data', () => {
+      expect(() => validateBulkImport(sampleBulkImportFileContent)).not.toThrow()
+    })
+
+    test('Should throw an error with invalid data', () => {
+      const invalidData = [
+        ...sampleBulkImportFileContent,
+        { ...sampleBulkImportFileContent[0], type: 'invalidType' }
+      ]
+      expect(() => validateBulkImport(invalidData)).toThrow()
+    })
+  })
 })

src/cli/workflows.js

@@ -2,6 +2,7 @@ const inquirer = require('inquirer').default
 const debug = require('debug')('cli:workflows')
 const { updateGithubOrgs, upsertGithubRepositories, runAllTheComplianceChecks, upsertOSSFScorecardAnalysis } = require('../workflows')
 const { generateReports } = require('../reports')
+const { bulkImport } = require('../importers')
 const { logger } = require('../utils')
 
 const commandList = [{
@@ -28,6 +29,10 @@ const commandList = [{
   name: 'generate-reports',
   description: 'Generate the reports for the stored data.',
   workflow: generateReports
+}, {
+  name: 'bulk-import',
+  description: 'Bulk import data from a CSV file.',
+  workflow: bulkImport
 }]
 
 const validCommandNames = commandList.map(({ name }) => name)
@@ -52,12 +57,17 @@ async function runWorkflowCommand (knex, options = {}) {
         message: 'What is the name of the workflow?',
         choices: validCommandNames,
         when: () => !options.name
+      }, {
+        type: 'input',
+        name: 'file',
+        message: 'Where is the input file for the workflow?',
+        when: ({ name }) => name === 'bulk-import'
       }
     ])
 
   const command = commandList.find(({ name }) => name === answers.name)
   debug(`Running workflow: ${command.name}`)
-  await command.workflow(knex)
+  await command.workflow(knex, answers.file)
 
   return answers
 }

src/importers/index.js

@@ -0,0 +1,51 @@
+const { logger } = require('../utils')
+const { validateBulkImport } = require('../schemas')
+const { initializeStore } = require('../store')
+const { simplifyObject } = require('@ulisesgascon/simplify-object')
+const fs = require('fs')
+
+const bulkImport = async (knex, filePath) => {
+  logger.info('Bulk importing data...')
+  const { upsertSoftwareDesignTraining } = initializeStore(knex)
+
+  if (!fs.existsSync(filePath)) {
+    logger.error(`File not found: ${filePath}`)
+    throw new Error('File not found')
+  }
+  // Try to read the file
+  let data
+  try {
+    data = JSON.parse(fs.readFileSync(filePath, 'utf8'))
+  } catch (error) {
+    logger.info('Check the documentation for the expected file format in https://openpathfinder.com/docs/visionBoard/importers')
+    logger.error(`Error reading file: ${error.message}`)
+    throw error
+  }
+
+  // Validate the data
+  try {
+    validateBulkImport(data)
+  } catch (error) {
+    logger.info('Check the data schema in https://github.com/OpenPathfinder/visionBoard/blob/main/src/schemas/bulkImport.json')
+    logger.error('Error validating the data')
+    throw error
+  }
+
+  validateBulkImport(data)
+
+  // Update de database
+  for await (const item of data) {
+    if (item.type === 'softwareDesignTraining') {
+      logger.info('Upserting software design training data...')
+      await upsertSoftwareDesignTraining(simplifyObject(item, {
+        exclude: ['type']
+      }))
+    }
+  }
+
+  logger.info('Bulk importing completed')
+}
+
+module.exports = {
+  bulkImport
+}

src/schemas/bulkImport.json

@@ -0,0 +1,31 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "array",
+  "minItems": 1,
+  "items": {
+    "type": "object",
+    "properties": {
+      "type": {
+        "type": "string",
+        "enum": ["softwareDesignTraining"]
+      },
+      "description": {
+        "type": "string"
+      },
+      "project_id": {
+        "type": "integer"
+      },
+      "implementation_status": {
+        "type": "string",
+        "enum": ["unknown", "pending", "completed"]
+      },
+      "training_date": {
+        "type": "string",
+        "format": "date",
+        "nullable": true
+      }
+    },
+    "required": ["type", "description", "project_id", "implementation_status"],
+    "additionalProperties": false
+  }
+}

src/schemas/index.js

@@ -4,6 +4,7 @@ const githubOrganizationSchema = require('./githubOrganization.json')
 const githubListOrgReposSchema = require('./githubListOrgRepos.json')
 const githubRepositorySchema = require('./githubRepository.json')
 const ossfScorecardResultSchema = require('./ossfScorecardResult.json')
+const bulkImportSchema = require('./bulkImport.json')
 
 const ajv = new Ajv()
 addFormats(ajv)
@@ -50,9 +51,20 @@ const validateOSSFResult = (data) => {
   return null
 }
 
+const validateBulkImport = (data) => {
+  const validate = ajv.compile(bulkImportSchema)
+  const valid = validate(data)
+  if (!valid) {
+    const readableErrors = getReadableErrors(validate)
+    throw new Error(`Error when validating the bulk import data: ${readableErrors}`)
+  }
+  return null
+}
+
 module.exports = {
   validateGithubOrg,
   validateGithubListOrgRepos,
   validateGithubRepository,
-  validateOSSFResult
+  validateOSSFResult,
+  validateBulkImport
 }

src/store/index.js

@@ -92,6 +92,15 @@ const upsertGithubRepository = (knex) => (repository, orgId) => upsertRecord({
   data: { ...repository, github_organization_id: orgId }
 })
 
+const upsertSoftwareDesignTraining = (knex) => (data) => upsertRecord({
+  table: 'software_design_training',
+  knex,
+  uniqueCriteria: {
+    project_id: data.project_id
+  },
+  data
+})
+
 const getAllChecksInChecklistById = (knex, checklistId) =>
   debug(`Fetching all checks in checklist by id (${checklistId})...`) ||
   knex('checklist_items')
@@ -158,6 +167,7 @@ const initializeStore = (knex) => {
     addOSSFScorecardResult: (ossf) => addTo('ossf_scorecard_results', ossf),
     upsertOSSFScorecard: upsertOSSFScorecard(knex),
     upsertComplianceCheckResult: upsertComplianceCheckResult(knex),
+    upsertSoftwareDesignTraining: upsertSoftwareDesignTraining(knex),
     getAllOSSFResults: () => getAll('ossf_scorecard_results')
   }
 }

visionboard.js

@@ -40,6 +40,7 @@ workflow
   .command('run')
   .description('Run a workflow')
   .option('--name <name>', 'Name of the workflow')
+  .option('--file <file>', 'Input file for the workflow')
   .action(async (options) => {
     try {
       await runWorkflowCommand(knex, options)