From aa208df7a11d993ba2e974fcae1a55b2253034ab Mon Sep 17 00:00:00 2001 From: Julien Delange Date: Wed, 9 Oct 2024 21:03:28 -0400 Subject: [PATCH 1/8] export secret validation in SARIF --- LICENSE-3rdparty.csv | 1 + crates/cli/src/file_utils.rs | 13 +- crates/cli/src/sarif/sarif_utils.rs | 169 +++++++++++++++++++--- crates/secrets/Cargo.toml | 1 + crates/secrets/src/model/secret_result.rs | 23 +++ crates/secrets/src/scanner.rs | 15 +- 6 files changed, 197 insertions(+), 25 deletions(-) diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index ba80e583..cc803917 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -4,6 +4,7 @@ base64,https://github.com/marshallpierce/rust-base64,Apache-2.0,Copyright (c) 20 bstr,https://github.com/BurntSushi/bstr,MIT,Copyright (c) 2018-2019 Andrew Gallant csv,https://github.com/BurntSushi/rust-csv,MIT,Copyright (c) 2015 Andrew Gallant deno-core,https://github.com/denoland/deno,MIT,Copyright 2018-2023 the Deno authors +futures,https://rust-lang.github.io/futures-rs/,MIT,Copyright (c) 2017 The Tokio Authors git2,https://crates.io/crates/git2,MIT,Copyright (c) 2014 Alex Crichton globset,https://crates.io/crates/globset,MIT,Copyright (c) 2015 Andrew Gallant graphviz-rust,https://github.com/besok/graphviz-rust,MIT,Copyright (c) 2013 Boris Zhguchev diff --git a/crates/cli/src/file_utils.rs b/crates/cli/src/file_utils.rs index ab8cae88..6b0f6091 100644 --- a/crates/cli/src/file_utils.rs +++ b/crates/cli/src/file_utils.rs @@ -9,10 +9,10 @@ use walkdir::WalkDir; use kernel::model::common::Language; use kernel::model::config_file::PathConfig; -use kernel::model::violation::Violation; use crate::model::cli_configuration::CliConfiguration; use crate::model::datadog_api::DiffAwareData; +use crate::sarif::sarif_utils::SarifViolation; static FILE_EXTENSIONS_PER_LANGUAGE_LIST: &[(Language, &[&str])] = &[ (Language::Csharp, &["cs"]), @@ -304,7 +304,7 @@ pub fn filter_files_by_diff_aware_info( /// SHA2( - - - ) pub fn get_fingerprint_for_violation( rule_name: String, - violation: &Violation, + violation: &SarifViolation, repository_root: &Path, file: &Path, use_debug: bool, @@ -394,7 +394,7 @@ mod tests { #[test] fn get_fingerprint_for_violation_success_single_region() { let d = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - let violation = Violation { + let violation = SarifViolation { start: Position { line: 10, col: 1 }, end: Position { line: 12, col: 1 }, message: "something bad happened".to_string(), @@ -402,6 +402,7 @@ mod tests { category: RuleCategory::Performance, fixes: vec![], taint_flow: None, + validation_status: None, }; let directory_string = d.into_os_string().into_string().unwrap(); let fingerprint = get_fingerprint_for_violation( @@ -438,7 +439,7 @@ mod tests { end: Position { line: 5, col: 30 }, }; let root_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - let violation = Violation { + let violation = SarifViolation { start: region0.start, end: region0.end, message: "flow violation".to_string(), @@ -446,6 +447,7 @@ mod tests { category: RuleCategory::Security, fixes: vec![], taint_flow: Some(vec![region0, region1]), + validation_status: None, }; let fingerprint = get_fingerprint_for_violation( "taint_flow_rule".to_string(), @@ -468,7 +470,7 @@ mod tests { let d = PathBuf::from(env!("CARGO_MANIFEST_DIR")); let path = "resources/test/gitignore/test1"; - let violation = Violation { + let violation = SarifViolation { start: Position { line: 10, col: 1 }, end: Position { line: 12, col: 1 }, message: "something bad happened".to_string(), @@ -476,6 +478,7 @@ mod tests { category: RuleCategory::Performance, fixes: vec![], taint_flow: None, + validation_status: None, }; let directory_string = d.into_os_string().into_string().unwrap(); diff --git a/crates/cli/src/sarif/sarif_utils.rs b/crates/cli/src/sarif/sarif_utils.rs index 2bffc4be..6b6c8bc0 100644 --- a/crates/cli/src/sarif/sarif_utils.rs +++ b/crates/cli/src/sarif/sarif_utils.rs @@ -5,20 +5,22 @@ use std::rc::Rc; use crate::constants::{SARIF_PROPERTY_DATADOG_FINGERPRINT, SARIF_PROPERTY_SHA}; use anyhow::Result; use base64::Engine; -use common::model::position::Position; use common::model::position::PositionBuilder; +use common::model::position::{Position, Region}; +use derive_builder::Builder; use git2::{BlameOptions, Repository}; use kernel::constants::CARGO_VERSION; use kernel::model::rule::{RuleCategory, RuleSeverity}; -use kernel::model::violation::Violation; +use kernel::model::violation::{Fix, Violation}; use kernel::model::{ rule::{Rule, RuleResult}, violation::{Edit, EditType}, }; use path_slash::PathExt; use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS}; -use secrets::model::secret_result::SecretResult; +use secrets::model::secret_result::{SecretResult, SecretValidationStatus}; use secrets::model::secret_rule::SecretRule; +use serde::{Deserialize, Serialize}; use serde_sarif::sarif::{ self, ArtifactChangeBuilder, ArtifactLocationBuilder, FixBuilder, LocationBuilder, MessageBuilder, PhysicalLocationBuilder, PropertyBagBuilder, RegionBuilder, Replacement, @@ -118,6 +120,45 @@ impl From for SarifRule { } } +/// Generic representation of a violation for both static analysis and secrets +#[derive(Deserialize, Debug, Serialize, Clone, Builder)] +pub struct SarifViolation { + pub start: Position, + pub end: Position, + pub message: String, + pub severity: RuleSeverity, + pub category: RuleCategory, + pub fixes: Vec, + pub taint_flow: Option>, + pub validation_status: Option, +} + +impl SarifViolation { + fn get_properties(&self) -> Vec { + if let Some(validation_status) = &self.validation_status { + match validation_status { + SecretValidationStatus::NotValidated => { + vec!["DATADOG_SECRET_VALIDATION_STATUS:NOT_VALIDATED".to_string()] + } + SecretValidationStatus::Valid => { + vec!["DATADOG_SECRET_VALIDATION_STATUS:VALID".to_string()] + } + SecretValidationStatus::Invalid => { + vec!["DATADOG_SECRET_VALIDATION_STATUS:INVALID".to_string()] + } + SecretValidationStatus::ValidationError => { + vec!["DATADOG_SECRET_VALIDATION_STATUS:VALIDATION_ERROR".to_string()] + } + SecretValidationStatus::NotAvailable => { + vec!["DATADOG_SECRET_VALIDATION_STATUS:NOT_AVAILABLE".to_string()] + } + } + } else { + vec![] + } + } +} + #[derive(Debug, Clone)] pub enum SarifRuleResult { StaticAnalysis(RuleResult), @@ -125,13 +166,26 @@ pub enum SarifRuleResult { } impl SarifRuleResult { - fn violations(&self) -> Vec { + fn violations(&self) -> Vec { match self { - SarifRuleResult::StaticAnalysis(r) => r.violations.clone(), + SarifRuleResult::StaticAnalysis(r) => r + .violations + .iter() + .map(|v| SarifViolation { + start: v.start, + end: v.end, + message: v.message.clone(), + severity: v.severity, + category: v.category, + fixes: v.fixes.clone(), + taint_flow: v.taint_flow.clone(), + validation_status: None, + }) + .collect::>(), SarifRuleResult::Secret(secret_result) => secret_result .matches .iter() - .map(|r| Violation { + .map(|r| SarifViolation { start: r.start, end: r.end, message: secret_result.message.clone(), @@ -139,8 +193,9 @@ impl SarifRuleResult { category: RuleCategory::Security, fixes: vec![], taint_flow: None, + validation_status: Some(r.validation_status.clone()), }) - .collect::>(), + .collect::>(), } } @@ -419,17 +474,17 @@ fn is_valid_position(position: &Position) -> bool { } /// Check that the violation is valid and must be included -fn is_valid_violation(violation: &Violation) -> bool { +fn is_valid_violation(violation: &SarifViolation) -> bool { if !is_valid_position(&violation.start) || !is_valid_position(&violation.end) { return false; } // make sure that no violation has an invalid fix - return violation.fixes.iter().all(|f| { + violation.fixes.iter().all(|f| { f.edits.iter().all(|e| { is_valid_position(&e.start) && e.end.map(|p| is_valid_position(&p)).unwrap_or(true) }) - }); + }) } /// Convert our severity enumeration into the corresponding SARIF values. @@ -694,7 +749,7 @@ fn generate_results( ) .properties( PropertyBagBuilder::default() - .tags(tags.clone()) + .tags([tags.clone(), violation.get_properties()].concat()) .build() .unwrap(), ) @@ -795,6 +850,8 @@ pub fn generate_sarif_file( #[cfg(test)] mod tests { + use super::*; + use crate::sarif::sarif_utils::SarifRule::SecretRule; use assert_json_diff::assert_json_eq; use common::model::position::{Position, PositionBuilder, Region}; use kernel::model::violation::Fix; @@ -803,11 +860,10 @@ mod tests { rule::{RuleBuilder, RuleCategory, RuleResultBuilder, RuleSeverity, RuleType}, violation::{EditBuilder, EditType, FixBuilder as RosieFixBuilder, ViolationBuilder}, }; + use secrets::model::secret_result::SecretResultMatch; use serde_json::{from_str, Value}; use valico::json_schema; - use super::*; - /// Validate JSON data against the SARIF schema fn validate_data(v: &Value) -> bool { let j_schema = from_str(include_str!("sarif-schema-2.1.0.json")).unwrap(); @@ -819,7 +875,7 @@ mod tests { #[test] fn test_is_valid_violation() { // bad location in the violation location - assert!(!is_valid_violation(&Violation { + assert!(!is_valid_violation(&SarifViolation { start: Position { line: 0, col: 1 }, end: Position { line: 42, col: 42 }, message: "bad stuff".to_string(), @@ -827,10 +883,11 @@ mod tests { category: RuleCategory::BestPractices, fixes: vec![], taint_flow: None, + validation_status: None, })); // good location in the violation location and no fixes - assert!(is_valid_violation(&Violation { + assert!(is_valid_violation(&SarifViolation { start: Position { line: 1, col: 1 }, end: Position { line: 42, col: 42 }, message: "bad stuff".to_string(), @@ -838,10 +895,11 @@ mod tests { category: RuleCategory::BestPractices, fixes: vec![], taint_flow: None, + validation_status: None, })); // bad location in the fixes location - assert!(!is_valid_violation(&Violation { + assert!(!is_valid_violation(&SarifViolation { start: Position { line: 1, col: 1 }, end: Position { line: 42, col: 42 }, message: "bad stuff".to_string(), @@ -857,10 +915,11 @@ mod tests { }] }], taint_flow: None, + validation_status: None, })); // good location everywhere - assert!(is_valid_violation(&Violation { + assert!(is_valid_violation(&SarifViolation { start: Position { line: 1, col: 1 }, end: Position { line: 42, col: 42 }, message: "bad stuff".to_string(), @@ -876,6 +935,7 @@ mod tests { }] }], taint_flow: None, + validation_status: None, })); } @@ -1230,6 +1290,81 @@ mod tests { assert!(validate_data(&sarif_report_to_string)); } + #[test] + fn test_generate_secret() { + let rule = secrets::model::secret_rule::SecretRule { + id: "secret-rule".to_string(), + name: "secret-rule".to_string(), + description: "myfile.py".to_string(), + pattern: "foobarbaz".to_string(), + default_included_keywords: vec![], + }; + + let secret_results = vec![SecretResult { + rule_id: "secret-rule".to_string(), + rule_name: "secret-rule".to_string(), + filename: "myfile.py".to_string(), + message: "some secret".to_string(), + matches: vec![ + SecretResultMatch { + start: Position { line: 1, col: 1 }, + end: Position { line: 2, col: 2 }, + validation_status: SecretValidationStatus::NotValidated, + }, + SecretResultMatch { + start: Position { line: 2, col: 2 }, + end: Position { line: 3, col: 3 }, + validation_status: SecretValidationStatus::Valid, + }, + SecretResultMatch { + start: Position { line: 3, col: 3 }, + end: Position { line: 4, col: 4 }, + validation_status: SecretValidationStatus::Invalid, + }, + SecretResultMatch { + start: Position { line: 5, col: 5 }, + end: Position { line: 6, col: 6 }, + validation_status: SecretValidationStatus::ValidationError, + }, + SecretResultMatch { + start: Position { line: 6, col: 6 }, + end: Position { line: 7, col: 7 }, + validation_status: SecretValidationStatus::NotAvailable, + }, + ], + }]; + + let sarif_secret_results = secret_results + .into_iter() + .map(SarifRuleResult::try_from) + .collect::, _>>() + .map_err(anyhow::Error::msg) + .expect("getting results"); + + let sarif_report = generate_sarif_report( + &[rule.into()], + &sarif_secret_results, + &"mydir".to_string(), + SarifReportMetadata { + add_git_info: false, + debug: false, + config_digest: "5d7273dec32b80788b4d3eac46c866f0".to_string(), + diff_aware_parameters: None, + execution_time_secs: 42, + }, + ) + .expect("generate sarif report"); + + let sarif_report_to_string = serde_json::to_value(sarif_report).unwrap(); + assert_json_eq!( + sarif_report_to_string, + serde_json::json!({"runs":[{"results":[{"fixes":[],"level":"error","locations":[{"physicalLocation":{"artifactLocation":{"uri":"myfile.py"},"region":{"endColumn":2,"endLine":2,"startColumn":1,"startLine":1}}}],"message":{"text":"some secret"},"partialFingerprints":{},"properties":{"tags":["DATADOG_CATEGORY:SECURITY","DATADOG_SECRET_VALIDATION_STATUS:NOT_VALIDATED"]},"ruleId":"secret-rule","ruleIndex":0},{"fixes":[],"level":"error","locations":[{"physicalLocation":{"artifactLocation":{"uri":"myfile.py"},"region":{"endColumn":3,"endLine":3,"startColumn":2,"startLine":2}}}],"message":{"text":"some secret"},"partialFingerprints":{},"properties":{"tags":["DATADOG_CATEGORY:SECURITY","DATADOG_SECRET_VALIDATION_STATUS:VALID"]},"ruleId":"secret-rule","ruleIndex":0},{"fixes":[],"level":"error","locations":[{"physicalLocation":{"artifactLocation":{"uri":"myfile.py"},"region":{"endColumn":4,"endLine":4,"startColumn":3,"startLine":3}}}],"message":{"text":"some secret"},"partialFingerprints":{},"properties":{"tags":["DATADOG_CATEGORY:SECURITY","DATADOG_SECRET_VALIDATION_STATUS:INVALID"]},"ruleId":"secret-rule","ruleIndex":0},{"fixes":[],"level":"error","locations":[{"physicalLocation":{"artifactLocation":{"uri":"myfile.py"},"region":{"endColumn":6,"endLine":6,"startColumn":5,"startLine":5}}}],"message":{"text":"some secret"},"partialFingerprints":{},"properties":{"tags":["DATADOG_CATEGORY:SECURITY","DATADOG_SECRET_VALIDATION_STATUS:VALIDATION_ERROR"]},"ruleId":"secret-rule","ruleIndex":0},{"fixes":[],"level":"error","locations":[{"physicalLocation":{"artifactLocation":{"uri":"myfile.py"},"region":{"endColumn":7,"endLine":7,"startColumn":6,"startLine":6}}}],"message":{"text":"some secret"},"partialFingerprints":{},"properties":{"tags":["DATADOG_CATEGORY:SECURITY","DATADOG_SECRET_VALIDATION_STATUS:NOT_AVAILABLE"]},"ruleId":"secret-rule","ruleIndex":0}],"tool":{"driver":{"informationUri":"https://www.datadoghq.com","name":"datadog-static-analyzer","properties":{"tags":["DATADOG_DIFF_AWARE_CONFIG_DIGEST:5d7273dec32b80788b4d3eac46c866f0","DATADOG_EXECUTION_TIME_SECS:42","DATADOG_DIFF_AWARE_ENABLED:false"]},"rules":[{"fullDescription":{"text":"myfile.py"},"id":"secret-rule","name":"secret-rule","properties":{"tags":["DATADOG_RULE_TYPE:SECRET"]},"shortDescription":{"text":"secret-rule"}}],"version":"0.4.4"}}}],"version":"2.1.0"}) + ); + + // validate the schema + assert!(validate_data(&sarif_report_to_string)); + } + // in this test, the rule in the violation cannot be found in the list // of rules and the rule index in the sarif report must be empty #[test] diff --git a/crates/secrets/Cargo.toml b/crates/secrets/Cargo.toml index a07ab30e..e2b716b8 100644 --- a/crates/secrets/Cargo.toml +++ b/crates/secrets/Cargo.toml @@ -11,6 +11,7 @@ itertools = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +futures = "0.3" lazy_static = "1.5.0" # remote diff --git a/crates/secrets/src/model/secret_result.rs b/crates/secrets/src/model/secret_result.rs index 858646d0..74f0083f 100644 --- a/crates/secrets/src/model/secret_result.rs +++ b/crates/secrets/src/model/secret_result.rs @@ -3,12 +3,35 @@ // Copyright 2024 Datadog, Inc. use common::model::position::Position; +use sds::MatchStatus; use serde::{Deserialize, Serialize}; +#[derive(Clone, Debug, PartialEq, Hash, Eq, Serialize, Deserialize)] +pub enum SecretValidationStatus { + NotValidated, + Valid, + Invalid, + ValidationError, + NotAvailable, +} + +impl From<&MatchStatus> for SecretValidationStatus { + fn from(value: &MatchStatus) -> Self { + match value { + MatchStatus::NotChecked => SecretValidationStatus::NotValidated, + MatchStatus::Valid => SecretValidationStatus::Valid, + MatchStatus::Invalid => SecretValidationStatus::Invalid, + MatchStatus::Error(_) => SecretValidationStatus::ValidationError, + MatchStatus::NotAvailable => SecretValidationStatus::NotAvailable, + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Hash, Eq)] pub struct SecretResultMatch { pub start: Position, pub end: Position, + pub validation_status: SecretValidationStatus, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Hash, Eq)] diff --git a/crates/secrets/src/scanner.rs b/crates/secrets/src/scanner.rs index 34d1b90a..2e5257f3 100644 --- a/crates/secrets/src/scanner.rs +++ b/crates/secrets/src/scanner.rs @@ -2,7 +2,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2024 Datadog, Inc. -use crate::model::secret_result::{SecretResult, SecretResultMatch}; +use crate::model::secret_result::{SecretResult, SecretResultMatch, SecretValidationStatus}; use crate::model::secret_rule::SecretRule; use anyhow::Error; use common::analysis_options::AnalysisOptions; @@ -32,16 +32,23 @@ pub fn find_secrets( sds_rules: &[SecretRule], filename: &str, code: &str, - _options: &AnalysisOptions, + options: &AnalysisOptions, ) -> Vec { struct Result { rule_index: usize, start: Position, end: Position, + validation_status: SecretValidationStatus, } let mut codemut = code.to_owned(); - let matches = scanner.scan(&mut codemut, vec![]); + let mut matches = scanner.scan(&mut codemut, vec![]); + + let matches_validation = futures::executor::block_on(scanner.validate_matches(&mut matches)); + + if matches_validation.is_err() && options.use_debug { + eprintln!("error when validating secrets for filename {}", filename) + } matches .iter() @@ -53,6 +60,7 @@ pub fn find_secrets( rule_index: sds_match.rule_index, start, end, + validation_status: SecretValidationStatus::from(&sds_match.match_status), }) }) .group_by(|v| v.rule_index) @@ -66,6 +74,7 @@ pub fn find_secrets( .map(|v| SecretResultMatch { start: v.start, end: v.end, + validation_status: v.validation_status, }) .collect(), }) From cf7263f5c866aa8e04e89352037e2f3b62cc79a9 Mon Sep 17 00:00:00 2001 From: Julien Delange Date: Wed, 9 Oct 2024 21:07:12 -0400 Subject: [PATCH 2/8] export secret validation in SARIF --- crates/cli/src/sarif/sarif_utils.rs | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/crates/cli/src/sarif/sarif_utils.rs b/crates/cli/src/sarif/sarif_utils.rs index 6b6c8bc0..dbb9c56b 100644 --- a/crates/cli/src/sarif/sarif_utils.rs +++ b/crates/cli/src/sarif/sarif_utils.rs @@ -185,15 +185,25 @@ impl SarifRuleResult { SarifRuleResult::Secret(secret_result) => secret_result .matches .iter() - .map(|r| SarifViolation { - start: r.start, - end: r.end, - message: secret_result.message.clone(), - severity: RuleSeverity::Error, - category: RuleCategory::Security, - fixes: vec![], - taint_flow: None, - validation_status: Some(r.validation_status.clone()), + .map(|r| { + let severity = match &r.validation_status { + SecretValidationStatus::NotValidated => RuleSeverity::Notice, + SecretValidationStatus::Valid => RuleSeverity::Error, + SecretValidationStatus::Invalid => RuleSeverity::None, + SecretValidationStatus::ValidationError => RuleSeverity::Warning, + SecretValidationStatus::NotAvailable => RuleSeverity::Warning, + }; + + SarifViolation { + start: r.start, + end: r.end, + message: secret_result.message.clone(), + severity, + category: RuleCategory::Security, + fixes: vec![], + taint_flow: None, + validation_status: Some(r.validation_status.clone()), + } }) .collect::>(), } From 19c67d055042cf3254e61021f16d92e57384e639 Mon Sep 17 00:00:00 2001 From: Julien Delange Date: Wed, 9 Oct 2024 21:17:39 -0400 Subject: [PATCH 3/8] export secret validation in SARIF --- crates/cli/src/sarif/sarif_utils.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/cli/src/sarif/sarif_utils.rs b/crates/cli/src/sarif/sarif_utils.rs index dbb9c56b..cfa2827f 100644 --- a/crates/cli/src/sarif/sarif_utils.rs +++ b/crates/cli/src/sarif/sarif_utils.rs @@ -593,6 +593,7 @@ fn generate_results( let category = format!("DATADOG_CATEGORY:{}", rule.category()).to_uppercase(); result_builder.rule_index(i64::try_from(rule_index).unwrap()); + result_builder.level(get_level_from_severity(rule.severity())); tags.push(category); @@ -747,6 +748,14 @@ fn generate_results( }; let mut sarif_result = result_builder.clone(); + + // For secrets, the level is set by violation depending on the validation + // status. We override it here. For static analysis, we report the + // severity of the rule before. + if let SarifRuleResult::Secret(_) = rule_result { + sarif_result.level(get_level_from_severity(violation.severity)); + } + sarif_result .rule_id(rule_result.rule_id()) .locations([location]) @@ -1368,7 +1377,7 @@ mod tests { let sarif_report_to_string = serde_json::to_value(sarif_report).unwrap(); assert_json_eq!( sarif_report_to_string, - serde_json::json!({"runs":[{"results":[{"fixes":[],"level":"error","locations":[{"physicalLocation":{"artifactLocation":{"uri":"myfile.py"},"region":{"endColumn":2,"endLine":2,"startColumn":1,"startLine":1}}}],"message":{"text":"some secret"},"partialFingerprints":{},"properties":{"tags":["DATADOG_CATEGORY:SECURITY","DATADOG_SECRET_VALIDATION_STATUS:NOT_VALIDATED"]},"ruleId":"secret-rule","ruleIndex":0},{"fixes":[],"level":"error","locations":[{"physicalLocation":{"artifactLocation":{"uri":"myfile.py"},"region":{"endColumn":3,"endLine":3,"startColumn":2,"startLine":2}}}],"message":{"text":"some secret"},"partialFingerprints":{},"properties":{"tags":["DATADOG_CATEGORY:SECURITY","DATADOG_SECRET_VALIDATION_STATUS:VALID"]},"ruleId":"secret-rule","ruleIndex":0},{"fixes":[],"level":"error","locations":[{"physicalLocation":{"artifactLocation":{"uri":"myfile.py"},"region":{"endColumn":4,"endLine":4,"startColumn":3,"startLine":3}}}],"message":{"text":"some secret"},"partialFingerprints":{},"properties":{"tags":["DATADOG_CATEGORY:SECURITY","DATADOG_SECRET_VALIDATION_STATUS:INVALID"]},"ruleId":"secret-rule","ruleIndex":0},{"fixes":[],"level":"error","locations":[{"physicalLocation":{"artifactLocation":{"uri":"myfile.py"},"region":{"endColumn":6,"endLine":6,"startColumn":5,"startLine":5}}}],"message":{"text":"some secret"},"partialFingerprints":{},"properties":{"tags":["DATADOG_CATEGORY:SECURITY","DATADOG_SECRET_VALIDATION_STATUS:VALIDATION_ERROR"]},"ruleId":"secret-rule","ruleIndex":0},{"fixes":[],"level":"error","locations":[{"physicalLocation":{"artifactLocation":{"uri":"myfile.py"},"region":{"endColumn":7,"endLine":7,"startColumn":6,"startLine":6}}}],"message":{"text":"some secret"},"partialFingerprints":{},"properties":{"tags":["DATADOG_CATEGORY:SECURITY","DATADOG_SECRET_VALIDATION_STATUS:NOT_AVAILABLE"]},"ruleId":"secret-rule","ruleIndex":0}],"tool":{"driver":{"informationUri":"https://www.datadoghq.com","name":"datadog-static-analyzer","properties":{"tags":["DATADOG_DIFF_AWARE_CONFIG_DIGEST:5d7273dec32b80788b4d3eac46c866f0","DATADOG_EXECUTION_TIME_SECS:42","DATADOG_DIFF_AWARE_ENABLED:false"]},"rules":[{"fullDescription":{"text":"myfile.py"},"id":"secret-rule","name":"secret-rule","properties":{"tags":["DATADOG_RULE_TYPE:SECRET"]},"shortDescription":{"text":"secret-rule"}}],"version":"0.4.4"}}}],"version":"2.1.0"}) + serde_json::json!({"runs":[{"results":[{"fixes":[],"level":"note","locations":[{"physicalLocation":{"artifactLocation":{"uri":"myfile.py"},"region":{"endColumn":2,"endLine":2,"startColumn":1,"startLine":1}}}],"message":{"text":"some secret"},"partialFingerprints":{},"properties":{"tags":["DATADOG_CATEGORY:SECURITY","DATADOG_SECRET_VALIDATION_STATUS:NOT_VALIDATED"]},"ruleId":"secret-rule","ruleIndex":0},{"fixes":[],"level":"error","locations":[{"physicalLocation":{"artifactLocation":{"uri":"myfile.py"},"region":{"endColumn":3,"endLine":3,"startColumn":2,"startLine":2}}}],"message":{"text":"some secret"},"partialFingerprints":{},"properties":{"tags":["DATADOG_CATEGORY:SECURITY","DATADOG_SECRET_VALIDATION_STATUS:VALID"]},"ruleId":"secret-rule","ruleIndex":0},{"fixes":[],"level":"none","locations":[{"physicalLocation":{"artifactLocation":{"uri":"myfile.py"},"region":{"endColumn":4,"endLine":4,"startColumn":3,"startLine":3}}}],"message":{"text":"some secret"},"partialFingerprints":{},"properties":{"tags":["DATADOG_CATEGORY:SECURITY","DATADOG_SECRET_VALIDATION_STATUS:INVALID"]},"ruleId":"secret-rule","ruleIndex":0},{"fixes":[],"level":"warning","locations":[{"physicalLocation":{"artifactLocation":{"uri":"myfile.py"},"region":{"endColumn":6,"endLine":6,"startColumn":5,"startLine":5}}}],"message":{"text":"some secret"},"partialFingerprints":{},"properties":{"tags":["DATADOG_CATEGORY:SECURITY","DATADOG_SECRET_VALIDATION_STATUS:VALIDATION_ERROR"]},"ruleId":"secret-rule","ruleIndex":0},{"fixes":[],"level":"warning","locations":[{"physicalLocation":{"artifactLocation":{"uri":"myfile.py"},"region":{"endColumn":7,"endLine":7,"startColumn":6,"startLine":6}}}],"message":{"text":"some secret"},"partialFingerprints":{},"properties":{"tags":["DATADOG_CATEGORY:SECURITY","DATADOG_SECRET_VALIDATION_STATUS:NOT_AVAILABLE"]},"ruleId":"secret-rule","ruleIndex":0}],"tool":{"driver":{"informationUri":"https://www.datadoghq.com","name":"datadog-static-analyzer","properties":{"tags":["DATADOG_DIFF_AWARE_CONFIG_DIGEST:5d7273dec32b80788b4d3eac46c866f0","DATADOG_EXECUTION_TIME_SECS:42","DATADOG_DIFF_AWARE_ENABLED:false"]},"rules":[{"fullDescription":{"text":"myfile.py"},"id":"secret-rule","name":"secret-rule","properties":{"tags":["DATADOG_RULE_TYPE:SECRET"]},"shortDescription":{"text":"secret-rule"}}],"version":"0.4.4"}}}],"version":"2.1.0"}) ); // validate the schema From 505d6b03116d87182c217653d45fa1b03ec51b8d Mon Sep 17 00:00:00 2001 From: Julien Delange Date: Wed, 9 Oct 2024 21:18:52 -0400 Subject: [PATCH 4/8] linting --- crates/cli/src/sarif/sarif_utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/cli/src/sarif/sarif_utils.rs b/crates/cli/src/sarif/sarif_utils.rs index cfa2827f..fbaa0869 100644 --- a/crates/cli/src/sarif/sarif_utils.rs +++ b/crates/cli/src/sarif/sarif_utils.rs @@ -11,7 +11,7 @@ use derive_builder::Builder; use git2::{BlameOptions, Repository}; use kernel::constants::CARGO_VERSION; use kernel::model::rule::{RuleCategory, RuleSeverity}; -use kernel::model::violation::{Fix, Violation}; +use kernel::model::violation::{Fix}; use kernel::model::{ rule::{Rule, RuleResult}, violation::{Edit, EditType}, From 42d81cb602c5ae1e6565d4ce3505e79bf429f4e0 Mon Sep 17 00:00:00 2001 From: Julien Delange Date: Wed, 9 Oct 2024 21:20:06 -0400 Subject: [PATCH 5/8] linting --- crates/cli/src/sarif/sarif_utils.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/cli/src/sarif/sarif_utils.rs b/crates/cli/src/sarif/sarif_utils.rs index fbaa0869..bea9238b 100644 --- a/crates/cli/src/sarif/sarif_utils.rs +++ b/crates/cli/src/sarif/sarif_utils.rs @@ -11,7 +11,7 @@ use derive_builder::Builder; use git2::{BlameOptions, Repository}; use kernel::constants::CARGO_VERSION; use kernel::model::rule::{RuleCategory, RuleSeverity}; -use kernel::model::violation::{Fix}; +use kernel::model::violation::Fix; use kernel::model::{ rule::{Rule, RuleResult}, violation::{Edit, EditType}, @@ -870,10 +870,9 @@ pub fn generate_sarif_file( #[cfg(test)] mod tests { use super::*; - use crate::sarif::sarif_utils::SarifRule::SecretRule; use assert_json_diff::assert_json_eq; use common::model::position::{Position, PositionBuilder, Region}; - use kernel::model::violation::Fix; + use kernel::model::violation::{Fix, Violation}; use kernel::model::{ common::Language, rule::{RuleBuilder, RuleCategory, RuleResultBuilder, RuleSeverity, RuleType}, From bb8816dbae1aec3e4e4166a9634d4cc8e5abbe06 Mon Sep 17 00:00:00 2001 From: Julien Delange Date: Thu, 10 Oct 2024 14:50:55 -0400 Subject: [PATCH 6/8] address feedback --- .../bins/src/bin/datadog-static-analyzer.rs | 20 +++- crates/cli/src/file_utils.rs | 18 ++- crates/cli/src/sarif/sarif_utils.rs | 112 ++++++++---------- crates/secrets/src/model/secret_result.rs | 2 +- crates/secrets/src/scanner.rs | 4 + 5 files changed, 76 insertions(+), 80 deletions(-) diff --git a/crates/bins/src/bin/datadog-static-analyzer.rs b/crates/bins/src/bin/datadog-static-analyzer.rs index bcc45d32..7c92b772 100644 --- a/crates/bins/src/bin/datadog-static-analyzer.rs +++ b/crates/bins/src/bin/datadog-static-analyzer.rs @@ -41,7 +41,7 @@ use kernel::model::common::OutputFormat; use kernel::model::config_file::{ConfigFile, ConfigMethod, PathConfig}; use kernel::model::rule::{Rule, RuleInternal, RuleResult, RuleSeverity}; use kernel::rule_config::RuleConfigProvider; -use secrets::model::secret_result::SecretResult; +use secrets::model::secret_result::{SecretResult, SecretValidationStatus}; use secrets::scanner::{build_sds_scanner, find_secrets}; use secrets::secret_files::should_ignore_file_for_secret; @@ -660,6 +660,16 @@ fn main() -> Result<()> { .collect(); let nb_secrets_found: u32 = secrets_results.iter().map(|x| x.matches.len() as u32).sum(); + let nb_secrets_validated: u32 = secrets_results + .iter() + .map(|x| { + x.matches + .iter() + .filter(|m| m.validation_status == SecretValidationStatus::Valid) + .collect::>() + .len() as u32 + }) + .sum(); if let Some(pb) = &progress_bar { pb.finish(); @@ -668,8 +678,12 @@ fn main() -> Result<()> { let secrets_execution_time_secs = secrets_start.elapsed().as_secs(); println!( - "Found {} secret(s) in {} file(s) using {} rule(s) within {} sec(s)", - nb_secrets_found, nb_secrets_files, nb_secrets_rules, secrets_execution_time_secs + "Found {} secret(s) ({} validated) in {} file(s) using {} rule(s) within {} sec(s)", + nb_secrets_found, + nb_secrets_validated, + nb_secrets_files, + nb_secrets_rules, + secrets_execution_time_secs ); } diff --git a/crates/cli/src/file_utils.rs b/crates/cli/src/file_utils.rs index 6b0f6091..1acb36ad 100644 --- a/crates/cli/src/file_utils.rs +++ b/crates/cli/src/file_utils.rs @@ -7,12 +7,11 @@ use anyhow::Result; use sha2::{Digest, Sha256}; use walkdir::WalkDir; -use kernel::model::common::Language; -use kernel::model::config_file::PathConfig; - use crate::model::cli_configuration::CliConfiguration; use crate::model::datadog_api::DiffAwareData; -use crate::sarif::sarif_utils::SarifViolation; +use kernel::model::common::Language; +use kernel::model::config_file::PathConfig; +use kernel::model::violation::Violation; static FILE_EXTENSIONS_PER_LANGUAGE_LIST: &[(Language, &[&str])] = &[ (Language::Csharp, &["cs"]), @@ -304,7 +303,7 @@ pub fn filter_files_by_diff_aware_info( /// SHA2( - - - ) pub fn get_fingerprint_for_violation( rule_name: String, - violation: &SarifViolation, + violation: &Violation, repository_root: &Path, file: &Path, use_debug: bool, @@ -394,7 +393,7 @@ mod tests { #[test] fn get_fingerprint_for_violation_success_single_region() { let d = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - let violation = SarifViolation { + let violation = Violation { start: Position { line: 10, col: 1 }, end: Position { line: 12, col: 1 }, message: "something bad happened".to_string(), @@ -402,7 +401,6 @@ mod tests { category: RuleCategory::Performance, fixes: vec![], taint_flow: None, - validation_status: None, }; let directory_string = d.into_os_string().into_string().unwrap(); let fingerprint = get_fingerprint_for_violation( @@ -439,7 +437,7 @@ mod tests { end: Position { line: 5, col: 30 }, }; let root_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - let violation = SarifViolation { + let violation = Violation { start: region0.start, end: region0.end, message: "flow violation".to_string(), @@ -447,7 +445,6 @@ mod tests { category: RuleCategory::Security, fixes: vec![], taint_flow: Some(vec![region0, region1]), - validation_status: None, }; let fingerprint = get_fingerprint_for_violation( "taint_flow_rule".to_string(), @@ -470,7 +467,7 @@ mod tests { let d = PathBuf::from(env!("CARGO_MANIFEST_DIR")); let path = "resources/test/gitignore/test1"; - let violation = SarifViolation { + let violation = Violation { start: Position { line: 10, col: 1 }, end: Position { line: 12, col: 1 }, message: "something bad happened".to_string(), @@ -478,7 +475,6 @@ mod tests { category: RuleCategory::Performance, fixes: vec![], taint_flow: None, - validation_status: None, }; let directory_string = d.into_os_string().into_string().unwrap(); diff --git a/crates/cli/src/sarif/sarif_utils.rs b/crates/cli/src/sarif/sarif_utils.rs index bea9238b..a951c9cf 100644 --- a/crates/cli/src/sarif/sarif_utils.rs +++ b/crates/cli/src/sarif/sarif_utils.rs @@ -7,11 +7,10 @@ use anyhow::Result; use base64::Engine; use common::model::position::PositionBuilder; use common::model::position::{Position, Region}; -use derive_builder::Builder; use git2::{BlameOptions, Repository}; use kernel::constants::CARGO_VERSION; use kernel::model::rule::{RuleCategory, RuleSeverity}; -use kernel::model::violation::Fix; +use kernel::model::violation::{Fix, Violation}; use kernel::model::{ rule::{Rule, RuleResult}, violation::{Edit, EditType}, @@ -31,6 +30,7 @@ use serde_sarif::sarif::{ use crate::file_utils::get_fingerprint_for_violation; use crate::model::cli_configuration::CliConfiguration; use crate::model::datadog_api::DiffAwareData; +use crate::sarif::sarif_utils::SarifViolation::{Secret, StaticAnalysis}; trait IntoSarif { type SarifType; @@ -121,38 +121,30 @@ impl From for SarifRule { } /// Generic representation of a violation for both static analysis and secrets -#[derive(Deserialize, Debug, Serialize, Clone, Builder)] -pub struct SarifViolation { - pub start: Position, - pub end: Position, - pub message: String, - pub severity: RuleSeverity, - pub category: RuleCategory, - pub fixes: Vec, - pub taint_flow: Option>, - pub validation_status: Option, +#[derive(Debug, Clone)] +pub enum SarifViolation { + StaticAnalysis(Violation), + Secret(Violation, SecretValidationStatus), } impl SarifViolation { + fn get_violation(&self) -> &Violation { + match self { + StaticAnalysis(v) => v, + Secret(v, _) => v, + } + } + fn get_properties(&self) -> Vec { - if let Some(validation_status) = &self.validation_status { - match validation_status { - SecretValidationStatus::NotValidated => { - vec!["DATADOG_SECRET_VALIDATION_STATUS:NOT_VALIDATED".to_string()] - } - SecretValidationStatus::Valid => { - vec!["DATADOG_SECRET_VALIDATION_STATUS:VALID".to_string()] - } - SecretValidationStatus::Invalid => { - vec!["DATADOG_SECRET_VALIDATION_STATUS:INVALID".to_string()] - } - SecretValidationStatus::ValidationError => { - vec!["DATADOG_SECRET_VALIDATION_STATUS:VALIDATION_ERROR".to_string()] - } - SecretValidationStatus::NotAvailable => { - vec!["DATADOG_SECRET_VALIDATION_STATUS:NOT_AVAILABLE".to_string()] - } - } + if let Secret(_, validation_status) = &self { + let status = match validation_status { + SecretValidationStatus::NotValidated => "NOT_VALIDATED", + SecretValidationStatus::Valid => "VALID", + SecretValidationStatus::Invalid => "INVALID", + SecretValidationStatus::ValidationError => "VALIDATION_ERROR", + SecretValidationStatus::NotAvailable => "NOT_AVAILABLE", + }; + vec![format!("DATADOG_SECRET_VALIDATION_STATUS:{}", status).to_string()] } else { vec![] } @@ -171,16 +163,7 @@ impl SarifRuleResult { SarifRuleResult::StaticAnalysis(r) => r .violations .iter() - .map(|v| SarifViolation { - start: v.start, - end: v.end, - message: v.message.clone(), - severity: v.severity, - category: v.category, - fixes: v.fixes.clone(), - taint_flow: v.taint_flow.clone(), - validation_status: None, - }) + .map(|v| StaticAnalysis(v.clone())) .collect::>(), SarifRuleResult::Secret(secret_result) => secret_result .matches @@ -191,19 +174,21 @@ impl SarifRuleResult { SecretValidationStatus::Valid => RuleSeverity::Error, SecretValidationStatus::Invalid => RuleSeverity::None, SecretValidationStatus::ValidationError => RuleSeverity::Warning, - SecretValidationStatus::NotAvailable => RuleSeverity::Warning, + SecretValidationStatus::NotAvailable => RuleSeverity::Error, }; - SarifViolation { - start: r.start, - end: r.end, - message: secret_result.message.clone(), - severity, - category: RuleCategory::Security, - fixes: vec![], - taint_flow: None, - validation_status: Some(r.validation_status.clone()), - } + Secret( + Violation { + start: r.start, + end: r.end, + message: secret_result.message.clone(), + severity, + category: RuleCategory::Security, + fixes: vec![], + taint_flow: None, + }, + r.validation_status, + ) }) .collect::>(), } @@ -484,7 +469,7 @@ fn is_valid_position(position: &Position) -> bool { } /// Check that the violation is valid and must be included -fn is_valid_violation(violation: &SarifViolation) -> bool { +fn is_valid_violation(violation: &Violation) -> bool { if !is_valid_position(&violation.start) || !is_valid_position(&violation.end) { return false; } @@ -612,8 +597,8 @@ fn generate_results( let violations = rule_result.violations(); violations .into_iter() - .filter(|violation| { - let is_valid = is_valid_violation(violation); + .filter(|sarif_violation| { + let is_valid = is_valid_violation(sarif_violation.get_violation()); if !is_valid && options_orig.debug { eprintln!( "Invalid violations detected, check the rule {}", @@ -622,7 +607,8 @@ fn generate_results( } is_valid }) - .map(move |violation| { + .map(move |sarif_violation| { + let violation = sarif_violation.get_violation(); // if we find the rule for this violation, get the id, level and category let location = LocationBuilder::default() .physical_location( @@ -768,7 +754,7 @@ fn generate_results( ) .properties( PropertyBagBuilder::default() - .tags([tags.clone(), violation.get_properties()].concat()) + .tags([tags.clone(), sarif_violation.get_properties()].concat()) .build() .unwrap(), ) @@ -893,7 +879,7 @@ mod tests { #[test] fn test_is_valid_violation() { // bad location in the violation location - assert!(!is_valid_violation(&SarifViolation { + assert!(!is_valid_violation(&Violation { start: Position { line: 0, col: 1 }, end: Position { line: 42, col: 42 }, message: "bad stuff".to_string(), @@ -901,11 +887,10 @@ mod tests { category: RuleCategory::BestPractices, fixes: vec![], taint_flow: None, - validation_status: None, })); // good location in the violation location and no fixes - assert!(is_valid_violation(&SarifViolation { + assert!(is_valid_violation(&Violation { start: Position { line: 1, col: 1 }, end: Position { line: 42, col: 42 }, message: "bad stuff".to_string(), @@ -913,11 +898,10 @@ mod tests { category: RuleCategory::BestPractices, fixes: vec![], taint_flow: None, - validation_status: None, })); // bad location in the fixes location - assert!(!is_valid_violation(&SarifViolation { + assert!(!is_valid_violation(&Violation { start: Position { line: 1, col: 1 }, end: Position { line: 42, col: 42 }, message: "bad stuff".to_string(), @@ -933,11 +917,10 @@ mod tests { }] }], taint_flow: None, - validation_status: None, })); // good location everywhere - assert!(is_valid_violation(&SarifViolation { + assert!(is_valid_violation(&Violation { start: Position { line: 1, col: 1 }, end: Position { line: 42, col: 42 }, message: "bad stuff".to_string(), @@ -953,7 +936,6 @@ mod tests { }] }], taint_flow: None, - validation_status: None, })); } @@ -1376,7 +1358,7 @@ mod tests { let sarif_report_to_string = serde_json::to_value(sarif_report).unwrap(); assert_json_eq!( sarif_report_to_string, - serde_json::json!({"runs":[{"results":[{"fixes":[],"level":"note","locations":[{"physicalLocation":{"artifactLocation":{"uri":"myfile.py"},"region":{"endColumn":2,"endLine":2,"startColumn":1,"startLine":1}}}],"message":{"text":"some secret"},"partialFingerprints":{},"properties":{"tags":["DATADOG_CATEGORY:SECURITY","DATADOG_SECRET_VALIDATION_STATUS:NOT_VALIDATED"]},"ruleId":"secret-rule","ruleIndex":0},{"fixes":[],"level":"error","locations":[{"physicalLocation":{"artifactLocation":{"uri":"myfile.py"},"region":{"endColumn":3,"endLine":3,"startColumn":2,"startLine":2}}}],"message":{"text":"some secret"},"partialFingerprints":{},"properties":{"tags":["DATADOG_CATEGORY:SECURITY","DATADOG_SECRET_VALIDATION_STATUS:VALID"]},"ruleId":"secret-rule","ruleIndex":0},{"fixes":[],"level":"none","locations":[{"physicalLocation":{"artifactLocation":{"uri":"myfile.py"},"region":{"endColumn":4,"endLine":4,"startColumn":3,"startLine":3}}}],"message":{"text":"some secret"},"partialFingerprints":{},"properties":{"tags":["DATADOG_CATEGORY:SECURITY","DATADOG_SECRET_VALIDATION_STATUS:INVALID"]},"ruleId":"secret-rule","ruleIndex":0},{"fixes":[],"level":"warning","locations":[{"physicalLocation":{"artifactLocation":{"uri":"myfile.py"},"region":{"endColumn":6,"endLine":6,"startColumn":5,"startLine":5}}}],"message":{"text":"some secret"},"partialFingerprints":{},"properties":{"tags":["DATADOG_CATEGORY:SECURITY","DATADOG_SECRET_VALIDATION_STATUS:VALIDATION_ERROR"]},"ruleId":"secret-rule","ruleIndex":0},{"fixes":[],"level":"warning","locations":[{"physicalLocation":{"artifactLocation":{"uri":"myfile.py"},"region":{"endColumn":7,"endLine":7,"startColumn":6,"startLine":6}}}],"message":{"text":"some secret"},"partialFingerprints":{},"properties":{"tags":["DATADOG_CATEGORY:SECURITY","DATADOG_SECRET_VALIDATION_STATUS:NOT_AVAILABLE"]},"ruleId":"secret-rule","ruleIndex":0}],"tool":{"driver":{"informationUri":"https://www.datadoghq.com","name":"datadog-static-analyzer","properties":{"tags":["DATADOG_DIFF_AWARE_CONFIG_DIGEST:5d7273dec32b80788b4d3eac46c866f0","DATADOG_EXECUTION_TIME_SECS:42","DATADOG_DIFF_AWARE_ENABLED:false"]},"rules":[{"fullDescription":{"text":"myfile.py"},"id":"secret-rule","name":"secret-rule","properties":{"tags":["DATADOG_RULE_TYPE:SECRET"]},"shortDescription":{"text":"secret-rule"}}],"version":"0.4.4"}}}],"version":"2.1.0"}) + serde_json::json!({"runs":[{"results":[{"fixes":[],"level":"note","locations":[{"physicalLocation":{"artifactLocation":{"uri":"myfile.py"},"region":{"endColumn":2,"endLine":2,"startColumn":1,"startLine":1}}}],"message":{"text":"some secret"},"partialFingerprints":{},"properties":{"tags":["DATADOG_CATEGORY:SECURITY","DATADOG_SECRET_VALIDATION_STATUS:NOT_VALIDATED"]},"ruleId":"secret-rule","ruleIndex":0},{"fixes":[],"level":"error","locations":[{"physicalLocation":{"artifactLocation":{"uri":"myfile.py"},"region":{"endColumn":3,"endLine":3,"startColumn":2,"startLine":2}}}],"message":{"text":"some secret"},"partialFingerprints":{},"properties":{"tags":["DATADOG_CATEGORY:SECURITY","DATADOG_SECRET_VALIDATION_STATUS:VALID"]},"ruleId":"secret-rule","ruleIndex":0},{"fixes":[],"level":"none","locations":[{"physicalLocation":{"artifactLocation":{"uri":"myfile.py"},"region":{"endColumn":4,"endLine":4,"startColumn":3,"startLine":3}}}],"message":{"text":"some secret"},"partialFingerprints":{},"properties":{"tags":["DATADOG_CATEGORY:SECURITY","DATADOG_SECRET_VALIDATION_STATUS:INVALID"]},"ruleId":"secret-rule","ruleIndex":0},{"fixes":[],"level":"warning","locations":[{"physicalLocation":{"artifactLocation":{"uri":"myfile.py"},"region":{"endColumn":6,"endLine":6,"startColumn":5,"startLine":5}}}],"message":{"text":"some secret"},"partialFingerprints":{},"properties":{"tags":["DATADOG_CATEGORY:SECURITY","DATADOG_SECRET_VALIDATION_STATUS:VALIDATION_ERROR"]},"ruleId":"secret-rule","ruleIndex":0},{"fixes":[],"level":"error","locations":[{"physicalLocation":{"artifactLocation":{"uri":"myfile.py"},"region":{"endColumn":7,"endLine":7,"startColumn":6,"startLine":6}}}],"message":{"text":"some secret"},"partialFingerprints":{},"properties":{"tags":["DATADOG_CATEGORY:SECURITY","DATADOG_SECRET_VALIDATION_STATUS:NOT_AVAILABLE"]},"ruleId":"secret-rule","ruleIndex":0}],"tool":{"driver":{"informationUri":"https://www.datadoghq.com","name":"datadog-static-analyzer","properties":{"tags":["DATADOG_DIFF_AWARE_CONFIG_DIGEST:5d7273dec32b80788b4d3eac46c866f0","DATADOG_EXECUTION_TIME_SECS:42","DATADOG_DIFF_AWARE_ENABLED:false"]},"rules":[{"fullDescription":{"text":"myfile.py"},"id":"secret-rule","name":"secret-rule","properties":{"tags":["DATADOG_RULE_TYPE:SECRET"]},"shortDescription":{"text":"secret-rule"}}],"version":"0.4.4"}}}],"version":"2.1.0"}) ); // validate the schema diff --git a/crates/secrets/src/model/secret_result.rs b/crates/secrets/src/model/secret_result.rs index 74f0083f..92c8379f 100644 --- a/crates/secrets/src/model/secret_result.rs +++ b/crates/secrets/src/model/secret_result.rs @@ -6,7 +6,7 @@ use common::model::position::Position; use sds::MatchStatus; use serde::{Deserialize, Serialize}; -#[derive(Clone, Debug, PartialEq, Hash, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Hash, Eq, Serialize, Deserialize)] pub enum SecretValidationStatus { NotValidated, Valid, diff --git a/crates/secrets/src/scanner.rs b/crates/secrets/src/scanner.rs index 2e5257f3..8e600167 100644 --- a/crates/secrets/src/scanner.rs +++ b/crates/secrets/src/scanner.rs @@ -44,6 +44,10 @@ pub fn find_secrets( let mut codemut = code.to_owned(); let mut matches = scanner.scan(&mut codemut, vec![]); + if matches.is_empty() { + return vec![]; + } + let matches_validation = futures::executor::block_on(scanner.validate_matches(&mut matches)); if matches_validation.is_err() && options.use_debug { From 2d3feae0e9bb08ec279fcffd65148b57ff5ba87a Mon Sep 17 00:00:00 2001 From: Julien Delange Date: Thu, 10 Oct 2024 14:55:04 -0400 Subject: [PATCH 7/8] address feedback --- crates/cli/src/sarif/sarif_utils.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/cli/src/sarif/sarif_utils.rs b/crates/cli/src/sarif/sarif_utils.rs index a951c9cf..dce89133 100644 --- a/crates/cli/src/sarif/sarif_utils.rs +++ b/crates/cli/src/sarif/sarif_utils.rs @@ -5,12 +5,12 @@ use std::rc::Rc; use crate::constants::{SARIF_PROPERTY_DATADOG_FINGERPRINT, SARIF_PROPERTY_SHA}; use anyhow::Result; use base64::Engine; +use common::model::position::Position; use common::model::position::PositionBuilder; -use common::model::position::{Position, Region}; use git2::{BlameOptions, Repository}; use kernel::constants::CARGO_VERSION; use kernel::model::rule::{RuleCategory, RuleSeverity}; -use kernel::model::violation::{Fix, Violation}; +use kernel::model::violation::Violation; use kernel::model::{ rule::{Rule, RuleResult}, violation::{Edit, EditType}, @@ -19,7 +19,6 @@ use path_slash::PathExt; use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS}; use secrets::model::secret_result::{SecretResult, SecretValidationStatus}; use secrets::model::secret_rule::SecretRule; -use serde::{Deserialize, Serialize}; use serde_sarif::sarif::{ self, ArtifactChangeBuilder, ArtifactLocationBuilder, FixBuilder, LocationBuilder, MessageBuilder, PhysicalLocationBuilder, PropertyBagBuilder, RegionBuilder, Replacement, @@ -711,7 +710,7 @@ fn generate_results( let fingerprint_option = get_fingerprint_for_violation( rule_result.rule_name().to_string(), - &violation, + violation, Path::new(options.repository_directory.as_str()), Path::new(rule_result.file_path().as_str()), options.debug, From afbe321590b0184a129f41c49ddadf899c862e8f Mon Sep 17 00:00:00 2001 From: Julien Delange Date: Thu, 10 Oct 2024 15:05:20 -0400 Subject: [PATCH 8/8] fix test secrets --- misc/integration-test-secrets.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/misc/integration-test-secrets.sh b/misc/integration-test-secrets.sh index 72d9afa2..149bdaf9 100755 --- a/misc/integration-test-secrets.sh +++ b/misc/integration-test-secrets.sh @@ -26,6 +26,20 @@ if [ "$RES" -ne "2" ]; then exit 1 fi +status1=`jq '.runs[0].results[0].properties.tags[1]' "${REPO_DIR}/results1.json"` + +if [ "$status1" != "\"DATADOG_SECRET_VALIDATION_STATUS:NOT_AVAILABLE\"" ]; then + echo "did not find DATADOG_SECRET_VALIDATION_STATUS:NOT_AVAILABLE in properties, found $status1" + exit 1 +fi + +status2=`jq '.runs[0].results[1].properties.tags[1]' "${REPO_DIR}/results1.json"` + +if [ "$status2" != "\"DATADOG_SECRET_VALIDATION_STATUS:NOT_AVAILABLE\"" ]; then + echo "did not find DATADOG_SECRET_VALIDATION_STATUS:NOT_AVAILABLE in properties, found $status2" + exit 1 +fi + echo "All tests passed" exit 0