SALTO-6720: (Salesforce) Store Profiles And PermissionSets broken pat…

…hs on fetch (#6631)

packages/salesforce-adapter/src/adapter.ts

@@ -106,6 +106,7 @@ import omitStandardFieldsNonDeployableValuesFilter from './filters/omit_standard
 import waveStaticFilesFilter from './filters/wave_static_files'
 import generatedDependenciesFilter from './filters/generated_dependencies'
 import extendTriggersMetadataFilter from './filters/extend_triggers_metadata'
+import profilesAndPermissionSetsBrokenPathsFilter from './filters/profiles_and_permission_sets_broken_paths'
 import { CUSTOM_REFS_CONFIG, FetchElements, FetchProfile, MetadataQuery, SalesforceConfig } from './types'
 import mergeProfilesWithSourceValuesFilter from './filters/merge_profiles_with_source_values'
 import flowCoordinatesFilter from './filters/flow_coordinates'
@@ -247,10 +248,12 @@ export const allFilters: Array<FilterCreator> = [
   omitStandardFieldsNonDeployableValuesFilter,
   // taskAndEventCustomFields should run before customTypeSplit
   taskAndEventCustomFields,
-  // customTypeSplit should run after omitStandardFieldsNonDeployableValuesFilter
-  customTypeSplit,
   mergeProfilesWithSourceValuesFilter,
-  // profileInstanceSplitFilter should run after mergeProfilesWithSourceValuesFilter
+  // profilesAndPermissionSetsBrokenPathsFilter should run after mergeProfilesWithSourceValuesFilter
+  profilesAndPermissionSetsBrokenPathsFilter,
+  // customTypeSplit should run after omitStandardFieldsNonDeployableValuesFilter and profilesAndPermissionSetsBrokenPathsFilter
+  customTypeSplit,
+  // profileInstanceSplitFilter should run after mergeProfilesWithSourceValuesFilter and profilesAndPermissionSetsBrokenPathsFilter
   profileInstanceSplitFilter,
   // Any filter that relies on _created_at or _changed_at should run after removeUnixTimeZero
   removeUnixTimeZeroFilter,

packages/salesforce-adapter/src/constants.ts

@@ -8,7 +8,7 @@
 import { client as clientUtils } from '@salto-io/adapter-components'
 import { types } from '@salto-io/lowerdash'
 import _ from 'lodash'
-import { ActionName, CORE_ANNOTATIONS, ElemID, ObjectType } from '@salto-io/adapter-api'
+import { ActionName, BuiltinTypes, CORE_ANNOTATIONS, ElemID, ListType, ObjectType } from '@salto-io/adapter-api'
 
 export const { RATE_LIMIT_UNLIMITED_MAX_CONCURRENT_REQUESTS } = clientUtils
 
@@ -440,6 +440,15 @@ export const CUSTOM_METADATA_META_TYPE = 'CustomMetadata'
 // Artificial Types
 export const CURRENCY_CODE_TYPE_NAME = 'CurrencyIsoCodes'
 export const CHANGED_AT_SINGLETON = 'ChangedAtSingleton'
+export const PROFILE_AND_PERMISSION_SETS_BROKEN_PATHS = 'ProfilesAndPermissionSetsBrokenPaths'
+export const PATHS_FIELD = 'paths'
+
+export const getTypePath = (name: string, isTopLevelType = true): string[] => [
+  SALESFORCE,
+  TYPES_PATH,
+  ...(isTopLevelType ? [] : [SUBTYPES_PATH]),
+  name,
+]
 
 export const ArtificialTypes = {
   [CHANGED_AT_SINGLETON]: new ObjectType({
@@ -450,6 +459,17 @@ export const ArtificialTypes = {
       [CORE_ANNOTATIONS.HIDDEN_VALUE]: true,
     },
   }),
+  [PROFILE_AND_PERMISSION_SETS_BROKEN_PATHS]: new ObjectType({
+    elemID: new ElemID(SALESFORCE, PROFILE_AND_PERMISSION_SETS_BROKEN_PATHS),
+    isSettings: true,
+    path: getTypePath(PROFILE_AND_PERMISSION_SETS_BROKEN_PATHS),
+    fields: {
+      [PATHS_FIELD]: { refType: new ListType(BuiltinTypes.STRING) },
+    },
+    annotations: {
+      [CORE_ANNOTATIONS.HIDDEN_VALUE]: true,
+    },
+  }),
 } as const
 
 // Standard Object Types

packages/salesforce-adapter/src/custom_references/profiles_and_permission_sets.ts

@@ -9,7 +9,7 @@
 import _, { Dictionary } from 'lodash'
 import { collections, promises } from '@salto-io/lowerdash'
 import { logger } from '@salto-io/logging'
-import { Element, ElemID, InstanceElement, ReferenceInfo, Values } from '@salto-io/adapter-api'
+import { Element, ElemID, InstanceElement, ReadOnlyElementsSource, ReferenceInfo, Values } from '@salto-io/adapter-api'
 import { invertNaclCase } from '@salto-io/adapter-utils'
 import { MetadataInstance, MetadataQuery, ProfileSection, WeakReferencesHandler } from '../types'
 import {
@@ -32,6 +32,7 @@ import {
   ENDS_WITH_CUSTOM_SUFFIX_REGEX,
   extractFlatCustomObjectFields,
   getNamespaceFromString,
+  getProfilesAndPermissionSetsBrokenPaths,
   isInstanceOfTypeSync,
 } from '../filters/utils'
 import { buildMetadataQuery } from '../fetch_profile/metadata_query'
@@ -43,7 +44,7 @@ const log = logger(module)
 
 const FIELD_NO_ACCESS = 'NoAccess'
 
-const isProfileOrPermissionSetInstance = isInstanceOfTypeSync(
+export const isProfileOrPermissionSetInstance = isInstanceOfTypeSync(
   PROFILE_METADATA_TYPE,
   PERMISSION_SET_METADATA_TYPE,
   MUTING_PERMISSION_SET_METADATA_TYPE,
@@ -250,7 +251,7 @@ const instanceEntriesTargets = (instance: InstanceElement, metadataQuery?: Metad
       target,
     ]),
   )
-    .filter(([, target]) => metadataQuery?.isInstanceMatch(target) ?? true)
+    .filter(([, target]) => metadataQuery?.isInstanceIncluded(target) ?? true)
     .fromPairs()
     .value()
 
@@ -288,38 +289,57 @@ export const buildElemIDMetadataQuery = (metadataQuery: MetadataQuery): Metadata
   }
 }
 
+export const getProfilesAndPsBrokenReferenceFields = async ({
+  profilesAndPermissionSets,
+  elementsSource,
+  metadataQuery,
+}: {
+  profilesAndPermissionSets: InstanceElement[]
+  elementsSource: ReadOnlyElementsSource
+  metadataQuery: MetadataQuery
+}): Promise<{ paths: string[]; entriesTargets: Record<string, ElemID> }> => {
+  const entriesTargets: Dictionary<ElemID> = _.merge(
+    {},
+    ...profilesAndPermissionSets.map(instance =>
+      instanceEntriesTargets(instance, buildElemIDMetadataQuery(metadataQuery)),
+    ),
+  )
+  const elementNames = new Set(
+    await awu(await elementsSource.getAll())
+      .flatMap(extractFlatCustomObjectFields)
+      .map(elem => elem.elemID.getFullName())
+      .toArray(),
+  )
+  const brokenPaths = new Set(await getProfilesAndPermissionSetsBrokenPaths(elementsSource))
+  const paths = Object.keys(await pickAsync(entriesTargets, async target => !elementNames.has(target.getFullName())))
+    // Ignore broken paths that were calculated in fetch
+    .filter(path => !brokenPaths.has(path))
+    // fieldPermissions may contain standard values that are not referring to any field, we shouldn't omit these
+    .filter(path => !isStandardFieldPermissionsPath(path))
+    // Some standard objects are not managed in the metadata API and won't exist in the workspace.
+    .filter(path => !isStandardObjectPermissionsPath(path))
+  return { paths, entriesTargets }
+}
+
 const removeWeakReferences: WeakReferencesHandler['removeWeakReferences'] =
   ({ elementsSource, config }) =>
   async elements => {
-    const metadataQuery = buildElemIDMetadataQuery(buildMetadataQuery({ fetchParams: config.fetch ?? {} }))
-    const instances = elements.filter(isProfileOrPermissionSetInstance)
-    const entriesTargets: Dictionary<ElemID> = _.merge(
-      {},
-      ...instances.map(instance => instanceEntriesTargets(instance, metadataQuery)),
-    )
-    const elementNames = new Set(
-      await awu(await elementsSource.getAll())
-        .flatMap(extractFlatCustomObjectFields)
-        .map(elem => elem.elemID.getFullName())
-        .toArray(),
-    )
-    const brokenReferenceFields = Object.keys(
-      await pickAsync(entriesTargets, async target => !elementNames.has(target.getFullName())),
-      // fieldPermissions may contain standard values that are not referring to any field, we shouldn't omit these
-    )
-      .filter(path => !isStandardFieldPermissionsPath(path))
-      // Some standard objects are not managed in the metadata API and won't exist in the workspace.
-      .filter(path => !isStandardObjectPermissionsPath(path))
-    const instancesWithBrokenReferences = instances.filter(instance =>
-      brokenReferenceFields.some(field => _(instance.value).has(field)),
+    const profilesAndPermissionSets = elements.filter(isProfileOrPermissionSetInstance)
+    const { paths: brokenReferencePaths, entriesTargets } = await getProfilesAndPsBrokenReferenceFields({
+      profilesAndPermissionSets,
+      elementsSource,
+      metadataQuery: buildMetadataQuery({ fetchParams: config.fetch ?? {} }),
+    })
+    const instancesWithBrokenReferences = profilesAndPermissionSets.filter(instance =>
+      brokenReferencePaths.some(field => _(instance.value).has(field)),
     )
     const fixedElements = instancesWithBrokenReferences.map(instance => {
       const fixed = instance.clone()
-      fixed.value = _.omit(fixed.value, brokenReferenceFields)
+      fixed.value = _.omit(fixed.value, brokenReferencePaths)
       return fixed
     })
     const errors = instancesWithBrokenReferences.map(instance => {
-      const instanceBrokenReferenceFields = brokenReferenceFields
+      const instanceBrokenReferenceFields = brokenReferencePaths
         .filter(field => _(instance.value).has(field))
         .map(field => entriesTargets[field].getFullName())
         .sort()

packages/salesforce-adapter/src/fetch_profile/fetch_profile.ts

@@ -50,6 +50,7 @@ const optionalFeaturesDefaultValues: OptionalFeaturesDefaultValues = {
   logDiffsFromParsingXmlNumbers: true,
   extendTriggersMetadata: false,
   removeReferenceFromFilterItemToRecordType: false,
+  storeProfilesAndPermissionSetsBrokenPaths: true,
 }
 
 type BuildFetchProfileParams = {

packages/salesforce-adapter/src/filters/currency_iso_code.ts

@@ -27,8 +27,9 @@ import {
   CUSTOM_VALUE,
   CURRENCY_CODE_TYPE_NAME,
   CURRENCY_ISO_CODE,
+  getTypePath,
 } from '../constants'
-import { Types, getTypePath } from '../transformers/transformer'
+import { Types } from '../transformers/transformer'
 
 const currencyCodeType = new ObjectType({
   elemID: new ElemID(SALESFORCE, CURRENCY_CODE_TYPE_NAME),

packages/salesforce-adapter/src/filters/organization_settings.ts

@@ -11,8 +11,8 @@ import { BuiltinTypes, CORE_ANNOTATIONS, ElemID, InstanceElement, ObjectType, Va
 import { logger } from '@salto-io/logging'
 import { FilterCreator } from '../filter'
 import { ensureSafeFilterFetch, queryClient, safeApiName } from './utils'
-import { getSObjectFieldElement, getTypePath } from '../transformers/transformer'
-import { API_NAME, ORGANIZATION_SETTINGS, RECORDS_PATH, SALESFORCE, SETTINGS_PATH } from '../constants'
+import { getSObjectFieldElement } from '../transformers/transformer'
+import { API_NAME, getTypePath, ORGANIZATION_SETTINGS, RECORDS_PATH, SALESFORCE, SETTINGS_PATH } from '../constants'
 import SalesforceClient from '../client/client'
 import { FetchProfile } from '../types'
 

packages/salesforce-adapter/src/filters/profiles_and_permission_sets_broken_paths.ts

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2024 Salto Labs Ltd.
+ * Licensed under the Salto Terms of Use (the "License");
+ * You may not use this file except in compliance with the License.  You may obtain a copy of the License at https://www.salto.io/terms-of-use
+ *
+ * CERTAIN THIRD PARTY SOFTWARE MAY BE CONTAINED IN PORTIONS OF THE SOFTWARE. See NOTICE FILE AT https://github.com/salto-io/salto/blob/main/NOTICES
+ */
+import _ from 'lodash'
+import { logger } from '@salto-io/logging'
+import { inspectValue } from '@salto-io/adapter-utils'
+import { ElemID, InstanceElement } from '@salto-io/adapter-api'
+import { ArtificialTypes } from '../constants'
+import { FilterCreator } from '../filter'
+import { buildElementsSourceForFetch, ensureSafeFilterFetch, getProfilesAndPermissionSetsBrokenPaths } from './utils'
+import {
+  getProfilesAndPsBrokenReferenceFields,
+  isProfileOrPermissionSetInstance,
+} from '../custom_references/profiles_and_permission_sets'
+
+const log = logger(module)
+
+const filter: FilterCreator = ({ config }) => ({
+  name: 'profilesAndPermissionSetsBrokenPaths',
+  onFetch: ensureSafeFilterFetch({
+    config,
+    warningMessage: 'Error occurred while calculating Profiles and PermissionSets broken paths',
+    fetchFilterFunc: async elements => {
+      const elementsSource = buildElementsSourceForFetch(elements, config)
+      const profilesAndPermissionSets = elements.filter(isProfileOrPermissionSetInstance)
+      if (profilesAndPermissionSets.length === 0) {
+        return
+      }
+      const { paths } = await getProfilesAndPsBrokenReferenceFields({
+        elementsSource,
+        profilesAndPermissionSets,
+        metadataQuery: config.fetchProfile.metadataQuery,
+      })
+      if (paths.length === 0) {
+        return
+      }
+      const uniquePaths = _.uniq(
+        paths.concat(
+          // We should concat the existing broken paths in case of partial fetch, and override them in full fetch
+          config.fetchProfile.metadataQuery.isPartialFetch()
+            ? await getProfilesAndPermissionSetsBrokenPaths(elementsSource)
+            : [],
+        ),
+      )
+      log.debug('Profiles and PermissionSets broken paths: %s', inspectValue(uniquePaths, { maxArrayLength: 100 }))
+      elements.push(
+        new InstanceElement(ElemID.CONFIG_NAME, ArtificialTypes.ProfilesAndPermissionSetsBrokenPaths, {
+          paths: uniquePaths,
+        }),
+      )
+    },
+  }),
+})
+
+export default filter

packages/salesforce-adapter/src/filters/utils.ts

@@ -93,6 +93,8 @@ import {
   SALESFORCE_CUSTOM_SUFFIX,
   TASK_CUSTOM_OBJECT,
   EVENT_CUSTOM_OBJECT,
+  PROFILE_AND_PERMISSION_SETS_BROKEN_PATHS,
+  PATHS_FIELD,
 } from '../constants'
 import { CustomField, CustomObject, JSONBool, SalesforceRecord } from '../client/types'
 import * as transformer from '../transformers/transformer'
@@ -954,3 +956,15 @@ export const isCustomField = (field: Field): boolean => field.name.endsWith(SALE
 
 export const isFieldOfTaskOrEvent = ({ parent }: Field): boolean =>
   isCustomObjectSync(parent) && [TASK_CUSTOM_OBJECT, EVENT_CUSTOM_OBJECT].includes(apiNameSync(parent) ?? '')
+
+export const getProfilesAndPermissionSetsBrokenPaths = async (
+  elementsSource: ReadOnlyElementsSource,
+): Promise<string[]> => {
+  const instance = await elementsSource.get(
+    new ElemID(SALESFORCE, PROFILE_AND_PERMISSION_SETS_BROKEN_PATHS, 'instance', ElemID.CONFIG_NAME),
+  )
+  if (!isInstanceElement(instance)) {
+    return []
+  }
+  return instance.value[PATHS_FIELD] ?? []
+}