Fix validation perf. Fixes #1428. Remove duplicate LoadOneById method.

controller/db/api_session_store.go

@@ -257,7 +257,7 @@ func (store *apiSessionStoreImpl) initializeLinked() {
 func (store *apiSessionStoreImpl) LoadOneByToken(tx *bbolt.Tx, token string) (*ApiSession, error) {
 	id := store.indexToken.Read(tx, []byte(token))
 	if id != nil {
-		return store.LoadOneById(tx, string(id))
+		return store.LoadById(tx, string(id))
 	}
 	return nil, boltz.NewNotFoundError(store.GetSingularEntityType(), "token", token)
 }

controller/db/api_session_store_test.go

@@ -178,11 +178,11 @@ func (ctx *TestContext) testUpdateApiSessions(t *testing.T) {
 	mutateCtx := change.New().NewMutateContext()
 	err := ctx.GetDb().Update(mutateCtx, func(mutateCtx boltz.MutateContext) error {
 		tx := mutateCtx.Tx()
-		original, err := ctx.stores.ApiSession.LoadOneById(tx, entities.apiSession1.Id)
+		original, err := ctx.stores.ApiSession.LoadById(tx, entities.apiSession1.Id)
 		ctx.NoError(err)
 		ctx.NotNil(original)
 
-		apiSession, err := ctx.stores.ApiSession.LoadOneById(tx, entities.apiSession1.Id)
+		apiSession, err := ctx.stores.ApiSession.LoadById(tx, entities.apiSession1.Id)
 		ctx.NoError(err)
 		ctx.NotNil(apiSession)
 
@@ -196,7 +196,7 @@ func (ctx *TestContext) testUpdateApiSessions(t *testing.T) {
 
 		err = ctx.stores.ApiSession.Update(mutateCtx, apiSession, nil)
 		ctx.NoError(err)
-		loaded, err := ctx.stores.ApiSession.LoadOneById(tx, entities.apiSession1.Id)
+		loaded, err := ctx.stores.ApiSession.LoadById(tx, entities.apiSession1.Id)
 		ctx.NoError(err)
 		ctx.NotNil(loaded)
 		ctx.EqualValues(original.CreatedAt, loaded.CreatedAt)

controller/db/base_store.go

@@ -34,7 +34,6 @@ type initializableStore interface {
 type Store[E boltz.ExtEntity] interface {
 	boltz.EntityStore[E]
 	initializableStore
-	LoadOneById(tx *bbolt.Tx, id string) (E, error)
 }
 
 type baseStore[E boltz.ExtEntity] struct {
@@ -51,25 +50,6 @@ func (store *baseStore[E]) initializeIndexes(tx *bbolt.Tx, errorHolder errorz.Er
 	store.InitializeIndexes(tx, errorHolder)
 }
 
-func (store *baseStore[E]) LoadOneById(tx *bbolt.Tx, id string) (E, error) {
-	entity := store.NewStoreEntity()
-	if err := store.baseLoadOneById(tx, id, entity); err != nil {
-		return *new(E), err
-	}
-	return entity, nil
-}
-
-func (store *baseStore[E]) baseLoadOneById(tx *bbolt.Tx, id string, entity E) error {
-	found, err := store.LoadEntity(tx, id, entity)
-	if err != nil {
-		return err
-	}
-	if !found {
-		return boltz.NewNotFoundError(store.GetSingularEntityType(), "id", id)
-	}
-	return nil
-}
-
 func (store *baseStore[E]) deleteEntityReferences(tx *bbolt.Tx, entity boltz.NamedExtEntity, rolesSymbol boltz.EntitySetSymbol) error {
 	idRef := entityRef(entity.GetId())
 

controller/db/config_store_test.go

@@ -64,7 +64,7 @@ func (ctx *TestContext) testConfigCrud(*testing.T) {
 	boltztest.ValidateBaseline(ctx, config)
 
 	err = ctx.GetDb().View(func(tx *bbolt.Tx) error {
-		testConfig, err := ctx.stores.Config.LoadOneById(tx, config.Id)
+		testConfig, err := ctx.stores.Config.LoadById(tx, config.Id)
 		ctx.NoError(err)
 		ctx.NotNil(testConfig)
 		ctx.Equal(config.Name, testConfig.Name)

controller/db/config_type_store.go

@@ -120,7 +120,7 @@ func (store *configTypeStoreImpl) PersistEntity(entity *ConfigType, ctx *boltz.P
 func (store *configTypeStoreImpl) LoadOneByName(tx *bbolt.Tx, name string) (*ConfigType, error) {
 	id := store.indexName.Read(tx, []byte(name))
 	if id != nil {
-		return store.LoadOneById(tx, string(id))
+		return store.LoadById(tx, string(id))
 	}
 	return nil, nil
 }

controller/db/edge_router_policy_store.go

@@ -164,7 +164,7 @@ func (store *edgeRouterPolicyStoreImpl) identityRolesUpdated(persistCtx *boltz.P
 }
 
 func (store *edgeRouterPolicyStoreImpl) DeleteById(ctx boltz.MutateContext, id string) error {
-	policy, err := store.LoadOneById(ctx.Tx(), id)
+	policy, err := store.LoadById(ctx.Tx(), id)
 	if err != nil {
 		return err
 	}

controller/db/edge_router_policy_store_test.go

@@ -38,7 +38,7 @@ func (ctx *TestContext) testCreateEdgeRouterPolicy(t *testing.T) {
 		ctx.Equal(0, len(ctx.stores.EdgeRouterPolicy.GetRelatedEntitiesIdList(tx, policy.Id, EntityTypeRouters)))
 		ctx.Equal(0, len(ctx.stores.EdgeRouterPolicy.GetRelatedEntitiesIdList(tx, policy.Id, EntityTypeIdentities)))
 
-		testPolicy, err := ctx.stores.EdgeRouterPolicy.LoadOneById(tx, policy.Id)
+		testPolicy, err := ctx.stores.EdgeRouterPolicy.LoadById(tx, policy.Id)
 		ctx.NoError(err)
 		ctx.NotNil(testPolicy)
 		ctx.Equal(policy.Name, testPolicy.Name)

controller/db/edge_router_store.go

@@ -231,7 +231,7 @@ func (store *edgeRouterStoreImpl) GetNameIndex() boltz.ReadIndex {
 }
 
 func (store *edgeRouterStoreImpl) cleanupEdgeRouter(ctx boltz.MutateContext, id string) error {
-	if entity, _ := store.LoadOneById(ctx.Tx(), id); entity != nil {
+	if entity, _ := store.LoadById(ctx.Tx(), id); entity != nil {
 		// Remove entity from EdgeRouterRoles in edge router policies
 		if err := store.deleteEntityReferences(ctx.Tx(), entity, store.stores.edgeRouterPolicy.symbolEdgeRouterRoles); err != nil {
 			return err
@@ -311,7 +311,7 @@ func (self *routerIdentityConstraint) ProcessAfterUpdate(ctx *boltz.IndexingCont
 	}
 
 	if !createEntities && currentTunnelerEnabled && oldName != name {
-		identity, err := self.stores.identity.LoadOneById(ctx.Tx(), routerId)
+		identity, err := self.stores.identity.LoadById(ctx.Tx(), routerId)
 		if err != nil {
 			if boltz.IsErrNotFoundErr(err) {
 				logrus.Errorf("identity for router with id %v not found", routerId)

controller/db/edge_service_store.go

@@ -251,7 +251,7 @@ func (store *edgeServiceStoreImpl) Update(ctx boltz.MutateContext, entity *EdgeS
 }
 
 func (store *edgeServiceStoreImpl) cleanupEdgeService(ctx boltz.MutateContext, id string) error {
-	if entity, _ := store.LoadOneById(ctx.Tx(), id); entity != nil {
+	if entity, _ := store.LoadById(ctx.Tx(), id); entity != nil {
 		// Remove entity from ServiceRoles in service policies
 		if err := store.deleteEntityReferences(ctx.Tx(), entity, store.stores.servicePolicy.symbolServiceRoles); err != nil {
 			return err

controller/db/edge_service_store_test.go

@@ -181,7 +181,7 @@ func (ctx *TestContext) testLoadQueryEdgeServices(_ *testing.T) {
 	entities := ctx.createEdgeServiceTestEntities()
 
 	err := ctx.GetDb().View(func(tx *bbolt.Tx) error {
-		service, err := ctx.stores.EdgeService.LoadOneById(tx, entities.service1.Id)
+		service, err := ctx.stores.EdgeService.LoadById(tx, entities.service1.Id)
 		ctx.NoError(err)
 		ctx.NotNil(service)
 		ctx.EqualValues(entities.service1.Id, service.Id)
@@ -217,11 +217,11 @@ func (ctx *TestContext) testUpdateEdgeServices(_ *testing.T) {
 	mutateCtx := change.New().NewMutateContext()
 	err := ctx.GetDb().Update(mutateCtx, func(mutateCtx boltz.MutateContext) error {
 		tx := mutateCtx.Tx()
-		original, err := ctx.stores.EdgeService.LoadOneById(tx, entities.service1.Id)
+		original, err := ctx.stores.EdgeService.LoadById(tx, entities.service1.Id)
 		ctx.NoError(err)
 		ctx.NotNil(original)
 
-		service, err := ctx.stores.EdgeService.LoadOneById(tx, entities.service1.Id)
+		service, err := ctx.stores.EdgeService.LoadById(tx, entities.service1.Id)
 		ctx.NoError(err)
 		ctx.NotNil(service)
 
@@ -234,7 +234,7 @@ func (ctx *TestContext) testUpdateEdgeServices(_ *testing.T) {
 
 		err = ctx.stores.EdgeService.Update(mutateCtx, service, nil)
 		ctx.NoError(err)
-		loaded, err := ctx.stores.EdgeService.LoadOneById(tx, entities.service1.Id)
+		loaded, err := ctx.stores.EdgeService.LoadById(tx, entities.service1.Id)
 		ctx.NoError(err)
 		ctx.NotNil(loaded)
 		ctx.EqualValues(original.CreatedAt, loaded.CreatedAt)

controller/db/enrollment_store.go

@@ -142,7 +142,7 @@ func (store *enrollmentStoreImpl) PersistEntity(entity *Enrollment, ctx *boltz.P
 func (store *enrollmentStoreImpl) LoadOneByToken(tx *bbolt.Tx, token string) (*Enrollment, error) {
 	id := store.tokenIndex.Read(tx, []byte(token))
 	if id != nil {
-		return store.LoadOneById(tx, string(id))
+		return store.LoadById(tx, string(id))
 	}
 	return nil, nil
 }

controller/db/eventual_eventer.go

@@ -489,7 +489,7 @@ func (a *EventualEventerBbolt) getEventualEvents() ([]string, []*EventualEvent,
 		}
 
 		for _, id := range ids {
-			event, err := a.store.LoadOneById(tx, id)
+			event, err := a.store.LoadById(tx, id)
 
 			if err != nil {
 				pfxlog.Logger().WithField("id", id).WithError(err).Errorf("error could not load event id %s", id)

controller/db/identity_store.go

@@ -400,7 +400,7 @@ func (store *identityStoreImpl) DeleteById(ctx boltz.MutateContext, id string) e
 		}
 	}
 
-	if entity, _ := store.LoadOneById(ctx.Tx(), id); entity != nil {
+	if entity, _ := store.LoadById(ctx.Tx(), id); entity != nil {
 		if entity.IsDefaultAdmin {
 			return errorz.NewEntityCanNotBeDeleted()
 		}
@@ -444,7 +444,7 @@ func (store *identityStoreImpl) AssignServiceConfigs(tx *bbolt.Tx, identityId st
 	}
 	configTypes := map[string]struct{}{}
 	for _, serviceConfig := range serviceConfigs {
-		config, err := store.stores.config.LoadOneById(tx, serviceConfig.ConfigId)
+		config, err := store.stores.config.LoadById(tx, serviceConfig.ConfigId)
 		if err != nil {
 			return err
 		}
@@ -625,7 +625,7 @@ func (store *identityStoreImpl) LoadServiceConfigsByServiceAndType(tx *bbolt.Tx,
 				_, wantsType = configTypes[configTypeId]
 			}
 			if wantsType {
-				if config, _ := store.stores.config.LoadOneById(tx, configId); config != nil {
+				if config, _ := store.stores.config.LoadById(tx, configId); config != nil {
 					serviceMap, ok := result[serviceId]
 					if !ok {
 						serviceMap = map[string]map[string]interface{}{}

controller/db/migration_v16.go

@@ -15,15 +15,15 @@ func (m *Migrations) removeOrphanedOttCaEnrollments(step *boltz.MigrationStep) {
 		current := cursor.Current()
 		currentEnrollmentId := string(current)
 
-		enrollment, err := m.stores.Enrollment.LoadOneById(step.Ctx.Tx(), currentEnrollmentId)
+		enrollment, err := m.stores.Enrollment.LoadById(step.Ctx.Tx(), currentEnrollmentId)
 
 		if err != nil {
 			step.SetError(fmt.Errorf("error iterating ids of enrollments, enrollment [%s]: %v", currentEnrollmentId, err))
 			return
 		}
 
 		if enrollment.CaId != nil && *enrollment.CaId != "" {
-			_, err := m.stores.Ca.LoadOneById(step.Ctx.Tx(), *enrollment.CaId)
+			_, err := m.stores.Ca.LoadById(step.Ctx.Tx(), *enrollment.CaId)
 
 			if err != nil && boltz.IsErrNotFoundErr(err) {
 				enrollmentsToDelete = append(enrollmentsToDelete, currentEnrollmentId)

controller/db/migration_v18.go

@@ -8,7 +8,7 @@ import (
 // Primes API Session's lastActivityAt proper to their previous updatedAt value
 func (m *Migrations) setLastActivityAt(step *boltz.MigrationStep) {
 	for cursor := m.stores.ApiSession.IterateIds(step.Ctx.Tx(), ast.BoolNodeTrue); cursor.IsValid(); cursor.Next() {
-		if apiSession, err := m.stores.ApiSession.LoadOneById(step.Ctx.Tx(), string(cursor.Current())); err == nil {
+		if apiSession, err := m.stores.ApiSession.LoadById(step.Ctx.Tx(), string(cursor.Current())); err == nil {
 			apiSession.LastActivityAt = apiSession.UpdatedAt
 			step.SetError(m.stores.ApiSession.Update(step.Ctx, apiSession, UpdateLastActivityAtChecker{}))
 		} else {

controller/db/migration_v19.go

@@ -35,7 +35,7 @@ func (m *Migrations) updateIdentityTypes(step *boltz.MigrationStep) {
 
 	for cursor := identityStore.IterateIds(step.Ctx.Tx(), ast.BoolNodeTrue); cursor.IsValid(); cursor.Next() {
 		current := cursor.Current()
-		identity, err := identityStore.LoadOneById(step.Ctx.Tx(), string(current))
+		identity, err := identityStore.LoadById(step.Ctx.Tx(), string(current))
 		if step.SetError(err) {
 			return
 		}
@@ -64,7 +64,7 @@ func (m *Migrations) updateIdentityTypes(step *boltz.MigrationStep) {
 }
 
 func (m *Migrations) migrateIdentityType(step *boltz.MigrationStep, id string) {
-	idType, err := m.stores.IdentityType.LoadOneById(step.Ctx.Tx(), id)
+	idType, err := m.stores.IdentityType.LoadById(step.Ctx.Tx(), id)
 	if step.SetError(err) {
 		return
 	}

controller/db/migration_v24.go

@@ -15,7 +15,7 @@ func (m *Migrations) addIdentityIdToSessions(step *boltz.MigrationStep) {
 
 	for cursor.IsValid() {
 		sessionId := string(cursor.Current())
-		session, err := m.stores.Session.LoadOneById(step.Ctx.Tx(), sessionId)
+		session, err := m.stores.Session.LoadById(step.Ctx.Tx(), sessionId)
 
 		if err != nil {
 			step.SetError(fmt.Errorf("could no load session by id [%s]: %v", sessionId, err))
@@ -28,7 +28,7 @@ func (m *Migrations) addIdentityIdToSessions(step *boltz.MigrationStep) {
 		}
 
 		if session.IdentityId == "" {
-			if apiSession, err := m.stores.ApiSession.LoadOneById(step.Ctx.Tx(), session.ApiSessionId); err == nil {
+			if apiSession, err := m.stores.ApiSession.LoadById(step.Ctx.Tx(), session.ApiSessionId); err == nil {
 				if apiSession != nil {
 					session.IdentityId = apiSession.IdentityId
 					if err = m.stores.Session.Update(step.Ctx, session, fieldChecker); err != nil {

controller/db/policy_common.go

@@ -1,7 +1,6 @@
 package db
 
 import (
-	"bytes"
 	"fmt"
 	"github.com/michaelquigley/pfxlog"
 	"github.com/openziti/foundation/v2/errorz"
@@ -311,24 +310,36 @@ type denormCheckCtx struct {
 
 func validatePolicyDenormalization(ctx *denormCheckCtx) error {
 	tx := ctx.mutateCtx.Tx()
+
+	links := map[string]map[string]int{}
+
+	for policyCursor := ctx.policyStore.IterateIds(tx, ast.BoolNodeTrue); policyCursor.IsValid(); policyCursor.Next() {
+		policyId := policyCursor.Current()
+		if ctx.policyFilter == nil || ctx.policyFilter(policyId) {
+			for sourceCursor := ctx.sourceCollection.IterateLinks(tx, policyId); sourceCursor.IsValid(); sourceCursor.Next() {
+				sourceId := string(sourceCursor.Current())
+				for destCursor := ctx.targetCollection.IterateLinks(tx, policyId); destCursor.IsValid(); destCursor.Next() {
+					destId := string(destCursor.Current())
+					destMap := links[sourceId]
+					if destMap == nil {
+						destMap = map[string]int{}
+						links[sourceId] = destMap
+					}
+					destMap[destId] = destMap[destId] + 1
+				}
+			}
+		}
+	}
+
 	for sourceCursor := ctx.sourceStore.IterateIds(tx, ast.BoolNodeTrue); sourceCursor.IsValid(); sourceCursor.Next() {
 		sourceEntityId := sourceCursor.Current()
 		for targetCursor := ctx.targetStore.IterateIds(tx, ast.BoolNodeTrue); targetCursor.IsValid(); targetCursor.Next() {
 			targetEntityId := targetCursor.Current()
 
-			var relatedPolicies []string
-
-			for policyCursor := ctx.policyStore.IterateIds(tx, ast.BoolNodeTrue); policyCursor.IsValid(); policyCursor.Next() {
-				policyId := policyCursor.Current()
-				if ctx.policyFilter == nil || ctx.policyFilter(policyId) {
-					sourceRelated := isRelatedByLinkCollection(tx, ctx.sourceCollection, policyId, sourceEntityId)
-					targetRelated := isRelatedByLinkCollection(tx, ctx.targetCollection, policyId, targetEntityId)
-					if sourceRelated && targetRelated {
-						relatedPolicies = append(relatedPolicies, string(policyId))
-					}
-				}
+			linkCount := 0
+			if destMap, ok := links[string(sourceEntityId)]; ok {
+				linkCount = destMap[string(targetEntityId)]
 			}
-			linkCount := len(relatedPolicies)
 			var sourceLinkCount, targetLinkCount *int32
 			var err error
 			if ctx.repair {
@@ -367,9 +378,3 @@ func logDiscrepencies(ctx *denormCheckCtx, count int, sourceId, targetId []byte,
 		ctx.errorSink(err, ctx.repair)
 	}
 }
-
-func isRelatedByLinkCollection(tx *bbolt.Tx, linkCollection boltz.LinkCollection, entityId, relatedId []byte) bool {
-	cursor := linkCollection.IterateLinks(tx, entityId)
-	cursor.Seek(relatedId)
-	return bytes.Equal(cursor.Current(), relatedId)
-}

controller/db/posture_check_store.go

@@ -83,7 +83,6 @@ func (entity *PostureCheck) GetEntityType() string {
 
 type PostureCheckStore interface {
 	Store[*PostureCheck]
-	LoadOneById(tx *bbolt.Tx, id string) (*PostureCheck, error)
 	GetRoleAttributesIndex() boltz.SetReadIndex
 	GetRoleAttributesCursorProvider(filters []string, semantic string) (ast.SetCursorProvider, error)
 }
@@ -180,7 +179,7 @@ func (store *postureCheckStoreImpl) GetNameIndex() boltz.ReadIndex {
 }
 
 func (store *postureCheckStoreImpl) DeleteById(ctx boltz.MutateContext, id string) error {
-	if entity, _ := store.LoadOneById(ctx.Tx(), id); entity != nil {
+	if entity, _ := store.LoadById(ctx.Tx(), id); entity != nil {
 		// Remove entity from PostureCheckRoles in service policies
 		if err := store.deleteEntityReferences(ctx.Tx(), entity, store.stores.servicePolicy.symbolPostureCheckRoles); err != nil {
 			return err

controller/db/service_edge_router_policy_store.go

@@ -163,7 +163,7 @@ func (store *serviceEdgeRouterPolicyStoreImpl) serviceRolesUpdated(persistCtx *b
 }
 
 func (store *serviceEdgeRouterPolicyStoreImpl) DeleteById(ctx boltz.MutateContext, id string) error {
-	policy, err := store.LoadOneById(ctx.Tx(), id)
+	policy, err := store.LoadById(ctx.Tx(), id)
 	if err != nil {
 		return err
 	}

controller/db/service_edge_router_policy_store_test.go

@@ -33,7 +33,7 @@ func (ctx *TestContext) testCreateServiceEdgeRouterPolicy(_ *testing.T) {
 		ctx.Equal(0, len(ctx.stores.ServiceEdgeRouterPolicy.GetRelatedEntitiesIdList(tx, policy.Id, EntityTypeRouters)))
 		ctx.Equal(0, len(ctx.stores.ServiceEdgeRouterPolicy.GetRelatedEntitiesIdList(tx, policy.Id, EntityTypeServices)))
 
-		testPolicy, err := ctx.stores.ServiceEdgeRouterPolicy.LoadOneById(tx, policy.Id)
+		testPolicy, err := ctx.stores.ServiceEdgeRouterPolicy.LoadById(tx, policy.Id)
 		ctx.NoError(err)
 		ctx.NotNil(testPolicy)
 		ctx.Equal(policy.Name, testPolicy.Name)

controller/db/service_policy_store.go

@@ -274,7 +274,7 @@ func (store *servicePolicyStoreImpl) postureCheckRolesUpdated(persistCtx *boltz.
 }
 
 func (store *servicePolicyStoreImpl) DeleteById(ctx boltz.MutateContext, id string) error {
-	policy, err := store.LoadOneById(ctx.Tx(), id)
+	policy, err := store.LoadById(ctx.Tx(), id)
 	if err != nil {
 		return err
 	}

controller/db/service_policy_store_test.go

@@ -49,7 +49,7 @@ func (ctx *TestContext) testCreateServicePolicy(_ *testing.T) {
 		ctx.Equal(0, len(ctx.stores.ServicePolicy.GetRelatedEntitiesIdList(tx, policy.Id, EntityTypeServices)))
 		ctx.Equal(0, len(ctx.stores.ServicePolicy.GetRelatedEntitiesIdList(tx, policy.Id, EntityTypeIdentities)))
 
-		testPolicy, err := ctx.stores.ServicePolicy.LoadOneById(tx, policy.Id)
+		testPolicy, err := ctx.stores.ServicePolicy.LoadById(tx, policy.Id)
 		ctx.NoError(err)
 		ctx.NotNil(testPolicy)
 		ctx.Equal(policy.Name, testPolicy.Name)