diff --git a/app.arc b/app.arc index 28dd2106b7..d6338aa1bf 100644 --- a/app.arc +++ b/app.arc @@ -94,6 +94,16 @@ legacy_users email *String PointInTimeRecovery true +kafka_acls + topicName *String + cognitoGroup **String + PointInTimeRecovery true + +kafka_acl_log + partitionKey *Number + syncedOn **Number + PointInTimeRecovery ture + @tables-indexes email_notification_subscription topic *String @@ -143,6 +153,11 @@ synonyms synonymId *String name synonymsByUuid +kafka_acls + cognitoGroup *String + permissionType **String + name aclsByGroup + @aws runtime nodejs20.x region us-east-1 diff --git a/app/components/NoticeTypeCheckboxes/NoticeTypeCheckboxes.tsx b/app/components/NoticeTypeCheckboxes/NoticeTypeCheckboxes.tsx index de9d8ac1f0..067bc56aee 100644 --- a/app/components/NoticeTypeCheckboxes/NoticeTypeCheckboxes.tsx +++ b/app/components/NoticeTypeCheckboxes/NoticeTypeCheckboxes.tsx @@ -215,12 +215,14 @@ interface NoticeTypeCheckboxProps { defaultSelected?: string[] selectedFormat?: NoticeFormat validationFunction?: (arg: any) => void + filterSet?: string[] } export function NoticeTypeCheckboxes({ defaultSelected, selectedFormat = 'text', validationFunction, + filterSet, }: NoticeTypeCheckboxProps) { const [userSelected, setUserSelected] = useState(new Set()) const [selectedCounter, setSelectedCounter] = useState(0) @@ -257,6 +259,18 @@ export function NoticeTypeCheckboxes({ } } + const filteredJsonNoticeTypes = { + ...Object.keys(JsonNoticeTypes) + .filter((key) => + JsonNoticeTypes[key].every((topic) => filterSet?.includes(topic)) + ) + .map((key) => { + return { key: JsonNoticeTypes[key] } + }), + } + + // There must be a better way, but i will worry about that at another time + console.log(filteredJsonNoticeTypes) // { '0': { key: [ 'gcn.circulars' ] } }, not quite right yet return ( <> + x.startsWith('gcn.nasa.gov/kafka-') + ) + const db = await tables() + const items = ( + await Promise.all([ + ...userGroups.map((cognitoGroup) => + db.kafka_acls.query({ + IndexName: 'aclsByGroup', + KeyConditionExpression: + 'cognitoGroup = :group AND permissionType = :permission', + ProjectionExpression: 'topicName', + ExpressionAttributeValues: { + ':group': cognitoGroup, + ':permission': 'consumer', + }, + }) + ), + ]) + ) + .filter((x) => x.Count && x.Count > 0) + .flatMap((x) => x.Items) + .map((x) => x.topicName) + + return items +} +export async function getAclsFromBrokers() { + const adminClient = adminKafka.admin() + await adminClient.connect() + const acls = await adminClient.describeAcls({ + resourceType: AclResourceTypes.TOPIC, + host: '*', + permissionType: AclPermissionTypes.ALLOW, + operation: AclOperationTypes.ANY, + resourcePatternType: ResourcePatternTypes.ANY, + }) + await adminClient.disconnect() + const results: KafkaACL[] = [] + for (const item of acls.resources) { + const topicName = item.resourceName + const prefixed = item.resourcePatternType === ResourcePatternTypes.PREFIXED + const producerRules = producerOperations.every((op) => + item.acls.map((x) => x.operation).includes(op) + ) + const producerGroup = + producerRules && + [ + ...new Set( + item.acls + .filter((acl) => producerOperations.includes(acl.operation)) + .map((x) => x.principal) + ), + ][0]?.replace('User:', '') + const consumerRules = consumerOperations.every((op) => + item.acls.map((x) => x.operation).includes(op) + ) + const consumerGroup = + consumerRules && + [ + ...new Set( + item.acls + .filter((acl) => consumerOperations.includes(acl.operation)) + .map((x) => x.principal) + ), + ][0]?.replace('User:', '') + if (producerRules && producerGroup) + results.push({ + topicName, + permissionType: 'producer', + cognitoGroup: producerGroup, + prefixed, + }) + if (consumerRules && consumerGroup) + results.push({ + topicName, + permissionType: 'consumer', + cognitoGroup: consumerGroup, + prefixed, + }) + } + return results +} + +export async function deleteKafkaACL(user: User, acl: KafkaACL) { + validateUser(user) + const db = await tables() + await db.kafka_acls.delete({ + topicName: acl.topicName, + cognitoGroup: acl.cognitoGroup, + }) + + const acls = + acl.permissionType == 'producer' + ? createProducerAcls(acl) + : createConsumerAcls(acl) + + const adminClient = adminKafka.admin() + await adminClient.connect() + await adminClient.deleteAcls({ filters: acls }) + await adminClient.disconnect() +} + +function createProducerAcls(acl: KafkaACL): AclEntry[] { + // Create, Write, and Describe operations + return mapAclAndOperations(acl, producerOperations) +} + +function createConsumerAcls(acl: KafkaACL): AclEntry[] { + // Read and Describe operations + return mapAclAndOperations(acl, consumerOperations) +} + +function mapAclAndOperations(acl: KafkaACL, operations: AclOperationTypes[]) { + return operations.map((operation) => { + return { + resourceType: AclResourceTypes.TOPIC, + resourceName: acl.topicName, + resourcePatternType: acl.prefixed + ? ResourcePatternTypes.PREFIXED + : ResourcePatternTypes.LITERAL, + principal: `User:${acl.cognitoGroup}`, + host: '*', + operation, + permissionType: AclPermissionTypes.ALLOW, + } + }) +} + +export async function updateBrokersFromDb(user: User) { + const dbDefinedAcls = await getKafkaACLsFromDynamoDB(user) + const mappedAcls = dbDefinedAcls.flatMap((x) => + x.permissionType === 'producer' + ? createProducerAcls(x) + : createConsumerAcls(x) + ) + + const adminClient = adminKafka.admin() + await adminClient.connect() + await adminClient.createAcls({ acl: mappedAcls }) + await adminClient.disconnect() +} + +export async function updateDbFromBrokers(user: User) { + const kafkaDefinedAcls = await getAclsFromBrokers() + const db = await tables() + await Promise.all([ + ...kafkaDefinedAcls.map((acl) => db.kafka_acls.put(acl)), + db.kafka_acl_log.put({ + partitionKey: 1, + syncedOn: Date.now(), + syncedBy: user.email, + }), + ]) +} + +type KafkaAclSyncLog = { + partitionKey: number + syncedOn: number + syncedBy: string +} + +export async function getLastSyncDate(): Promise { + const db = await tables() + return ( + await db.kafka_acl_log.query({ + KeyConditionExpression: 'partitionKey = :1', + ExpressionAttributeValues: { ':1': 1 }, + ScanIndexForward: false, + Limit: 1, + }) + ).Items.pop() as KafkaAclSyncLog +} diff --git a/app/root.tsx b/app/root.tsx index 10ac78d99a..2ddbb06d45 100644 --- a/app/root.tsx +++ b/app/root.tsx @@ -46,6 +46,7 @@ import { useSpinDelay } from 'spin-delay' import invariant from 'tiny-invariant' import { features, getEnvOrDieInProduction, origin } from './lib/env.server' +import { adminGroup } from './lib/kafka.server' import { DevBanner } from './root/DevBanner' import { Footer } from './root/Footer' import NewsBanner from './root/NewsBanner' @@ -119,6 +120,7 @@ export async function loader({ request }: LoaderFunctionArgs) { const recaptchaSiteKey = getEnvOrDieInProduction('RECAPTCHA_SITE_KEY') const userIsMod = user?.groups.includes(moderatorGroup) const userIsVerifiedSubmitter = user?.groups.includes(submitterGroup) + const userIsAdmin = user?.groups.includes(adminGroup) return { origin, @@ -129,6 +131,7 @@ export async function loader({ request }: LoaderFunctionArgs) { idp, userIsMod, userIsVerifiedSubmitter, + userIsAdmin, } } @@ -168,6 +171,11 @@ export function useSubmitterStatus() { return userIsVerifiedSubmitter } +export function useAdminStatus() { + const { userIsAdmin } = useLoaderDataRoot() + return userIsAdmin +} + export function useRecaptchaSiteKey() { const { recaptchaSiteKey } = useLoaderDataRoot() return recaptchaSiteKey diff --git a/app/root/header/Header.tsx b/app/root/header/Header.tsx index dde3947adc..61e3e9fcd7 100644 --- a/app/root/header/Header.tsx +++ b/app/root/header/Header.tsx @@ -17,7 +17,7 @@ import { useEffect, useState } from 'react' import { useClickAnyWhere, useWindowSize } from 'usehooks-ts' import { Meatball } from '~/components/meatball/Meatball' -import { useEmail, useUserIdp } from '~/root' +import { useAdminStatus, useEmail, useUserIdp } from '~/root' import styles from './header.module.css' @@ -74,6 +74,7 @@ export function Header() { const [expanded, setExpanded] = useState(false) const [userMenuIsOpen, setUserMenuIsOpen] = useState(false) const isMobile = useWindowSize().width < 1024 + const userIsAdmin = useAdminStatus() function toggleMobileNav() { setExpanded((expanded) => !expanded) @@ -162,6 +163,11 @@ export function Header() { Profile , + userIsAdmin && ( + + Admin + + ), Peer Endorsements , diff --git a/app/routes/admin.kafka._index.tsx b/app/routes/admin.kafka._index.tsx new file mode 100644 index 0000000000..482799e435 --- /dev/null +++ b/app/routes/admin.kafka._index.tsx @@ -0,0 +1,264 @@ +/*! + * Copyright © 2023 United States Government as represented by the + * Administrator of the National Aeronautics and Space Administration. + * All Rights Reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ +import type { ActionFunctionArgs, LoaderFunctionArgs } from '@remix-run/node' +import { useFetcher, useLoaderData } from '@remix-run/react' +import type { ModalRef } from '@trussworks/react-uswds' +import { + Button, + Grid, + Icon, + Label, + Modal, + ModalFooter, + ModalHeading, + ModalToggleButton, + TextInput, +} from '@trussworks/react-uswds' +import { useEffect, useRef, useState } from 'react' + +import { getUser } from './_auth/user.server' +import HeadingWithAddButton from '~/components/HeadingWithAddButton' +import SegmentedCards from '~/components/SegmentedCards' +import Spinner from '~/components/Spinner' +import TimeAgo from '~/components/TimeAgo' +import type { KafkaACL, PermissionType } from '~/lib/kafka.server' +import { + adminGroup, + createKafkaACL, + deleteKafkaACL, + getKafkaACLsFromDynamoDB, + getLastSyncDate, + updateDbFromBrokers, +} from '~/lib/kafka.server' +import { getFormDataString } from '~/lib/utils' + +export async function loader({ request }: LoaderFunctionArgs) { + const user = await getUser(request) + if (!user || !user.groups.includes(adminGroup)) + throw new Response(null, { status: 403 }) + const { aclFilter } = Object.fromEntries(new URL(request.url).searchParams) + const dynamoDbAclData = await getKafkaACLsFromDynamoDB(user, aclFilter) + const latestSync = await getLastSyncDate() + return { dynamoDbAclData, latestSync } +} + +export async function action({ request }: ActionFunctionArgs) { + const user = await getUser(request) + if (!user?.groups.includes(adminGroup)) + throw new Response(null, { status: 403 }) + const data = await request.formData() + const intent = getFormDataString(data, 'intent') + if (intent === 'migrateFromBroker') { + await updateDbFromBrokers(user) + return null + } + const topicName = getFormDataString(data, 'topicName') + const permissionType = getFormDataString( + data, + 'permissionType' + ) as PermissionType + const group = getFormDataString(data, 'group') + const includePrefixed = getFormDataString(data, 'includePrefixed') + if (!topicName || !permissionType || !group) + throw new Response(null, { status: 400 }) + const promises = [] + + switch (intent) { + case 'delete': + promises.push( + deleteKafkaACL(user, { + topicName, + permissionType, + cognitoGroup: group, + prefixed: false, + }) + ) + break + case 'create': + promises.push( + createKafkaACL(user, { + topicName, + permissionType, + cognitoGroup: group, + prefixed: false, + }) + ) + + if (includePrefixed) + promises.push( + createKafkaACL(user, { + topicName: `${topicName}.`, + permissionType, + cognitoGroup: group, + prefixed: true, + }) + ) + break + default: + break + } + await Promise.all(promises) + + return null +} + +export default function Index() { + const { dynamoDbAclData, latestSync } = useLoaderData() + const [aclData, setAclData] = useState(dynamoDbAclData) + const updateFetcher = useFetcher() + const aclFetcher = useFetcher() + + useEffect(() => { + setAclData(aclFetcher.data?.dynamoDbAclData ?? aclData) + }, [aclFetcher.data, aclData]) + + return ( + <> + Kafka +

Kafka ACLs

+

+ Kafka Access Control Lists (ACLs) are a security mechanism used to + control access to resources within a Kafka cluster. They define which + users or client applications have permissions to perform specific + operations on Kafka resources, such as topics, consumer groups, and + broker resources. ACLs specify who can produce (write) or consume (read) + data from topics, create or delete topics, manage consumer groups, and + perform administrative tasks. +

+ + + + {updateFetcher.state !== 'idle' && ( + + Updating... + + )} + + {latestSync && ( +

+ Last synced by {latestSync.syncedBy}{' '} + +

+ )} + {aclData && ( + <> + + + + + {aclFetcher.state !== 'idle' && ( + + Loading... + + )} + + + {aclData + .sort((a, b) => a.topicName.localeCompare(b.topicName)) + .map((x, index) => ( + + ))} + + + )} + + ) +} + +function KafkaAclCard({ acl }: { acl: KafkaACL }) { + const ref = useRef(null) + const fetcher = useFetcher() + const disabled = fetcher.state !== 'idle' + + return ( + <> + +
+
+ + Topic: {acl.topicName} + +
+
+ + Permission Type: {acl.permissionType} + +
+
+ + Group: {acl.cognitoGroup} + +
+
+
+ + + Delete + +
+
+ + + + + + + Delete Kafka ACL + + + + + Cancel + + + + + + + ) +} diff --git a/app/routes/admin.kafka.edit.tsx b/app/routes/admin.kafka.edit.tsx new file mode 100644 index 0000000000..7d9af231a1 --- /dev/null +++ b/app/routes/admin.kafka.edit.tsx @@ -0,0 +1,95 @@ +/*! + * Copyright © 2023 United States Government as represented by the + * Administrator of the National Aeronautics and Space Administration. + * All Rights Reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ +import type { LoaderFunctionArgs } from '@remix-run/node' +import { Form, useLoaderData } from '@remix-run/react' +import { + Button, + Checkbox, + Label, + Select, + TextInput, +} from '@trussworks/react-uswds' + +import { getUser } from './_auth/user.server' +import { getGroups } from '~/lib/cognito.server' + +export async function loader({ request }: LoaderFunctionArgs) { + const user = await getUser(request) + if (!user) throw new Response(null, { status: 403 }) + const userGroups = (await getGroups()) + .filter((group) => group.GroupName?.startsWith('gcn.nasa.gov/')) + .map((group) => group.GroupName) + + return { userGroups } +} + +export default function Kafka() { + const { userGroups } = useLoaderData() + return +} + +function KafkaAclForm({ groups }: { groups: string[] }) { + return ( + <> +

Create Kafka ACLs

+
+ + console.log(e.target.value)} + /> + + + + Producer will generate ACLs for the Create, Write, and Describe + operations. Consumer will generate ACLs for the Read and Describe + operations + + + + +
+ + If yes, submission will also trigger th generation of ACLs for the + provided topic name as a PREFIXED topic with a period included at + the end. For example, if checked, a topic of `gcn.notices.icecube` + will result in ACLs for both `gcn.notices.icecube` (literal) and + `gcn.notices.icecube.` (prefixed). + +
+ + + + ) +} diff --git a/app/routes/admin.tsx b/app/routes/admin.tsx index a385863072..0531f8c358 100644 --- a/app/routes/admin.tsx +++ b/app/routes/admin.tsx @@ -27,6 +27,9 @@ export default function () {
+ Kafka + , Users , diff --git a/app/routes/user.email.edit.tsx b/app/routes/user.email.edit.tsx index 8bd33530bb..60f44adf84 100644 --- a/app/routes/user.email.edit.tsx +++ b/app/routes/user.email.edit.tsx @@ -34,6 +34,7 @@ import { import { type NoticeFormat, NoticeFormatInput } from '~/components/NoticeFormat' import { NoticeTypeCheckboxes } from '~/components/NoticeTypeCheckboxes/NoticeTypeCheckboxes' import { ReCAPTCHA, verifyRecaptcha } from '~/components/ReCAPTCHA' +import { getKafkaTopicsForUser } from '~/lib/kafka.server' import { formatAndNoticeTypeToTopic } from '~/lib/utils' import { useFeature, useRecaptchaSiteKey } from '~/root' import type { BreadcrumbHandle } from '~/root/Title' @@ -111,11 +112,14 @@ export async function loader({ request }: LoaderFunctionArgs) { intent = 'update' } const format = notification.format as NoticeFormat - return { notification, intent, format } + const subscribeableTopics = await getKafkaTopicsForUser(user) + console.log(subscribeableTopics) + return { notification, intent, format, subscribeableTopics } } export default function () { - const { notification, format } = useLoaderData() + const { notification, format, subscribeableTopics } = + useLoaderData() const defaultNameValid = Boolean(notification.name) const [nameValid, setNameValid] = useState(defaultNameValid) const defaultRecipientValid = Boolean(notification.recipient) @@ -175,6 +179,7 @@ export default function () { selectedFormat={defaultFormat} defaultSelected={notification.noticeTypes} validationFunction={setAlertsValid} + filterSet={subscribeableTopics} /> { diff --git a/playwright.config.ts b/playwright.config.ts index 5aa5401b45..76bf15d536 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -4,7 +4,6 @@ import { defineConfig, devices } from '@playwright/test' * Read environment variables from file. * https://github.com/motdotla/dotenv */ -// require('dotenv').config(); const deviceList = ['Desktop Firefox', 'Desktop Chrome', 'Desktop Safari'] @@ -69,7 +68,6 @@ export default defineConfig({ command: 'npm run dev', url: 'http://localhost:3333', reuseExistingServer: !process.env.CI, - stdout: 'pipe', timeout: 120 * 1000, // 120 Seconds timeout on webServer }, }) diff --git a/sandbox-seed.json b/sandbox-seed.json index 35534487e4..448b71353e 100644 --- a/sandbox-seed.json +++ b/sandbox-seed.json @@ -5155,5 +5155,19 @@ "requestorEmail": "example@example.com", "subject": "Optical Observations for GRB 971227" } + ], + "kafka_acls": [ + { + "topicName": "test_topic_created_from_website", + "permissionType": "consumer", + "cognitoGroup": "gcn.nasa.gov/kafka-gcn-test-consumer", + "prefixed": false + }, + { + "topicName": "test_topic_created_from_website", + "permissionType": "producer", + "cognitoGroup": "gcn.nasa.gov/kafka-gcn-test-producer", + "prefixed": false + } ] }