executors: Add support for private docker registries (#45488)

This PR adds support for private registries to executors, so for code intel auto-indexing and server-side batch changes.
It does so by introducing a new environment variable and a magic secret DOCKER_AUTH_CONFIG that can be set to authenticate to a protected docker registry. See the inline docs change for how this works exactly. I've also added storybooks coverage for the new components, so make sure to check out the UI review check.

CHANGELOG.md

@@ -53,6 +53,7 @@ All notable changes to Sourcegraph are documented in this file.
 - [search.largeFiles](https://docs.sourcegraph.com/admin/config/site_config#search-largeFiles) accepts an optional prefix `!` to negate a pattern. The order of the patterns within search.largeFiles is honored such that the last pattern matching overrides preceding patterns. For patterns that begin with a literal `!` prefix with a backslash, for example, `\!fileNameStartsWithExcl!.txt`. Previously indexed files that become excluded due to this change will remain in the index until the next reindex [#45318](https://github.com/sourcegraph/sourcegraph/pull/45318)
 - [Webhooks](https://docs.sourcegraph.com/admin/config/webhooks) have been overhauled completely and can now be found under **Site admin > Repositories > Incoming webhooks**. Webhooks that were added via code host configuration are [deprecated](https://docs.sourcegraph.com/admin/config/webhooks#deprecation-notice) and will be removed in 4.6.0.
 - Added support for receiving webhook `push` events from GitHub which will trigger Sourcegraph to fetch the latest commit rather than relying on polling.
+- Added support for private container registries in Sourcegraph executors. [Using private registries](https://docs.sourcegraph.com/admin/deploy_executors#using-private-registries)
 
 ### Changed
 

client/web/src/enterprise/batches/list/BatchChangesChangelogAlert.tsx

@@ -11,17 +11,17 @@ import styles from './BatchChangesListIntro.module.scss'
 export const BatchChangesChangelogAlert: React.FunctionComponent<React.PropsWithChildren<unknown>> = () => (
     <DismissibleAlert
         className={styles.batchChangesListIntroAlert}
-        partialStorageKey="batch-changes-list-intro-changelog-4.3"
+        partialStorageKey="batch-changes-list-intro-changelog-4.4"
     >
         <Card className={classNames(styles.batchChangesListIntroCard, 'h-100')}>
             <CardBody>
-                <H4 as={H3}>Batch Changes updates in version 4.3</H4>
+                <H4 as={H3}>Batch Changes updates in version 4.4</H4>
                 <ul className="mb-0 pl-3">
                     <li>
-                        <Link to="/help/batch_changes/how-tos/server_side_file_mounts" rel="noopener" target="_blank">
-                            Mounted files
+                        <Link to="/help/admin/deploy_executors#using-private-registries" rel="noopener" target="_blank">
+                            Using private container registries
                         </Link>{' '}
-                        are now accessible via the UI.
+                        is now supported in server-side batch changes.
                     </li>
                 </ul>
             </CardBody>

client/web/src/enterprise/executors/secrets/AddSecretModal.story.tsx

@@ -36,3 +36,20 @@ export const GitHub: Story = () => (
 )
 
 GitHub.storyName = 'Add secret'
+
+export const DockerAuthConfig: Story = () => (
+    <WebStory>
+        {props => (
+            <AddSecretModal
+                {...props}
+                namespaceID="user-id-1"
+                scope={ExecutorSecretScope.BATCHES}
+                afterCreate={noop}
+                onCancel={noop}
+                initialKey="DOCKER_AUTH_CONFIG"
+            />
+        )}
+    </WebStory>
+)
+
+DockerAuthConfig.storyName = 'Docker auth config'

client/web/src/enterprise/executors/secrets/AddSecretModal.tsx

@@ -3,7 +3,7 @@ import React, { useCallback, useState } from 'react'
 import { ErrorAlert } from '@sourcegraph/branded/src/components/alerts'
 import { Form } from '@sourcegraph/branded/src/components/Form'
 import { logger } from '@sourcegraph/common'
-import { Button, Modal, Input, H3, Text } from '@sourcegraph/wildcard'
+import { Button, Modal, Input, H3, Text, Alert, Link } from '@sourcegraph/wildcard'
 
 import { LoaderButton } from '../../../components/LoaderButton'
 import { ExecutorSecretScope, Scalars } from '../../../graphql-operations'
@@ -15,17 +15,21 @@ export interface AddSecretModalProps {
     afterCreate: () => void
     namespaceID: Scalars['ID'] | null
     scope: ExecutorSecretScope
+
+    /** For testing only */
+    initialKey?: string
 }
 
 export const AddSecretModal: React.FunctionComponent<React.PropsWithChildren<AddSecretModalProps>> = ({
     onCancel,
     afterCreate,
     namespaceID,
     scope,
+    initialKey = '',
 }) => {
     const labelId = 'addSecret'
 
-    const [key, setKey] = useState<string>('')
+    const [key, setKey] = useState<string>(initialKey)
     const onChangeKey = useCallback<React.ChangeEventHandler<HTMLInputElement>>(event => {
         setKey(event.target.value)
     }, [])
@@ -85,11 +89,25 @@ export const AddSecretModal: React.FunctionComponent<React.PropsWithChildren<Add
                         message={
                             <>
                                 Must be uppercase characters, digits and underscores only. Must start with an uppercase
-                                character.
+                                character.{' '}
+                                <Link
+                                    to="/help/admin/deploy_executors#using-private-registries"
+                                    rel="noopener"
+                                    target="_blank"
+                                >
+                                    DOCKER_AUTH_CONFIG will be used to authenticate with private registries
+                                </Link>
+                                .
                             </>
                         }
                         label="Key"
                     />
+                    {key === 'DOCKER_AUTH_CONFIG' && (
+                        <Alert variant="info" className="mt-2">
+                            This secret value will be used to configure docker client authentication with private
+                            registries.
+                        </Alert>
+                    )}
                 </div>
                 <div className="form-group">
                     <Input

client/web/src/enterprise/executors/secrets/ExecutorSecretNode.tsx

@@ -1,9 +1,9 @@
 import React, { useCallback, useRef, useState } from 'react'
 
+import { mdiDocker, mdiLock } from '@mdi/js'
 import classNames from 'classnames'
-import LockIcon from 'mdi-react/LockIcon'
 
-import { Badge, Button, Icon, H3, Link, Text } from '@sourcegraph/wildcard'
+import { Badge, Button, Icon, H3, Link, Text, Tooltip } from '@sourcegraph/wildcard'
 
 import { ExecutorSecretFields, Scalars } from '../../../graphql-operations'
 
@@ -62,7 +62,19 @@ export const ExecutorSecretNode: React.FunctionComponent<React.PropsWithChildren
                 >
                     <div className="d-flex align-items-center">
                         <H3 className="text-nowrap mb-0 mr-2">
-                            <Icon className="mx-2" aria-hidden={true} as={LockIcon} /> {node.key}
+                            {node.key === 'DOCKER_AUTH_CONFIG' ? (
+                                <Tooltip content="This secret value will be used to configure docker client authentication with private registries.">
+                                    <Icon
+                                        className="mx-2"
+                                        svgPath={mdiDocker}
+                                        aria-label="This secret value will be used to configure docker client authentication with
+                                    private registries."
+                                    />
+                                </Tooltip>
+                            ) : (
+                                <Icon className="mx-2" aria-hidden={true} svgPath={mdiLock} />
+                            )}{' '}
+                            {node.key}
                         </H3>
                         {node.namespace === null && (
                             <span>

client/web/src/enterprise/executors/secrets/ExecutorSecretsListPage.story.tsx

@@ -43,7 +43,7 @@ const EXECUTOR_SECRET_LIST_MOCK: MockedResponse<UserExecutorSecretsResult> = {
                 __typename: 'User',
                 executorSecrets: {
                     pageInfo: { hasNextPage: false, endCursor: null },
-                    totalCount: 4,
+                    totalCount: 5,
                     nodes: [
                         // Global secret.
                         {
@@ -126,6 +126,24 @@ const EXECUTOR_SECRET_LIST_MOCK: MockedResponse<UserExecutorSecretsResult> = {
                             createdAt: subDays(new Date(), 1).toISOString(),
                             updatedAt: subDays(new Date(), 1).toISOString(),
                         },
+                        // Docker auth secret.
+                        {
+                            __typename: 'ExecutorSecret',
+                            id: 'secret5',
+                            creator: {
+                                __typename: 'User',
+                                id: 'user1',
+                                displayName: 'John Doe',
+                                url: '/users/jdoe',
+                                username: 'jdoe',
+                            },
+                            key: 'DOCKER_AUTH_CONFIG',
+                            namespace: null,
+                            overwritesGlobalSecret: false,
+                            scope: ExecutorSecretScope.BATCHES,
+                            createdAt: subDays(new Date(), 1).toISOString(),
+                            updatedAt: subDays(new Date(), 1).toISOString(),
+                        },
                     ],
                 },
             },

client/web/src/enterprise/executors/secrets/UpdateSecretModal.story.tsx

@@ -51,3 +51,35 @@ export const Update: Story = () => (
         )}
     </WebStory>
 )
+
+export const DockerAuthConfig: Story = () => (
+    <WebStory>
+        {props => (
+            <UpdateSecretModal
+                {...props}
+                secret={{
+                    __typename: 'ExecutorSecret',
+                    id: 'secret1',
+                    creator: {
+                        __typename: 'User',
+                        username: 'test',
+                        displayName: 'Test user',
+                        id: 'testID',
+                        url: '/users/test',
+                    },
+                    key: 'DOCKER_AUTH_CONFIG',
+                    scope: ExecutorSecretScope.BATCHES,
+                    overwritesGlobalSecret: false,
+                    // Global secret.
+                    namespace: null,
+                    createdAt: subDays(new Date(), 1).toISOString(),
+                    updatedAt: subHours(new Date(), 12).toISOString(),
+                }}
+                onCancel={noop}
+                afterUpdate={noop}
+            />
+        )}
+    </WebStory>
+)
+
+DockerAuthConfig.storyName = 'Docker auth config'

client/web/src/enterprise/executors/secrets/UpdateSecretModal.tsx

@@ -3,7 +3,7 @@ import React, { useCallback, useState } from 'react'
 import { ErrorAlert } from '@sourcegraph/branded/src/components/alerts'
 import { Form } from '@sourcegraph/branded/src/components/Form'
 import { logger } from '@sourcegraph/common'
-import { Button, Modal, Input, H3, Text } from '@sourcegraph/wildcard'
+import { Button, Modal, Input, H3, Text, Alert, Link } from '@sourcegraph/wildcard'
 
 import { LoaderButton } from '../../../components/LoaderButton'
 import { ExecutorSecretFields } from '../../../graphql-operations'
@@ -59,6 +59,15 @@ export const UpdateSecretModal: React.FunctionComponent<React.PropsWithChildren<
                 Executor secrets are available to executor jobs as environment variables. They will never appear in
                 logs.
             </Text>
+            {secret.key === 'DOCKER_AUTH_CONFIG' && (
+                <Alert variant="info" className="mt-2">
+                    This secret value will be used to{' '}
+                    <Link to="/help/admin/deploy_executors#using-private-registries" rel="noopener" target="_blank">
+                        configure docker client authentication with private registries
+                    </Link>
+                    .
+                </Alert>
+            )}
 
             {error && <ErrorAlert error={error} />}
 

cmd/frontend/graphqlbackend/executor_secrets.go

@@ -1,7 +1,9 @@
 package graphqlbackend
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"strings"
 
 	"github.com/grafana/regexp"
@@ -62,16 +64,17 @@ func (r *schemaResolver) CreateExecutorSecret(ctx context.Context, args CreateEx
 		return nil, errors.New("invalid key format, should be a valid env var name")
 	}
 
-	if len(args.Value) == 0 {
-		return nil, errors.New("value cannot be empty string")
-	}
-
 	secret := &database.ExecutorSecret{
 		Key:             args.Key,
 		CreatorID:       a.UID,
 		NamespaceUserID: userID,
 		NamespaceOrgID:  orgID,
 	}
+
+	if err := validateExecutorSecret(secret, args.Value); err != nil {
+		return nil, err
+	}
+
 	if err := store.Create(ctx, args.Scope.ToDatabaseScope(), secret, args.Value); err != nil {
 		if err == database.ErrDuplicateExecutorSecret {
 			return nil, &ErrDuplicateExecutorSecret{}
@@ -113,10 +116,6 @@ func (r *schemaResolver) UpdateExecutorSecret(ctx context.Context, args UpdateEx
 		return nil, errors.New("scope mismatch")
 	}
 
-	if len(args.Value) == 0 {
-		return nil, errors.New("value cannot be empty string")
-	}
-
 	store := r.db.ExecutorSecrets(keyring.Default().ExecutorSecretKey)
 
 	tx, err := store.Transact(ctx)
@@ -135,6 +134,10 @@ func (r *schemaResolver) UpdateExecutorSecret(ctx context.Context, args UpdateEx
 		return nil, err
 	}
 
+	if err := validateExecutorSecret(secret, args.Value); err != nil {
+		return nil, err
+	}
+
 	if err := tx.Update(ctx, args.Scope.ToDatabaseScope(), secret, args.Value); err != nil {
 		return nil, err
 	}
@@ -283,3 +286,46 @@ func checkNamespaceAccess(ctx context.Context, db database.DB, namespaceUserID,
 
 	return auth.CheckCurrentUserIsSiteAdmin(ctx, db)
 }
+
+// validateExecutorSecret validates that the secret value is non-empty and if the
+// secret key is DOCKER_AUTH_CONFIG that the value is acceptable.
+func validateExecutorSecret(secret *database.ExecutorSecret, value string) error {
+	if len(value) == 0 {
+		return errors.New("value cannot be empty string")
+	}
+	// Validate a docker auth config is correctly formatted before storing it to avoid
+	// confusion and broken config.
+	if secret.Key == "DOCKER_AUTH_CONFIG" {
+		var dac dockerAuthConfig
+		dec := json.NewDecoder(strings.NewReader(value))
+		dec.DisallowUnknownFields()
+		if err := dec.Decode(&dac); err != nil {
+			return errors.Wrap(err, "failed to unmarshal docker auth config for validation")
+		}
+		if len(dac.CredHelpers) > 0 {
+			return errors.New("cannot use credential helpers in docker auth config set via secrets")
+		}
+		if dac.CredsStore != "" {
+			return errors.New("cannot use credential stores in docker auth config set via secrets")
+		}
+		for key, auth := range dac.Auths {
+			if !bytes.Contains(auth.Auth, []byte(":")) {
+				return errors.Newf("invalid credential in auths section for %q format has to be base64(username:password)", key)
+			}
+		}
+	}
+
+	return nil
+}
+
+type dockerAuthConfig struct {
+	Auths       dockerAuthConfigAuths `json:"auths"`
+	CredsStore  string                `json:"credsStore"`
+	CredHelpers map[string]string     `json:"credHelpers"`
+}
+
+type dockerAuthConfigAuths map[string]dockerAuthConfigAuth
+
+type dockerAuthConfigAuth struct {
+	Auth []byte `json:"auth"`
+}