diff --git a/Dockerfile b/Dockerfile index 65f0f32cf..8647a619e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,7 +38,8 @@ RUN apk add --no-cache ca-certificates iptables ip6tables fuse3 git openssh pigz RUN adduser -D acorn RUN mkdir /wd && \ chown acorn /wd && \ - mkdir /etc/coredns + mkdir /etc/coredns && \ + mkdir /lib/modules RUN --mount=from=binfmt,src=/usr/bin,target=/usr/src for i in aarch64 x86_64; do if [ -e /usr/src/qemu-$i ]; then cp /usr/src/qemu-$i /usr/bin; fi; done RUN --mount=from=buildkit,src=/usr/bin,target=/usr/src for i in aarch64 x86_64; do if [ -e /usr/src/buildkit-qemu-$i ]; then cp /usr/src/buildkit-qemu-$i /usr/bin; fi; done COPY --from=binfmt /usr/bin/binfmt /usr/local/bin @@ -64,7 +65,6 @@ COPY /scripts/acorn-job-get-output /usr/local/bin COPY /scripts/k3s-config.yaml /etc/rancher/k3s/config.yaml CMD [] WORKDIR /wd -VOLUME /var/lib/buildkit VOLUME /var/lib/rancher/k3s STOPSIGNAL SIGTERM ENTRYPOINT ["/usr/local/bin/acorn"] diff --git a/Makefile b/Makefile index d2a620dd4..23efe969b 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,10 @@ build: tidy: go mod tidy +dev-reset: build + docker build -t localdev . + ACORN_IMAGE=localdev ACORN_LOCAL_PORT=6442 acorn local start --delete + dev-install: [ -e .dev-image ] && go mod vendor ; go run main.go install --dev "$$(cat .dev-image)"; rm -rf vendor diff --git a/pkg/cli/local.go b/pkg/cli/local.go index 10be7d9e6..da40d0cdd 100644 --- a/pkg/cli/local.go +++ b/pkg/cli/local.go @@ -12,12 +12,10 @@ func NewLocal(c CommandContext) *cobra.Command { Hidden: true, }) cmd.AddCommand(NewLocalServer(c)) - cmd.AddCommand(NewLocalCreate(c)) cmd.AddCommand(NewLocalLogs(c)) cmd.AddCommand(NewLocalRm(c)) cmd.AddCommand(NewLocalStart(c)) cmd.AddCommand(NewLocalStop(c)) - cmd.AddCommand(NewLocalReset(c)) return cmd } diff --git a/pkg/cli/local_create.go b/pkg/cli/local_create.go deleted file mode 100644 index ea4e15d06..000000000 --- a/pkg/cli/local_create.go +++ /dev/null @@ -1,35 +0,0 @@ -package cli - -import ( - "fmt" - - cli "github.com/acorn-io/runtime/pkg/cli/builder" - "github.com/acorn-io/runtime/pkg/local" - "github.com/acorn-io/runtime/pkg/system" - "github.com/spf13/cobra" -) - -func NewLocalCreate(c CommandContext) *cobra.Command { - cmd := cli.Command(&Create{}, cobra.Command{ - SilenceUsage: true, - Short: "Create local development server", - }) - return cmd -} - -type Create struct { - Upgrade bool `usage:"Upgrade if runtime already exists"` -} - -func (a *Create) Run(cmd *cobra.Command, args []string) error { - c, err := local.NewContainer(cmd.Context()) - if err != nil { - return err - } - - if _, err := c.Create(cmd.Context(), a.Upgrade); err != nil { - return err - } - fmt.Println("running", system.DefaultImage()) - return nil -} diff --git a/pkg/cli/local_reset.go b/pkg/cli/local_reset.go deleted file mode 100644 index 6e9cac540..000000000 --- a/pkg/cli/local_reset.go +++ /dev/null @@ -1,34 +0,0 @@ -package cli - -import ( - "fmt" - - cli "github.com/acorn-io/runtime/pkg/cli/builder" - "github.com/acorn-io/runtime/pkg/local" - "github.com/acorn-io/runtime/pkg/system" - "github.com/spf13/cobra" -) - -func NewLocalReset(c CommandContext) *cobra.Command { - cmd := cli.Command(&Reset{}, cobra.Command{ - SilenceUsage: true, - Short: "Reset local development server, deleting all data", - }) - return cmd -} - -type Reset struct { -} - -func (a *Reset) Run(cmd *cobra.Command, args []string) error { - c, err := local.NewContainer(cmd.Context()) - if err != nil { - return err - } - - if err := c.Reset(cmd.Context()); err != nil { - return err - } - fmt.Println("running", system.DefaultImage()) - return nil -} diff --git a/pkg/cli/local_rm.go b/pkg/cli/local_rm.go index 5bb844eac..8444574ea 100644 --- a/pkg/cli/local_rm.go +++ b/pkg/cli/local_rm.go @@ -19,7 +19,7 @@ func NewLocalRm(c CommandContext) *cobra.Command { } type LocalRm struct { - State bool `usage:"Include associated state (acorns and acorn data)"` + State bool `usage:"Include associated state (acorns, secrets and volume data)"` } func (a *LocalRm) Run(cmd *cobra.Command, args []string) error { diff --git a/pkg/cli/local_start.go b/pkg/cli/local_start.go index aa31da267..66d303b07 100644 --- a/pkg/cli/local_start.go +++ b/pkg/cli/local_start.go @@ -1,8 +1,6 @@ package cli import ( - "fmt" - cli "github.com/acorn-io/runtime/pkg/cli/builder" "github.com/acorn-io/runtime/pkg/local" "github.com/spf13/cobra" @@ -19,22 +17,22 @@ func NewLocalStart(c CommandContext) *cobra.Command { } type LocalStart struct { + Reset bool `usage:"Delete existing server and all data before starting"` + Delete bool `usage:"Delete existing server before starting"` } -func (a *LocalStart) Run(cmd *cobra.Command, args []string) error { +func (a *LocalStart) Run(cmd *cobra.Command, args []string) (err error) { c, err := local.NewContainer(cmd.Context()) if err != nil { return err } - if _, err := c.Create(cmd.Context(), false); err != nil { - return err - } - - if err := c.Start(cmd.Context()); err != nil { - return err + if a.Reset { + return c.Reset(cmd.Context(), true) + } else if a.Delete { + return c.Reset(cmd.Context(), false) } - fmt.Println("started") - return nil + _, _, err = c.Upgrade(cmd.Context(), false) + return err } diff --git a/pkg/client/client.go b/pkg/client/client.go index f66a91da1..87917ef2c 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -428,7 +428,9 @@ func generateKubeConfig(restConfig *rest.Config) ([]byte, error) { }, AuthInfos: map[string]*clientcmdapi.AuthInfo{ "auth": { - Token: restConfig.BearerToken, + Token: restConfig.BearerToken, + ClientCertificateData: restConfig.TLSClientConfig.CertData, + ClientKeyData: restConfig.TLSClientConfig.KeyData, }, }, Contexts: map[string]*clientcmdapi.Context{ diff --git a/pkg/controller/appdefinition/router.go b/pkg/controller/appdefinition/router.go index 4ca7c98bb..0092f957c 100644 --- a/pkg/controller/appdefinition/router.go +++ b/pkg/controller/appdefinition/router.go @@ -84,6 +84,10 @@ func toRouter(appInstance *v1.AppInstance, routerName string, router v1.Router, "daemon off;", }, VolumeMounts: []corev1.VolumeMount{ + { + Name: "confd", + MountPath: "/etc/nginx/conf.d", + }, { Name: "conf", ReadOnly: true, @@ -121,6 +125,14 @@ func toRouter(appInstance *v1.AppInstance, routerName string, router v1.Router, }, }, Volumes: []corev1.Volume{ + { + Name: "confd", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: corev1.StorageMediumMemory, + }, + }, + }, { Name: "conf", VolumeSource: corev1.VolumeSource{ diff --git a/pkg/controller/appdefinition/testdata/router/expected.golden b/pkg/controller/appdefinition/testdata/router/expected.golden index 4cb43ede6..921438236 100644 --- a/pkg/controller/appdefinition/testdata/router/expected.golden +++ b/pkg/controller/appdefinition/testdata/router/expected.golden @@ -54,6 +54,8 @@ spec: port: 8080 resources: {} volumeMounts: + - mountPath: /etc/nginx/conf.d + name: confd - mountPath: /etc/nginx/conf.d/nginx.conf name: conf readOnly: true @@ -65,6 +67,9 @@ spec: - key: taints.acorn.io/workload operator: Exists volumes: + - emptyDir: + medium: Memory + name: confd - configMap: name: router-name-30019cec name: conf diff --git a/pkg/controller/data.go b/pkg/controller/data.go index 1070bc90b..069641dbd 100644 --- a/pkg/controller/data.go +++ b/pkg/controller/data.go @@ -97,5 +97,15 @@ func (c *Controller) initData(ctx context.Context) error { if err != nil { return err } + if system.IsLocal() { + err = c.apply.Ensure(ctx, &v1.ProjectInstance{ + ObjectMeta: metav1.ObjectMeta{ + Name: "local", + }, + }) + if err != nil { + return err + } + } return config.Init(ctx, c.client) } diff --git a/pkg/controller/local/pods.go b/pkg/controller/local/pods.go index 2d4cc87e2..3215adc00 100644 --- a/pkg/controller/local/pods.go +++ b/pkg/controller/local/pods.go @@ -13,5 +13,10 @@ func DeletePods(req router.Request, resp router.Response) error { return req.Client.Delete(req.Ctx, pod) } } + for _, container := range pod.Spec.InitContainers { + if container.Image == system.LocalImage { + return req.Client.Delete(req.Ctx, pod) + } + } return nil } diff --git a/pkg/controller/local/services.go b/pkg/controller/local/services.go index 532cbbc12..8bd66ddec 100644 --- a/pkg/controller/local/services.go +++ b/pkg/controller/local/services.go @@ -14,6 +14,7 @@ import ( "github.com/docker/docker/api/types/container" "github.com/docker/docker/client" "github.com/docker/go-connections/nat" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -57,7 +58,7 @@ func (c *Handler) ProvisionPorts(req router.Request, resp router.Response) error continue } - pod := &corev1.Pod{ + deploy := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: system.Namespace, @@ -65,47 +66,62 @@ func (c *Handler) ProvisionPorts(req router.Request, resp router.Response) error "app": "klipper-lb", }, }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "klipper-lb", - Command: []string{"klipper-lb"}, - Image: system.LocalImage, - SecurityContext: &corev1.SecurityContext{ - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{ - "NET_ADMIN", - }, - }, - }, - Env: []corev1.EnvVar{ - { - Name: "SRC_PORT", - Value: fmt.Sprint(port.Port), - }, - { - Name: "SRC_RANGES", - Value: "0.0.0.0/0", - }, - { - Name: "DEST_PROTO", - Value: string(port.Protocol), - }, - { - Name: "DEST_PORT", - Value: fmt.Sprint(port.Port), - }, - { - Name: "DEST_IPS", - Value: svc.Spec.ClusterIP, - }, + Spec: appsv1.DeploymentSpec{ + Selector: metav1.SetAsLabelSelector(map[string]string{ + "app": "klipper-lb", + }), + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RecreateDeploymentStrategyType, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "klipper-lb", }, - Ports: []corev1.ContainerPort{ + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ { - Name: "port", - HostPort: port.Port, - ContainerPort: port.Port, - Protocol: port.Protocol, + Name: "klipper-lb", + Command: []string{"klipper-lb"}, + Image: system.LocalImage, + SecurityContext: &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{ + "NET_ADMIN", + }, + }, + }, + Env: []corev1.EnvVar{ + { + Name: "SRC_PORT", + Value: fmt.Sprint(port.Port), + }, + { + Name: "SRC_RANGES", + Value: "0.0.0.0/0", + }, + { + Name: "DEST_PROTO", + Value: string(port.Protocol), + }, + { + Name: "DEST_PORT", + Value: fmt.Sprint(port.Port), + }, + { + Name: "DEST_IPS", + Value: svc.Spec.ClusterIP, + }, + }, + Ports: []corev1.ContainerPort{ + { + Name: "port", + HostPort: port.Port, + ContainerPort: port.Port, + Protocol: port.Protocol, + }, + }, }, }, }, @@ -114,8 +130,8 @@ func (c *Handler) ProvisionPorts(req router.Request, resp router.Response) error } // Patch inline now, otherwise baaah will fight with the changes the webhook makes - webhook.PatchPodSpec(&pod.Spec) - resp.Objects(pod) + webhook.PatchPodSpec(&deploy.Spec.Template.Spec) + resp.Objects(deploy) svc.Status.LoadBalancer.Ingress = []corev1.LoadBalancerIngress{ { diff --git a/pkg/controller/local/storage.go b/pkg/controller/local/storage.go index 7b668a0cb..775410185 100644 --- a/pkg/controller/local/storage.go +++ b/pkg/controller/local/storage.go @@ -60,6 +60,10 @@ func createFolder(req router.Request, resp router.Response) error { return err } + if err := os.Chmod(path, 0777); err != nil { + return err + } + err := apply.New(req.Client).Ensure(req.Ctx, &corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: pvName, diff --git a/pkg/edit/edit.go b/pkg/edit/edit.go index 4c497db26..67749f546 100644 --- a/pkg/edit/edit.go +++ b/pkg/edit/edit.go @@ -78,7 +78,7 @@ func editSecret(ctx context.Context, c client.Client, secret *apiv1.Secret) erro editor := editor.NewDefaultEditor(envs) for { - buf, file, err := editor.LaunchTempFile("acorn", "secret", bytes.NewReader(spec)) + buf, file, err := editor.LaunchTempFile("acorn", "secret.acorn", bytes.NewReader(spec)) if file != "" { _ = os.Remove(file) } @@ -128,7 +128,7 @@ func editApp(ctx context.Context, c client.Client, app *apiv1.App) error { editor := editor.NewDefaultEditor(envs) for { - buf, file, err := editor.LaunchTempFile("acorn", "app", bytes.NewReader(spec)) + buf, file, err := editor.LaunchTempFile("acorn", "acorn.acorn", bytes.NewReader(spec)) if file != "" { _ = os.Remove(file) } diff --git a/pkg/imagesystem/buildertemplate.go b/pkg/imagesystem/buildertemplate.go index dcb43ce44..303789c95 100644 --- a/pkg/imagesystem/buildertemplate.go +++ b/pkg/imagesystem/buildertemplate.go @@ -32,12 +32,20 @@ func BuilderObjects(name, namespace, forNamespace, buildKitImage, pub, privKey, }, } + var strategy appsv1.DeploymentStrategyType + if system.IsLocal() { + strategy = appsv1.RecreateDeploymentStrategyType + } + deployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: namespace, }, Spec: appsv1.DeploymentSpec{ + Strategy: appsv1.DeploymentStrategy{ + Type: strategy, + }, Selector: &metav1.LabelSelector{ MatchLabels: labels.ManagedByApp(namespace, name, "app", name), }, @@ -50,6 +58,7 @@ func BuilderObjects(name, namespace, forNamespace, buildKitImage, pub, privKey, ServiceAccountName: "acorn-builder", EnableServiceLinks: new(bool), TerminationGracePeriodSeconds: z.Pointer[int64](10), + Hostname: "builder", Containers: []corev1.Container{ { Name: "buildkitd", diff --git a/pkg/install/install.go b/pkg/install/install.go index 8bc8d95f8..653f53f51 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -279,7 +279,7 @@ func Install(ctx context.Context, image string, opts *Options) error { return err } - if err := waitAPI(ctx, opts.Progress, *opts.APIServerReplicas, image, c); err != nil { + if err := WaitAPI(ctx, opts.Progress, *opts.APIServerReplicas, image, c); err != nil { return err } @@ -486,7 +486,7 @@ func waitRegistry(ctx context.Context, p progress.Builder, image string, client return s.Fail(waitDeployment(ctx, s, client, image, system.RegistryName, system.ImagesNamespace, 1)) } -func waitAPI(ctx context.Context, p progress.Builder, replicas int, image string, client kclient.WithWatch) error { +func WaitAPI(ctx context.Context, p progress.Builder, replicas int, image string, client kclient.WithWatch) error { s := p.New("Waiting for API server deployment to be available") if err := waitDeployment(ctx, s, client, image, "acorn-api", system.Namespace, int32(replicas)); err != nil { return s.Fail(err) diff --git a/pkg/install/local.yaml b/pkg/install/local.yaml index 2ad47d43f..119e06f4a 100644 --- a/pkg/install/local.yaml +++ b/pkg/install/local.yaml @@ -60,10 +60,4 @@ metadata: storageclass.kubernetes.io/is-default-class: "true" provisioner: acorn.io/local-storage reclaimPolicy: Delete -volumeBindingMode: WaitForFirstConsumer - ---- -kind: Project -apiVersion: api.acorn.io/v1 -metadata: - name: local \ No newline at end of file +volumeBindingMode: WaitForFirstConsumer \ No newline at end of file diff --git a/pkg/local/docker.go b/pkg/local/docker.go index 2ae87668a..23f296a3e 100644 --- a/pkg/local/docker.go +++ b/pkg/local/docker.go @@ -11,8 +11,12 @@ import ( "time" "github.com/acorn-io/baaah/pkg/restconfig" + "github.com/acorn-io/baaah/pkg/watcher" + v1 "github.com/acorn-io/runtime/pkg/apis/api.acorn.io/v1" + "github.com/acorn-io/runtime/pkg/install" "github.com/acorn-io/runtime/pkg/scheme" "github.com/acorn-io/runtime/pkg/system" + "github.com/acorn-io/runtime/pkg/term" "github.com/acorn-io/z" "github.com/docker/cli/cli/streams" "github.com/docker/docker/api/types" @@ -26,6 +30,7 @@ import ( "github.com/docker/go-connections/nat" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + kclient "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -47,28 +52,16 @@ func NewContainer(_ context.Context) (*Container, error) { }, nil } -func (c *Container) Ensure(ctx context.Context) (*rest.Config, error) { - con, err := c.c.ContainerInspect(ctx, ContainerName) - if client.IsErrNotFound(err) { - var id string - id, err = c.Upgrade(ctx) - if err != nil { - return nil, err - } - con, err = c.c.ContainerInspect(ctx, id) - } - if err != nil { - return nil, err - } - +func (c *Container) getKubeconfig(ctx context.Context, port string) (*rest.Config, error) { var ( + err error out io.ReadCloser ) for i := 0; ; i++ { if i > 20 { return nil, fmt.Errorf("timeout trying to launch %s container", ContainerName) } - out, _, err = c.c.CopyFromContainer(ctx, con.ID, "/etc/rancher/k3s/k3s.yaml") + out, _, err = c.c.CopyFromContainer(ctx, ContainerName, "/etc/rancher/k3s/k3s.yaml") if client.IsErrNotFound(err) { time.Sleep(500 * time.Millisecond) continue @@ -94,12 +87,21 @@ func (c *Container) Ensure(ctx context.Context) (*rest.Config, error) { if err != nil { return nil, err } - cfg.Host = fmt.Sprintf("https://localhost:%s", con.NetworkSettings.Ports["6443/tcp"][0].HostPort) + cfg.Host = fmt.Sprintf("https://localhost:%s", port) restconfig.SetScheme(cfg, scheme.Scheme) return cfg, waitFor(ctx, cfg) } +func (c *Container) Ensure(ctx context.Context) (*rest.Config, error) { + _, port, err := c.Upgrade(ctx, true) + if err != nil { + return nil, err + } + + return c.getKubeconfig(ctx, port) +} + func waitFor(ctx context.Context, cfg *rest.Config) error { ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) defer cancel() @@ -138,7 +140,8 @@ func (c *Container) DeletePorts(ctx context.Context) error { for _, name := range con.Names { if strings.HasPrefix(name, "/"+ContainerName+"-") { if err := c.c.ContainerRemove(ctx, name, types.ContainerRemoveOptions{ - Force: true, + RemoveVolumes: true, + Force: true, }); err != nil { return err } @@ -153,7 +156,6 @@ func (c *Container) DeletePorts(ctx context.Context) error { func (c *Container) Delete(ctx context.Context, data bool) error { err := c.c.ContainerRemove(ctx, ContainerName, types.ContainerRemoveOptions{ RemoveVolumes: data, - RemoveLinks: false, Force: true, }) if client.IsErrNotFound(err) { @@ -171,34 +173,147 @@ func (c *Container) Delete(ctx context.Context, data bool) error { return c.DeletePorts(ctx) } -func (c *Container) Upgrade(ctx context.Context) (string, error) { +func (c *Container) Upgrade(ctx context.Context, ignoreLocal bool) (string, string, error) { con, err := c.c.ContainerInspect(ctx, ContainerName) if client.IsErrNotFound(err) { - return c.Create(ctx, false) + if _, err := c.Create(ctx); err != nil { + return "", "", err + } + con, err = c.c.ContainerInspect(ctx, ContainerName) + if err != nil { + return "", "", err + } } else if err != nil { - return "", err + return "", "", err } - if con.Config.Image == system.DefaultImage() { - return con.ID, c.Start(ctx) + if con.Config.Image == system.DefaultImage() || (ignoreLocal && con.Config.Image == "localdev") { + return con.ID, con.NetworkSettings.Ports["6443/tcp"][0].HostPort, c.Start(ctx) } if err := c.Delete(ctx, false); err != nil { - return "", err + return "", "", err } - return c.Create(ctx, false) + return c.Upgrade(ctx, ignoreLocal) } -func (c *Container) Reset(ctx context.Context) error { - if err := c.Delete(ctx, true); err != nil { +func (c *Container) Reset(ctx context.Context, data bool) error { + if err := c.Delete(ctx, data); err != nil { return err } - _, err := c.Create(ctx, false) + _, err := c.Create(ctx) return err } -func (c *Container) Create(ctx context.Context, upgrade bool) (string, error) { +func (c *Container) Wait(ctx context.Context) error { + pb := &term.Builder{} + + imageStatus := pb.New("Image pulled") + imageStatus.Infof("Pulling image %s", system.DefaultImage()) + if err := c.pull(ctx); err != nil { + return imageStatus.Fail(err) + } + imageStatus.Success() + + conStatus := pb.New("Container created") + conStatus.Infof("Creating") + + for { + _, err := c.c.ContainerInspect(ctx, ContainerName) + if client.IsErrNotFound(err) { + } else if err != nil { + return conStatus.Fail(err) + } else { + conStatus.Success() + break + } + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(500 * time.Millisecond): + } + } + + running := pb.New("Container running") + running.Infof("Starting") + + var port string + + for { + con, err := c.c.ContainerInspect(ctx, ContainerName) + if err != nil { + return running.Fail(err) + } + + if con.State == nil || !con.State.Running { + select { + case <-ctx.Done(): + case <-time.After(500 * time.Millisecond): + continue + } + } + + port = con.NetworkSettings.Ports["6443/tcp"][0].HostPort + break + } + + running.Success() + + restConfig, err := c.getKubeconfig(ctx, port) + if err != nil { + return err + } + + kc, err := kclient.NewWithWatch(restConfig, kclient.Options{ + Scheme: scheme.Scheme, + }) + if err != nil { + return err + } + + if err := install.WaitAPI(ctx, pb, 1, system.LocalImageBind, kc); err != nil { + return err + } + + ns := pb.New("Local project created") + ns.Infof("Waiting for local project") + w := watcher.New[*v1.Project](kc) + for { + _, err = w.ByName(ctx, "", "local", func(obj *v1.Project) (bool, error) { + return true, nil + }) + if err != nil { + ns.Infof("Waiting for local project: %v", err) + select { + case <-ctx.Done(): + return ns.Fail(ctx.Err()) + case <-time.After(1 * time.Second): + } + continue + } + break + } + ns.Success() + return nil +} + +func (c *Container) pull(ctx context.Context) error { + _, _, err := c.c.ImageInspectWithRaw(ctx, system.DefaultImage()) + if err == nil { + return nil + } + + resp, err := c.c.ImagePull(ctx, system.DefaultImage(), types.ImagePullOptions{}) + if err != nil { + return err + } + out := streams.NewOut(os.Stdout) + return jsonmessage.DisplayJSONMessagesToStream(resp, out, nil) +} + +func (c *Container) Create(ctx context.Context) (string, error) { v, err := c.c.VolumeCreate(ctx, volume.CreateOptions{ Name: volumeName, }) @@ -226,7 +341,7 @@ func (c *Container) Create(ctx context.Context, upgrade bool) (string, error) { "6443/tcp": { { HostIP: "0.0.0.0", - HostPort: "", + HostPort: os.Getenv("ACORN_LOCAL_PORT"), }, }, }, @@ -240,34 +355,40 @@ func (c *Container) Create(ctx context.Context, upgrade bool) (string, error) { Source: v.Name, Target: "/var/lib/rancher/k3s", }, + { + Type: mount.TypeVolume, + Source: v.Name, + Target: "/var/lib/buildkit", + }, { Type: mount.TypeBind, Source: "/var/run/docker.sock", Target: "/var/run/docker.sock", }, + { + Type: mount.TypeBind, + Source: "/lib/modules", + Target: "/lib/modules", + ReadOnly: true, + }, }, }, nil, nil, ContainerName) if client.IsErrNotFound(err) { - resp, err := c.c.ImagePull(ctx, system.DefaultImage(), types.ImagePullOptions{}) - if err != nil { - return "", err - } - out := streams.NewOut(os.Stdout) - if err := jsonmessage.DisplayJSONMessagesToStream(resp, out, nil); err != nil { + if err := c.pull(ctx); err != nil { return "", err } - return c.Create(ctx, false) + return c.Create(ctx) } else if errdefs.IsConflict(err) { - if upgrade { - return c.Upgrade(ctx) - } else { - return con.ID, c.Start(ctx) - } + return con.ID, c.Start(ctx) } else if err != nil { return "", err } - return con.ID, c.Start(ctx) + if err := c.Start(ctx); err != nil { + return "", err + } + + return con.ID, c.Wait(ctx) } func (c *Container) Start(ctx context.Context) error { diff --git a/pkg/local/server_linux.go b/pkg/local/server_linux.go index 43b672079..1ab70ad46 100644 --- a/pkg/local/server_linux.go +++ b/pkg/local/server_linux.go @@ -1,19 +1,24 @@ package local import ( + "archive/tar" "bytes" "context" _ "embed" "encoding/json" "fmt" + "io" "os" "os/exec" + "syscall" "github.com/acorn-io/baaah/pkg/yaml" + apiv1 "github.com/acorn-io/runtime/pkg/apis/api.acorn.io/v1" "github.com/acorn-io/runtime/pkg/install" "github.com/acorn-io/runtime/pkg/local/webhook" "github.com/acorn-io/runtime/pkg/scheme" "github.com/acorn-io/runtime/pkg/system" + "github.com/acorn-io/z" "github.com/google/go-containerregistry/pkg/name" ggcrv1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" @@ -27,9 +32,11 @@ func ServerRun(ctx context.Context) error { if os.Getuid() != 0 { return fmt.Errorf("must run as root") } + if _, err := os.Stat("/.dockerenv"); err != nil { return fmt.Errorf("must be ran in a docker container: %w", err) } + if f, err := os.Open("/dev/kmsg"); err != nil { return fmt.Errorf("must be ran in a privileged docker container: %w", err) } else { @@ -53,6 +60,11 @@ func ServerRun(ctx context.Context) error { err = install.PrintObjects("acorn-local", &install.Options{ Output: buf, IncludeLocalEnvResources: true, + Config: apiv1.Config{ + IngressClassName: z.Pointer("traefik"), + SetPodSecurityEnforceProfile: z.Pointer(false), + IgnoreResourceRequirements: z.Pointer(true), + }, }) if err != nil { return err @@ -106,7 +118,6 @@ func ServerRun(ctx context.Context) error { cmd := exec.Command("/bin/sh", "-c", ` mkdir -p /sys/fs/cgroup/init busybox xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || : -# enable controllers sed -e 's/ / +/g' -e 's/^/+/' <"/sys/fs/cgroup/cgroup.controllers" >"/sys/fs/cgroup/cgroup.subtree_control" `) cmd.Stderr = os.Stderr @@ -121,6 +132,46 @@ sed -e 's/ / +/g' -e 's/^/+/' <"/sys/fs/cgroup/cgroup.controllers" >"/sys/fs/cgr return err } + img, err := buildImage(ctx) + if err != nil { + return err + } + + if err := os.MkdirAll("/var/lib/rancher/k3s/agent/images", 0755); err != nil { + return err + } + + if err := tarball.WriteToFile("/var/lib/rancher/k3s/agent/images/empty.tar", ref, img); err != nil { + return err + } + + return syscall.Exec("/bin/k3s", []string{"k3s", "server"}, os.Environ()) +} + +func buildImage(ctx context.Context) (ggcrv1.Image, error) { + layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { + out := &bytes.Buffer{} + t := tar.NewWriter(out) + for _, dir := range []string{"wd", "tmp", "var", "var/lib", "etc", "etc/nginx", "var/log", "var/log/nginx", + "var/cache", "var/cache/nginx"} { + err := t.WriteHeader(&tar.Header{ + Typeflag: tar.TypeDir, + Name: dir, + Mode: 0777, + }) + if err != nil { + return nil, err + } + } + if err := t.Close(); err != nil { + return nil, err + } + return io.NopCloser(out), nil + }) + if err != nil { + return nil, err + } + img, err := mutate.Config(empty.Image, ggcrv1.Config{ Entrypoint: []string{ "/usr/local/bin/acorn", @@ -136,20 +187,8 @@ sed -e 's/ / +/g' -e 's/^/+/' <"/sys/fs/cgroup/cgroup.controllers" >"/sys/fs/cgr StopSignal: "SIGTERM", }) if err != nil { - return err - } - - if err := os.MkdirAll("/var/lib/rancher/k3s/agent/images", 0755); err != nil { - return err - } - - if err := tarball.WriteToFile("/var/lib/rancher/k3s/agent/images/empty.tar", ref, img); err != nil { - return err + return nil, err } - cmd := exec.Command("k3s", "server") - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Stdin = os.Stdin - return cmd.Run() + return mutate.AppendLayers(img, layer) } diff --git a/pkg/local/webhook/patch.go b/pkg/local/webhook/patch.go index c65cffc3c..412b9fb7b 100644 --- a/pkg/local/webhook/patch.go +++ b/pkg/local/webhook/patch.go @@ -19,8 +19,20 @@ func PatchPodSpec(podSpec *corev1.PodSpec) bool { var ( modified bool paths = []string{ - "/etc", "/lib", "/bin", "/sbin", "/usr", "docker-entrypoint.d", "docker-entrypoint.sh", "/wd", - "/var/run/docker.sock", "/var/lib/rancher/k3s/storage", + "/etc/passwd", + "/etc/group", + "/etc/docker", + "/etc/nginx", + "/lib", + "/bin", + "/sbin", + "/usr", + "docker-entrypoint.d", + "docker-entrypoint.sh", + "/etc/ssl/certs/ca-certificates.crt", + "/var/run/docker.sock", + "/var/lib/rancher/k3s/storage", + "/var/lib/buildkit", } mounts []corev1.VolumeMount existing = map[string]bool{} @@ -37,8 +49,10 @@ func PatchPodSpec(podSpec *corev1.PodSpec) bool { continue } mounts = append(mounts, corev1.VolumeMount{ - Name: "acorn-local-host", - ReadOnly: path != "/wd" && path != "/sbin" && path != "/var/lib/rancher/k3s/storage", + Name: "acorn-local-host", + ReadOnly: path != "/sbin" && + path != "/var/lib/rancher/k3s/storage" && + path != "/var/lib/buildkit", MountPath: path, SubPath: strings.TrimPrefix(path, "/"), }) @@ -60,6 +74,16 @@ func PatchPodSpec(podSpec *corev1.PodSpec) bool { } } + for i, container := range podSpec.InitContainers { + if container.Image == "acorn-local" { + modified = true + container.Image = system.LocalImageBind + container.ImagePullPolicy = corev1.PullIfNotPresent + container.VolumeMounts = append(container.VolumeMounts, mounts...) + podSpec.InitContainers[i] = container + } + } + if modified { podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{ Name: "acorn-local-host", diff --git a/scripts/k3s-config.yaml b/scripts/k3s-config.yaml index b8e91a520..8b9616323 100644 --- a/scripts/k3s-config.yaml +++ b/scripts/k3s-config.yaml @@ -11,7 +11,6 @@ disable-network-policy: true disable-helm-controller: true disable-scheduler: true kube-apiserver-arg: - - etcd-servers-overrides=coordination.k8s.io/leases#unix:///tmp/kine.sock,/apiServerIPInfo#unix:///tmp/kine.sock - enable-priority-and-fairness=false - feature-gates=AllBeta=false - feature-gates=AllAlpha=false @@ -58,7 +57,6 @@ kubelet-arg: - feature-gates=AllAlpha=false - feature-gates=AllBeta=false - log-flush-frequency=25s - - node-status-update-frequency=10m kube-proxy-arg: - log-flush-frequency=25s kube-cloud-controller-arg: