From b9f00bdfe5bb86bce01e86cda0eeff36c5eefb15 Mon Sep 17 00:00:00 2001 From: Al Davidson Date: Mon, 27 Mar 2023 10:30:55 +0100 Subject: [PATCH 01/10] add dependency on postgresql for content-store --- projects/content-store/docker-compose.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/projects/content-store/docker-compose.yml b/projects/content-store/docker-compose.yml index 7d45d529..53bde3c3 100644 --- a/projects/content-store/docker-compose.yml +++ b/projects/content-store/docker-compose.yml @@ -22,9 +22,11 @@ services: depends_on: # In production this uses Mongo 2.6, however there is not a published Mongo 2.6 image compatible with Arm64 - mongo-3.6 + - postgres-13 environment: MONGODB_URI: "mongodb://mongo-3.6/content-store" TEST_MONGODB_URI: "mongodb://mongo-3.6/content-store-test" + DATABASE_URL: "postgresql://postgres@postgres-13/content-store" content-store-app: &content-store-app <<: *content-store @@ -32,10 +34,13 @@ services: - mongo-3.6 - router-api-app - nginx-proxy + - postgres-13 environment: MONGODB_URI: "mongodb://mongo-3.6/content-store" VIRTUAL_HOST: content-store.dev.gov.uk BINDING: 0.0.0.0 + DATABASE_URL: "postgresql://postgres@postgres-13/content-store" + expose: - "3000" command: bin/rails s --restart From ddeca34c084891dd71d4407aa4431e793f1c5744 Mon Sep 17 00:00:00 2001 From: Al Davidson Date: Mon, 27 Mar 2023 10:34:54 +0100 Subject: [PATCH 02/10] remove mongo config for content-store --- projects/content-store/docker-compose.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/projects/content-store/docker-compose.yml b/projects/content-store/docker-compose.yml index 53bde3c3..f2a58fc4 100644 --- a/projects/content-store/docker-compose.yml +++ b/projects/content-store/docker-compose.yml @@ -20,18 +20,14 @@ services: content-store-lite: <<: *content-store depends_on: - # In production this uses Mongo 2.6, however there is not a published Mongo 2.6 image compatible with Arm64 - - mongo-3.6 - postgres-13 environment: - MONGODB_URI: "mongodb://mongo-3.6/content-store" - TEST_MONGODB_URI: "mongodb://mongo-3.6/content-store-test" + TEST_DATABASE_URL: "postgresql://postgres@postgres-13/content-store-test" DATABASE_URL: "postgresql://postgres@postgres-13/content-store" content-store-app: &content-store-app <<: *content-store depends_on: - - mongo-3.6 - router-api-app - nginx-proxy - postgres-13 From 8b2c5159c6d4cb8b5dae3a500bf62b9be39c4cba Mon Sep 17 00:00:00 2001 From: Al Davidson Date: Wed, 5 Apr 2023 08:14:04 +0100 Subject: [PATCH 03/10] add TEST_DATABASE_URL for content-store --- projects/content-store/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/content-store/docker-compose.yml b/projects/content-store/docker-compose.yml index f2a58fc4..6fe88e1c 100644 --- a/projects/content-store/docker-compose.yml +++ b/projects/content-store/docker-compose.yml @@ -36,6 +36,7 @@ services: VIRTUAL_HOST: content-store.dev.gov.uk BINDING: 0.0.0.0 DATABASE_URL: "postgresql://postgres@postgres-13/content-store" + TEST_DATABASE_URL: "postgresql://postgres@postgres-13/content-store-test" expose: - "3000" From 0b727fd70b7ca927f6ee922c2e23ffd7e6cc8105 Mon Sep 17 00:00:00 2001 From: Al Davidson Date: Thu, 27 Apr 2023 11:14:09 +0100 Subject: [PATCH 04/10] first draft docker setup for content-store-on-postgresql --- .../content-store-on-postgresql/Dockerfile | 30 +++++++++++++ projects/content-store-on-postgresql/Makefile | 2 + .../docker-compose.yml | 42 +++++++++++++++++++ 3 files changed, 74 insertions(+) create mode 100644 projects/content-store-on-postgresql/Dockerfile create mode 100644 projects/content-store-on-postgresql/Makefile create mode 100644 projects/content-store-on-postgresql/docker-compose.yml diff --git a/projects/content-store-on-postgresql/Dockerfile b/projects/content-store-on-postgresql/Dockerfile new file mode 100644 index 00000000..97c186c9 --- /dev/null +++ b/projects/content-store-on-postgresql/Dockerfile @@ -0,0 +1,30 @@ +# Standard Dockerfile for GOV.UK projects. Random project-specific stuff should *not* +# go in this file. Instead, you should copy this file into the directory for the +# project and make any changes to it there. + +# Install packages for building ruby +FROM buildpack-deps + +# Install chromium browser and its webdriver +RUN apt-get update -qq && apt-get install -y chromium chromium-driver + +# Enable no-sandbox for chrome so that it can run as a root user +ENV GOVUK_TEST_CHROME_NO_SANDBOX 1 + +# Install node / yarn +RUN curl -sL https://deb.nodesource.com/setup_16.x | bash - +RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - +RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list +RUN apt-get update -qq && apt-get install -y yarn nodejs +RUN yarn config set cache-folder /root/.yarn/ + +# Install rbenv to manage ruby versions +RUN git clone https://github.com/rbenv/rbenv.git /rbenv +RUN git clone https://github.com/rbenv/ruby-build.git /rbenv/plugins/ruby-build +RUN /rbenv/plugins/ruby-build/install.sh +ENV PATH /root/.rbenv/shims:/rbenv/bin:$PATH +ENV BUNDLE_SILENCE_ROOT_WARNING 1 + +# Install `psql` for 'rails dbconsole' +# and `less` for paginated output in 'rails console' and 'binding.pry' +RUN apt-get update -qq && apt-get install -y postgresql-client less diff --git a/projects/content-store-on-postgresql/Makefile b/projects/content-store-on-postgresql/Makefile new file mode 100644 index 00000000..07308980 --- /dev/null +++ b/projects/content-store-on-postgresql/Makefile @@ -0,0 +1,2 @@ +content-store-on-postgresql: bundle-content-store router-api publishing-api + $(GOVUK_DOCKER) run $@-lite bin/rake db:setup diff --git a/projects/content-store-on-postgresql/docker-compose.yml b/projects/content-store-on-postgresql/docker-compose.yml new file mode 100644 index 00000000..c691f48d --- /dev/null +++ b/projects/content-store-on-postgresql/docker-compose.yml @@ -0,0 +1,42 @@ +version: '3.7' + +volumes: + content-store-on-postgresql-tmp: + +x-content-store-on-postgresql: &content-store-on-postgresql + build: + context: . + dockerfile: projects/content-store-on-postgresql/Dockerfile + image: content-store-on-postgresql + stdin_open: true + tty: true + volumes: + - ${GOVUK_ROOT_DIR:-~/govuk}:/govuk:delegated + - root-home:/root + - content-store-on-postgresql-tmp:/govuk/content-store-on-postgresql/tmp + working_dir: /govuk/content-store-on-postgresql + +services: + content-store-on-postgresql-lite: + <<: *content-store-on-postgresql + depends_on: + - postgres-13 + environment: + TEST_DATABASE_URL: "postgresql://postgres@postgres-13/content-store-test" + DATABASE_URL: "postgresql://postgres@postgres-13/content-store" + + content-store-on-postgresql-app: &content-store-on-postgresql-app + <<: *content-store-on-postgresql + depends_on: + - router-api-app + - nginx-proxy + - postgres-13 + environment: + VIRTUAL_HOST: content-store-on-postgresql.dev.gov.uk + BINDING: 0.0.0.0 + DATABASE_URL: "postgresql://postgres@postgres-13/content-store" + TEST_DATABASE_URL: "postgresql://postgres@postgres-13/content-store-test" + + expose: + - "3000" + command: bin/rails s --restart From ee565fdbe5df5b28f0d887e8a7f060e7af4c5f01 Mon Sep 17 00:00:00 2001 From: Al Davidson Date: Thu, 27 Apr 2023 17:57:27 +0100 Subject: [PATCH 05/10] refactor to allow dual-running of content-store and content-store-on-postgresql apps --- .../content-store-on-postgresql/Dockerfile | 30 ------------------- projects/content-store-on-postgresql/Makefile | 4 +-- .../docker-compose.yml | 2 +- projects/content-store/docker-compose.yml | 12 ++++---- 4 files changed, 8 insertions(+), 40 deletions(-) delete mode 100644 projects/content-store-on-postgresql/Dockerfile diff --git a/projects/content-store-on-postgresql/Dockerfile b/projects/content-store-on-postgresql/Dockerfile deleted file mode 100644 index 97c186c9..00000000 --- a/projects/content-store-on-postgresql/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -# Standard Dockerfile for GOV.UK projects. Random project-specific stuff should *not* -# go in this file. Instead, you should copy this file into the directory for the -# project and make any changes to it there. - -# Install packages for building ruby -FROM buildpack-deps - -# Install chromium browser and its webdriver -RUN apt-get update -qq && apt-get install -y chromium chromium-driver - -# Enable no-sandbox for chrome so that it can run as a root user -ENV GOVUK_TEST_CHROME_NO_SANDBOX 1 - -# Install node / yarn -RUN curl -sL https://deb.nodesource.com/setup_16.x | bash - -RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - -RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list -RUN apt-get update -qq && apt-get install -y yarn nodejs -RUN yarn config set cache-folder /root/.yarn/ - -# Install rbenv to manage ruby versions -RUN git clone https://github.com/rbenv/rbenv.git /rbenv -RUN git clone https://github.com/rbenv/ruby-build.git /rbenv/plugins/ruby-build -RUN /rbenv/plugins/ruby-build/install.sh -ENV PATH /root/.rbenv/shims:/rbenv/bin:$PATH -ENV BUNDLE_SILENCE_ROOT_WARNING 1 - -# Install `psql` for 'rails dbconsole' -# and `less` for paginated output in 'rails console' and 'binding.pry' -RUN apt-get update -qq && apt-get install -y postgresql-client less diff --git a/projects/content-store-on-postgresql/Makefile b/projects/content-store-on-postgresql/Makefile index 07308980..8e3c4320 100644 --- a/projects/content-store-on-postgresql/Makefile +++ b/projects/content-store-on-postgresql/Makefile @@ -1,2 +1,2 @@ -content-store-on-postgresql: bundle-content-store router-api publishing-api - $(GOVUK_DOCKER) run $@-lite bin/rake db:setup +content-store-on-postgresql: bundle-content-store-on-postgresql router-api publishing-api + $(GOVUK_DOCKER) run $@-lite bin/rake db:prepare diff --git a/projects/content-store-on-postgresql/docker-compose.yml b/projects/content-store-on-postgresql/docker-compose.yml index c691f48d..e1b530ef 100644 --- a/projects/content-store-on-postgresql/docker-compose.yml +++ b/projects/content-store-on-postgresql/docker-compose.yml @@ -6,7 +6,7 @@ volumes: x-content-store-on-postgresql: &content-store-on-postgresql build: context: . - dockerfile: projects/content-store-on-postgresql/Dockerfile + dockerfile: Dockerfile.govuk-base image: content-store-on-postgresql stdin_open: true tty: true diff --git a/projects/content-store/docker-compose.yml b/projects/content-store/docker-compose.yml index 6fe88e1c..7d45d529 100644 --- a/projects/content-store/docker-compose.yml +++ b/projects/content-store/docker-compose.yml @@ -20,24 +20,22 @@ services: content-store-lite: <<: *content-store depends_on: - - postgres-13 + # In production this uses Mongo 2.6, however there is not a published Mongo 2.6 image compatible with Arm64 + - mongo-3.6 environment: - TEST_DATABASE_URL: "postgresql://postgres@postgres-13/content-store-test" - DATABASE_URL: "postgresql://postgres@postgres-13/content-store" + MONGODB_URI: "mongodb://mongo-3.6/content-store" + TEST_MONGODB_URI: "mongodb://mongo-3.6/content-store-test" content-store-app: &content-store-app <<: *content-store depends_on: + - mongo-3.6 - router-api-app - nginx-proxy - - postgres-13 environment: MONGODB_URI: "mongodb://mongo-3.6/content-store" VIRTUAL_HOST: content-store.dev.gov.uk BINDING: 0.0.0.0 - DATABASE_URL: "postgresql://postgres@postgres-13/content-store" - TEST_DATABASE_URL: "postgresql://postgres@postgres-13/content-store-test" - expose: - "3000" command: bin/rails s --restart From 288bc24f57e0f4dd8e6224afd2b48f168dda18e8 Mon Sep 17 00:00:00 2001 From: Al Davidson Date: Thu, 27 Apr 2023 18:14:19 +0100 Subject: [PATCH 06/10] add nginx-proxy alias for content-store-on-postgresql --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 86096f32..b41724f4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -118,6 +118,7 @@ services: - content-data-api.dev.gov.uk - content-data-admin.dev.gov.uk - content-store.dev.gov.uk + - content-store-on-postgresql.dev.gov.uk - content-tagger.dev.gov.uk - draft-origin.dev.gov.uk - draft-router.dev.gov.uk From 1a0db0fb836f0fc0a8ac2230dcc1f35f37f7c6dc Mon Sep 17 00:00:00 2001 From: Al Davidson Date: Mon, 15 May 2023 09:31:25 +0100 Subject: [PATCH 07/10] WIP content-store-proxy container added, but hangs on start --- docker-compose.yml | 1 + projects/content-store-proxy/Dockerfile | 14 ++++++ projects/content-store-proxy/Makefile | 1 + .../content-store-proxy/docker-compose.yml | 44 +++++++++++++++++++ 4 files changed, 60 insertions(+) create mode 100644 projects/content-store-proxy/Dockerfile create mode 100644 projects/content-store-proxy/Makefile create mode 100644 projects/content-store-proxy/docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml index b41724f4..8bc37e20 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -119,6 +119,7 @@ services: - content-data-admin.dev.gov.uk - content-store.dev.gov.uk - content-store-on-postgresql.dev.gov.uk + - content-store-proxy.dev.gov.uk - content-tagger.dev.gov.uk - draft-origin.dev.gov.uk - draft-router.dev.gov.uk diff --git a/projects/content-store-proxy/Dockerfile b/projects/content-store-proxy/Dockerfile new file mode 100644 index 00000000..492d571e --- /dev/null +++ b/projects/content-store-proxy/Dockerfile @@ -0,0 +1,14 @@ +# content-store-proxy is a very lightweight Sinatra app +# with no database, and no cucumber/other end-to-end tests. +# So it doesn't need many of the standard packages. + +# Install packages for building ruby +FROM buildpack-deps + + +# Install rbenv to manage ruby versions +RUN git clone https://github.com/rbenv/rbenv.git /rbenv +RUN git clone https://github.com/rbenv/ruby-build.git /rbenv/plugins/ruby-build +RUN /rbenv/plugins/ruby-build/install.sh +ENV PATH /root/.rbenv/shims:/rbenv/bin:$PATH +ENV BUNDLE_SILENCE_ROOT_WARNING 1 diff --git a/projects/content-store-proxy/Makefile b/projects/content-store-proxy/Makefile new file mode 100644 index 00000000..ab62f7e7 --- /dev/null +++ b/projects/content-store-proxy/Makefile @@ -0,0 +1 @@ +content-store-proxy: bundle-content-store-proxy content-store content-store-on-postgresql \ No newline at end of file diff --git a/projects/content-store-proxy/docker-compose.yml b/projects/content-store-proxy/docker-compose.yml new file mode 100644 index 00000000..acd48fe3 --- /dev/null +++ b/projects/content-store-proxy/docker-compose.yml @@ -0,0 +1,44 @@ +version: '3.7' + +volumes: + content-store-proxy-tmp: + +x-content-store-proxy: &content-store-proxy + build: + context: . + dockerfile: projects/content-store-proxy/Dockerfile + image: content-store-proxy + stdin_open: true + tty: true + volumes: + - ${GOVUK_ROOT_DIR:-~/govuk}:/govuk:delegated + - root-home:/root + - content-store-proxy-tmp:/govuk/content-store-proxy/tmp + working_dir: /govuk/content-store-proxy + +services: + content-store-proxy-lite: + <<: *content-store-proxy + depends_on: + - nginx-proxy + + environment: + PRIMARY_UPSTREAM: http://content-store.dev.gov.uk + SECONDARY_UPSTREAM: http://content-store-on-postgresql.dev.gov.uk + expose: + - "3000" + command: bundle exec rackup config.ru -p 3000 > /var/log/rackup.log + + content-store-proxy-app: &content-store-proxy-app + <<: *content-store-proxy + depends_on: + - content-store-app + - content-store-on-postgresql-app + - nginx-proxy + environment: + VIRTUAL_HOST: content-store-proxy.dev.gov.uk + BINDING: 0.0.0.0 + PRIMARY_UPSTREAM: http://content-store.dev.gov.uk + SECONDARY_UPSTREAM: http://content-store-on-postgresql.dev.gov.uk + + From 0b4510cf84615c2e121be1299e977402a13cd845 Mon Sep 17 00:00:00 2001 From: Al Davidson Date: Mon, 15 May 2023 10:23:50 +0100 Subject: [PATCH 08/10] expose content-store-proxy-app to the rest of the govuk-docker virtual network --- projects/content-store-proxy/docker-compose.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/projects/content-store-proxy/docker-compose.yml b/projects/content-store-proxy/docker-compose.yml index acd48fe3..f983c23d 100644 --- a/projects/content-store-proxy/docker-compose.yml +++ b/projects/content-store-proxy/docker-compose.yml @@ -21,16 +21,13 @@ services: <<: *content-store-proxy depends_on: - nginx-proxy - environment: PRIMARY_UPSTREAM: http://content-store.dev.gov.uk SECONDARY_UPSTREAM: http://content-store-on-postgresql.dev.gov.uk - expose: - - "3000" - command: bundle exec rackup config.ru -p 3000 > /var/log/rackup.log content-store-proxy-app: &content-store-proxy-app <<: *content-store-proxy + command: bundle exec rackup config.ru -p 3000 -o 0.0.0.0 > /var/log/rackup.log depends_on: - content-store-app - content-store-on-postgresql-app @@ -40,5 +37,7 @@ services: BINDING: 0.0.0.0 PRIMARY_UPSTREAM: http://content-store.dev.gov.uk SECONDARY_UPSTREAM: http://content-store-on-postgresql.dev.gov.uk + expose: + - "3000" From c33e1b9c1c5c0a4254ce59eb9fb97dae1df434bd Mon Sep 17 00:00:00 2001 From: Al Davidson Date: Mon, 15 May 2023 15:15:49 +0100 Subject: [PATCH 09/10] wip get content-store-proxy to respond on content-store.dev.gov.uk, and content-store to respond on content-store-on-mongodb.dev.gov.uk --- docker-compose.yml | 1 + projects/content-store-proxy/docker-compose.yml | 9 ++++++--- projects/content-store/docker-compose.yml | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8bc37e20..7da83e3e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -118,6 +118,7 @@ services: - content-data-api.dev.gov.uk - content-data-admin.dev.gov.uk - content-store.dev.gov.uk + - content-store-on-mongodb-dev.gov.uk - content-store-on-postgresql.dev.gov.uk - content-store-proxy.dev.gov.uk - content-tagger.dev.gov.uk diff --git a/projects/content-store-proxy/docker-compose.yml b/projects/content-store-proxy/docker-compose.yml index f983c23d..869f4897 100644 --- a/projects/content-store-proxy/docker-compose.yml +++ b/projects/content-store-proxy/docker-compose.yml @@ -22,7 +22,7 @@ services: depends_on: - nginx-proxy environment: - PRIMARY_UPSTREAM: http://content-store.dev.gov.uk + PRIMARY_UPSTREAM: http://content-store-on-mongodb.dev.gov.uk SECONDARY_UPSTREAM: http://content-store-on-postgresql.dev.gov.uk content-store-proxy-app: &content-store-proxy-app @@ -33,9 +33,12 @@ services: - content-store-on-postgresql-app - nginx-proxy environment: - VIRTUAL_HOST: content-store-proxy.dev.gov.uk + # the proxy pretends to be the real thing, so that we don't + # have to make either a) global changes to plek, or b) lots + # of changes to lots of things + VIRTUAL_HOST: content-store.dev.gov.uk BINDING: 0.0.0.0 - PRIMARY_UPSTREAM: http://content-store.dev.gov.uk + PRIMARY_UPSTREAM: http://content-store-on-mongodb.dev.gov.uk SECONDARY_UPSTREAM: http://content-store-on-postgresql.dev.gov.uk expose: - "3000" diff --git a/projects/content-store/docker-compose.yml b/projects/content-store/docker-compose.yml index 7d45d529..dfe89617 100644 --- a/projects/content-store/docker-compose.yml +++ b/projects/content-store/docker-compose.yml @@ -34,7 +34,7 @@ services: - nginx-proxy environment: MONGODB_URI: "mongodb://mongo-3.6/content-store" - VIRTUAL_HOST: content-store.dev.gov.uk + VIRTUAL_HOST: content-store-on-mongodb.dev.gov.uk BINDING: 0.0.0.0 expose: - "3000" From 1d1c04da0974f05839ddb426c456bda7a2f54028 Mon Sep 17 00:00:00 2001 From: Al Davidson Date: Tue, 16 May 2023 09:43:24 +0100 Subject: [PATCH 10/10] typo fix --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7da83e3e..4d2f9401 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -118,7 +118,7 @@ services: - content-data-api.dev.gov.uk - content-data-admin.dev.gov.uk - content-store.dev.gov.uk - - content-store-on-mongodb-dev.gov.uk + - content-store-on-mongodb.dev.gov.uk - content-store-on-postgresql.dev.gov.uk - content-store-proxy.dev.gov.uk - content-tagger.dev.gov.uk