Merge pull request #300 from thomas-robinson/container2step

Container2step

fre/make/create_docker_script.py

@@ -52,14 +52,16 @@ def dockerfile_create(yamlfile, platform, target, execute):
             ## Check for type of build
             if platform["container"] is True:
                 image=modelYaml.platforms.getContainerImage(platformName)
+                stage2image = modelYaml.platforms.getContainer2base(platformName)
                 bldDir = platform["modelRoot"] + "/" + fremakeYaml["experiment"] + "/exec"
                 tmpDir = "tmp/"+platformName
                 dockerBuild = buildDocker.container(base = image,
                                               exp = fremakeYaml["experiment"],
                                               libs = fremakeYaml["container_addlibs"],
                                               RUNenv = platform["RUNenv"],
                                               target = targetObject,
-                                              mkTemplate = platform["mkTemplate"])
+                                              mkTemplate = platform["mkTemplate"],
+                                              stage2base = stage2image)
                 dockerBuild.writeDockerfileCheckout("checkout.sh", tmpDir+"/checkout.sh")
                 dockerBuild.writeDockerfileMakefile(tmpDir+"/Makefile", tmpDir+"/linkline.sh")
 

fre/make/gfdlfremake/buildDocker.py

@@ -18,8 +18,12 @@ class container():
         - RUNenv : The commands that have to be run at
                    the beginning of a RUN in the dockerfile
                    to set up the environment
+        - target : The FRE target
+        - mkTemplate: The mkmf template to use
+        - stage2base: The base for the second stage. Empty
+                      string if there is no second stage
     """
-    def __init__(self,base,exp,libs,RUNenv,target,mkTemplate):
+    def __init__(self,base,exp,libs,RUNenv,target,mkTemplate,stage2base):
         """
         Initialize variables and write to the dockerfile
         """
@@ -31,6 +35,7 @@ def __init__(self,base,exp,libs,RUNenv,target,mkTemplate):
         self.mkmf = True
         self.target = target
         self.template = mkTemplate
+        self.stage2base = stage2base
 
         # Set up spack loads in RUN commands in dockerfile
         if RUNenv == "":
@@ -54,15 +59,15 @@ def __init__(self,base,exp,libs,RUNenv,target,mkTemplate):
                        " && src_dir="+self.src+" \\ \n",
                        " && mkmf_template="+self.template+ " \\ \n"]
         self.d=open("Dockerfile","w")
-        self.d.writelines("FROM "+self.base+" \n")
-        if self.base == "ecpe4s/noaa-intel-prototype:2023.09.25":
-            self.prebuild = '''RUN 
-            '''
-            self.postbuild = '''
-            '''
-            self.secondstage = '''
-            '''
-
+        self.d.writelines("FROM "+self.base+" as builder\n")
+        ## Set up the second stage build list of lines to add
+        if self.stage2base == "":
+            self.secondstage = ["\n"]
+        else:
+            self.secondstage = [f"FROM {self.stage2base} as final\n",
+                                f"COPY --from=builder  {self.src} {self.src}\n",
+                                f"COPY --from=builder {self.bld} {self.bld}\n",
+                                f"ENV PATH=$PATH:{self.bld}\n"]
     def writeDockerfileCheckout(self, cScriptName, cOnDisk):
         """
         Brief: writes to the checkout part of the Dockerfile and sets up the compile
@@ -201,6 +206,9 @@ def writeRunscript(self,RUNenv,containerRun,runOnDisk):
             self.d.write(" cd "+self.bld+" && make -j 4 "+self.target.getmakeline_add()+"\n")
         else:
             self.d.write(" && cd "+self.bld+" && make -j 4 "+self.target.getmakeline_add()+"\n")
+        ## Write any second stage lines here 
+        for l in self.secondstage:
+            self.d.write(l)
         self.d.write('ENTRYPOINT ["/bin/bash"]')
         self.d.close()
 

fre/make/gfdlfremake/platformfre.py

@@ -41,35 +41,49 @@ def __init__(self,platforminfo):
             ## Check if we are working with a container and get the info for that
             try:
                 p["container"]
+            ## When not doing a container build, this should all be set to empty strings and Falses
             except:
                 p["container"] = False
                 p["RUNenv"] = [""]
                 p["containerBuild"] = ""
                 p["containerRun"] = ""
-                p["containerViews"] = False
                 p["containerBase"] = ""
-                p["container2step"] = ""
+                p["container2step"] = False
+                p["container2base"] = ""
             if p["container"]:
                 ## Check the container builder
                 try:
                     p["containerBuild"]
                 except:
-                    raise Exception("You must specify the program used to build the container (containerBuild) on the "+p["name"]+" platform in the file "+fname+"\n")
+                    raise Exception("Platform "+p["name"]+": You must specify the program used to build the container (containerBuild) on the "+p["name"]+" platform in the file "+fname+"\n")
                 if p["containerBuild"] != "podman" and p["containerBuild"] != "docker":
-                    raise ValueError("Container builds only supported with docker or podman, but you listed "+p["containerBuild"]+"\n")
-## Check for container environment set up for RUN commands
+                    raise ValueError("Platform "+p["name"]+": Container builds only supported with docker or podman, but you listed "+p["containerBuild"]+"\n")
+                ## Get the name of the base container
                 try:
                     p["containerBase"]
-                except NameError:
-                    print("You must specify the base container you wish to use to build your application")
-                try:
-                    p["containerViews"]
                 except:
-                    p["containerViews"] = False
+                    raise NameError("Platform "+p["name"]+": You must specify the base container you wish to use to build your application")
+                ## Check if this is a 2 step (multi stage) build
                 try:
                     p["container2step"]
                 except:
-                    p["container2step"] = ""
+                    p["container2step"] = False
+                ## Get the base for the second stage of the build
+                if p["container2step"]:
+                    try:
+                        p["container2base"]
+                    except:
+                        raise NameError ("Platform "+p["name"]+": container2step is True, so you must define a container2base\n")
+                    ## Check if there is anything special to copy over
+                else:
+                    ## There should not be a second base if this is not a 2 step build
+                    try:
+                        p["container2base"]
+                    except:
+                        p["container2base"] = ""
+                    else:
+                        raise ValueError ("Platform "+p["name"]+": You defined container2base "+p["container2base"]+" but container2step is False\n")
+                ## Get any commands to execute in the dockerfile RUN command
                 try:
                     p["RUNenv"]
                 except:
@@ -82,6 +96,7 @@ def __init__(self,platforminfo):
                 if p["containerRun"] != "apptainer" and p["containerRun"] != "singularity":
                     raise ValueError("Container builds only supported with apptainer, but you listed "+p["containerRun"]+"\n")
             else:
+                ## Find the location of the mkmf template
                 try:
                     p["mkTemplate"]
                 except:
@@ -119,7 +134,6 @@ def getContainerInfoFromName(self,name):
                 p["RUNenv"], \
                 p["containerBuild"], \
                 p["containerRun"], \
-                p["containerViews"], \
                 p["containerBase"], \
                 p["container2step"])
     def isContainer(self, name):
@@ -136,3 +150,10 @@ def getContainerImage(self,name):
         for p in self.yaml:
             if p["name"] == name:
                 return p["containerBase"]
+    def getContainer2base(self,name):
+        """
+        Brief: returns the image to be used in the second step of the Dockerfile
+        """
+        for p in self.yaml:
+            if p["name"] == name:
+                return p["container2base"]

fre/make/run_fremake_script.py

@@ -147,6 +147,7 @@ def fremake_run(yamlfile, platform, target, parallel, jobs, no_parallel_checkout
                 ###################### container stuff below #######################################
                 ## Run the checkout script
                 image=modelYaml.platforms.getContainerImage(platformName)
+                stage2image = modelYaml.platforms.getContainer2base(platformName)
                 srcDir = platform["modelRoot"] + "/" + fremakeYaml["experiment"] + "/src"
                 bldDir = platform["modelRoot"] + "/" + fremakeYaml["experiment"] + "/exec"
                 tmpDir = "tmp/"+platformName
@@ -174,7 +175,8 @@ def fremake_run(yamlfile, platform, target, parallel, jobs, no_parallel_checkout
                                                     libs = fremakeYaml["container_addlibs"],
                                                     RUNenv = platform["RUNenv"],
                                                     target = target,
-                                                    mkTemplate = platform["mkTemplate"])
+                                                    mkTemplate = platform["mkTemplate"],
+                                                    stage2base = stage2image)
                 dockerBuild.writeDockerfileCheckout("checkout.sh", tmpDir+"/checkout.sh")
                 dockerBuild.writeDockerfileMakefile(freMakefile.getTmpDir() + "/Makefile",
                                                     freMakefile.getTmpDir() + "/linkline.sh")

fre/make/tests/null_example/platforms.yaml

@@ -18,3 +18,14 @@ platforms:
      compiler: gnu
      mkTemplate: /__w/fre-cli/fre-cli/mkmf/templates/linux-ubuntu-xenial-gnu.mk
      modelRoot: ${TEST_BUILD_DIR}/fremake_canopy/test
+   - name: con.twostep
+     compiler: intel
+     RUNenv: [". /spack/share/spack/setup-env.sh", "spack load libyaml", "spack load [email protected]", "spack load [email protected]"]
+     modelRoot: /apps
+     container: True
+     containerBuild: "podman"
+     containerRun: "apptainer"
+     containerBase: "ecpe4s/noaa-intel-prototype:2023.09.25"
+     mkTemplate: "/apps/mkmf/templates/hpcme-intel21.mk"
+     container2step: True
+     container2base: "ecpe4s/noaa-intel-prototype:2023.09.25"

fre/make/tests/test_create_makefile.py

@@ -12,6 +12,7 @@
 YAMLFILE = "null_model.yaml"
 BM_PLATFORM = ["ncrc5.intel23"]
 CONTAINER_PLATFORM = ["hpcme.2023"]
+CONTAINER_PLAT2 = ["con.twostep"]
 TARGET = ["debug"]
 EXPERIMENT = "null_model_full"
 
@@ -73,3 +74,13 @@ def test_container_makefile_creation():
     create_makefile_script.makefile_create(yamlfile_path,CONTAINER_PLATFORM,TARGET)
 
     assert Path(f"tmp/{container_plat}/Makefile").exists()
+
+def test_container2step_makefile_creation():
+    """
+    Check the makefile is created when the two stage container is used
+    """
+    container_plat = CONTAINER_PLAT2[0]
+    yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}"
+    create_makefile_script.makefile_create(yamlfile_path,CONTAINER_PLAT2,TARGET)
+
+    assert Path(f"tmp/{container_plat}/Makefile").exists()

fre/make/tests/test_run_fremake.py

@@ -19,6 +19,7 @@
 YAMLPATH = f"{YAMLDIR}/{YAMLFILE}"
 PLATFORM = [ "ci.gnu" ]
 CONTAINER_PLATFORM = ["hpcme.2023"]
+CONTAINER_PLAT2 = ["con.twostep"]
 TARGET = ["debug"]
 BADOPT = ["foo"]
 EXPERIMENT = "null_model_full"
@@ -125,6 +126,31 @@ def test_run_fremake_run_script_creation_container():
     ''' checks (internal) container run script creation from previous test '''
     assert Path(f"tmp/{CONTAINER_PLATFORM[0]}/execrunscript.sh").exists()
 
+# tests container 2 stage build script/makefile/dockerfile creation
+def test_run_fremake_container():
+    '''run run-fremake with options for containerized build'''
+    run_fremake_script.fremake_run(YAMLPATH, CONTAINER_PLAT2, TARGET, False, 1, True, False, VERBOSE)
+
+def test_run_fremake_build_script_creation_container():
+    ''' checks container build script creation from previous test '''
+    assert Path("createContainer.sh").exists()
+
+def test_run_fremake_dockerfile_creation_container():
+    ''' checks dockerfile creation from previous test '''
+    assert Path("Dockerfile").exists()
+
+def test_run_fremake_checkout_script_creation_container():
+    ''' checks checkout script creation from previous test '''
+    assert Path(f"tmp/{CONTAINER_PLAT2[0]}/checkout.sh").exists()
+
+def test_run_fremake_makefile_creation_container():
+    ''' checks makefile creation from previous test '''
+    assert Path(f"tmp/{CONTAINER_PLAT2[0]}/Makefile").exists()
+
+def test_run_fremake_run_script_creation_container():
+    ''' checks (internal) container run script creation from previous test '''
+    assert Path(f"tmp/{CONTAINER_PLAT2[0]}/execrunscript.sh").exists()
+
 # tests for builds with multiple targets
 
 def test_run_fremake_bad_target():

fre/yamltools/tests/AM5_example/compile_yamls/platforms.yaml

@@ -3,24 +3,18 @@ platforms:
      compiler: intel
      modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"]
      modules: [ !join [*INTEL, "/2022.2.1"],"fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3]
-     fc: ftn
-     cc: cc
      mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/$(INTEL).mk"
      modelRoot: ${HOME}/fremake_canopy/test
    - name: ncrc5.intel23
      compiler: intel
      modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"]
      modules: [!join [*INTEL, "/2023.1.0"],"fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3]
-     fc: ftn
-     cc: cc
      mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/$(INTEL).mk"
      modelRoot: ${HOME}/fremake_canopy/test
    - name: hpcme.2023
      compiler: intel
      RUNenv: [". /spack/share/spack/setup-env.sh", "spack load libyaml", "spack load [email protected]", "spack load [email protected]"]
      modelRoot: /apps
-     fc: mpiifort
-     cc: mpiicc
      container: True
      containerBuild: "podman"
      containerRun: "apptainer"