From 41b393d11478cf18634393774cbce01a51c81686 Mon Sep 17 00:00:00 2001 From: Tom Robinson Date: Mon, 23 Dec 2024 07:04:17 -0500 Subject: [PATCH 01/13] Adds platform updates for 2 stage container build --- fre/make/gfdlfremake/platformfre.py | 65 +++++++++++++++++++++++++---- 1 file changed, 56 insertions(+), 9 deletions(-) diff --git a/fre/make/gfdlfremake/platformfre.py b/fre/make/gfdlfremake/platformfre.py index 69820d46..b462a883 100644 --- a/fre/make/gfdlfremake/platformfre.py +++ b/fre/make/gfdlfremake/platformfre.py @@ -41,14 +41,16 @@ def __init__(self,platforminfo): ## Check if we are working with a container and get the info for that try: p["container"] + ## When not doing a container build, this should all be set to empty strings and Falses except: p["container"] = False p["RUNenv"] = [""] p["containerBuild"] = "" p["containerRun"] = "" - p["containerViews"] = False p["containerBase"] = "" - p["container2step"] = "" + p["container2step"] = False + p["container2base"] = "" + p["container2copy"] = [""] if p["container"]: ## Check the container builder try: @@ -58,19 +60,42 @@ def __init__(self,platforminfo): if p["containerBuild"] != "podman" and p["containerBuild"] != "docker": raise ValueError("Container builds only supported with docker or podman, but you listed "+p["containerBuild"]+"\n") print (p["containerBuild"]) -## Check for container environment set up for RUN commands + ## Get the name of the base container try: p["containerBase"] except NameError: print("You must specify the base container you wish to use to build your application") - try: - p["containerViews"] - except: - p["containerViews"] = False + ## Check if this is a 2 step (multi stage) build try: p["container2step"] except: - p["container2step"] = "" + p["container2step"] = False + ## Get the base for the second stage of the build + if p["container2step"]: + try: + p["container2base"] + except TypeError: + print ("If container2step is True, you must include a container2base\n") + ## Check if there is anything special to copy over + try: + p["container2copy"] + except: + p["container2copy"] = "" + else: + ## There should not be a second base if this is not a 2 step build + try: + p["container2base"] + except: + p["container2base"] = "" + else: + raise ValueError ("You defined container2base "+p["container2base"]+" but container2step is false\n") + try: + p["container2copy"] + except: + p["container2copy"] = "" + else: + raise ValueError ("You defined container2copy "+p["container2copy"]+" but container2step is false\n") + ## Get any commands to execute in the dockerfile RUN command try: p["RUNenv"] except: @@ -83,6 +108,7 @@ def __init__(self,platforminfo): if p["containerRun"] != "apptainer" and p["containerRun"] != "singularity": raise ValueError("Container builds only supported with apptainer, but you listed "+p["containerRun"]+"\n") else: + ## Find the location of the mkmf template try: p["mkTemplate"] except: @@ -120,7 +146,6 @@ def getContainerInfoFromName(self,name): p["RUNenv"], \ p["containerBuild"], \ p["containerRun"], \ - p["containerViews"], \ p["containerBase"], \ p["container2step"]) def isContainer(self, name): @@ -137,3 +162,25 @@ def getContainerImage(self,name): for p in self.yaml: if p["name"] == name: return p["containerBase"] + def is2stepContainer(self,name): + """ + Brief: Returns True if this is a 2 step container + """ + for p in self.yaml: + if p["name"] == name: + return p["container2step"] + def getContainer2base(self,name): + """ + Brief: returns the image to be used in the second step of the Dockerfile + """ + for p in self.yaml: + if p["name"] == name: + return p["container2base"] + def getContainer2copy(self,name): + """ + Brief: returns anything copied from the first stage to the second stage of the Dockerfile + """ + for p in self.yaml: + if p["name"] == name: + return p["container2copy"] + From 69a490ea6a2452bf059e69167ac5e405765f091e Mon Sep 17 00:00:00 2001 From: Tom Robinson Date: Mon, 23 Dec 2024 07:45:33 -0500 Subject: [PATCH 02/13] FIxes error reporting in container platforms Improves error messages in container platform section --- fre/make/gfdlfremake/platformfre.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/fre/make/gfdlfremake/platformfre.py b/fre/make/gfdlfremake/platformfre.py index b462a883..f8d0e2b0 100644 --- a/fre/make/gfdlfremake/platformfre.py +++ b/fre/make/gfdlfremake/platformfre.py @@ -56,15 +56,14 @@ def __init__(self,platforminfo): try: p["containerBuild"] except: - raise Exception("You must specify the program used to build the container (containerBuild) on the "+p["name"]+" platform in the file "+fname+"\n") + raise Exception("Platform "+p["name"]+": You must specify the program used to build the container (containerBuild) on the "+p["name"]+" platform in the file "+fname+"\n") if p["containerBuild"] != "podman" and p["containerBuild"] != "docker": - raise ValueError("Container builds only supported with docker or podman, but you listed "+p["containerBuild"]+"\n") - print (p["containerBuild"]) + raise ValueError("Platform "+p["name"]+": Container builds only supported with docker or podman, but you listed "+p["containerBuild"]+"\n") ## Get the name of the base container try: p["containerBase"] - except NameError: - print("You must specify the base container you wish to use to build your application") + except: + raise NameError("Platform "+p["name"]+": You must specify the base container you wish to use to build your application") ## Check if this is a 2 step (multi stage) build try: p["container2step"] @@ -74,8 +73,8 @@ def __init__(self,platforminfo): if p["container2step"]: try: p["container2base"] - except TypeError: - print ("If container2step is True, you must include a container2base\n") + except: + raise NameError ("Platform "+p["name"]+": container2step is True, so you must define a container2base\n") ## Check if there is anything special to copy over try: p["container2copy"] @@ -88,7 +87,7 @@ def __init__(self,platforminfo): except: p["container2base"] = "" else: - raise ValueError ("You defined container2base "+p["container2base"]+" but container2step is false\n") + raise ValueError ("Platform "+p["name"]+": You defined container2base "+p["container2base"]+" but container2step is False\n") try: p["container2copy"] except: From 7e511b3cb0618aa7444951208ffcb33fae1c2683 Mon Sep 17 00:00:00 2001 From: Tom Robinson Date: Mon, 23 Dec 2024 08:56:13 -0500 Subject: [PATCH 03/13] Adds the ability to perform a 2 stage container build --- fre/make/create_docker_script.py | 4 +++- fre/make/gfdlfremake/buildDocker.py | 28 ++++++++++++++++++---------- fre/make/run_fremake_script.py | 4 +++- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/fre/make/create_docker_script.py b/fre/make/create_docker_script.py index e785dc82..7d21364e 100644 --- a/fre/make/create_docker_script.py +++ b/fre/make/create_docker_script.py @@ -52,6 +52,7 @@ def dockerfile_create(yamlfile,platform,target,execute): ## Check for type of build if platform["container"] is True: image=modelYaml.platforms.getContainerImage(platformName) + stage2image = modelYaml.platforms.getContainer2base(platformName) bldDir = platform["modelRoot"] + "/" + fremakeYaml["experiment"] + "/exec" tmpDir = "tmp/"+platformName dockerBuild = buildDocker.container(base = image, @@ -59,7 +60,8 @@ def dockerfile_create(yamlfile,platform,target,execute): libs = fremakeYaml["container_addlibs"], RUNenv = platform["RUNenv"], target = targetObject, - mkTemplate = platform["mkTemplate"]) + mkTemplate = platform["mkTemplate"], + stage2base = stage2image) dockerBuild.writeDockerfileCheckout("checkout.sh", tmpDir+"/checkout.sh") dockerBuild.writeDockerfileMakefile(tmpDir+"/Makefile", tmpDir+"/linkline.sh") diff --git a/fre/make/gfdlfremake/buildDocker.py b/fre/make/gfdlfremake/buildDocker.py index 491082ef..450c61bd 100644 --- a/fre/make/gfdlfremake/buildDocker.py +++ b/fre/make/gfdlfremake/buildDocker.py @@ -17,8 +17,12 @@ class container(): - RUNenv : The commands that have to be run at the beginning of a RUN in the dockerfile to set up the environment + - target : The FRE target + - mkTemplate: The mkmf template to use + - stage2base: The base for the second stage. Empty + string if there is no second stage """ - def __init__(self,base,exp,libs,RUNenv,target,mkTemplate): + def __init__(self,base,exp,libs,RUNenv,target,mkTemplate,stage2base): """ Initialize variables and write to the dockerfile """ @@ -30,6 +34,7 @@ def __init__(self,base,exp,libs,RUNenv,target,mkTemplate): self.mkmf = True self.target = target self.template = mkTemplate + self.stage2base = stage2base # Set up spack loads in RUN commands in dockerfile if RUNenv == "": @@ -53,15 +58,15 @@ def __init__(self,base,exp,libs,RUNenv,target,mkTemplate): " && src_dir="+self.src+" \\ \n", " && mkmf_template="+self.template+ " \\ \n"] self.d=open("Dockerfile","w") - self.d.writelines("FROM "+self.base+" \n") - if self.base == "ecpe4s/noaa-intel-prototype:2023.09.25": - self.prebuild = '''RUN - ''' - self.postbuild = ''' - ''' - self.secondstage = ''' - ''' - + self.d.writelines("FROM "+self.base+" as builder\n") + ## Set up the second stage build list of lines to add + if self.stage2base == "": + self.secondstage = ["\n"] + else: + self.secondstage = ["FROM "+self.stage2base+" as final\n", + "COPY --from=builder "+self.src+" "+self.src+"\n", + "COPY --from=builder "+self.bld+" "+self.bld+"\n", + "ENV PATH=$PATH:"+self.bld+"\n"] def writeDockerfileCheckout(self, cScriptName, cOnDisk): """ Brief: writes to the checkout part of the Dockerfile and sets up the compile @@ -200,6 +205,9 @@ def writeRunscript(self,RUNenv,containerRun,runOnDisk): self.d.write(" cd "+self.bld+" && make -j 4 "+self.target.getmakeline_add()+"\n") else: self.d.write(" && cd "+self.bld+" && make -j 4 "+self.target.getmakeline_add()+"\n") + ## Write any second stage lines here + for l in self.secondstage: + self.d.write(l) self.d.write('ENTRYPOINT ["/bin/bash"]') self.d.close() diff --git a/fre/make/run_fremake_script.py b/fre/make/run_fremake_script.py index 0d0ea534..996f88a4 100644 --- a/fre/make/run_fremake_script.py +++ b/fre/make/run_fremake_script.py @@ -148,6 +148,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec ###################### container stuff below ####################################### ## Run the checkout script image=modelYaml.platforms.getContainerImage(platformName) + stage2image = modelYaml.platforms.getContainer2base(platformName) srcDir = platform["modelRoot"] + "/" + fremakeYaml["experiment"] + "/src" bldDir = platform["modelRoot"] + "/" + fremakeYaml["experiment"] + "/exec" tmpDir = "tmp/"+platformName @@ -175,7 +176,8 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec libs = fremakeYaml["container_addlibs"], RUNenv = platform["RUNenv"], target = target, - mkTemplate = platform["mkTemplate"]) + mkTemplate = platform["mkTemplate"], + stage2base = stage2image) dockerBuild.writeDockerfileCheckout("checkout.sh", tmpDir+"/checkout.sh") dockerBuild.writeDockerfileMakefile(freMakefile.getTmpDir() + "/Makefile", freMakefile.getTmpDir() + "/linkline.sh") From 5fcc5bec14fb7765aaaf27d127fd621167a8e3c2 Mon Sep 17 00:00:00 2001 From: Tom Robinson Date: Mon, 23 Dec 2024 09:11:28 -0500 Subject: [PATCH 04/13] Updates tests for two stage build --- fre/make/tests/null_example/platforms.yaml | 11 +++++++++ fre/make/tests/test_create_makefile.py | 11 +++++++++ fre/make/tests/test_run_fremake.py | 26 ++++++++++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/fre/make/tests/null_example/platforms.yaml b/fre/make/tests/null_example/platforms.yaml index e7d9e5ae..6f35db7c 100644 --- a/fre/make/tests/null_example/platforms.yaml +++ b/fre/make/tests/null_example/platforms.yaml @@ -18,3 +18,14 @@ platforms: compiler: gnu mkTemplate: /__w/fre-cli/fre-cli/mkmf/templates/linux-ubuntu-xenial-gnu.mk modelRoot: ${TEST_BUILD_DIR}/fremake_canopy/test + - name: con.twostep + compiler: intel + RUNenv: [". /spack/share/spack/setup-env.sh", "spack load libyaml", "spack load netcdf-fortran@4.5.4", "spack load hdf5@1.14.0"] + modelRoot: /apps + container: True + containerBuild: "podman" + containerRun: "apptainer" + containerBase: "ecpe4s/noaa-intel-prototype:2023.09.25" + mkTemplate: "/apps/mkmf/templates/hpcme-intel21.mk" + container2step: True + container2base: "ecpe4s/noaa-intel-prototype:2023.09.25" diff --git a/fre/make/tests/test_create_makefile.py b/fre/make/tests/test_create_makefile.py index dd180262..736a662e 100644 --- a/fre/make/tests/test_create_makefile.py +++ b/fre/make/tests/test_create_makefile.py @@ -12,6 +12,7 @@ YAMLFILE = "null_model.yaml" BM_PLATFORM = ["ncrc5.intel23"] CONTAINER_PLATFORM = ["hpcme.2023"] +CONTAINER_PLAT2 = ["con.twostep"] TARGET = ["debug"] EXPERIMENT = "null_model_full" @@ -73,3 +74,13 @@ def test_container_makefile_creation(): create_makefile_script.makefile_create(yamlfile_path,CONTAINER_PLATFORM,TARGET) assert Path(f"tmp/{container_plat}/Makefile").exists() + +def test_container2step_makefile_creation(): + """ + Check the makefile is created when the two stage container is used + """ + container_plat = CONTAINER_PLAT2[0] + yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" + create_makefile_script.makefile_create(yamlfile_path,container_plat,TARGET) + + assert Path(f"tmp/{container_plat}/Makefile").exists() diff --git a/fre/make/tests/test_run_fremake.py b/fre/make/tests/test_run_fremake.py index 4447acf6..f5b75547 100644 --- a/fre/make/tests/test_run_fremake.py +++ b/fre/make/tests/test_run_fremake.py @@ -19,6 +19,7 @@ YAMLPATH = f"{YAMLDIR}/{YAMLFILE}" PLATFORM = [ "ci.gnu" ] CONTAINER_PLATFORM = ["hpcme.2023"] +CONTAINER_PLAT2 = = ["con.twostep"] TARGET = ["debug"] BADOPT = ["foo"] EXPERIMENT = "null_model_full" @@ -125,6 +126,31 @@ def test_run_fremake_run_script_creation_container(): ''' checks (internal) container run script creation from previous test ''' assert Path(f"tmp/{CONTAINER_PLATFORM[0]}/execrunscript.sh").exists() +# tests container 2 stage build script/makefile/dockerfile creation +def test_run_fremake_container(): + '''run run-fremake with options for containerized build''' + run_fremake_script.fremake_run(YAMLPATH, CONTAINER_PLAT2, TARGET, False, 1, True, False, VERBOSE) + +def test_run_fremake_build_script_creation_container(): + ''' checks container build script creation from previous test ''' + assert Path("createContainer.sh").exists() + +def test_run_fremake_dockerfile_creation_container(): + ''' checks dockerfile creation from previous test ''' + assert Path("Dockerfile").exists() + +def test_run_fremake_checkout_script_creation_container(): + ''' checks checkout script creation from previous test ''' + assert Path(f"tmp/{CONTAINER_PLAT2[0]}/checkout.sh").exists() + +def test_run_fremake_makefile_creation_container(): + ''' checks makefile creation from previous test ''' + assert Path(f"tmp/{CONTAINER_PLAT2[0]}/Makefile").exists() + +def test_run_fremake_run_script_creation_container(): + ''' checks (internal) container run script creation from previous test ''' + assert Path(f"tmp/{CONTAINER_PLAT2[0]}/execrunscript.sh").exists() + # tests for builds with multiple targets def test_run_fremake_bad_target(): From c47471dabe410ae4d55657cf0154b455035473be Mon Sep 17 00:00:00 2001 From: Tom Robinson Date: Mon, 23 Dec 2024 10:13:22 -0500 Subject: [PATCH 05/13] Removes container2copy as it was unused --- fre/make/gfdlfremake/platformfre.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/fre/make/gfdlfremake/platformfre.py b/fre/make/gfdlfremake/platformfre.py index f8d0e2b0..c9096f5e 100644 --- a/fre/make/gfdlfremake/platformfre.py +++ b/fre/make/gfdlfremake/platformfre.py @@ -50,7 +50,6 @@ def __init__(self,platforminfo): p["containerBase"] = "" p["container2step"] = False p["container2base"] = "" - p["container2copy"] = [""] if p["container"]: ## Check the container builder try: @@ -76,10 +75,6 @@ def __init__(self,platforminfo): except: raise NameError ("Platform "+p["name"]+": container2step is True, so you must define a container2base\n") ## Check if there is anything special to copy over - try: - p["container2copy"] - except: - p["container2copy"] = "" else: ## There should not be a second base if this is not a 2 step build try: @@ -88,12 +83,6 @@ def __init__(self,platforminfo): p["container2base"] = "" else: raise ValueError ("Platform "+p["name"]+": You defined container2base "+p["container2base"]+" but container2step is False\n") - try: - p["container2copy"] - except: - p["container2copy"] = "" - else: - raise ValueError ("You defined container2copy "+p["container2copy"]+" but container2step is false\n") ## Get any commands to execute in the dockerfile RUN command try: p["RUNenv"] @@ -175,11 +164,3 @@ def getContainer2base(self,name): for p in self.yaml: if p["name"] == name: return p["container2base"] - def getContainer2copy(self,name): - """ - Brief: returns anything copied from the first stage to the second stage of the Dockerfile - """ - for p in self.yaml: - if p["name"] == name: - return p["container2copy"] - From a1655ff242d133167c056622745226d09ae460b9 Mon Sep 17 00:00:00 2001 From: Tom Robinson Date: Mon, 23 Dec 2024 10:20:06 -0500 Subject: [PATCH 06/13] Fixes typo in test_run_fremake.py = = --- fre/make/tests/test_run_fremake.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/make/tests/test_run_fremake.py b/fre/make/tests/test_run_fremake.py index f5b75547..099604d2 100644 --- a/fre/make/tests/test_run_fremake.py +++ b/fre/make/tests/test_run_fremake.py @@ -19,7 +19,7 @@ YAMLPATH = f"{YAMLDIR}/{YAMLFILE}" PLATFORM = [ "ci.gnu" ] CONTAINER_PLATFORM = ["hpcme.2023"] -CONTAINER_PLAT2 = = ["con.twostep"] +CONTAINER_PLAT2 = ["con.twostep"] TARGET = ["debug"] BADOPT = ["foo"] EXPERIMENT = "null_model_full" From 9dddc1777f05a70e1b5979ac6875a71866bd07bb Mon Sep 17 00:00:00 2001 From: Tom Robinson Date: Mon, 23 Dec 2024 13:00:24 -0500 Subject: [PATCH 07/13] fre make test fixes --- fre/make/tests/test_create_makefile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/make/tests/test_create_makefile.py b/fre/make/tests/test_create_makefile.py index 736a662e..adb7ab08 100644 --- a/fre/make/tests/test_create_makefile.py +++ b/fre/make/tests/test_create_makefile.py @@ -81,6 +81,6 @@ def test_container2step_makefile_creation(): """ container_plat = CONTAINER_PLAT2[0] yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" - create_makefile_script.makefile_create(yamlfile_path,container_plat,TARGET) + create_makefile_script.makefile_create(yamlfile_path,CONTAINER_PLAT2,TARGET) assert Path(f"tmp/{container_plat}/Makefile").exists() From c5a6de81aae0ecb10724163c4c54b2f6f7e44e1a Mon Sep 17 00:00:00 2001 From: Tom Robinson Date: Tue, 21 Jan 2025 05:34:35 -0500 Subject: [PATCH 08/13] Updates schema submodule for use with 2 step build --- fre/gfdl_msd_schemas | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/gfdl_msd_schemas b/fre/gfdl_msd_schemas index 32b9d7ca..2a19e666 160000 --- a/fre/gfdl_msd_schemas +++ b/fre/gfdl_msd_schemas @@ -1 +1 @@ -Subproject commit 32b9d7ca00aa314b781341dda4d241d48e588d18 +Subproject commit 2a19e666d8152b95a7c8825b412e89556ea873ac From 709207c6b058a5b33fdb2c24d27fbbc99fd22d83 Mon Sep 17 00:00:00 2001 From: Tom Robinson Date: Tue, 21 Jan 2025 06:38:34 -0500 Subject: [PATCH 09/13] Fixes test yaml to adhere to new schema --- .../tests/AM5_example/compile_yamls/platforms.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/fre/yamltools/tests/AM5_example/compile_yamls/platforms.yaml b/fre/yamltools/tests/AM5_example/compile_yamls/platforms.yaml index 7e1b9f49..cf52e27f 100644 --- a/fre/yamltools/tests/AM5_example/compile_yamls/platforms.yaml +++ b/fre/yamltools/tests/AM5_example/compile_yamls/platforms.yaml @@ -3,24 +3,18 @@ platforms: compiler: intel modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"] modules: [ !join [*INTEL, "/2022.2.1"],"fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3] - fc: ftn - cc: cc mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/$(INTEL).mk" modelRoot: ${HOME}/fremake_canopy/test - name: ncrc5.intel23 compiler: intel modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"] modules: [!join [*INTEL, "/2023.1.0"],"fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3] - fc: ftn - cc: cc mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/$(INTEL).mk" modelRoot: ${HOME}/fremake_canopy/test - name: hpcme.2023 compiler: intel RUNenv: [". /spack/share/spack/setup-env.sh", "spack load libyaml", "spack load netcdf-fortran@4.5.4", "spack load hdf5@1.14.0"] modelRoot: /apps - fc: mpiifort - cc: mpiicc container: True containerBuild: "podman" containerRun: "apptainer" From 3f788ab5c9fda856e26f8453d47fe0d79633400b Mon Sep 17 00:00:00 2001 From: Tom Robinson Date: Tue, 21 Jan 2025 08:16:04 -0500 Subject: [PATCH 10/13] Fixes for new schema in platform yaml tests --- fre/make/tests/AM5_example/yaml_include/platforms.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/fre/make/tests/AM5_example/yaml_include/platforms.yaml b/fre/make/tests/AM5_example/yaml_include/platforms.yaml index 0c8bac45..de8661e8 100644 --- a/fre/make/tests/AM5_example/yaml_include/platforms.yaml +++ b/fre/make/tests/AM5_example/yaml_include/platforms.yaml @@ -3,24 +3,18 @@ platforms: compiler: intel modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"] modules: [ !join [*INTEL, "/2022.2.1"],"fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3] - fc: ftn - cc: cc mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/", *INTEL, ".mk"] modelRoot: ${HOME}/fremake_canopy/test - name: ncrc5.intel23 compiler: intel modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"] modules: [!join [*INTEL, "/2023.1.0"],"fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3] - fc: ftn - cc: cc mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/", *INTEL, ".mk"] modelRoot: ${HOME}/fremake_canopy/test - name: hpcme.2023 compiler: intel RUNenv: [". /spack/share/spack/setup-env.sh", "spack load libyaml", "spack load netcdf-fortran@4.5.4", "spack load hdf5@1.14.0"] modelRoot: /apps - fc: mpiifort - cc: mpiicc container: True containerBuild: "podman" containerRun: "apptainer" From 86d4aa32f79dfc7e68629216231a3c3810458f5c Mon Sep 17 00:00:00 2001 From: Tom Robinson Date: Tue, 28 Jan 2025 12:46:08 -0500 Subject: [PATCH 11/13] Adds fstrings eventhough it was fine --- fre/make/gfdlfremake/buildDocker.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fre/make/gfdlfremake/buildDocker.py b/fre/make/gfdlfremake/buildDocker.py index 1305c2e8..6d10ee55 100644 --- a/fre/make/gfdlfremake/buildDocker.py +++ b/fre/make/gfdlfremake/buildDocker.py @@ -64,10 +64,10 @@ def __init__(self,base,exp,libs,RUNenv,target,mkTemplate,stage2base): if self.stage2base == "": self.secondstage = ["\n"] else: - self.secondstage = ["FROM "+self.stage2base+" as final\n", - "COPY --from=builder "+self.src+" "+self.src+"\n", - "COPY --from=builder "+self.bld+" "+self.bld+"\n", - "ENV PATH=$PATH:"+self.bld+"\n"] + self.secondstage = [f"FROM {self.stage2base} as final\n", + f"COPY --from=builder {self.src} {self.src}\n", + f"COPY --from=builder {self.bld} {self.bld}\n", + f"ENV PATH=$PATH:{self.bld}\n"] def writeDockerfileCheckout(self, cScriptName, cOnDisk): """ Brief: writes to the checkout part of the Dockerfile and sets up the compile From 321e368de95d083af3b9e7a47c85b4b746b66e4e Mon Sep 17 00:00:00 2001 From: Tom Robinson Date: Tue, 28 Jan 2025 12:50:45 -0500 Subject: [PATCH 12/13] Removes unused function --- fre/make/gfdlfremake/platformfre.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/fre/make/gfdlfremake/platformfre.py b/fre/make/gfdlfremake/platformfre.py index c9096f5e..356dbe02 100644 --- a/fre/make/gfdlfremake/platformfre.py +++ b/fre/make/gfdlfremake/platformfre.py @@ -150,13 +150,6 @@ def getContainerImage(self,name): for p in self.yaml: if p["name"] == name: return p["containerBase"] - def is2stepContainer(self,name): - """ - Brief: Returns True if this is a 2 step container - """ - for p in self.yaml: - if p["name"] == name: - return p["container2step"] def getContainer2base(self,name): """ Brief: returns the image to be used in the second step of the Dockerfile From e68f49df3fcbdd7139e3dea4ecb1b569d7f37e8f Mon Sep 17 00:00:00 2001 From: Tom Robinson Date: Tue, 28 Jan 2025 14:21:07 -0500 Subject: [PATCH 13/13] Adds back in platformfre.py --- fre/make/gfdlfremake/platformfre.py | 159 ++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 fre/make/gfdlfremake/platformfre.py diff --git a/fre/make/gfdlfremake/platformfre.py b/fre/make/gfdlfremake/platformfre.py new file mode 100644 index 00000000..356dbe02 --- /dev/null +++ b/fre/make/gfdlfremake/platformfre.py @@ -0,0 +1,159 @@ +import yaml + +class platforms (): + def __init__(self,platforminfo): + """ + Param: + - self The platform yaml object + - platforminfo dictionary with platform information + from the combined yaml + """ + self.yaml = platforminfo + + ## Check the yaml for errors/omissions + ## Loop through the platforms + for p in self.yaml: + ## Check the platform name + try: + p["name"] + except: + raise Exception("At least one of the platforms is missing a name in "+fname+"\n") + ## Check the compiler + try: + p["compiler"] + except: + raise Exception("You must specify a compiler in your "+p["name"]+" platform in the file "+fname+"\n") + ## Check for modules to load + try: + p["modules"] + except: + p["modules"]=[""] + ## Check for modulesInit to set up the modules environment + try: + p["modulesInit"] + except: + p["modulesInit"]=[""] + ## Get the root for the build + try: + p["modelRoot"] + except: + p["modelRoot"] = "/apps" + ## Check if we are working with a container and get the info for that + try: + p["container"] + ## When not doing a container build, this should all be set to empty strings and Falses + except: + p["container"] = False + p["RUNenv"] = [""] + p["containerBuild"] = "" + p["containerRun"] = "" + p["containerBase"] = "" + p["container2step"] = False + p["container2base"] = "" + if p["container"]: + ## Check the container builder + try: + p["containerBuild"] + except: + raise Exception("Platform "+p["name"]+": You must specify the program used to build the container (containerBuild) on the "+p["name"]+" platform in the file "+fname+"\n") + if p["containerBuild"] != "podman" and p["containerBuild"] != "docker": + raise ValueError("Platform "+p["name"]+": Container builds only supported with docker or podman, but you listed "+p["containerBuild"]+"\n") + ## Get the name of the base container + try: + p["containerBase"] + except: + raise NameError("Platform "+p["name"]+": You must specify the base container you wish to use to build your application") + ## Check if this is a 2 step (multi stage) build + try: + p["container2step"] + except: + p["container2step"] = False + ## Get the base for the second stage of the build + if p["container2step"]: + try: + p["container2base"] + except: + raise NameError ("Platform "+p["name"]+": container2step is True, so you must define a container2base\n") + ## Check if there is anything special to copy over + else: + ## There should not be a second base if this is not a 2 step build + try: + p["container2base"] + except: + p["container2base"] = "" + else: + raise ValueError ("Platform "+p["name"]+": You defined container2base "+p["container2base"]+" but container2step is False\n") + ## Get any commands to execute in the dockerfile RUN command + try: + p["RUNenv"] + except: + p["RUNenv"] = "" + ## Check the container runner + try: + p["containerRun"] + except: + raise Exception("You must specify the program used to run the container (containerRun) on the "+p["name"]+" platform in the file "+fname+"\n") + if p["containerRun"] != "apptainer" and p["containerRun"] != "singularity": + raise ValueError("Container builds only supported with apptainer, but you listed "+p["containerRun"]+"\n") + else: + ## Find the location of the mkmf template + try: + p["mkTemplate"] + except: + raise ValueError("The platform "+p["name"]+" must specify a mkTemplate \n") + + def hasPlatform(self,name): + """ + Brief: Checks if the platform yaml has the named platform + """ + for p in self.yaml: + if p["name"] == name: + return True + return False + + def getPlatformsYaml(self): + """ + Brief: Get the platform yaml + """ + return self.yaml + + def getPlatformFromName(self,name): + """ + Brief: Get the platform information from the name of the platform + """ + for p in self.yaml: + if p["name"] == name: + return p + def getContainerInfoFromName(self,name): + """ + Brief: Return a tuple of the container information + """ + for p in self.yaml: + if p["name"] == name: + return (p["container"], \ + p["RUNenv"], \ + p["containerBuild"], \ + p["containerRun"], \ + p["containerBase"], \ + p["container2step"]) + def isContainer(self, name): + """ + Brief: Returns boolean of if this platform is a container based on the name + """ + for p in self.yaml: + if p["name"] == name: + return p["container"] + def getContainerImage(self,name): + """ + Brief: Returns the image name from the platform + """ + for p in self.yaml: + if p["name"] == name: + return p["containerBase"] + def getContainer2base(self,name): + """ + Brief: returns the image to be used in the second step of the Dockerfile + """ + for p in self.yaml: + if p["name"] == name: + return p["container2base"]