modify for build-tool-chain-13.2.0 and test run oe demo

README.md

@@ -55,6 +55,7 @@ Follow the instructions in openeuler riscv gitee to compile uboot for OE-23.X.
 ```
 # Fetch the uboot submodule
 git submodule update --init --recursive
+docker run -v $(pwd):/home/penglai/penglai-enclave -w /home/penglai/penglai-enclave --network=host --rm -it fly0307/penglai-enclave:v0.5 bash
 cd ./u-boot
 make qemu-riscv64_smode_defconfig
 make ARCH=riscv CROSS_COMPILE=riscv64-unknown-linux-gnu- -j$(nproc)
@@ -79,26 +80,41 @@ For oe versions greater than 23, you can access the source code after [Run openE
 ### Build OpenSBI (with Penglai supports)
 **For openEuler version $\lt$ 23:**
 
+When compiling opensbi-0.9/1.0, you can use the toolchain `CROSS_COMPILE=/home/penglai/toolchain-720/bin/riscv64-unkown-linux-gnu-`. For opensbi-1.2, you can use `CROSS_COMPILE=riscv64-unknown-linux-gnu-`.
+
+
 	cp openeuler-kernel/arch/riscv/boot/Image .
-	docker run -v $(pwd):/home/penglai/penglai-enclave -w /home/penglai/penglai-enclave --rm -it ddnirvana/penglai-enclave:v0.5 bash
+	docker run -v $(pwd):/home/penglai/penglai-enclave -w /home/penglai/penglai-enclave --rm -it fly0307/penglai-enclave:v0.5 bash
 	# In the docker image
 	cd /home/penglai/penglai-enclave/opensbi-0.9
 	mkdir -p build-oe/qemu-virt
-	CROSS_COMPILE=riscv64-unknown-linux-gnu- make O=build-oe/qemu-virt PLATFORM=generic FW_PAYLOAD=y FW_PAYLOAD_PATH=/home/penglai/penglai-enclave/Image
+	CROSS_COMPILE=/home/penglai/toolchain-720/bin/riscv64-unkown-linux-gnu- make O=build-oe/qemu-virt PLATFORM=generic FW_PAYLOAD=y FW_PAYLOAD_PATH=/home/penglai/penglai-enclave/Image
 Note: the /home/penglai/penglai-enclave/Image is the image compiled openEuler Kernel Image.
 
+A simple way:
+For opensbi-0.9 and oe-2203(oe $\lt$ 23 )
+
+```
+./docker_cmd.sh opensbi-0.9
+```
+
 **For openEuler version $\ge$ 23:**
 
 ```
-docker run -v $(pwd):/home/penglai/penglai-enclave -w /home/penglai/penglai-enclave --rm -it ddnirvana/penglai-enclave:v0.5 bash
+docker run -v $(pwd):/home/penglai/penglai-enclave -w /home/penglai/penglai-enclave --rm -it fly0307/penglai-enclave:v0.5 bash
 cd /home/penglai/penglai-enclave/opensbi-1.2
 rm -rf build-oe/qemu-virt
 mkdir -p build-oe/qemu-virt
 CROSS_COMPILE=riscv64-unknown-linux-gnu- make O=build-oe/qemu-virt PLATFORM=generic FW_PAYLOAD=y FW_PAYLOAD_PATH=/home/penglai/penglai-enclave/u-boot/u-boot.bin -j$(nproc)
 ```
 
 A simpler way:
+For opensbi-1.2 and oe-2303(oe $\ge$ 23 )
+```
+./docker_cmd.sh opensbi-1.2 
+```
 
+For others
 ```
 ./docker_cmd.sh docker
 #In the docker image，build opensbi 1.2 for OE20.03
@@ -140,7 +156,7 @@ When penglai.ko is completed,following the commnads to build user-level sdk and
 
 You should download the disk image of openEuler (i.e., openEuler-preview.riscv64.qcow2) and raname image file to openEuler-xxxx-qemu-riscv64.qcow2.
 
-You can download OE 2303 from [openEuler-23.03-V1-riscv64](https://mirror.iscas.ac.cn/openeuler-sig-riscv/openEuler-RISC-V/preview/openEuler-23.03-V1-riscv64/QEMU/)(i.e., openEuler-23.03-V1-base-qemu-preview.qcow2）or download openEuler 20.03 from [here](http://pan.sjtu.edu.cn/web/share/4440d1d40d859f141d9e6cf18b89bb4d).
+You can download OE 2303 from [openEuler-23.03-V1-riscv64](https://mirror.iscas.ac.cn/openeuler-sig-riscv/openEuler-RISC-V/preview/openEuler-23.03-V1-riscv64/QEMU/)(i.e., openEuler-23.03-V1-base-qemu-preview.qcow2)or download openEuler 20.03 from [here](http://pan.sjtu.edu.cn/web/share/4440d1d40d859f141d9e6cf18b89bb4d).
 
 ```
 wget https://mirror.iscas.ac.cn/openeuler-sig-riscv/openEuler-RISC-V/preview/openEuler-23.03-V1-riscv64/QEMU/openEuler-23.03-V1-base-qemu-preview.qcow2.zst
@@ -238,7 +254,7 @@ Copy penglai-enclave-driver to the root/ directory of the oe VM:
 
 ```
 #in host
-scp -P -r 12055 penglai-enclave-driver root@localhost:~/
+scp -P 12055 -r penglai-enclave-driver root@localhost:~/
 scp -P 12055 sdk/demo/host/host root@localhost:~/
 scp -P 12055 sdk/demo/prime/prime root@localhost:~/
 ```

docker_cmd.sh

@@ -18,7 +18,7 @@ function print_usage() {
 # no arguments
 if [ $# == 0 ]; then
 	echo "Default: building penglai demo image"
-	docker run -v $(pwd):/home/penglai/penglai-enclave -w /home/penglai/penglai-enclave --rm -it fly0307/penglai-enclave:v0.1 bash scripts/build_opensbi.sh -v 1.2 -k 2203
+	docker run -v $(pwd):/home/penglai/penglai-enclave -w /home/penglai/penglai-enclave --rm -it fly0307/penglai-enclave:v0.5 bash -c "export PATH=\$PATH:/home/penglai/toolchain/bin && /home/penglai/penglai-enclave/scripts/build_opensbi.sh -v1.2 -k2203"
 	exit 0
 fi
 
@@ -27,10 +27,17 @@ if [[ $1 == *"help"* ]]; then
 	exit 0
 fi
 
+# build penglai-0.9/1.0
+if [[ $1 == "opensbi-0.9" ]]; then
+	echo "Build: build-opensbi-0.9/1.0"
+	docker run -v $(pwd):/home/penglai/penglai-enclave -w /home/penglai/penglai-enclave --rm -it fly0307/penglai-enclave:v0.5 bash -c "export PATH=\$PATH:/home/penglai/toolchain-720/bin && /home/penglai/penglai-enclave/scripts/build_opensbi.sh -v0.9 -k2203"
+	exit 0
+fi
+
 # build penglai
-if [[ $1 == *"build"* ]]; then
-	echo "Build: building penglai demo image"
-	docker run -v $(pwd):/home/penglai/penglai-enclave -w /home/penglai/penglai-enclave --rm -it fly0307/penglai-enclave:v0.1 bash scripts/build_opensbi.sh
+if [[ $1 == "opensbi-1.2" ]]; then
+	echo "Build: build-opensbi-1.2"
+	docker run -v $(pwd):/home/penglai/penglai-enclave -w /home/penglai/penglai-enclave --rm -it fly0307/penglai-enclave:v0.5 bash -c "export PATH=\$PATH:/home/penglai/toolchain/bin && /home/penglai/penglai-enclave/scripts/build_opensbi.sh -v1.2 -k2303"
 	exit 0
 fi
 
@@ -45,7 +52,7 @@ fi
 if [[ $1 == *"docker"* ]]; then
 	echo "Run: run docker"
 	#sudo docker run --privileged --cap-add=ALL  -v $(pwd):/home/penglai/penglai-enclave -w /home/penglai/penglai-enclave --rm -it ddnirvana/penglai-enclave:v0.1
-	docker run -v $(pwd):/home/penglai/penglai-enclave -w /home/penglai/penglai-enclave --network=host --rm -it fly0307/penglai-enclave:v0.1 bash
+	docker run -v $(pwd):/home/penglai/penglai-enclave -w /home/penglai/penglai-enclave --network=host --rm -it fly0307/penglai-enclave:v0.5 bash
 	exit 0
 fi
 

openeuler-kernel/arch/riscv/Makefile

@@ -41,6 +41,12 @@ riscv-march-$(CONFIG_ARCH_RV32I)	:= rv32ima
 riscv-march-$(CONFIG_ARCH_RV64I)	:= rv64ima
 riscv-march-$(CONFIG_FPU)		:= $(riscv-march-y)fd
 riscv-march-$(CONFIG_RISCV_ISA_C)	:= $(riscv-march-y)c
+
+# Newer binutils versions default to ISA spec version 20191213 which moves some
+# instructions from the I extension to the Zicsr and Zifencei extensions.
+toolchain-need-zicsr-zifencei := $(call cc-option-yn, -march=$(riscv-march-y)_zicsr_zifencei)
+riscv-march-$(toolchain-need-zicsr-zifencei) := $(riscv-march-y)_zicsr_zifencei
+
 KBUILD_CFLAGS += -march=$(subst fd,,$(riscv-march-y))
 KBUILD_AFLAGS += -march=$(riscv-march-y)
 

opensbi-0.9/Makefile

@@ -157,7 +157,11 @@ ifndef PLATFORM_RISCV_ABI
 endif
 ifndef PLATFORM_RISCV_ISA
   ifneq ($(PLATFORM_RISCV_TOOLCHAIN_DEFAULT), 1)
-    PLATFORM_RISCV_ISA = rv$(PLATFORM_RISCV_XLEN)imafdc
+    ifeq ($(CC_SUPPORT_ZICSR_ZIFENCEI), y)
+      PLATFORM_RISCV_ISA = rv$(PLATFORM_RISCV_XLEN)imafdc_zicsr_zifencei
+    else
+      PLATFORM_RISCV_ISA = rv$(PLATFORM_RISCV_XLEN)imafdc
+    endif
   else
     PLATFORM_RISCV_ISA = $(OPENSBI_CC_ISA)
   endif

opensbi-1.2/lib/sbi/sbi_pmp.c

@@ -90,6 +90,7 @@ static void sbi_pmp_sync(struct sbi_scratch *scratch)
 
 	u32 remote_hartid = curr_skip_hartid;
 
+	// if (remote_hartid != -1UL && (wait_for_sync[remote_hartid] == IPI_TLB || wait_for_sync[remote_hartid] == IPI_PMP)){
 	if (remote_hartid != -1 && (wait_for_sync[remote_hartid] == IPI_TLB || waiting_for_spinlock[remote_hartid] == 1)){
 		if (SYNC_DEBUG)
 			sbi_printf("hart %ld skip wait %u sync pmp\n", hartid,

opensbi-1.2/lib/sbi/sm/platform/pmp/enclave_mm.c

@@ -959,13 +959,6 @@ static int insert_mm_region(int region_idx, struct mm_list_t* mm_region, int mer
 	}
 
 	//found the exact mm_list
-
-	if(merge && mm_regions[region_idx].mm_list_head == mm_list_head && mm_list_head->mm_list == mm_region){
-		//An entire pmp region is reclaimed
-		if(mm_region->order == mm_list_head->order){
-			return 0;
-		}
-	}
 	int ret_val = 0;
 	struct mm_list_head_t *new_list_head = (struct mm_list_head_t*)MM_LIST_2_PADDR(mm_region);
 	if(mm_list_head && mm_list_head->order == mm_region->order)
@@ -1235,10 +1228,10 @@ int mm_free_clear(void* req_paddr, unsigned long free_size)
 
 		mm_list_head = mm_list_head->next_list_head;
 	}
-	// if(mm_list_head)
-	// {
-	// 	goto mm_free_out;
-	// }
+	if(mm_list_head)
+	{
+		goto mm_free_out;
+	}
 
 	//insert with merge
 	ret_val = insert_mm_region(region_idx, mm_region, 1);

opensbi-1.2/lib/sbi/sm/sm.c

@@ -9,6 +9,7 @@
 #include <sbi/riscv_locks.h>
 
 extern volatile int print_m_mode;
+extern volatile unsigned long pmp_bitmap;
 
 //static int sm_initialized = 0;
 //static spinlock_t sm_init_lock = SPINLOCK_INIT;
@@ -331,7 +332,7 @@ uintptr_t sm_do_timer_irq(uintptr_t *regs, uintptr_t mcause, uintptr_t mepc)
 	return ret;
 }
 /**
- * \brief Used to clear pmp settings when uninstalling kernel modules
+ * \brief Used to clear pmp settings when uninstalling kernel modules\
  * 
  * \param size_ptr Used to pass the size of the freed memory to the driver
  * \param flag Select whether to clear a specific pmp
@@ -354,13 +355,15 @@ uintptr_t sm_free_enclave_mem(uintptr_t size_ptr, unsigned long flag)
 			}
 
 			if (pmp_idx == 0) {
-				sbi_printf("M mode:Finish free and there is no mem to reclaim\r\n");
+				sbi_printf("M mode: sm_free_enclave_mem: There is no mem to reclaim\r\n");
 				dump_pmps();
 				size = 0;
 				ret  = 0;
 				break;
 			}
-			mm_free_clear((void *)pmp_config.paddr, pmp_config.size);
+
+			clear_pmp_and_sync(pmp_idx);
+			pmp_bitmap &= ~(1 << pmp_idx);
 			ret  = pmp_config.paddr;
 			size = pmp_config.size;
 

run_openeuler.sh

@@ -47,6 +47,9 @@ then
 elif (($kernel_version == 2303))
 then
 	imagepath=openEuler-2303-qemu-riscv64.qcow2
+elif (($kernel_version == 2309))
+then
+	imagepath=openEuler-2309-qemu-riscv64.qcow2
 else
 	echo "Invalid kernel version: $kernel_version" >&2
 	print_usage

scripts/build_opensbi.sh

@@ -5,20 +5,29 @@
 ## 		For older version (e.g., v0.6), please find other scripts
 opensbi_version=1.2
 kernel_version=2003
+set_cross_compile=''
+
+function set_cross_compile() {
+    if [ $(echo "$opensbi_version > 1.0" | bc -l) -eq 1 ]; then
+        set_cross_compile="riscv64-unknown-linux-gnu-"
+    else
+        set_cross_compile="/home/penglai/toolchain-720/bin/riscv64-unknown-linux-gnu-"
+    fi
+}
 
 function build_opensbi_1() {
     # build opensbi
     cd /home/penglai/penglai-enclave/opensbi-${1}
 	rm -rf build-oe/qemu-virt
 	mkdir -p build-oe/qemu-virt
-	CROSS_COMPILE=riscv64-unknown-linux-gnu- make O=build-oe/qemu-virt PLATFORM=generic FW_PAYLOAD=y FW_PAYLOAD_PATH=/home/penglai/penglai-enclave/Image
+	CROSS_COMPILE=$set_cross_compile make O=build-oe/qemu-virt PLATFORM=generic FW_PAYLOAD=y FW_PAYLOAD_PATH=/home/penglai/penglai-enclave/Image
 }
 
 function build_opensbi_2() {
     cd /home/penglai/penglai-enclave/opensbi-${1}
     rm -rf build-oe/qemu-virt
     mkdir -p build-oe/qemu-virt
-    CROSS_COMPILE=riscv64-unknown-linux-gnu- make O=build-oe/qemu-virt PLATFORM=generic FW_PAYLOAD=y FW_PAYLOAD_PATH=/home/penglai/penglai-enclave/u-boot/u-boot.bin -j$(nproc)
+    CROSS_COMPILE=$set_cross_compile make O=build-oe/qemu-virt PLATFORM=generic FW_PAYLOAD=y FW_PAYLOAD_PATH=/home/penglai/penglai-enclave/u-boot/u-boot.bin -j$(nproc)
 }
 
 function print_usage() {
@@ -63,9 +72,11 @@ done
 
 if [ $(echo "$kernel_version < 2303" | bc -l) -eq 1 ]
 then
+	set_cross_compile
 	build_opensbi_1  $opensbi_version
 	exit 0
 else
+	set_cross_compile
 	build_opensbi_2  $opensbi_version
 	exit 0
 fi