diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 368222a68..ef750daef 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -3,7 +3,7 @@ Thank you for reporting an issue. *IMPORTANT* - Please search our issue tracker *before* making a new issue. If you cannot find a similar issue, then create a new issue. -https://github.com/LLNL/UnifyCR/issues +https://github.com/LLNL/UnifyFS/issues Please fill in as much of the template as possible. --> @@ -15,7 +15,7 @@ Type | Version/Name Operating System | OS Version | Architecture | -UnifyCR Version | +UnifyFS Version | ### Describe the problem you're observing diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 76a40f96d..a6b6861f2 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -25,7 +25,7 @@ ### Checklist: -- [ ] My code follows the UnifyCR code style requirements. +- [ ] My code follows the UnifyFS code style requirements. - [ ] I have updated the documentation accordingly. - [ ] I have read the **CONTRIBUTING** document. - [ ] I have added tests to cover my changes. diff --git a/.gitignore b/.gitignore index c929c3dfa..c851036c6 100644 --- a/.gitignore +++ b/.gitignore @@ -47,13 +47,13 @@ log *_unittest # project files -server/src/unifycrd -util/unifycr/src/unifycr +server/src/unifyfsd +util/unifyfs/src/unifyfs # generated package files -client/unifycr-config -client/unifycr.pc -extras/unifycr.conf +client/unifyfs-config +client/unifyfs.pc +extras/unifyfs.conf libtool m4/libtool.m4 m4/ltoptions.m4 @@ -64,24 +64,13 @@ meta/src/Mlog2/.dirstamp t/sys/.dirstamp # test files -client/unifycr-runtime-config.h -examples/src/app-btio-gotcha -examples/src/app-btio-static -examples/src/app-mpiio-gotcha -examples/src/app-mpiio-static -examples/src/sysio-read-gotcha -examples/src/sysio-read-static -examples/src/app-tileio-gotcha -examples/src/app-tileio-static -examples/src/sysio-write-gotcha -examples/src/sysio-write-static -examples/src/sysio-writeread-gotcha -examples/src/sysio-writeread-static -examples/src/sysio-writeread2-gotcha -examples/src/sysio-writeread2-static -examples/src/app-hdf5-create-gotcha -examples/src/app-hdf5-writeread-gotcha +client/unifyfs-runtime-config.h +examples/src/*-gotcha +examples/src/*-posix +examples/src/*-static t/sys/open.t t/test-results/ -t/unifycr_unmount.t +t/unifyfs_unmount.t t/test_run_env.sh +deps +install diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..1351251ea --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,132 @@ +stages: + - build + - test-unit + - test-integ + +cache: + paths: + - spack_ci/ + +##### Templates ##### + +.quartz-template: &quartz_template + tags: + - quartz + - shell + variables: + LLNL_SERVICE_USER: "unifysrv" + retry: + max: 1 + when: + - unknown_failure + - stuck_or_timeout_failure + +.butte-template: &butte_template + tags: + - butte + - shell + variables: + LLNL_SERVICE_USER: "unifysrv" + retry: + max: 1 + when: + - unknown_failure + - stuck_or_timeout_failure + +.build-template: &build_template + stage: build + script: + - ./autogen.sh + - mkdir -p unifyfs-build unifyfs-install && cd unifyfs-build + - ../configure --prefix=$CI_PROJECT_DIR/unifyfs-install --enable-fortran --disable-silent-rules + - make V=1 + - make V=1 install + artifacts: + name: "${CI_JOB_NAME}-${CI_PIPELINE_ID}" + untracked: true + expire_in: 1 hour + paths: + - unifyfs-build/ + - unifyfs-install/ + +.unit-test-template: &unit_test_template + stage: test-unit + script: + - cd unifyfs-build/t && make check + after_script: + - rm -rf /tmp/unify* /tmp/tmp.* /tmp/mdhim* /tmp/na_sm + +.quartz-batch-variables: + variables: &quartz_batch_variables + LLNL_SLURM_SCHEDULER_PARAMETERS: "-N $NNODES -p pbatch -t $WALL_TIME" + LLNL_SERVICE_USER: "unifysrv" + CI_PROJDIR: "$CI_PROJECT_DIR" + UNIFYFS_INSTALL: "$CI_PROJECT_DIR/unifyfs-install" + CI_NPROCS: "$NPROCS" + +.butte-batch-variables: + variables: &butte_batch_variables + LLNL_LSF_SCHEDULER_PARAMETERS: "-nnodes $NNODES -q pbatch -W $WALL_TIME" + LLNL_SERVICE_USER: "unifysrv" + CI_PROJDIR: "$CI_PROJECT_DIR" + UNIFYFS_INSTALL: "$CI_PROJECT_DIR/unifyfs-install" + CI_NPROCS: "$NPROCS" + +##### Jobs ##### + +before_script: + # HERE BE DRAGONS!: Since on HPC and running as user, Spack might already + # exist and can get complicated if we install it again. + # + # check for sourced spack || check for unsourced spack in $HOME/spack and + # source it || check for cached spack, clone if none, and source it + - which spack || ((cd $HOME/spack && git describe) && . $HOME/spack/share/spack/setup-env.sh) || (((cd spack_ci && git describe) || git clone https://github.com/CamStan/spack spack_ci) && . spack_ci/share/spack/setup-env.sh) + - SPACK_ARCH=$(spack arch) + - spack install leveldb && spack load leveldb arch=$SPACK_ARCH + - spack install gotcha@0.0.2 && spack load gotcha@0.0.2 arch=$SPACK_ARCH + - spack install flatcc && spack load flatcc arch=$SPACK_ARCH + - spack install margo^mercury+bmi~boostsys && spack load argobots arch=$SPACK_ARCH && spack load mercury arch=$SPACK_ARCH && spack load margo arch=$SPACK_ARCH + +build-quartz: + <<: *quartz_template + <<: *build_template + +build-butte: + <<: *butte_template + <<: *build_template + +unit-test-quartz: + <<: *quartz_template + <<: *unit_test_template + dependencies: + - build-quartz + +unit-test-butte: + <<: *butte_template + <<: *unit_test_template + dependencies: + - build-butte + +#integ-test-quartz: +# <<: *quartz_template +# stage: test-integ +# tags: +# - quartz +# - batch +# variables: *quartz_batch_variables +# script: +# - cd t/ci && prove -v RUN_CI_TESTS.sh +# dependencies: +# - build-quartz + +integ-test-butte: + <<: *butte_template + stage: test-integ + tags: + - butte + - batch + variables: *butte_batch_variables + script: + - cd t/ci && prove -v RUN_CI_TESTS.sh + dependencies: + - build-butte diff --git a/.travis.yml b/.travis.yml index 1d5caaec8..f9de90d0c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,27 +1,61 @@ language: c sudo: required +dist: xenial + +addons: + apt: + update: true + packages: + - autoconf + - automake + - build-essential + - cmake + - gfortran + - libhdf5-openmpi-dev + - libopenmpi-dev + - libtool-bin + - m4 + - openmpi-bin before_install: # The default environment variable $CC is known to interfere with # MPI projects. - test -n $CC && unset CC - - sudo apt-get -qq update - - sudo apt-get install --yes -qq build-essential autoconf libtool cmake - - sudo apt-get install --yes -qq libopenmpi-dev openmpi-bin - - sudo apt-get install --yes -qq libhdf5-openmpi-dev - (cd $HOME/spack; git describe) || git clone https://github.com/spack/spack $HOME/spack + # Create packages.yaml to prevent building dependencies that time out + - | + test -f $HOME/spack/etc/spack/packages.yaml || cat > $HOME/spack/etc/spack/packages.yaml << ' EOF' + packages: + autoconf: + buildable: False + paths: + autoconf@2.69: /usr + automake: + buildable: False + paths: + automake@1.15: /usr + cmake: + buildable: False + paths: + cmake@3.12.4: /usr/local/cmake-3.12.4 + libtool: + buildable: False + paths: + libtool@2.4.6: /usr + m4: + buildable: False + paths: + m4@4.17: /usr + EOF + +install: - $HOME/spack/bin/spack install environment-modules - - $HOME/spack/bin/spack install leveldb - - $HOME/spack/bin/spack install gotcha@0.0.2 - - $HOME/spack/bin/spack install flatcc - # insall margo with spack - - git config --global http.sslVerify false - - git clone https://xgitlab.cels.anl.gov/sds/sds-repo.git $HOME/sds-repo.git - - $HOME/spack/bin/spack repo add $HOME/sds-repo.git - - $HOME/spack/bin/spack install margo - - git config --global http.sslVerify true - # prepare build environment - . $HOME/spack/share/spack/setup-env.sh + - spack install leveldb + - spack install gotcha@0.0.2 + - spack install flatcc + - spack install margo^mercury+bmi~boostsys + # prepare build environment - spack load environment-modules - source <(spack module tcl loads leveldb gotcha@0.0.2 flatcc mercury argobots margo) - eval $(./scripts/git_log_test_env.sh) @@ -39,7 +73,7 @@ script: # Force git to update the shallow clone and include tags so git-describe works - git fetch --unshallow --tags - sh autogen.sh - - ./configure || cat config.log + - ./configure --enable-fortran || cat config.log - make -k && make distcheck - ./scripts/checkpatch.sh || test "$TEST_CHECKPATCH_ALLOW_FAILURE" = yes diff --git a/LICENSE b/LICENSE index ce3d2916a..c0f119e66 100644 --- a/LICENSE +++ b/LICENSE @@ -7,8 +7,8 @@ Copyright 2017, UT-Battelle, LLC. LLNL-CODE-741539 All rights reserved. -This is the license for UnifyCR. For details, see -https://github.com/LLNL/UnifyCR. +This is the license for UnifyFS. For details, see +https://github.com/LLNL/UnifyFS. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Makefile.am b/Makefile.am index 36e4f093b..76addc56e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -5,7 +5,7 @@ CONFIG = ordered #ACLOCAL_AMFLAGS = -I m4 pkgconfigdir = @pkgconfigdir@ -pkgconfig_DATA = client/unifycr.pc +pkgconfig_DATA = client/unifyfs.pc CLEANFILES = diff --git a/README.md b/README.md index 3e970453c..678a894f2 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# UnifyCR: A Distributed Burst Buffer File System - 0.1.0 +# UnifyFS: A Distributed Burst Buffer File System - 0.1.0 Node-local burst buffers are becoming an indispensable hardware resource on large-scale supercomputers to buffer the bursty I/O from scientific @@ -8,26 +8,26 @@ across different batch jobs. In addition, burst buffers need to cope with a variety of challenging I/O patterns from data-intensive scientific applications. -UnifyCR is a user-level burst buffer file system under active development. -UnifyCR supports scalable and efficient aggregation of I/O bandwidth from burst -buffers while having the same life cycle as a batch-submitted job. While UnifyCR -is designed for N-N write/read, UnifyCR compliments its functionality with the +UnifyFS is a user-level burst buffer file system under active development. +UnifyFS supports scalable and efficient aggregation of I/O bandwidth from burst +buffers while having the same life cycle as a batch-submitted job. While UnifyFS +is designed for N-N write/read, UnifyFS compliments its functionality with the support for N-1 write/read. It efficiently accelerates scientific I/O based on scalable metadata indexing, co-located I/O delegation, and server-side read clustering and pipelining. ## Documentation -Full UnifyCR documentation is contained [here](https://unifycr.readthedocs.io). +Full UnifyFS documentation is contained [here](https://unifyfs.readthedocs.io). -Use [Build & I/O Interception](http://unifycr.readthedocs.io/en/dev/build-intercept.html) -for instructions on how to build and install UnifyCR. +Use [Build & I/O Interception](http://unifyfs.readthedocs.io/en/dev/build-intercept.html) +for instructions on how to build and install UnifyFS. ## Build Status -The current status of the UnifyCR dev branch is: +The current status of the UnifyFS dev branch is: -[![Build Status](https://api.travis-ci.org/LLNL/UnifyCR.png?branch=dev)](https://travis-ci.org/LLNL/UnifyCR) -[![Read the Docs](https://readthedocs.org/projects/unifycr/badge/?version=dev)](https://unifycr.readthedocs.io) +[![Build Status](https://api.travis-ci.org/LLNL/UnifyFS.png?branch=dev)](https://travis-ci.org/LLNL/UnifyFS) +[![Read the Docs](https://readthedocs.org/projects/unifyfs/badge/?version=dev)](https://unifyfs.readthedocs.io) ## Contribute and Develop Contributing guidelines can be found in our -[documentation](https://unifycr.readthedocs.io/en/dev/style-guides.html). +[documentation](https://unifyfs.readthedocs.io/en/dev/style-guides.html). diff --git a/autogen.sh b/autogen.sh index 4667aba94..77822e92a 100755 --- a/autogen.sh +++ b/autogen.sh @@ -1,7 +1,7 @@ #!/bin/sh echo -echo ... UnifyCR autogen ... +echo ... UnifyFS autogen ... echo ## Check all dependencies are present @@ -43,14 +43,14 @@ fi env libtoolize --version > /dev/null 2>&1 if [ $? -eq 0 ]; then # libtoolize was found, so use it - TOOL=libtoolize + LIBTOOLIZE=libtoolize else # libtoolize wasn't found, so check for glibtoolize env glibtoolize --version > /dev/null 2>&1 if [ $? -eq 0 ]; then - TOOL=glibtoolize + LIBTOOLIZE=glibtoolize else - MISSING="$MISSING libtoolize/glibtoolize" + MISSING="$MISSING libtoolize" fi fi @@ -76,16 +76,25 @@ if [ "x$MISSING" != "x" ]; then fi ## Do the autogeneration -echo Running ${ACLOCAL}... -$ACLOCAL -echo Running ${AUTOHEADER}... -$AUTOHEADER -echo Running ${TOOL}... -$TOOL --automake --copy --force -echo Running ${AUTOCONF}... -$AUTOCONF -echo Running ${AUTOMAKE}... -$AUTOMAKE --add-missing --force-missing --copy --foreign +env autoreconf --version > /dev/null 2>&1 +if [ $? -eq 0 ]; then + echo Running ${LIBTOOLIZE}... + $LIBTOOLIZE --automake --copy --force + echo Running autoreconf... + autoreconf --force --install --verbose +else + echo Running ${LIBTOOLIZE}... + $LIBTOOLIZE --automake --copy --force + echo Running ${ACLOCAL}... + $ACLOCAL + echo Running ${AUTOHEADER}... + $AUTOHEADER + echo Running ${AUTOCONF}... + $AUTOCONF + echo Running ${AUTOMAKE}... + $AUTOMAKE --add-missing --force-missing --copy --foreign +fi + # Instruct user on next steps echo diff --git a/bootstrap.sh b/bootstrap.sh new file mode 100755 index 000000000..ff66a2e99 --- /dev/null +++ b/bootstrap.sh @@ -0,0 +1,114 @@ +#!/bin/bash +# +# This is an easy-bake script to download and build all UnifyFS's dependencies. +# + +ROOT="$(pwd)" + +mkdir -p deps +mkdir -p install +INSTALL_DIR=$ROOT/install + +cd deps + +repos=( https://xgitlab.cels.anl.gov/sds/bmi.git + https://github.com/google/leveldb.git + https://github.com/LLNL/GOTCHA.git + https://github.com/pmodels/argobots.git + https://github.com/mercury-hpc/mercury.git + https://xgitlab.cels.anl.gov/sds/margo.git + https://github.com/dvidelabs/flatcc.git +) + +for i in "${repos[@]}" ; do + # Get just the name of the project (like "mercury") + name=$(basename $i | sed 's/\.git//g') + if [ -d $name ] ; then + echo "$name already exists, skipping it" + else + if [ "$name" == "mercury" ] ; then + git clone --recurse-submodules $i + else + git clone $i + fi + fi +done + +echo "### building bmi ###" +cd bmi +./prepare && ./configure --enable-shared --enable-bmi-only \ + --prefix="$INSTALL_DIR" +make -j $(nproc) && make install +cd .. + +echo "### building leveldb ###" +cd leveldb +git checkout 1.22 +mkdir -p build && cd build +cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" \ + -DBUILD_SHARED_LIBS=yes .. +make -j $(nproc) && make install +cd .. +cd .. + +echo "### building GOTCHA ###" +cd GOTCHA +# Unify won't build against latest GOTCHA, so use a known compatible version. +git checkout 0.0.2 +mkdir -p build && cd build +cmake -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" .. +make -j $(nproc) && make install +cd .. +cd .. + +echo "### building argobots ###" +cd argobots +./autogen.sh && CC=gcc ./configure --prefix="$INSTALL_DIR" +make -j $(nproc) && make install +cd .. + +echo "### building mercury ###" +cd mercury +mkdir -p build && cd build +cmake -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" \ + -DMERCURY_USE_BOOST_PP=ON \ + -DMERCURY_USE_CHECKSUMS=ON \ + -DMERCURY_USE_EAGER_BULK=ON \ + -DMERCURY_USE_SYSTEM_MCHECKSUM=OFF \ + -DNA_USE_BMI=ON \ + -DMERCURY_USE_XDR=OFF \ + -DBUILD_SHARED_LIBS=on .. +make -j $(nproc) && make install +cd .. +cd .. + +echo "### building margo ###" +cd margo +git checkout v0.4.3 +export PKG_CONFIG_PATH="$INSTALL_DIR/lib/pkgconfig" +./prepare.sh +./configure --prefix="$INSTALL_DIR" +make -j $(nproc) && make install +cd .. + +echo "### building flatcc ###" +cd flatcc +# need -DBUILD_SHARED_LIBS=ye +mkdir -p build && cd build +cmake -DBUILD_SHARED_LIBS=on -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DFLATCC_INSTALL=on .. +make -j $(nproc) && make install +cd .. +cd .. + +cd "$ROOT" + +echo "*************************************************************************" +echo "Dependencies are all built. You can now build Unify with:" +echo "" +echo " export PKG_CONFIG_PATH=$INSTALL_DIR/lib/pkgconfig" +echo " ./autogen.sh && ./configure --with-leveldb=$INSTALL_DIR" \ + "--with-gotcha=$INSTALL_DIR --with-flatcc=$INSTALL_DIR" +echo " make" +echo "" +echo "*************************************************************************" diff --git a/client/Makefile.am b/client/Makefile.am index 4636a01d9..7f2c8147e 100644 --- a/client/Makefile.am +++ b/client/Makefile.am @@ -1,5 +1,5 @@ SUBDIRS = src -bin_SCRIPTS = unifycr-config +bin_SCRIPTS = unifyfs-config CLEANFILES = $(bin_SCRIPTS) diff --git a/client/check_fns/README.md b/client/check_fns/README.md index 0b4a587de..20dc2c187 100644 --- a/client/check_fns/README.md +++ b/client/check_fns/README.md @@ -2,16 +2,16 @@ The files in this folder help manage the set of wrappers that are implemented. In particluar, they are used to enable a tool that -detects I/O routines used by an application that are not yet supported in UnifyCR. +detects I/O routines used by an application that are not yet supported in UnifyFS. They are also used to generate the code required for GOTCHA. - fakechroot_list.txt - lists I/O routines from fakechroot - gnulibc_list.txt - I/O routines from libc - cstdio_list.txt - I/O routines from stdio - posix_list.txt - I/O routines in POSIX -- unifycr_list.txt - list of wrappers in UnifyCR -- unifycr_unsupported_list.txt - list of wrappers in UnifyCR that implemented, but not supported +- unifyfs_list.txt - list of wrappers in UnifyFS +- unifyfs_unsupported_list.txt - list of wrappers in UnifyFS that implemented, but not supported -Our [Wrapper Guide](https://unifycr.readthedocs.io/en/dev/wrappers.html) -has the full documentation on running the unifycr_check_fns tool, building the +Our [Wrapper Guide](https://unifyfs.readthedocs.io/en/dev/wrappers.html) +has the full documentation on running the unifyfs_check_fns tool, building the GOTCHA list, and building the other .txt files here. diff --git a/client/check_fns/unifycr_list.txt b/client/check_fns/unifycr_list.txt deleted file mode 100644 index 7d17a8f8a..000000000 --- a/client/check_fns/unifycr_list.txt +++ /dev/null @@ -1,76 +0,0 @@ -int UNIFYCR_WRAP(access)(const char *path, int mode) -int UNIFYCR_WRAP(mkdir)(const char *path, mode_t mode) -int UNIFYCR_WRAP(rmdir)(const char *path) -int UNIFYCR_WRAP(rename)(const char *oldpath, const char *newpath) -int UNIFYCR_WRAP(truncate)(const char* path, off_t length) -int UNIFYCR_WRAP(unlink)(const char *path) -int UNIFYCR_WRAP(remove)(const char *path) -int UNIFYCR_WRAP(stat)(const char *path, struct stat *buf) -int UNIFYCR_WRAP(__xstat)(int vers, const char *path, struct stat *buf) -int UNIFYCR_WRAP(__lxstat)(int vers, const char *path, struct stat *buf) -int UNIFYCR_WRAP(creat)(const char* path, mode_t mode) -int UNIFYCR_WRAP(creat64)(const char* path, mode_t mode) -int UNIFYCR_WRAP(open)(const char *path, int flags, ...) -int UNIFYCR_WRAP(open64)(const char* path, int flags, ...) -off_t UNIFYCR_WRAP(lseek)(int fd, off_t offset, int whence) -off64_t UNIFYCR_WRAP(lseek64)(int fd, off64_t offset, int whence) -int UNIFYCR_WRAP(posix_fadvise)(int fd, off_t offset, off_t len, int advice) -ssize_t UNIFYCR_WRAP(read)(int fd, void *buf, size_t count) -ssize_t UNIFYCR_WRAP(write)(int fd, const void *buf, size_t count) -ssize_t UNIFYCR_WRAP(readv)(int fd, const struct iovec *iov, int iovcnt) -ssize_t UNIFYCR_WRAP(writev)(int fd, const struct iovec *iov, int iovcnt) -ssize_t UNIFYCR_WRAP(pread)(int fd, void *buf, size_t count, off_t offset) -ssize_t UNIFYCR_WRAP(pread64)(int fd, void *buf, size_t count, off64_t offset) -ssize_t UNIFYCR_WRAP(pwrite)(int fd, const void *buf, size_t count, off_t offset) -ssize_t UNIFYCR_WRAP(pwrite64)(int fd, const void *buf, size_t count, off64_t offset) -int UNIFYCR_WRAP(ftruncate)(int fd, off_t length) -int UNIFYCR_WRAP(fsync)(int fd) -int UNIFYCR_WRAP(fdatasync)(int fd) -int UNIFYCR_WRAP(flock)(int fd, int operation) -void* UNIFYCR_WRAP(mmap)(void *addr, size_t length, int prot, int flags, int fd, off_t offset) -int UNIFYCR_WRAP(msync)(void *addr, size_t length, int flags) -void* UNIFYCR_WRAP(mmap64)(void *addr, size_t length, int prot, int flags, int fd, off_t offset) -int UNIFYCR_WRAP(__fxstat)(int vers, int fd, struct stat *buf) -int UNIFYCR_WRAP(close)(int fd) -FILE* UNIFYCR_WRAP(fopen)(const char *path, const char *mode) -FILE* UNIFYCR_WRAP(freopen)(const char *path, const char *mode, FILE *stream) -int UNIFYCR_WRAP(setvbuf)(FILE* stream, char* buf, int type, size_t size) -void* UNIFYCR_WRAP(setbuf)(FILE* stream, char* buf) -int UNIFYCR_WRAP(ungetc)(int c, FILE *stream) -int UNIFYCR_WRAP(fgetc)(FILE *stream) -int UNIFYCR_WRAP(fputc)(int c, FILE *stream) -int UNIFYCR_WRAP(getc)(FILE *stream) -int UNIFYCR_WRAP(putc)(int c, FILE *stream) -char* UNIFYCR_WRAP(fgets)(char* s, int n, FILE* stream) -int UNIFYCR_WRAP(fputs)(const char* s, FILE* stream) -size_t UNIFYCR_WRAP(fread)(void *ptr, size_t size, size_t nitems, FILE *stream) -size_t UNIFYCR_WRAP(fwrite)(const void *ptr, size_t size, size_t nitems, FILE *stream) -int UNIFYCR_WRAP(fprintf)(FILE *stream, const char* format, ...) -int UNIFYCR_WRAP(vfprintf)(FILE *stream, const char* format, va_list ap) -int UNIFYCR_WRAP(fscanf)(FILE *stream, const char* format, ...) -int UNIFYCR_WRAP(vfscanf)(FILE *stream, const char* format, va_list ap) -int UNIFYCR_WRAP(fseek)(FILE *stream, long offset, int whence) -int UNIFYCR_WRAP(fseeko)(FILE *stream, off_t offset, int whence) -long UNIFYCR_WRAP(ftell)(FILE *stream) -off_t UNIFYCR_WRAP(ftello)(FILE *stream) -void UNIFYCR_WRAP(rewind)(FILE* stream) -int UNIFYCR_WRAP(fgetpos)(FILE* stream, fpos_t* pos) -int UNIFYCR_WRAP(fsetpos)(FILE* stream, const fpos_t* pos) -int UNIFYCR_WRAP(fflush)(FILE* stream) -int UNIFYCR_WRAP(feof)(FILE *stream) -int UNIFYCR_WRAP(ferror)(FILE* stream) -void UNIFYCR_WRAP(clearerr)(FILE* stream) -int UNIFYCR_WRAP(fileno)(FILE *stream) -int UNIFYCR_WRAP(fclose)(FILE *stream) -int UNIFYCR_WRAP(fwprintf)(FILE *stream, const wchar_t* format, ...) -int UNIFYCR_WRAP(fwscanf)(FILE *stream, const wchar_t* format, ...) -int UNIFYCR_WRAP(vfwprintf)(FILE *stream, const wchar_t* format, va_list arg) -int UNIFYCR_WRAP(vfwscanf)(FILE *stream, const wchar_t* format, va_list arg) -wint_t UNIFYCR_WRAP(fgetwc)(FILE *stream) -wchar_t* UNIFYCR_WRAP(fgetws)(wchar_t* s, int n, FILE *stream) -wint_t UNIFYCR_WRAP(fputwc)(wchar_t wc, FILE *stream) -int UNIFYCR_WRAP(fputws)(const wchar_t* s, FILE *stream) -int UNIFYCR_WRAP(fwide)(FILE *stream, int mode) -wint_t UNIFYCR_WRAP(getwc)(FILE *stream) -wint_t UNIFYCR_WRAP(putwc)(wchar_t c, FILE *stream) -wint_t UNIFYCR_WRAP(ungetwc)(wint_t c, FILE *stream) diff --git a/client/check_fns/unifycr_check_fns b/client/check_fns/unifyfs_check_fns similarity index 83% rename from client/check_fns/unifycr_check_fns rename to client/check_fns/unifyfs_check_fns index 32a6e8029..6b63e43f4 100644 --- a/client/check_fns/unifycr_check_fns +++ b/client/check_fns/unifyfs_check_fns @@ -12,28 +12,28 @@ use strict; # hand-wrote cstdio list from # http://www.open-std.org/jtc1/sc22/wg14/www/docs/n1124.pdf # -# hand-wrote unifycr unsupported list (wrapped calls that aren't supported) +# hand-wrote unifyfs unsupported list (wrapped calls that aren't supported) # -# commands to build unifycr list -# grep UNIFYCR_WRAP ../runtime/lib/*.c > unifycr_list.txt +# commands to build unifyfs list +# grep UNIFYFS_WRAP ../runtime/lib/*.c > unifyfs_list.txt my $fakechroot_list = "fakechroot_list.txt"; my $gnulibc_list = "gnulibc_list.txt"; my $cstdio_list = "cstdio_list.txt"; my $posix_list = "posix_list.txt"; -my $unsupported_list = "unifycr_unsupported_list.txt"; -my $unifycr_list = "unifycr_list.txt"; +my $unsupported_list = "unifyfs_unsupported_list.txt"; +my $unifyfs_list = "unifyfs_list.txt"; if (@ARGV != 1) { - print "Usage: unifycr_check_fns \n"; + print "Usage: unifyfs_check_fns \n"; exit 1; } my $exe = shift @ARGV; my %any_libc = (); my %unsupported = (); -my %unifycr = (); +my %unifyfs = (); open(IN, $fakechroot_list) || die "failed to open $fakechroot_list\n"; while (my $line = ) { @@ -111,14 +111,14 @@ while (my $line = ) { } close(IN); -open(IN, $unifycr_list) || die "failed to open $unifycr_list\n"; +open(IN, $unifyfs_list) || die "failed to open $unifyfs_list\n"; while (my $line = ) { # drop trailing newline chomp $line; - # pick off item in parens in UNIFYCR_DELC(fn_name) + # pick off item in parens in UNIFYFS_DELC(fn_name) my $fn = undef; - if ($line =~ /UNIFYCR_WRAP\((.*?)\)/) { + if ($line =~ /UNIFYFS_WRAP\((.*?)\)/) { $fn = $1; } else { next; @@ -128,7 +128,7 @@ while (my $line = ) { $fn =~ s/\s*//g; #print "wrapped: $fn\n"; - $unifycr{$fn} = 1; + $unifyfs{$fn} = 1; } close(IN); @@ -149,7 +149,7 @@ foreach my $line (@fns_list) { my $supported = 0; if (defined $any_libc{$fn}) { $used = 1; - if (defined $unifycr{$fn} and not $unsupported{$fn}) { + if (defined $unifyfs{$fn} and not $unsupported{$fn}) { $supported = 1; } } diff --git a/client/check_fns/unifyfs_list.txt b/client/check_fns/unifyfs_list.txt new file mode 100644 index 000000000..343f8bfc8 --- /dev/null +++ b/client/check_fns/unifyfs_list.txt @@ -0,0 +1,78 @@ +int UNIFYFS_WRAP(access)(const char *path, int mode) +int UNIFYFS_WRAP(chmod)(const char *path, mode_t mode) +int UNIFYFS_WRAP(fchmod)(int fd, mode_t mode) +int UNIFYFS_WRAP(mkdir)(const char *path, mode_t mode) +int UNIFYFS_WRAP(rmdir)(const char *path) +int UNIFYFS_WRAP(rename)(const char *oldpath, const char *newpath) +int UNIFYFS_WRAP(truncate)(const char* path, off_t length) +int UNIFYFS_WRAP(unlink)(const char *path) +int UNIFYFS_WRAP(remove)(const char *path) +int UNIFYFS_WRAP(stat)(const char *path, struct stat *buf) +int UNIFYFS_WRAP(__xstat)(int vers, const char *path, struct stat *buf) +int UNIFYFS_WRAP(__lxstat)(int vers, const char *path, struct stat *buf) +int UNIFYFS_WRAP(creat)(const char* path, mode_t mode) +int UNIFYFS_WRAP(creat64)(const char* path, mode_t mode) +int UNIFYFS_WRAP(open)(const char *path, int flags, ...) +int UNIFYFS_WRAP(open64)(const char* path, int flags, ...) +off_t UNIFYFS_WRAP(lseek)(int fd, off_t offset, int whence) +off64_t UNIFYFS_WRAP(lseek64)(int fd, off64_t offset, int whence) +int UNIFYFS_WRAP(posix_fadvise)(int fd, off_t offset, off_t len, int advice) +ssize_t UNIFYFS_WRAP(read)(int fd, void *buf, size_t count) +ssize_t UNIFYFS_WRAP(write)(int fd, const void *buf, size_t count) +ssize_t UNIFYFS_WRAP(readv)(int fd, const struct iovec *iov, int iovcnt) +ssize_t UNIFYFS_WRAP(writev)(int fd, const struct iovec *iov, int iovcnt) +ssize_t UNIFYFS_WRAP(pread)(int fd, void *buf, size_t count, off_t offset) +ssize_t UNIFYFS_WRAP(pread64)(int fd, void *buf, size_t count, off64_t offset) +ssize_t UNIFYFS_WRAP(pwrite)(int fd, const void *buf, size_t count, off_t offset) +ssize_t UNIFYFS_WRAP(pwrite64)(int fd, const void *buf, size_t count, off64_t offset) +int UNIFYFS_WRAP(ftruncate)(int fd, off_t length) +int UNIFYFS_WRAP(fsync)(int fd) +int UNIFYFS_WRAP(fdatasync)(int fd) +int UNIFYFS_WRAP(flock)(int fd, int operation) +void* UNIFYFS_WRAP(mmap)(void *addr, size_t length, int prot, int flags, int fd, off_t offset) +int UNIFYFS_WRAP(msync)(void *addr, size_t length, int flags) +void* UNIFYFS_WRAP(mmap64)(void *addr, size_t length, int prot, int flags, int fd, off_t offset) +int UNIFYFS_WRAP(__fxstat)(int vers, int fd, struct stat *buf) +int UNIFYFS_WRAP(close)(int fd) +FILE* UNIFYFS_WRAP(fopen)(const char *path, const char *mode) +FILE* UNIFYFS_WRAP(freopen)(const char *path, const char *mode, FILE *stream) +int UNIFYFS_WRAP(setvbuf)(FILE* stream, char* buf, int type, size_t size) +void* UNIFYFS_WRAP(setbuf)(FILE* stream, char* buf) +int UNIFYFS_WRAP(ungetc)(int c, FILE *stream) +int UNIFYFS_WRAP(fgetc)(FILE *stream) +int UNIFYFS_WRAP(fputc)(int c, FILE *stream) +int UNIFYFS_WRAP(getc)(FILE *stream) +int UNIFYFS_WRAP(putc)(int c, FILE *stream) +char* UNIFYFS_WRAP(fgets)(char* s, int n, FILE* stream) +int UNIFYFS_WRAP(fputs)(const char* s, FILE* stream) +size_t UNIFYFS_WRAP(fread)(void *ptr, size_t size, size_t nitems, FILE *stream) +size_t UNIFYFS_WRAP(fwrite)(const void *ptr, size_t size, size_t nitems, FILE *stream) +int UNIFYFS_WRAP(fprintf)(FILE *stream, const char* format, ...) +int UNIFYFS_WRAP(vfprintf)(FILE *stream, const char* format, va_list ap) +int UNIFYFS_WRAP(fscanf)(FILE *stream, const char* format, ...) +int UNIFYFS_WRAP(vfscanf)(FILE *stream, const char* format, va_list ap) +int UNIFYFS_WRAP(fseek)(FILE *stream, long offset, int whence) +int UNIFYFS_WRAP(fseeko)(FILE *stream, off_t offset, int whence) +long UNIFYFS_WRAP(ftell)(FILE *stream) +off_t UNIFYFS_WRAP(ftello)(FILE *stream) +void UNIFYFS_WRAP(rewind)(FILE* stream) +int UNIFYFS_WRAP(fgetpos)(FILE* stream, fpos_t* pos) +int UNIFYFS_WRAP(fsetpos)(FILE* stream, const fpos_t* pos) +int UNIFYFS_WRAP(fflush)(FILE* stream) +int UNIFYFS_WRAP(feof)(FILE *stream) +int UNIFYFS_WRAP(ferror)(FILE* stream) +void UNIFYFS_WRAP(clearerr)(FILE* stream) +int UNIFYFS_WRAP(fileno)(FILE *stream) +int UNIFYFS_WRAP(fclose)(FILE *stream) +int UNIFYFS_WRAP(fwprintf)(FILE *stream, const wchar_t* format, ...) +int UNIFYFS_WRAP(fwscanf)(FILE *stream, const wchar_t* format, ...) +int UNIFYFS_WRAP(vfwprintf)(FILE *stream, const wchar_t* format, va_list arg) +int UNIFYFS_WRAP(vfwscanf)(FILE *stream, const wchar_t* format, va_list arg) +wint_t UNIFYFS_WRAP(fgetwc)(FILE *stream) +wchar_t* UNIFYFS_WRAP(fgetws)(wchar_t* s, int n, FILE *stream) +wint_t UNIFYFS_WRAP(fputwc)(wchar_t wc, FILE *stream) +int UNIFYFS_WRAP(fputws)(const wchar_t* s, FILE *stream) +int UNIFYFS_WRAP(fwide)(FILE *stream, int mode) +wint_t UNIFYFS_WRAP(getwc)(FILE *stream) +wint_t UNIFYFS_WRAP(putwc)(wchar_t c, FILE *stream) +wint_t UNIFYFS_WRAP(ungetwc)(wint_t c, FILE *stream) diff --git a/client/check_fns/unifycr_translate.py b/client/check_fns/unifyfs_translate.py similarity index 71% rename from client/check_fns/unifycr_translate.py rename to client/check_fns/unifyfs_translate.py index 4ec591170..87dbf716b 100755 --- a/client/check_fns/unifycr_translate.py +++ b/client/check_fns/unifyfs_translate.py @@ -2,26 +2,26 @@ import sys import re -''' a modified version of Elsa's translate.py script here: - https://github.com/LLNL/GOTCHA-tracer/blob/master/modules/translate.py - this one just writes the gotcha struct to a header file - and writes out the UNIFYCR_DEF macros +''' a modified version of Elsa's translate.py script here: + https://github.com/LLNL/GOTCHA-tracer/blob/master/modules/translate.py + this one just writes the gotcha struct to a header file + and writes out the UNIFYFS_DEF macros - to generate gotcha header run the following: python unifycr_translate.py unifycr_list + to generate gotcha header run the following: python unifyfs_translate.py unifyfs_list ''' class func: def __init__(self, prototype): l = prototype.strip().split('(') x = filter(None, l[0].split(' ')) - p = l[2:] + p = l[2:] self.params = [elem.strip("),") for elem in p] # set up useful vars - self.ret_type = ' '.join(x[0:-1]) + self.ret_type = ' '.join(x[0:-1]) self.wrap_macro = x[-1] - self.real_macro = "UNIFYCR_REAL" - self.macro_def = "UNIFYCR_DEF" + self.real_macro = "UNIFYFS_REAL" + self.macro_def = "UNIFYFS_DEF" self.name = l[1].strip(")") self.name_quote = "\"" + self.name + "\"" @@ -43,10 +43,10 @@ def write_gotcha_file(filename, func_list, modulename): m = modulename.split('_')[0] f = open(filename, 'w') - # write each function definition in the list + # write each function definition in the list for function in func_list: - f.write(function.macro_def + "(" + function.name + ", " + function.ret_type + - "," " (" + function.params[0] + "));" + "\n") + f.write(function.macro_def + "(" + function.name + ", " + function.ret_type + + "," " (" + function.params[0] + "));" + "\n") # write map struct f.write("struct gotcha_binding_t wrap_" + modulename + "[] = {\n") @@ -55,7 +55,7 @@ def write_gotcha_file(filename, func_list, modulename): f.write("};\n") # define number of functions we are wrapping for gotcha struct - f.write("\n#define GOTCHA_NFUNCS (sizeof(wrap_unifycr_list) / sizeof(wrap_unifycr_list[0]))\n") + f.write("\n#define GOTCHA_NFUNCS (sizeof(wrap_unifyfs_list) / sizeof(wrap_unifyfs_list[0]))\n") # close struct and file f.close() diff --git a/client/check_fns/unifycr_unsupported_list.txt b/client/check_fns/unifyfs_unsupported_list.txt similarity index 100% rename from client/check_fns/unifycr_unsupported_list.txt rename to client/check_fns/unifyfs_unsupported_list.txt diff --git a/client/src/Makefile.am b/client/src/Makefile.am index 2b73da1f3..399eb80fb 100644 --- a/client/src/Makefile.am +++ b/client/src/Makefile.am @@ -1,80 +1,85 @@ -lib_LTLIBRARIES = libunifycr.la libunifycr_gotcha.la +lib_LTLIBRARIES = libunifyfs.la libunifyfs_gotcha.la -libunifycrdir = $(includedir) -libunifycr_gotchadir = $(includedir) +if HAVE_FORTRAN +lib_LTLIBRARIES += libunifyfsf.la +endif + +libunifyfsdir = $(includedir) +libunifyfs_gotchadir = $(includedir) AM_CFLAGS = -Wall -Wno-strict-aliasing -include_HEADERS = unifycr.h - -libunifycr_la_SOURCES = \ - unifycr.h \ - unifycr-fixed.c \ - unifycr-fixed.h \ - unifycr-stack.c \ - unifycr-stack.h \ - unifycr-stdio.c \ - unifycr-stdio.h \ - unifycr-sysio.c \ - unifycr-sysio.h \ - unifycr-dirops.h \ - unifycr-dirops.c \ - unifycr.c \ - unifycr.h \ - unifycr-internal.h \ - unifycr_clientcalls_rpc.h \ - unifycr_client.c \ - unifycr_client.h \ - flatbuffers_common_builder.h \ - flatbuffers_common_reader.h \ - ucr_read_builder.h \ - ucr_read_reader.h \ - uthash.h \ - utlist.h +include_HEADERS = unifyfs.h -libunifycr_la_CPPFLAGS = \ +if HAVE_FORTRAN +include_HEADERS += unifyfsf.h +endif + +CLIENT_COMMON_CPPFLAGS = \ -I$(top_builddir)/client \ -I$(top_srcdir)/common/src -libunifycr_la_CFLAGS = $(MPI_CFLAGS) $(MERCURY_CFLAGS) $(ARGOBOTS_CFLAGS) $(MARGO_CFLAGS) $(FLATCC_CFLAGS) -libunifycr_la_LDFLAGS = -version-info $(LIBUNIFYCR_LT_VERSION) $(MPI_CLDFLAGS) $(MERCURY_LDFLAGS) $(MERCURY_LIBS) $(ARGOBOTS_LDFLAGS) $(ARGOBOTS_LIBS) $(MARGO_LDFLAGS) $(MARGO_LIBS) $(FLATCC_LDFLAGS) $(FLATCC_LIBS) -libunifycr_la_LIBADD = $(top_builddir)/common/src/libunifycr_common.la \ - -lcrypto -lrt -lpthread - -libunifycr_gotcha_la_SOURCES = \ - unifycr.h \ - unifycr-fixed.c \ - unifycr-fixed.h \ - unifycr-stack.c \ - unifycr-stack.h \ - unifycr-stdio.c \ - unifycr-stdio.h \ - unifycr-sysio.c \ - unifycr-sysio.h \ - unifycr-dirops.h \ - unifycr-dirops.c \ - unifycr.c \ - unifycr.h \ - gotcha_map_unifycr_list.h \ - unifycr-internal.h \ - unifycr_clientcalls_rpc.h \ - unifycr_client.c \ - unifycr_client.h \ - flatbuffers_common_builder.h \ - flatbuffers_common_reader.h \ - ucr_read_builder.h \ - ucr_read_reader.h \ +CLIENT_COMMON_CFLAGS = \ + $(MPI_CFLAGS) \ + $(MERCURY_CFLAGS) \ + $(ARGOBOTS_CFLAGS) \ + $(MARGO_CFLAGS) \ + $(FLATCC_CFLAGS) + +CLIENT_COMMON_LDFLAGS = \ + -version-info $(LIBUNIFYFS_LT_VERSION) \ + $(MPI_CLDFLAGS) \ + $(MARGO_LDFLAGS) \ + $(FLATCC_LDFLAGS) + +CLIENT_COMMON_LIBADD = \ + $(top_builddir)/common/src/libunifyfs_common.la \ + $(MARGO_LIBS) \ + $(FLATCC_LIBS) \ + -lcrypto -lrt -lpthread + +CLIENT_COMMON_SOURCES = \ + margo_client.c \ + margo_client.h \ + unifyfs.c \ + unifyfs.h \ + unifyfs-dirops.h \ + unifyfs-dirops.c \ + unifyfs-fixed.c \ + unifyfs-fixed.h \ + unifyfs-internal.h \ + unifyfs-stack.c \ + unifyfs-stack.h \ + unifyfs-stdio.c \ + unifyfs-stdio.h \ + unifyfs-sysio.c \ + unifyfs-sysio.h \ uthash.h \ utlist.h -libunifycr_gotcha_la_CPPFLAGS = \ - -DUNIFYCR_GOTCHA \ - -I$(top_builddir)/client \ - -I$(top_srcdir)/common/src +if USE_PMPI_WRAPPERS +CLIENT_COMMON_SOURCES += \ + pmpi_wrappers.c \ + pmpi_wrappers.h +endif + +libunifyfs_la_SOURCES = $(CLIENT_COMMON_SOURCES) +libunifyfs_la_CPPFLAGS = $(CLIENT_COMMON_CPPFLAGS) +libunifyfs_la_CFLAGS = $(CLIENT_COMMON_CFLAGS) +libunifyfs_la_LDFLAGS = $(CLIENT_COMMON_LDFLAGS) +libunifyfs_la_LIBADD = $(CLIENT_COMMON_LIBADD) + +libunifyfs_gotcha_la_SOURCES = $(CLIENT_COMMON_SOURCES) gotcha_map_unifyfs_list.h +libunifyfs_gotcha_la_CPPFLAGS = $(CLIENT_COMMON_CPPFLAGS) -DUNIFYFS_GOTCHA +libunifyfs_gotcha_la_CFLAGS = $(CLIENT_COMMON_CFLAGS) $(GOTCHA_CFLAGS) +libunifyfs_gotcha_la_LDFLAGS = $(CLIENT_COMMON_LDFLAGS) $(GOTCHA_LDFLAGS) +libunifyfs_gotcha_la_LIBADD = $(CLIENT_COMMON_LIBADD) -lgotcha -libunifycr_gotcha_la_CFLAGS = $(GOTCHA_CFLAGS) $(MPI_CFLAGS) $(MERCURY_CFLAGS) $(ARGOBOTS_CFLAGS) $(MARGO_CFLAGS) $(FLATCC_CFLAGS) -libunifycr_gotcha_la_LDFLAGS = -version-info $(LIBUNIFYCR_LT_VERSION) $(GOTCHA_LDFLAGS) $(MPI_CLDFLAGS) $(MERCURY_LDFLAGS) $(MERCURY_LIBS) $(ARGOBOTS_LDFLAGS) $(ARGOBOTS_LIBS) $(MARGO_LDFLAGS) $(MARGO_LIBS) $(FLATCC_LDFLAGS) $(FLATCC_LIBS) -libunifycr_gotcha_la_LIBADD = $(top_builddir)/common/src/libunifycr_common.la \ - -lgotcha -lcrypto -lrt -lpthread +if HAVE_FORTRAN +libunifyfsf_la_SOURCES = unifyfsf.c +libunifyfsf_la_CPPFLAGS = $(CLIENT_COMMON_CPPFLAGS) +libunifyfsf_la_CFLAGS = $(AM_CFLAGS) $(CLIENT_COMMON_CFLAGS) +libunifyfsf_la_LIBADD = libunifyfs_gotcha.la +endif diff --git a/client/src/gotcha_map_unifycr_list.h b/client/src/gotcha_map_unifycr_list.h deleted file mode 100644 index 22f22f19d..000000000 --- a/client/src/gotcha_map_unifycr_list.h +++ /dev/null @@ -1,198 +0,0 @@ -//Generated by translate.py - -#include -#include - -UNIFYCR_DEF(access, int, (const char* path, int mode)); -UNIFYCR_DEF(mkdir, int, (const char* path, mode_t mode)); -UNIFYCR_DEF(rmdir, int, (const char* path)); -UNIFYCR_DEF(rename, int, (const char* oldpath, const char* newpath)); -UNIFYCR_DEF(truncate, int, (const char* path, off_t length)); -UNIFYCR_DEF(unlink, int, (const char* path)); -UNIFYCR_DEF(remove, int, (const char* path)); -UNIFYCR_DEF(stat, int, (const char* path, struct stat* buf)); -UNIFYCR_DEF(__xstat, int, (int vers, const char* path, struct stat* buf)); -UNIFYCR_DEF(__lxstat, int, (int vers, const char* path, struct stat* buf)); -UNIFYCR_DEF(creat, int, (const char* path, mode_t mode)); -UNIFYCR_DEF(creat64, int, (const char* path, mode_t mode)); -UNIFYCR_DEF(open, int, (const char* path, int flags, ...)); -UNIFYCR_DEF(open64, int, (const char* path, int flags, ...)); -UNIFYCR_DEF(__open_2, int, (const char* path, int flags, ...)); -UNIFYCR_DEF(lio_listio, int, (int mode, struct aiocb* const aiocb_list[], - int nitems, struct sigevent* sevp)); -UNIFYCR_DEF(lseek, off_t, (int fd, off_t offset, int whence)); -UNIFYCR_DEF(lseek64, off64_t, (int fd, off64_t offset, int whence)); -UNIFYCR_DEF(posix_fadvise, int, (int fd, off_t offset, off_t len, int advice)); -UNIFYCR_DEF(read, ssize_t, (int fd, void* buf, size_t count)); -UNIFYCR_DEF(write, ssize_t, (int fd, const void* buf, size_t count)); -UNIFYCR_DEF(readv, ssize_t, (int fd, const struct iovec* iov, int iovcnt)); -UNIFYCR_DEF(writev, ssize_t, (int fd, const struct iovec* iov, int iovcnt)); -UNIFYCR_DEF(pread, ssize_t, (int fd, void* buf, size_t count, off_t offset)); -UNIFYCR_DEF(pread64, ssize_t, (int fd, void* buf, size_t count, - off64_t offset)); -UNIFYCR_DEF(pwrite, ssize_t, (int fd, const void* buf, size_t count, - off_t offset)); -UNIFYCR_DEF(pwrite64, ssize_t, (int fd, const void* buf, size_t count, - off64_t offset)); -UNIFYCR_DEF(ftruncate, int, (int fd, off_t length)); -UNIFYCR_DEF(fsync, int, (int fd)); -UNIFYCR_DEF(fdatasync, int, (int fd)); -UNIFYCR_DEF(flock, int, (int fd, int operation)); -UNIFYCR_DEF(mmap, void*, (void* addr, size_t length, int prot, int flags, - int fd, off_t offset)); -UNIFYCR_DEF(msync, int, (void* addr, size_t length, int flags)); -UNIFYCR_DEF(munmap, int, (void* addr, size_t length)); -UNIFYCR_DEF(mmap64, void*, (void* addr, size_t length, int prot, int flags, - int fd, off_t offset)); -UNIFYCR_DEF(__fxstat, int, (int vers, int fd, struct stat* buf)); -UNIFYCR_DEF(close, int, (int fd)); -UNIFYCR_DEF(opendir, DIR*, (const char* name)); -UNIFYCR_DEF(fdopendir, DIR*, (int fd)); -UNIFYCR_DEF(closedir, int, (DIR* dirp)); -UNIFYCR_DEF(readdir, struct dirent*, (DIR* dirp)); -UNIFYCR_DEF(rewinddir, void, (DIR* dirp)); -UNIFYCR_DEF(dirfd, int, (DIR* dirp)); -UNIFYCR_DEF(telldir, long, (DIR* dirp)); -UNIFYCR_DEF(scandir, int, (const char* dirp, struct dirent** namelist, - int (*filter)(const struct dirent*), - int (*compar)(const struct dirent**, - const struct dirent**))); -UNIFYCR_DEF(seekdir, void, (DIR* dirp, long loc)); -UNIFYCR_DEF(fopen, FILE*, (const char* path, const char* mode)); -UNIFYCR_DEF(freopen, FILE*, (const char* path, const char* mode, - FILE* stream)); -UNIFYCR_DEF(setvbuf, int, (FILE* stream, char* buf, int type, size_t size)); -UNIFYCR_DEF(setbuf, void, (FILE* stream, char* buf)); -UNIFYCR_DEF(ungetc, int, (int c, FILE* stream)); -UNIFYCR_DEF(fgetc, int, (FILE* stream)); -UNIFYCR_DEF(fputc, int, (int c, FILE* stream)); -UNIFYCR_DEF(getc, int, (FILE* stream)); -UNIFYCR_DEF(putc, int, (int c, FILE* stream)); -UNIFYCR_DEF(fgets, char*, (char* s, int n, FILE* stream)); -UNIFYCR_DEF(fputs, int, (const char* s, FILE* stream)); -UNIFYCR_DEF(fread, size_t, (void* ptr, size_t size, size_t nitems, - FILE* stream)); -UNIFYCR_DEF(fwrite, size_t, (const void* ptr, size_t size, size_t nitems, - FILE* stream)); -UNIFYCR_DEF(fprintf, int, (FILE* stream, const char* format, ...)); -UNIFYCR_DEF(vfprintf, int, (FILE* stream, const char* format, va_list ap)); -UNIFYCR_DEF(fscanf, int, (FILE* stream, const char* format, ...)); -UNIFYCR_DEF(vfscanf, int, (FILE* stream, const char* format, va_list ap)); -UNIFYCR_DEF(fseek, int, (FILE* stream, long offset, int whence)); -UNIFYCR_DEF(fseeko, int, (FILE* stream, off_t offset, int whence)); -UNIFYCR_DEF(ftell, long, (FILE* stream)); -UNIFYCR_DEF(ftello, off_t, (FILE* stream)); -UNIFYCR_DEF(rewind, void, (FILE* stream)); -UNIFYCR_DEF(fgetpos, int, (FILE* stream, fpos_t* pos)); -UNIFYCR_DEF(fsetpos, int, (FILE* stream, const fpos_t* pos)); -UNIFYCR_DEF(fflush, int, (FILE* stream)); -UNIFYCR_DEF(feof, int, (FILE* stream)); -UNIFYCR_DEF(ferror, int, (FILE* stream)); -UNIFYCR_DEF(clearerr, void, (FILE* stream)); -UNIFYCR_DEF(fileno, int, (FILE* stream)); -UNIFYCR_DEF(fclose, int, (FILE* stream)); -UNIFYCR_DEF(fwprintf, int, (FILE* stream, const wchar_t* format, ...)); -UNIFYCR_DEF(fwscanf, int, (FILE* stream, const wchar_t* format, ...)); -UNIFYCR_DEF(vfwprintf, int, (FILE* stream, const wchar_t* format, va_list arg)); -UNIFYCR_DEF(vfwscanf, int, (FILE* stream, const wchar_t* format, va_list arg)); -UNIFYCR_DEF(fgetwc, wint_t, (FILE* stream)); -UNIFYCR_DEF(fgetws, wchar_t*, (wchar_t* s, int n, FILE* stream)); -UNIFYCR_DEF(fputwc, wint_t, (wchar_t wc, FILE* stream)); -UNIFYCR_DEF(fputws, int, (const wchar_t* s, FILE* stream)); -UNIFYCR_DEF(fwide, int, (FILE* stream, int mode)); -UNIFYCR_DEF(getwc, wint_t, (FILE* stream)); -UNIFYCR_DEF(putwc, wint_t, (wchar_t c, FILE* stream)); -UNIFYCR_DEF(ungetwc, wint_t, (wint_t c, FILE* stream)); - -struct gotcha_binding_t wrap_unifycr_list[] = { - { "access", UNIFYCR_WRAP(access), &UNIFYCR_REAL(access) }, - { "mkdir", UNIFYCR_WRAP(mkdir), &UNIFYCR_REAL(mkdir) }, - { "rmdir", UNIFYCR_WRAP(rmdir), &UNIFYCR_REAL(rmdir) }, - { "rename", UNIFYCR_WRAP(rename), &UNIFYCR_REAL(rename) }, - { "truncate", UNIFYCR_WRAP(truncate), &UNIFYCR_REAL(truncate) }, - { "unlink", UNIFYCR_WRAP(unlink), &UNIFYCR_REAL(unlink) }, - { "remove", UNIFYCR_WRAP(remove), &UNIFYCR_REAL(remove) }, - { "stat", UNIFYCR_WRAP(stat), &UNIFYCR_REAL(stat) }, - { "__xstat", UNIFYCR_WRAP(__xstat), &UNIFYCR_REAL(__xstat) }, - { "__lxstat", UNIFYCR_WRAP(__lxstat), &UNIFYCR_REAL(__lxstat) }, - { "creat", UNIFYCR_WRAP(creat), &UNIFYCR_REAL(creat) }, - { "creat64", UNIFYCR_WRAP(creat64), &UNIFYCR_REAL(creat64) }, - { "open", UNIFYCR_WRAP(open), &UNIFYCR_REAL(open) }, - { "open64", UNIFYCR_WRAP(open64), &UNIFYCR_REAL(open64) }, - { "__open_2", UNIFYCR_WRAP(__open_2), &UNIFYCR_REAL(__open_2) }, - { "lio_listio", UNIFYCR_WRAP(lio_listio), &UNIFYCR_REAL(lio_listio) }, - { "lseek", UNIFYCR_WRAP(lseek), &UNIFYCR_REAL(lseek) }, - { "lseek64", UNIFYCR_WRAP(lseek64), &UNIFYCR_REAL(lseek64) }, - { "posix_fadvise", UNIFYCR_WRAP(posix_fadvise), &UNIFYCR_REAL(posix_fadvise) }, - { "read", UNIFYCR_WRAP(read), &UNIFYCR_REAL(read) }, - { "write", UNIFYCR_WRAP(write), &UNIFYCR_REAL(write) }, - { "readv", UNIFYCR_WRAP(readv), &UNIFYCR_REAL(readv) }, - { "writev", UNIFYCR_WRAP(writev), &UNIFYCR_REAL(writev) }, - { "pread", UNIFYCR_WRAP(pread), &UNIFYCR_REAL(pread) }, - { "pread64", UNIFYCR_WRAP(pread64), &UNIFYCR_REAL(pread64) }, - { "pwrite", UNIFYCR_WRAP(pwrite), &UNIFYCR_REAL(pwrite) }, - { "pwrite64", UNIFYCR_WRAP(pwrite64), &UNIFYCR_REAL(pwrite64) }, - { "ftruncate", UNIFYCR_WRAP(ftruncate), &UNIFYCR_REAL(ftruncate) }, - { "fsync", UNIFYCR_WRAP(fsync), &UNIFYCR_REAL(fsync) }, - { "fdatasync", UNIFYCR_WRAP(fdatasync), &UNIFYCR_REAL(fdatasync) }, - { "flock", UNIFYCR_WRAP(flock), &UNIFYCR_REAL(flock) }, - { "mmap", UNIFYCR_WRAP(mmap), &UNIFYCR_REAL(mmap) }, - { "msync", UNIFYCR_WRAP(msync), &UNIFYCR_REAL(msync) }, - { "munmap", UNIFYCR_WRAP(munmap), &UNIFYCR_REAL(munmap) }, - { "mmap64", UNIFYCR_WRAP(mmap64), &UNIFYCR_REAL(mmap64) }, - { "__fxstat", UNIFYCR_WRAP(__fxstat), &UNIFYCR_REAL(__fxstat) }, - { "close", UNIFYCR_WRAP(close), &UNIFYCR_REAL(close) }, - { "opendir", UNIFYCR_WRAP(opendir), &UNIFYCR_REAL(opendir) }, - { "fdopendir", UNIFYCR_WRAP(fdopendir), &UNIFYCR_REAL(fdopendir) }, - { "closedir", UNIFYCR_WRAP(closedir), &UNIFYCR_REAL(closedir) }, - { "readdir", UNIFYCR_WRAP(readdir), &UNIFYCR_REAL(readdir) }, - { "rewinddir", UNIFYCR_WRAP(rewinddir), &UNIFYCR_REAL(rewinddir) }, - { "dirfd", UNIFYCR_WRAP(dirfd), &UNIFYCR_REAL(dirfd) }, - { "telldir", UNIFYCR_WRAP(telldir), &UNIFYCR_REAL(telldir) }, - { "scandir", UNIFYCR_WRAP(scandir), &UNIFYCR_REAL(scandir) }, - { "seekdir", UNIFYCR_WRAP(seekdir), &UNIFYCR_REAL(seekdir) }, - { "fopen", UNIFYCR_WRAP(fopen), &UNIFYCR_REAL(fopen) }, - { "freopen", UNIFYCR_WRAP(freopen), &UNIFYCR_REAL(freopen) }, - { "setvbuf", UNIFYCR_WRAP(setvbuf), &UNIFYCR_REAL(setvbuf) }, - { "setbuf", UNIFYCR_WRAP(setbuf), &UNIFYCR_REAL(setbuf) }, - { "ungetc", UNIFYCR_WRAP(ungetc), &UNIFYCR_REAL(ungetc) }, - { "fgetc", UNIFYCR_WRAP(fgetc), &UNIFYCR_REAL(fgetc) }, - { "fputc", UNIFYCR_WRAP(fputc), &UNIFYCR_REAL(fputc) }, - { "getc", UNIFYCR_WRAP(getc), &UNIFYCR_REAL(getc) }, - { "putc", UNIFYCR_WRAP(putc), &UNIFYCR_REAL(putc) }, - { "fgets", UNIFYCR_WRAP(fgets), &UNIFYCR_REAL(fgets) }, - { "fputs", UNIFYCR_WRAP(fputs), &UNIFYCR_REAL(fputs) }, - { "fread", UNIFYCR_WRAP(fread), &UNIFYCR_REAL(fread) }, - { "fwrite", UNIFYCR_WRAP(fwrite), &UNIFYCR_REAL(fwrite) }, - { "fprintf", UNIFYCR_WRAP(fprintf), &UNIFYCR_REAL(fprintf) }, - { "vfprintf", UNIFYCR_WRAP(vfprintf), &UNIFYCR_REAL(vfprintf) }, - { "fscanf", UNIFYCR_WRAP(fscanf), &UNIFYCR_REAL(fscanf) }, - { "vfscanf", UNIFYCR_WRAP(vfscanf), &UNIFYCR_REAL(vfscanf) }, - { "fseek", UNIFYCR_WRAP(fseek), &UNIFYCR_REAL(fseek) }, - { "fseeko", UNIFYCR_WRAP(fseeko), &UNIFYCR_REAL(fseeko) }, - { "ftell", UNIFYCR_WRAP(ftell), &UNIFYCR_REAL(ftell) }, - { "ftello", UNIFYCR_WRAP(ftello), &UNIFYCR_REAL(ftello) }, - { "rewind", UNIFYCR_WRAP(rewind), &UNIFYCR_REAL(rewind) }, - { "fgetpos", UNIFYCR_WRAP(fgetpos), &UNIFYCR_REAL(fgetpos) }, - { "fsetpos", UNIFYCR_WRAP(fsetpos), &UNIFYCR_REAL(fsetpos) }, - { "fflush", UNIFYCR_WRAP(fflush), &UNIFYCR_REAL(fflush) }, - { "feof", UNIFYCR_WRAP(feof), &UNIFYCR_REAL(feof) }, - { "ferror", UNIFYCR_WRAP(ferror), &UNIFYCR_REAL(ferror) }, - { "clearerr", UNIFYCR_WRAP(clearerr), &UNIFYCR_REAL(clearerr) }, - { "fileno", UNIFYCR_WRAP(fileno), &UNIFYCR_REAL(fileno) }, - { "fclose", UNIFYCR_WRAP(fclose), &UNIFYCR_REAL(fclose) }, - { "fwprintf", UNIFYCR_WRAP(fwprintf), &UNIFYCR_REAL(fwprintf) }, - { "fwscanf", UNIFYCR_WRAP(fwscanf), &UNIFYCR_REAL(fwscanf) }, - { "vfwprintf", UNIFYCR_WRAP(vfwprintf), &UNIFYCR_REAL(vfwprintf) }, - { "vfwscanf", UNIFYCR_WRAP(vfwscanf), &UNIFYCR_REAL(vfwscanf) }, - { "fgetwc", UNIFYCR_WRAP(fgetwc), &UNIFYCR_REAL(fgetwc) }, - { "fgetws", UNIFYCR_WRAP(fgetws), &UNIFYCR_REAL(fgetws) }, - { "fputwc", UNIFYCR_WRAP(fputwc), &UNIFYCR_REAL(fputwc) }, - { "fputws", UNIFYCR_WRAP(fputws), &UNIFYCR_REAL(fputws) }, - { "fwide", UNIFYCR_WRAP(fwide), &UNIFYCR_REAL(fwide) }, - { "getwc", UNIFYCR_WRAP(getwc), &UNIFYCR_REAL(getwc) }, - { "putwc", UNIFYCR_WRAP(putwc), &UNIFYCR_REAL(putwc) }, - { "ungetwc", UNIFYCR_WRAP(ungetwc), &UNIFYCR_REAL(ungetwc) }, -}; - -#define GOTCHA_NFUNCS (sizeof(wrap_unifycr_list) / sizeof(wrap_unifycr_list[0])) diff --git a/client/src/gotcha_map_unifyfs_list.h b/client/src/gotcha_map_unifyfs_list.h new file mode 100644 index 000000000..412b560f9 --- /dev/null +++ b/client/src/gotcha_map_unifyfs_list.h @@ -0,0 +1,202 @@ +//Generated by translate.py + +#include +#include + +UNIFYFS_DEF(access, int, (const char* path, int mode)); +UNIFYFS_DEF(chmod, int, (const char* path, mode_t mode)); +UNIFYFS_DEF(fchmod, int, (int fd, mode_t mode)); +UNIFYFS_DEF(mkdir, int, (const char* path, mode_t mode)); +UNIFYFS_DEF(rmdir, int, (const char* path)); +UNIFYFS_DEF(rename, int, (const char* oldpath, const char* newpath)); +UNIFYFS_DEF(truncate, int, (const char* path, off_t length)); +UNIFYFS_DEF(unlink, int, (const char* path)); +UNIFYFS_DEF(remove, int, (const char* path)); +UNIFYFS_DEF(stat, int, (const char* path, struct stat* buf)); +UNIFYFS_DEF(__xstat, int, (int vers, const char* path, struct stat* buf)); +UNIFYFS_DEF(__lxstat, int, (int vers, const char* path, struct stat* buf)); +UNIFYFS_DEF(creat, int, (const char* path, mode_t mode)); +UNIFYFS_DEF(creat64, int, (const char* path, mode_t mode)); +UNIFYFS_DEF(open, int, (const char* path, int flags, ...)); +UNIFYFS_DEF(open64, int, (const char* path, int flags, ...)); +UNIFYFS_DEF(__open_2, int, (const char* path, int flags, ...)); +UNIFYFS_DEF(lio_listio, int, (int mode, struct aiocb* const aiocb_list[], + int nitems, struct sigevent* sevp)); +UNIFYFS_DEF(lseek, off_t, (int fd, off_t offset, int whence)); +UNIFYFS_DEF(lseek64, off64_t, (int fd, off64_t offset, int whence)); +UNIFYFS_DEF(posix_fadvise, int, (int fd, off_t offset, off_t len, int advice)); +UNIFYFS_DEF(read, ssize_t, (int fd, void* buf, size_t count)); +UNIFYFS_DEF(write, ssize_t, (int fd, const void* buf, size_t count)); +UNIFYFS_DEF(readv, ssize_t, (int fd, const struct iovec* iov, int iovcnt)); +UNIFYFS_DEF(writev, ssize_t, (int fd, const struct iovec* iov, int iovcnt)); +UNIFYFS_DEF(pread, ssize_t, (int fd, void* buf, size_t count, off_t offset)); +UNIFYFS_DEF(pread64, ssize_t, (int fd, void* buf, size_t count, + off64_t offset)); +UNIFYFS_DEF(pwrite, ssize_t, (int fd, const void* buf, size_t count, + off_t offset)); +UNIFYFS_DEF(pwrite64, ssize_t, (int fd, const void* buf, size_t count, + off64_t offset)); +UNIFYFS_DEF(ftruncate, int, (int fd, off_t length)); +UNIFYFS_DEF(fsync, int, (int fd)); +UNIFYFS_DEF(fdatasync, int, (int fd)); +UNIFYFS_DEF(flock, int, (int fd, int operation)); +UNIFYFS_DEF(mmap, void*, (void* addr, size_t length, int prot, int flags, + int fd, off_t offset)); +UNIFYFS_DEF(msync, int, (void* addr, size_t length, int flags)); +UNIFYFS_DEF(munmap, int, (void* addr, size_t length)); +UNIFYFS_DEF(mmap64, void*, (void* addr, size_t length, int prot, int flags, + int fd, off_t offset)); +UNIFYFS_DEF(__fxstat, int, (int vers, int fd, struct stat* buf)); +UNIFYFS_DEF(close, int, (int fd)); +UNIFYFS_DEF(opendir, DIR*, (const char* name)); +UNIFYFS_DEF(fdopendir, DIR*, (int fd)); +UNIFYFS_DEF(closedir, int, (DIR* dirp)); +UNIFYFS_DEF(readdir, struct dirent*, (DIR* dirp)); +UNIFYFS_DEF(rewinddir, void, (DIR* dirp)); +UNIFYFS_DEF(dirfd, int, (DIR* dirp)); +UNIFYFS_DEF(telldir, long, (DIR* dirp)); +UNIFYFS_DEF(scandir, int, (const char* dirp, struct dirent** namelist, + int (*filter)(const struct dirent*), + int (*compar)(const struct dirent**, + const struct dirent**))); +UNIFYFS_DEF(seekdir, void, (DIR* dirp, long loc)); +UNIFYFS_DEF(fopen, FILE*, (const char* path, const char* mode)); +UNIFYFS_DEF(freopen, FILE*, (const char* path, const char* mode, + FILE* stream)); +UNIFYFS_DEF(setvbuf, int, (FILE* stream, char* buf, int type, size_t size)); +UNIFYFS_DEF(setbuf, void, (FILE* stream, char* buf)); +UNIFYFS_DEF(ungetc, int, (int c, FILE* stream)); +UNIFYFS_DEF(fgetc, int, (FILE* stream)); +UNIFYFS_DEF(fputc, int, (int c, FILE* stream)); +UNIFYFS_DEF(getc, int, (FILE* stream)); +UNIFYFS_DEF(putc, int, (int c, FILE* stream)); +UNIFYFS_DEF(fgets, char*, (char* s, int n, FILE* stream)); +UNIFYFS_DEF(fputs, int, (const char* s, FILE* stream)); +UNIFYFS_DEF(fread, size_t, (void* ptr, size_t size, size_t nitems, + FILE* stream)); +UNIFYFS_DEF(fwrite, size_t, (const void* ptr, size_t size, size_t nitems, + FILE* stream)); +UNIFYFS_DEF(fprintf, int, (FILE* stream, const char* format, ...)); +UNIFYFS_DEF(vfprintf, int, (FILE* stream, const char* format, va_list ap)); +UNIFYFS_DEF(fscanf, int, (FILE* stream, const char* format, ...)); +UNIFYFS_DEF(vfscanf, int, (FILE* stream, const char* format, va_list ap)); +UNIFYFS_DEF(fseek, int, (FILE* stream, long offset, int whence)); +UNIFYFS_DEF(fseeko, int, (FILE* stream, off_t offset, int whence)); +UNIFYFS_DEF(ftell, long, (FILE* stream)); +UNIFYFS_DEF(ftello, off_t, (FILE* stream)); +UNIFYFS_DEF(rewind, void, (FILE* stream)); +UNIFYFS_DEF(fgetpos, int, (FILE* stream, fpos_t* pos)); +UNIFYFS_DEF(fsetpos, int, (FILE* stream, const fpos_t* pos)); +UNIFYFS_DEF(fflush, int, (FILE* stream)); +UNIFYFS_DEF(feof, int, (FILE* stream)); +UNIFYFS_DEF(ferror, int, (FILE* stream)); +UNIFYFS_DEF(clearerr, void, (FILE* stream)); +UNIFYFS_DEF(fileno, int, (FILE* stream)); +UNIFYFS_DEF(fclose, int, (FILE* stream)); +UNIFYFS_DEF(fwprintf, int, (FILE* stream, const wchar_t* format, ...)); +UNIFYFS_DEF(fwscanf, int, (FILE* stream, const wchar_t* format, ...)); +UNIFYFS_DEF(vfwprintf, int, (FILE* stream, const wchar_t* format, va_list arg)); +UNIFYFS_DEF(vfwscanf, int, (FILE* stream, const wchar_t* format, va_list arg)); +UNIFYFS_DEF(fgetwc, wint_t, (FILE* stream)); +UNIFYFS_DEF(fgetws, wchar_t*, (wchar_t* s, int n, FILE* stream)); +UNIFYFS_DEF(fputwc, wint_t, (wchar_t wc, FILE* stream)); +UNIFYFS_DEF(fputws, int, (const wchar_t* s, FILE* stream)); +UNIFYFS_DEF(fwide, int, (FILE* stream, int mode)); +UNIFYFS_DEF(getwc, wint_t, (FILE* stream)); +UNIFYFS_DEF(putwc, wint_t, (wchar_t c, FILE* stream)); +UNIFYFS_DEF(ungetwc, wint_t, (wint_t c, FILE* stream)); + +struct gotcha_binding_t wrap_unifyfs_list[] = { + { "access", UNIFYFS_WRAP(access), &UNIFYFS_REAL(access) }, + { "chmod", UNIFYFS_WRAP(chmod), &UNIFYFS_REAL(chmod) }, + { "fchmod", UNIFYFS_WRAP(fchmod), &UNIFYFS_REAL(fchmod) }, + { "mkdir", UNIFYFS_WRAP(mkdir), &UNIFYFS_REAL(mkdir) }, + { "rmdir", UNIFYFS_WRAP(rmdir), &UNIFYFS_REAL(rmdir) }, + { "rename", UNIFYFS_WRAP(rename), &UNIFYFS_REAL(rename) }, + { "truncate", UNIFYFS_WRAP(truncate), &UNIFYFS_REAL(truncate) }, + { "unlink", UNIFYFS_WRAP(unlink), &UNIFYFS_REAL(unlink) }, + { "remove", UNIFYFS_WRAP(remove), &UNIFYFS_REAL(remove) }, + { "stat", UNIFYFS_WRAP(stat), &UNIFYFS_REAL(stat) }, + { "__xstat", UNIFYFS_WRAP(__xstat), &UNIFYFS_REAL(__xstat) }, + { "__lxstat", UNIFYFS_WRAP(__lxstat), &UNIFYFS_REAL(__lxstat) }, + { "creat", UNIFYFS_WRAP(creat), &UNIFYFS_REAL(creat) }, + { "creat64", UNIFYFS_WRAP(creat64), &UNIFYFS_REAL(creat64) }, + { "open", UNIFYFS_WRAP(open), &UNIFYFS_REAL(open) }, + { "open64", UNIFYFS_WRAP(open64), &UNIFYFS_REAL(open64) }, + { "__open_2", UNIFYFS_WRAP(__open_2), &UNIFYFS_REAL(__open_2) }, + { "lio_listio", UNIFYFS_WRAP(lio_listio), &UNIFYFS_REAL(lio_listio) }, + { "lseek", UNIFYFS_WRAP(lseek), &UNIFYFS_REAL(lseek) }, + { "lseek64", UNIFYFS_WRAP(lseek64), &UNIFYFS_REAL(lseek64) }, + { "posix_fadvise", UNIFYFS_WRAP(posix_fadvise), &UNIFYFS_REAL(posix_fadvise) }, + { "read", UNIFYFS_WRAP(read), &UNIFYFS_REAL(read) }, + { "write", UNIFYFS_WRAP(write), &UNIFYFS_REAL(write) }, + { "readv", UNIFYFS_WRAP(readv), &UNIFYFS_REAL(readv) }, + { "writev", UNIFYFS_WRAP(writev), &UNIFYFS_REAL(writev) }, + { "pread", UNIFYFS_WRAP(pread), &UNIFYFS_REAL(pread) }, + { "pread64", UNIFYFS_WRAP(pread64), &UNIFYFS_REAL(pread64) }, + { "pwrite", UNIFYFS_WRAP(pwrite), &UNIFYFS_REAL(pwrite) }, + { "pwrite64", UNIFYFS_WRAP(pwrite64), &UNIFYFS_REAL(pwrite64) }, + { "ftruncate", UNIFYFS_WRAP(ftruncate), &UNIFYFS_REAL(ftruncate) }, + { "fsync", UNIFYFS_WRAP(fsync), &UNIFYFS_REAL(fsync) }, + { "fdatasync", UNIFYFS_WRAP(fdatasync), &UNIFYFS_REAL(fdatasync) }, + { "flock", UNIFYFS_WRAP(flock), &UNIFYFS_REAL(flock) }, + { "mmap", UNIFYFS_WRAP(mmap), &UNIFYFS_REAL(mmap) }, + { "msync", UNIFYFS_WRAP(msync), &UNIFYFS_REAL(msync) }, + { "munmap", UNIFYFS_WRAP(munmap), &UNIFYFS_REAL(munmap) }, + { "mmap64", UNIFYFS_WRAP(mmap64), &UNIFYFS_REAL(mmap64) }, + { "__fxstat", UNIFYFS_WRAP(__fxstat), &UNIFYFS_REAL(__fxstat) }, + { "close", UNIFYFS_WRAP(close), &UNIFYFS_REAL(close) }, + { "opendir", UNIFYFS_WRAP(opendir), &UNIFYFS_REAL(opendir) }, + { "fdopendir", UNIFYFS_WRAP(fdopendir), &UNIFYFS_REAL(fdopendir) }, + { "closedir", UNIFYFS_WRAP(closedir), &UNIFYFS_REAL(closedir) }, + { "readdir", UNIFYFS_WRAP(readdir), &UNIFYFS_REAL(readdir) }, + { "rewinddir", UNIFYFS_WRAP(rewinddir), &UNIFYFS_REAL(rewinddir) }, + { "dirfd", UNIFYFS_WRAP(dirfd), &UNIFYFS_REAL(dirfd) }, + { "telldir", UNIFYFS_WRAP(telldir), &UNIFYFS_REAL(telldir) }, + { "scandir", UNIFYFS_WRAP(scandir), &UNIFYFS_REAL(scandir) }, + { "seekdir", UNIFYFS_WRAP(seekdir), &UNIFYFS_REAL(seekdir) }, + { "fopen", UNIFYFS_WRAP(fopen), &UNIFYFS_REAL(fopen) }, + { "freopen", UNIFYFS_WRAP(freopen), &UNIFYFS_REAL(freopen) }, + { "setvbuf", UNIFYFS_WRAP(setvbuf), &UNIFYFS_REAL(setvbuf) }, + { "setbuf", UNIFYFS_WRAP(setbuf), &UNIFYFS_REAL(setbuf) }, + { "ungetc", UNIFYFS_WRAP(ungetc), &UNIFYFS_REAL(ungetc) }, + { "fgetc", UNIFYFS_WRAP(fgetc), &UNIFYFS_REAL(fgetc) }, + { "fputc", UNIFYFS_WRAP(fputc), &UNIFYFS_REAL(fputc) }, + { "getc", UNIFYFS_WRAP(getc), &UNIFYFS_REAL(getc) }, + { "putc", UNIFYFS_WRAP(putc), &UNIFYFS_REAL(putc) }, + { "fgets", UNIFYFS_WRAP(fgets), &UNIFYFS_REAL(fgets) }, + { "fputs", UNIFYFS_WRAP(fputs), &UNIFYFS_REAL(fputs) }, + { "fread", UNIFYFS_WRAP(fread), &UNIFYFS_REAL(fread) }, + { "fwrite", UNIFYFS_WRAP(fwrite), &UNIFYFS_REAL(fwrite) }, + { "fprintf", UNIFYFS_WRAP(fprintf), &UNIFYFS_REAL(fprintf) }, + { "vfprintf", UNIFYFS_WRAP(vfprintf), &UNIFYFS_REAL(vfprintf) }, + { "fscanf", UNIFYFS_WRAP(fscanf), &UNIFYFS_REAL(fscanf) }, + { "vfscanf", UNIFYFS_WRAP(vfscanf), &UNIFYFS_REAL(vfscanf) }, + { "fseek", UNIFYFS_WRAP(fseek), &UNIFYFS_REAL(fseek) }, + { "fseeko", UNIFYFS_WRAP(fseeko), &UNIFYFS_REAL(fseeko) }, + { "ftell", UNIFYFS_WRAP(ftell), &UNIFYFS_REAL(ftell) }, + { "ftello", UNIFYFS_WRAP(ftello), &UNIFYFS_REAL(ftello) }, + { "rewind", UNIFYFS_WRAP(rewind), &UNIFYFS_REAL(rewind) }, + { "fgetpos", UNIFYFS_WRAP(fgetpos), &UNIFYFS_REAL(fgetpos) }, + { "fsetpos", UNIFYFS_WRAP(fsetpos), &UNIFYFS_REAL(fsetpos) }, + { "fflush", UNIFYFS_WRAP(fflush), &UNIFYFS_REAL(fflush) }, + { "feof", UNIFYFS_WRAP(feof), &UNIFYFS_REAL(feof) }, + { "ferror", UNIFYFS_WRAP(ferror), &UNIFYFS_REAL(ferror) }, + { "clearerr", UNIFYFS_WRAP(clearerr), &UNIFYFS_REAL(clearerr) }, + { "fileno", UNIFYFS_WRAP(fileno), &UNIFYFS_REAL(fileno) }, + { "fclose", UNIFYFS_WRAP(fclose), &UNIFYFS_REAL(fclose) }, + { "fwprintf", UNIFYFS_WRAP(fwprintf), &UNIFYFS_REAL(fwprintf) }, + { "fwscanf", UNIFYFS_WRAP(fwscanf), &UNIFYFS_REAL(fwscanf) }, + { "vfwprintf", UNIFYFS_WRAP(vfwprintf), &UNIFYFS_REAL(vfwprintf) }, + { "vfwscanf", UNIFYFS_WRAP(vfwscanf), &UNIFYFS_REAL(vfwscanf) }, + { "fgetwc", UNIFYFS_WRAP(fgetwc), &UNIFYFS_REAL(fgetwc) }, + { "fgetws", UNIFYFS_WRAP(fgetws), &UNIFYFS_REAL(fgetws) }, + { "fputwc", UNIFYFS_WRAP(fputwc), &UNIFYFS_REAL(fputwc) }, + { "fputws", UNIFYFS_WRAP(fputws), &UNIFYFS_REAL(fputws) }, + { "fwide", UNIFYFS_WRAP(fwide), &UNIFYFS_REAL(fwide) }, + { "getwc", UNIFYFS_WRAP(getwc), &UNIFYFS_REAL(getwc) }, + { "putwc", UNIFYFS_WRAP(putwc), &UNIFYFS_REAL(putwc) }, + { "ungetwc", UNIFYFS_WRAP(ungetwc), &UNIFYFS_REAL(ungetwc) }, +}; + +#define GOTCHA_NFUNCS (sizeof(wrap_unifyfs_list) / sizeof(wrap_unifyfs_list[0])) diff --git a/client/src/margo_client.c b/client/src/margo_client.c new file mode 100644 index 000000000..ea3676e8f --- /dev/null +++ b/client/src/margo_client.c @@ -0,0 +1,512 @@ +/************************************************************************** + * margo_client.c - Implements the client-server RPC calls (shared-memory) + **************************************************************************/ + +#include "unifyfs-internal.h" +#include "unifyfs_rpc_util.h" +#include "margo_client.h" + +/* global rpc context */ +client_rpc_context_t* client_rpc_context; // = NULL + +/* register client RPCs */ +static void register_client_rpcs(void) +{ + margo_instance_id mid = client_rpc_context->mid; + + client_rpc_context->rpcs.read_id = + MARGO_REGISTER(mid, "unifyfs_read_rpc", + unifyfs_read_in_t, + unifyfs_read_out_t, + NULL); + + client_rpc_context->rpcs.mread_id = + MARGO_REGISTER(mid, "unifyfs_mread_rpc", + unifyfs_mread_in_t, + unifyfs_mread_out_t, + NULL); + + client_rpc_context->rpcs.mount_id = + MARGO_REGISTER(mid, "unifyfs_mount_rpc", + unifyfs_mount_in_t, + unifyfs_mount_out_t, + NULL); + + client_rpc_context->rpcs.unmount_id = + MARGO_REGISTER(mid, "unifyfs_unmount_rpc", + unifyfs_unmount_in_t, + unifyfs_unmount_out_t, + NULL); + + client_rpc_context->rpcs.metaget_id = + MARGO_REGISTER(mid, "unifyfs_metaget_rpc", + unifyfs_metaget_in_t, unifyfs_metaget_out_t, + NULL); + + client_rpc_context->rpcs.metaset_id = + MARGO_REGISTER(mid, "unifyfs_metaset_rpc", + unifyfs_metaset_in_t, unifyfs_metaset_out_t, + NULL); + + client_rpc_context->rpcs.fsync_id = + MARGO_REGISTER(mid, "unifyfs_fsync_rpc", + unifyfs_fsync_in_t, unifyfs_fsync_out_t, + NULL); + + client_rpc_context->rpcs.filesize_id = + MARGO_REGISTER(mid, "unifyfs_filesize_rpc", + unifyfs_filesize_in_t, + unifyfs_filesize_out_t, + NULL); +} + +/* initialize margo client-server rpc */ +int unifyfs_client_rpc_init(void) +{ + /* initialize margo */ + hg_return_t hret; + char addr_self_string[128]; + hg_size_t addr_self_string_sz = sizeof(addr_self_string); + client_rpc_context_t* rpc_ctx; + + rpc_ctx = calloc(1, sizeof(client_rpc_context_t)); + if (NULL == rpc_ctx) { + LOGERR("Failed to allocate client RPC context"); + return UNIFYFS_FAILURE; + } + + /* initialize margo */ + char* svr_addr_string = rpc_lookup_local_server_addr(); + if (svr_addr_string == NULL) { + LOGERR("Failed to find local margo RPC server address"); + return UNIFYFS_FAILURE; + } + char* proto = strdup(svr_addr_string); + char* colon = strchr(proto, ':'); + if (NULL != colon) { + *colon = '\0'; + } + LOGDBG("svr_addr:'%s' proto:'%s'", svr_addr_string, proto); + + rpc_ctx->mid = margo_init(proto, MARGO_SERVER_MODE, 1, 1); + assert(rpc_ctx->mid); + free(proto); + margo_diag_start(rpc_ctx->mid); + + /* get server margo address */ + rpc_ctx->svr_addr = HG_ADDR_NULL; + margo_addr_lookup(rpc_ctx->mid, svr_addr_string, + &(rpc_ctx->svr_addr)); + free(svr_addr_string); + if (rpc_ctx->svr_addr == HG_ADDR_NULL) { + LOGERR("Failed to resolve margo server RPC address"); + free(rpc_ctx); + return UNIFYFS_FAILURE; + } + + /* get client margo address */ + hret = margo_addr_self(rpc_ctx->mid, &(rpc_ctx->client_addr)); + if (hret != HG_SUCCESS) { + LOGERR("margo_addr_self()"); + margo_finalize(rpc_ctx->mid); + free(rpc_ctx); + return UNIFYFS_FAILURE; + } + + hret = margo_addr_to_string(rpc_ctx->mid, addr_self_string, + &addr_self_string_sz, rpc_ctx->client_addr); + if (hret != HG_SUCCESS) { + LOGERR("margo_addr_to_string()"); + margo_finalize(rpc_ctx->mid); + free(rpc_ctx); + return UNIFYFS_FAILURE; + } + rpc_ctx->client_addr_str = strdup(addr_self_string); + + client_rpc_context = rpc_ctx; + register_client_rpcs(); + + return UNIFYFS_SUCCESS; +} + +/* free resources allocated in corresponding call + * to unifyfs_client_rpc_init, frees structure + * allocated and sets pcontect to NULL */ +int unifyfs_client_rpc_finalize(void) +{ + if (client_rpc_context != NULL) { + /* define a temporary to refer to context */ + client_rpc_context_t* ctx = client_rpc_context; + client_rpc_context = NULL; + + /* free margo address for client */ + margo_addr_free(ctx->mid, ctx->client_addr); + + /* free margo address to server */ + margo_addr_free(ctx->mid, ctx->svr_addr); + + /* shut down margo */ + margo_finalize(ctx->mid); + + /* free memory allocated for context structure, + * and set caller's pointer to NULL */ + free(ctx->client_addr_str); + free(ctx); + } + return UNIFYFS_SUCCESS; +} + +/* invokes the mount rpc function by calling unifyfs_sync_to_del */ +int invoke_client_mount_rpc(void) +{ + hg_handle_t handle; + unifyfs_mount_in_t in; + unifyfs_mount_out_t out; + hg_return_t hret; + int32_t ret; + + if (NULL == client_rpc_context) { + return UNIFYFS_FAILURE; + } + + hret = margo_create(client_rpc_context->mid, + client_rpc_context->svr_addr, + client_rpc_context->rpcs.mount_id, &handle); + assert(hret == HG_SUCCESS); + + /* fill in input struct */ + fill_client_mount_info(&in); + in.client_addr_str = strdup(client_rpc_context->client_addr_str); + + LOGDBG("invoking the mount rpc function in client"); + hret = margo_forward(handle, &in); + assert(hret == HG_SUCCESS); + free((void*)in.external_spill_dir); + free((void*)in.client_addr_str); + + /* decode response */ + hret = margo_get_output(handle, &out); + assert(hret == HG_SUCCESS); + ret = out.ret; + LOGDBG("Got response ret=%" PRIi32, ret); + + unifyfs_key_slice_range = out.max_recs_per_slice; + LOGDBG("set unifyfs_key_slice_range=%zu", unifyfs_key_slice_range); + + margo_free_output(handle, &out); + margo_destroy(handle); + return ret; +} + +/* function invokes the unmount rpc */ +int invoke_client_unmount_rpc(void) +{ + hg_handle_t handle; + unifyfs_unmount_in_t in; + unifyfs_unmount_out_t out; + hg_return_t hret; + int32_t ret; + + if (NULL == client_rpc_context) { + return UNIFYFS_FAILURE; + } + + hret = margo_create(client_rpc_context->mid, + client_rpc_context->svr_addr, + client_rpc_context->rpcs.unmount_id, + &handle); + assert(hret == HG_SUCCESS); + + /* fill in input struct */ + in.app_id = app_id; + in.local_rank_idx = local_rank_idx; + + LOGDBG("invoking the unmount rpc function in client"); + hret = margo_forward(handle, &in); + assert(hret == HG_SUCCESS); + + /* decode response */ + hret = margo_get_output(handle, &out); + assert(hret == HG_SUCCESS); + ret = out.ret; + LOGDBG("Got response ret=%" PRIi32, ret); + + margo_free_output(handle, &out); + margo_destroy(handle); + return (int)ret; +} + +/* invokes the client metaset rpc function */ +int invoke_client_metaset_rpc(unifyfs_file_attr_t* f_meta) +{ + hg_handle_t handle; + unifyfs_metaset_in_t in; + unifyfs_metaset_out_t out; + hg_return_t hret; + int32_t ret; + + if (NULL == client_rpc_context) { + return UNIFYFS_FAILURE; + } + + hret = margo_create(client_rpc_context->mid, + client_rpc_context->svr_addr, + client_rpc_context->rpcs.metaset_id, + &handle); + assert(hret == HG_SUCCESS); + + /* fill in input struct */ + in.fid = f_meta->fid; + in.gfid = f_meta->gfid; + in.filename = f_meta->filename; + in.mode = f_meta->mode; + in.uid = f_meta->uid; + in.gid = f_meta->gid; + in.size = f_meta->size; + in.atime = f_meta->atime; + in.mtime = f_meta->mtime; + in.ctime = f_meta->ctime; + in.is_laminated = f_meta->is_laminated; + + LOGDBG("invoking the metaset rpc function in client"); + hret = margo_forward(handle, &in); + assert(hret == HG_SUCCESS); + + /* decode response */ + hret = margo_get_output(handle, &out); + assert(hret == HG_SUCCESS); + ret = out.ret; + LOGDBG("Got response ret=%" PRIi32, ret); + + margo_free_output(handle, &out); + margo_destroy(handle); + return (int)ret; +} + +/* invokes the client metaget rpc function */ +int invoke_client_metaget_rpc(int gfid, + unifyfs_file_attr_t* file_meta) +{ + hg_handle_t handle; + unifyfs_metaget_in_t in; + unifyfs_metaget_out_t out; + hg_return_t hret; + int32_t ret; + + if (NULL == client_rpc_context) { + return UNIFYFS_FAILURE; + } + + hret = margo_create(client_rpc_context->mid, + client_rpc_context->svr_addr, + client_rpc_context->rpcs.metaget_id, + &handle); + assert(hret == HG_SUCCESS); + + /* fill in input struct */ + in.gfid = (int32_t)gfid; + LOGDBG("invoking the metaget rpc function in client"); + hret = margo_forward(handle, &in); + assert(hret == HG_SUCCESS); + + /* decode response */ + hret = margo_get_output(handle, &out); + assert(hret == HG_SUCCESS); + ret = out.ret; + LOGDBG("Got response ret=%" PRIi32, ret); + + if (ret == (int32_t)UNIFYFS_SUCCESS) { + /* fill in results */ + memset(file_meta, 0, sizeof(unifyfs_file_attr_t)); + strcpy(file_meta->filename, out.filename); + file_meta->gfid = gfid; + file_meta->mode = out.mode; + file_meta->uid = out.uid; + file_meta->gid = out.gid; + file_meta->size = out.size; + file_meta->atime = out.atime; + file_meta->mtime = out.mtime; + file_meta->ctime = out.ctime; + file_meta->is_laminated = out.is_laminated; + } + + margo_free_output(handle, &out); + margo_destroy(handle); + return (int)ret; +} + +/* invokes the client fsync rpc function */ +int invoke_client_fsync_rpc(int gfid) +{ + hg_handle_t handle; + unifyfs_fsync_in_t in; + unifyfs_fsync_out_t out; + hg_return_t hret; + int32_t ret; + + if (NULL == client_rpc_context) { + return UNIFYFS_FAILURE; + } + + hret = margo_create(client_rpc_context->mid, + client_rpc_context->svr_addr, + client_rpc_context->rpcs.fsync_id, + &handle); + assert(hret == HG_SUCCESS); + + /* fill in input struct */ + in.app_id = (int32_t)app_id; + in.local_rank_idx = (int32_t)local_rank_idx; + in.gfid = (int32_t)gfid; + + LOGDBG("invoking the fsync rpc function in client"); + hret = margo_forward(handle, &in); + assert(hret == HG_SUCCESS); + + /* decode response */ + hret = margo_get_output(handle, &out); + assert(hret == HG_SUCCESS); + ret = out.ret; + LOGDBG("Got response ret=%" PRIi32, ret); + + margo_free_output(handle, &out); + margo_destroy(handle); + return (int)ret; +} + +/* invokes the client filesize rpc function */ +int invoke_client_filesize_rpc(int gfid, + size_t* outsize) +{ + int32_t ret; + hg_handle_t handle; + + if (NULL == client_rpc_context) { + return UNIFYFS_FAILURE; + } + + /* get handle to rpc function */ + hg_return_t hret = margo_create(client_rpc_context->mid, + client_rpc_context->svr_addr, + client_rpc_context->rpcs.filesize_id, + &handle); + assert(hret == HG_SUCCESS); + + /* fill in input struct */ + unifyfs_filesize_in_t in; + in.app_id = (int32_t)app_id; + in.local_rank_idx = (int32_t)local_rank_idx; + in.gfid = (int32_t)gfid; + + /* call rpc function */ + LOGDBG("invoking the filesize rpc function in client"); + hret = margo_forward(handle, &in); + assert(hret == HG_SUCCESS); + + /* decode response */ + unifyfs_filesize_out_t out; + hret = margo_get_output(handle, &out); + assert(hret == HG_SUCCESS); + ret = out.ret; + LOGDBG("Got response ret=%" PRIu32, ret); + + /* save output from function */ + *outsize = (size_t) out.filesize; + + /* free resources */ + margo_free_output(handle, &out); + margo_destroy(handle); + return (int)ret; +} + +/* invokes the client read rpc function */ +int invoke_client_read_rpc(int gfid, + size_t offset, + size_t length) +{ + hg_handle_t handle; + unifyfs_read_in_t in; + unifyfs_read_out_t out; + hg_return_t hret; + int32_t ret; + + if (NULL == client_rpc_context) { + return UNIFYFS_FAILURE; + } + + /* fill in input struct */ + hret = margo_create(client_rpc_context->mid, + client_rpc_context->svr_addr, + client_rpc_context->rpcs.read_id, + &handle); + assert(hret == HG_SUCCESS); + + /* fill in input struct */ + in.app_id = (int32_t)app_id; + in.local_rank_idx = (int32_t)local_rank_idx; + in.gfid = (int32_t)gfid; + in.offset = (hg_size_t)offset; + in.length = (hg_size_t)length; + + LOGDBG("invoking the read rpc function in client"); + hret = margo_forward(handle, &in); + assert(hret == HG_SUCCESS); + + /* decode response */ + hret = margo_get_output(handle, &out); + assert(hret == HG_SUCCESS); + ret = out.ret; + LOGDBG("Got response ret=%" PRIi32, ret); + + margo_free_output(handle, &out); + margo_destroy(handle); + return (int)ret; +} + +/* invokes the client mread rpc function */ +int invoke_client_mread_rpc(int read_count, + size_t size, + void* buffer) +{ + hg_handle_t handle; + unifyfs_mread_in_t in; + unifyfs_mread_out_t out; + hg_return_t hret; + int32_t ret; + + if (NULL == client_rpc_context) { + return UNIFYFS_FAILURE; + } + + /* fill in input struct */ + hret = margo_create(client_rpc_context->mid, + client_rpc_context->svr_addr, + client_rpc_context->rpcs.mread_id, + &handle); + assert(hret == HG_SUCCESS); + + hret = margo_bulk_create(client_rpc_context->mid, 1, &buffer, &size, + HG_BULK_READ_ONLY, &in.bulk_handle); + assert(hret == HG_SUCCESS); + + /* fill in input struct */ + in.app_id = (int32_t)app_id; + in.local_rank_idx = (int32_t)local_rank_idx; + in.read_count = (int32_t)read_count; + in.bulk_size = (hg_size_t)size; + + LOGDBG("invoking the read rpc function in client"); + hret = margo_forward(handle, &in); + assert(hret == HG_SUCCESS); + + /* decode response */ + hret = margo_get_output(handle, &out); + assert(hret == HG_SUCCESS); + ret = out.ret; + LOGDBG("Got response ret=%" PRIi32, ret); + + margo_bulk_free(in.bulk_handle); + margo_free_output(handle, &out); + margo_destroy(handle); + return (int)ret; +} diff --git a/client/src/margo_client.h b/client/src/margo_client.h new file mode 100644 index 000000000..d4b0861c7 --- /dev/null +++ b/client/src/margo_client.h @@ -0,0 +1,60 @@ +#ifndef _MARGO_CLIENT_H +#define _MARGO_CLIENT_H + +/******************************************** + * margo_client.h - client-server margo RPCs + ********************************************/ + +#include +#include "unifyfs_meta.h" +#include "unifyfs_client_rpcs.h" + +typedef struct ClientRpcIds { + hg_id_t filesize_id; + hg_id_t read_id; + hg_id_t mread_id; + hg_id_t mount_id; + hg_id_t unmount_id; + hg_id_t metaget_id; + hg_id_t metaset_id; + hg_id_t fsync_id; +} client_rpcs_t; + +typedef struct ClientRpcContext { + margo_instance_id mid; + char* client_addr_str; + hg_addr_t client_addr; + hg_addr_t svr_addr; + client_rpcs_t rpcs; +} client_rpc_context_t; + + +int unifyfs_client_rpc_init(void); + +int unifyfs_client_rpc_finalize(void); + +void fill_client_mount_info(unifyfs_mount_in_t* in); + +int invoke_client_mount_rpc(void); + +int invoke_client_unmount_rpc(void); + +int invoke_client_metaset_rpc(unifyfs_file_attr_t* f_meta); + +int invoke_client_metaget_rpc(int gfid, + unifyfs_file_attr_t* f_meta); + +int invoke_client_fsync_rpc(int gfid); + +int invoke_client_filesize_rpc(int gfid, + size_t* filesize); + +int invoke_client_read_rpc(int gfid, + size_t offset, + size_t length); + +int invoke_client_mread_rpc(int read_count, + size_t size, + void* buffer); + +#endif // MARGO_CLIENT_H diff --git a/client/src/pmpi_wrappers.c b/client/src/pmpi_wrappers.c new file mode 100644 index 000000000..ac86d39ba --- /dev/null +++ b/client/src/pmpi_wrappers.c @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2019, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#include "pmpi_wrappers.h" +#include "unifyfs.h" +#include +#include + +int unifyfs_mpi_init(int* argc, char*** argv) +{ + int rc, ret; + int rank; + int world_sz = 0; + int app_id = 0; + + //fprintf(stderr, "DEBUG: %s - before PMPI_Init()\n", __func__); + + ret = PMPI_Init(argc, argv); + + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &world_sz); + + //fprintf(stderr, "DEBUG: %s - after PMPI_Init(), rank=%d ret=%d\n", + // __func__, rank, ret); + + rc = unifyfs_mount("/unifyfs", rank, (size_t)world_sz, app_id); + if (UNIFYFS_SUCCESS != rc) { + fprintf(stderr, "UNIFYFS ERROR: unifyfs_mount() failed with '%s'\n", + unifyfs_error_enum_description((unifyfs_error_e)rc)); + } + + return ret; +} + +int MPI_Init(int* argc, char*** argv) +{ + return unifyfs_mpi_init(argc, argv); +} + +void mpi_init_(MPI_Fint* ierr) +{ + int argc = 0; + char** argv = NULL; + int rc = unifyfs_mpi_init(&argc, &argv); + + if (NULL != ierr) { + *ierr = (MPI_Fint)rc; + } +} + +int unifyfs_mpi_finalize(void) +{ + int rc, ret; + + rc = unifyfs_unmount(); + if (UNIFYFS_SUCCESS != rc) { + fprintf(stderr, "UNIFYFS ERROR: unifyfs_unmount() failed with '%s'\n", + unifyfs_error_enum_description((unifyfs_error_e)rc)); + } + + //fprintf(stderr, "DEBUG: %s - before PMPI_Finalize()\n", __func__); + + ret = PMPI_Finalize(); + + //fprintf(stderr, "DEBUG: %s - after PMPI_Finalize(), ret=%d\n", + // __func__, ret); + + return ret; +} + +int MPI_Finalize(void) +{ + return unifyfs_mpi_finalize(); +} + +void mpi_finalize_(MPI_Fint* ierr) +{ + int rc = unifyfs_mpi_finalize(); + + if (NULL != ierr) { + *ierr = (MPI_Fint)rc; + } +} diff --git a/client/src/pmpi_wrappers.h b/client/src/pmpi_wrappers.h new file mode 100644 index 000000000..b90057ce4 --- /dev/null +++ b/client/src/pmpi_wrappers.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2019, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#ifndef UNIFYFS_PMPI_WRAPPERS_H +#define UNIFYFS_PMPI_WRAPPERS_H + +/* MPI_Init PMPI wrapper */ +int unifyfs_mpi_init(int* argc, char*** argv); +int MPI_Init(int* argc, char*** argv); +void mpi_init_(MPI_Fint* ierr); + +/* MPI_Finalize PMPI wrapper */ +int unifyfs_mpi_finalize(void); +int MPI_Finalize(void); +void mpi_finalize_(MPI_Fint* ierr); + +#endif /* UNIFYFS_PMPI_WRAPPERS_H */ diff --git a/client/src/ucr_read_builder.h b/client/src/ucr_read_builder.h deleted file mode 100644 index 44402b4bf..000000000 --- a/client/src/ucr_read_builder.h +++ /dev/null @@ -1,74 +0,0 @@ -#ifndef UCR_READ_BUILDER_H -#define UCR_READ_BUILDER_H - -/* Generated by flatcc 0.5.3-pre FlatBuffers schema compiler for C by dvide.com */ - -#ifndef UCR_READ_READER_H -#include "ucr_read_reader.h" -#endif -#ifndef FLATBUFFERS_COMMON_BUILDER_H -#include "flatbuffers_common_builder.h" -#endif -#include "flatcc/flatcc_prologue.h" -#ifndef flatbuffers_identifier -#define flatbuffers_identifier 0 -#endif -#ifndef flatbuffers_extension -#define flatbuffers_extension ".bin" -#endif - -#define __unifycr_Extent_formal_args , uint32_t v0, uint64_t v1, uint64_t v2 -#define __unifycr_Extent_call_args , v0, v1, v2 -static inline unifycr_Extent_t *unifycr_Extent_assign(unifycr_Extent_t *p, uint32_t v0, uint64_t v1, uint64_t v2) -{ p->fid = v0; p->offset = v1; p->length = v2; - return p; } -static inline unifycr_Extent_t *unifycr_Extent_copy(unifycr_Extent_t *p, const unifycr_Extent_t *p2) -{ p->fid = p2->fid; p->offset = p2->offset; p->length = p2->length; - return p; } -static inline unifycr_Extent_t *unifycr_Extent_assign_to_pe(unifycr_Extent_t *p, uint32_t v0, uint64_t v1, uint64_t v2) -{ flatbuffers_uint32_assign_to_pe(&p->fid, v0); flatbuffers_uint64_assign_to_pe(&p->offset, v1); flatbuffers_uint64_assign_to_pe(&p->length, v2); - return p; } -static inline unifycr_Extent_t *unifycr_Extent_copy_to_pe(unifycr_Extent_t *p, const unifycr_Extent_t *p2) -{ flatbuffers_uint32_copy_to_pe(&p->fid, &p2->fid); flatbuffers_uint64_copy_to_pe(&p->offset, &p2->offset); flatbuffers_uint64_copy_to_pe(&p->length, &p2->length); - return p; } -static inline unifycr_Extent_t *unifycr_Extent_assign_from_pe(unifycr_Extent_t *p, uint32_t v0, uint64_t v1, uint64_t v2) -{ flatbuffers_uint32_assign_from_pe(&p->fid, v0); flatbuffers_uint64_assign_from_pe(&p->offset, v1); flatbuffers_uint64_assign_from_pe(&p->length, v2); - return p; } -static inline unifycr_Extent_t *unifycr_Extent_copy_from_pe(unifycr_Extent_t *p, const unifycr_Extent_t *p2) -{ flatbuffers_uint32_copy_from_pe(&p->fid, &p2->fid); flatbuffers_uint64_copy_from_pe(&p->offset, &p2->offset); flatbuffers_uint64_copy_from_pe(&p->length, &p2->length); - return p; } -__flatbuffers_build_struct(flatbuffers_, unifycr_Extent, 24, 8, unifycr_Extent_identifier, unifycr_Extent_type_identifier) - -static const flatbuffers_voffset_t __unifycr_ReadRequest_required[] = { 0 }; -typedef flatbuffers_ref_t unifycr_ReadRequest_ref_t; -static unifycr_ReadRequest_ref_t unifycr_ReadRequest_clone(flatbuffers_builder_t *B, unifycr_ReadRequest_table_t t); -__flatbuffers_build_table(flatbuffers_, unifycr_ReadRequest, 1) - -#define __unifycr_ReadRequest_formal_args , unifycr_Extent_vec_ref_t v0 -#define __unifycr_ReadRequest_call_args , v0 -static inline unifycr_ReadRequest_ref_t unifycr_ReadRequest_create(flatbuffers_builder_t *B __unifycr_ReadRequest_formal_args); -__flatbuffers_build_table_prolog(flatbuffers_, unifycr_ReadRequest, unifycr_ReadRequest_identifier, unifycr_ReadRequest_type_identifier) - -__flatbuffers_build_vector_field(0, flatbuffers_, unifycr_ReadRequest_extents, unifycr_Extent, unifycr_Extent_t, unifycr_ReadRequest) - -static inline unifycr_ReadRequest_ref_t unifycr_ReadRequest_create(flatbuffers_builder_t *B __unifycr_ReadRequest_formal_args) -{ - if (unifycr_ReadRequest_start(B) - || unifycr_ReadRequest_extents_add(B, v0)) { - return 0; - } - return unifycr_ReadRequest_end(B); -} - -static unifycr_ReadRequest_ref_t unifycr_ReadRequest_clone(flatbuffers_builder_t *B, unifycr_ReadRequest_table_t t) -{ - __flatbuffers_memoize_begin(B, t); - if (unifycr_ReadRequest_start(B) - || unifycr_ReadRequest_extents_pick(B, t)) { - return 0; - } - __flatbuffers_memoize_end(B, t, unifycr_ReadRequest_end(B)); -} - -#include "flatcc/flatcc_epilogue.h" -#endif /* UCR_READ_BUILDER_H */ diff --git a/client/src/ucr_read_reader.h b/client/src/ucr_read_reader.h deleted file mode 100644 index 2b9acbd9c..000000000 --- a/client/src/ucr_read_reader.h +++ /dev/null @@ -1,74 +0,0 @@ -#ifndef UCR_READ_READER_H -#define UCR_READ_READER_H - -/* Generated by flatcc 0.5.3-pre FlatBuffers schema compiler for C by dvide.com */ - -#ifndef FLATBUFFERS_COMMON_READER_H -#include "flatbuffers_common_reader.h" -#endif -#include "flatcc/flatcc_flatbuffers.h" -#ifndef __alignas_is_defined -#include -#endif -#include "flatcc/flatcc_prologue.h" -#ifndef flatbuffers_identifier -#define flatbuffers_identifier 0 -#endif -#ifndef flatbuffers_extension -#define flatbuffers_extension ".bin" -#endif - -typedef struct unifycr_Extent unifycr_Extent_t; -typedef const unifycr_Extent_t *unifycr_Extent_struct_t; -typedef unifycr_Extent_t *unifycr_Extent_mutable_struct_t; -typedef const unifycr_Extent_t *unifycr_Extent_vec_t; -typedef unifycr_Extent_t *unifycr_Extent_mutable_vec_t; - -typedef const struct unifycr_ReadRequest_table *unifycr_ReadRequest_table_t; -typedef const flatbuffers_uoffset_t *unifycr_ReadRequest_vec_t; -typedef flatbuffers_uoffset_t *unifycr_ReadRequest_mutable_vec_t; -#ifndef unifycr_Extent_identifier -#define unifycr_Extent_identifier flatbuffers_identifier -#endif -#define unifycr_Extent_type_hash ((flatbuffers_thash_t)0xfe153735) -#define unifycr_Extent_type_identifier "\x35\x37\x15\xfe" -#ifndef unifycr_ReadRequest_identifier -#define unifycr_ReadRequest_identifier flatbuffers_identifier -#endif -#define unifycr_ReadRequest_type_hash ((flatbuffers_thash_t)0x70b2f5ee) -#define unifycr_ReadRequest_type_identifier "\xee\xf5\xb2\x70" - - -struct unifycr_Extent { - alignas(8) uint32_t fid; - alignas(8) uint64_t offset; - alignas(8) uint64_t length; -}; -static_assert(sizeof(unifycr_Extent_t) == 24, "struct size mismatch"); - -static inline const unifycr_Extent_t *unifycr_Extent__const_ptr_add(const unifycr_Extent_t *p, size_t i) { return p + i; } -static inline unifycr_Extent_t *unifycr_Extent__ptr_add(unifycr_Extent_t *p, size_t i) { return p + i; } -static inline unifycr_Extent_struct_t unifycr_Extent_vec_at(unifycr_Extent_vec_t vec, size_t i) -__flatbuffers_struct_vec_at(vec, i) -static inline size_t unifycr_Extent__size() { return 24; } -static inline size_t unifycr_Extent_vec_len(unifycr_Extent_vec_t vec) -__flatbuffers_vec_len(vec) -__flatbuffers_struct_as_root(unifycr_Extent) - -__flatbuffers_define_struct_scalar_field(unifycr_Extent, fid, flatbuffers_uint32, uint32_t) -__flatbuffers_define_struct_scalar_field(unifycr_Extent, offset, flatbuffers_uint64, uint64_t) -__flatbuffers_define_struct_scalar_field(unifycr_Extent, length, flatbuffers_uint64, uint64_t) - - -struct unifycr_ReadRequest_table { uint8_t unused__; }; - -static inline size_t unifycr_ReadRequest_vec_len(unifycr_ReadRequest_vec_t vec) -__flatbuffers_vec_len(vec) -static inline unifycr_ReadRequest_table_t unifycr_ReadRequest_vec_at(unifycr_ReadRequest_vec_t vec, size_t i) -__flatbuffers_offset_vec_at(unifycr_ReadRequest_table_t, vec, i, 0) -__flatbuffers_table_as_root(unifycr_ReadRequest) - -__flatbuffers_define_vector_field(0, unifycr_ReadRequest, extents, unifycr_Extent_vec_t, 0) - -#include "flatcc/flatcc_epilogue.h" -#endif /* UCR_READ_READER_H */ diff --git a/client/src/unifycr-stdio.h b/client/src/unifycr-stdio.h deleted file mode 100644 index 5bfd00aac..000000000 --- a/client/src/unifycr-stdio.h +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * - * Copyright 2017, UT-Battelle, LLC. - * - * LLNL-CODE-741539 - * All rights reserved. - * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. - */ - -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * Copyright (c) 2017, Florida State University. Contributions from - * the Computer Architecture and Systems Research Laboratory (CASTL) - * at the Department of Computer Science. - * - * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror - * LLNL-CODE-728877. All rights reserved. - * - * This file is part of UnifyCR. - * For details, see https://github.com/llnl/unifycr - * Please read https://github.com/llnl/unifycr/LICENSE for full license text. - */ - -/* - * Copyright (c) 2013, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * code Written by - * Raghunath Rajachandrasekar - * Kathryn Mohror - * Adam Moody - * All rights reserved. - * This file is part of UNIFYCR. - * For details, see https://github.com/hpc/unifycr - * Please also read this file LICENSE.UNIFYCR - */ - -#ifndef UNIFYCR_STDIO_H -#define UNIFYCR_STDIO_H - -#include "unifycr-internal.h" - -/* TODO: declare the wrapper functions we define in unifycr-stdio.c - * so other routines can call them */ -UNIFYCR_DECL(fclose, int, (FILE* stream)); -UNIFYCR_DECL(fflush, int, (FILE* stream)); -UNIFYCR_DECL(fopen, FILE*, (const char* path, const char* mode)); -UNIFYCR_DECL(freopen, FILE*, (const char* path, const char* mode, - FILE* stream)); -UNIFYCR_DECL(setbuf, void, (FILE* stream, char* buf)); -UNIFYCR_DECL(setvbuf, int, (FILE* stream, char* buf, int type, size_t size)); - -UNIFYCR_DECL(fprintf, int, (FILE* stream, const char* format, ...)); -UNIFYCR_DECL(fscanf, int, (FILE* stream, const char* format, ...)); -UNIFYCR_DECL(vfprintf, int, (FILE* stream, const char* format, va_list ap)); -UNIFYCR_DECL(vfscanf, int, (FILE* stream, const char* format, va_list ap)); - -UNIFYCR_DECL(fgetc, int, (FILE* stream)); -UNIFYCR_DECL(fgets, char*, (char* s, int n, FILE* stream)); -UNIFYCR_DECL(fputc, int, (int c, FILE* stream)); -UNIFYCR_DECL(fputs, int, (const char* s, FILE* stream)); -UNIFYCR_DECL(getc, int, (FILE* stream)); -UNIFYCR_DECL(putc, int, (int c, FILE* stream)); -UNIFYCR_DECL(ungetc, int, (int c, FILE* stream)); - -UNIFYCR_DECL(fread, size_t, (void* ptr, size_t size, size_t nitems, - FILE* stream)); -UNIFYCR_DECL(fwrite, size_t, (const void* ptr, size_t size, size_t nitems, - FILE* stream)); - -UNIFYCR_DECL(fgetpos, int, (FILE* stream, fpos_t* pos)); -UNIFYCR_DECL(fseek, int, (FILE* stream, long offset, int whence)); -UNIFYCR_DECL(fsetpos, int, (FILE* stream, const fpos_t* pos)); -UNIFYCR_DECL(ftell, long, (FILE* stream)); -UNIFYCR_DECL(rewind, void, (FILE* stream)); - -UNIFYCR_DECL(clearerr, void, (FILE* stream)); -UNIFYCR_DECL(feof, int, (FILE* stream)); -UNIFYCR_DECL(ferror, int, (FILE* stream)); - -UNIFYCR_DECL(fseeko, int, (FILE* stream, off_t offset, int whence)); -UNIFYCR_DECL(ftello, off_t, (FILE* stream)); -UNIFYCR_DECL(fileno, int, (FILE* stream)); - -/* http://www.open-std.org/jtc1/sc22/wg14/www/docs/n1124.pdf 7.24 */ - -UNIFYCR_DECL(fwprintf, int, (FILE* stream, const wchar_t* format, ...)); -UNIFYCR_DECL(fwscanf, int, (FILE* stream, const wchar_t* format, ...)); -UNIFYCR_DECL(vfwprintf, int, (FILE* stream, const wchar_t* format, - va_list arg)); -UNIFYCR_DECL(vfwscanf, int, (FILE* stream, const wchar_t* format, - va_list arg)); -UNIFYCR_DECL(fgetwc, wint_t, (FILE* stream)); -UNIFYCR_DECL(fgetws, wchar_t*, (wchar_t* s, int n, FILE* stream)); -UNIFYCR_DECL(fputwc, wint_t, (wchar_t wc, FILE* stream)); -UNIFYCR_DECL(fputws, int, (const wchar_t* s, FILE* stream)); -UNIFYCR_DECL(fwide, int, (FILE* stream, int mode)); -UNIFYCR_DECL(getwc, wint_t, (FILE* stream)); -UNIFYCR_DECL(putwc, wint_t, (wchar_t c, FILE* stream)); -UNIFYCR_DECL(ungetwc, wint_t, (wint_t c, FILE* stream)); - -#endif /* UNIFYCR_STDIO_H */ diff --git a/client/src/unifycr-sysio.h b/client/src/unifycr-sysio.h deleted file mode 100644 index 35c7759a8..000000000 --- a/client/src/unifycr-sysio.h +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * - * Copyright 2017, UT-Battelle, LLC. - * - * LLNL-CODE-741539 - * All rights reserved. - * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. - */ - -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * Copyright (c) 2017, Florida State University. Contributions from - * the Computer Architecture and Systems Research Laboratory (CASTL) - * at the Department of Computer Science. - * - * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror - * LLNL-CODE-728877. All rights reserved. - * - * This file is part of UnifyCR. - * For details, see https://github.com/llnl/unifycr - * Please read https://github.com/llnl/unifycr/LICENSE for full license text. - */ - -/* - * Copyright (c) 2013, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * code Written by - * Raghunath Rajachandrasekar - * Kathryn Mohror - * Adam Moody - * All rights reserved. - * This file is part of UNIFYCR. - * For details, see https://github.com/hpc/unifycr - * Please also read this file LICENSE.UNIFYCR - */ - -#ifndef UNIFYCR_SYSIO_H -#define UNIFYCR_SYSIO_H - -#include "unifycr-internal.h" - -/* --------------------------------------- - * POSIX wrappers: paths - * --------------------------------------- */ - -UNIFYCR_DECL(access, int, (const char* pathname, int mode)); -UNIFYCR_DECL(mkdir, int, (const char* path, mode_t mode)); -UNIFYCR_DECL(rmdir, int, (const char* path)); -UNIFYCR_DECL(unlink, int, (const char* path)); -UNIFYCR_DECL(remove, int, (const char* path)); -UNIFYCR_DECL(rename, int, (const char* oldpath, const char* newpath)); -UNIFYCR_DECL(truncate, int, (const char* path, off_t length)); -UNIFYCR_DECL(stat, int, (const char* path, struct stat* buf)); -//UNIFYCR_DECL(fstat, int, (int fd, struct stat* buf)); -UNIFYCR_DECL(__xstat, int, (int vers, const char* path, struct stat* buf)); -UNIFYCR_DECL(__lxstat, int, (int vers, const char* path, struct stat* buf)); -UNIFYCR_DECL(__fxstat, int, (int vers, int fd, struct stat* buf)); - -/* --------------------------------------- - * POSIX wrappers: file descriptors - * --------------------------------------- */ - -UNIFYCR_DECL(creat, int, (const char* path, mode_t mode)); -UNIFYCR_DECL(creat64, int, (const char* path, mode_t mode)); -UNIFYCR_DECL(open, int, (const char* path, int flags, ...)); -UNIFYCR_DECL(open64, int, (const char* path, int flags, ...)); -UNIFYCR_DECL(__open_2, int, (const char* path, int flags, ...)); -UNIFYCR_DECL(read, ssize_t, (int fd, void* buf, size_t count)); -UNIFYCR_DECL(write, ssize_t, (int fd, const void* buf, size_t count)); -UNIFYCR_DECL(readv, ssize_t, (int fd, const struct iovec* iov, int iovcnt)); -UNIFYCR_DECL(writev, ssize_t, (int fd, const struct iovec* iov, int iovcnt)); -UNIFYCR_DECL(pread, ssize_t, (int fd, void* buf, size_t count, off_t offset)); -UNIFYCR_DECL(pread64, ssize_t, (int fd, void* buf, size_t count, - off64_t offset)); -UNIFYCR_DECL(pwrite, ssize_t, (int fd, const void* buf, size_t count, - off_t offset)); -UNIFYCR_DECL(pwrite64, ssize_t, (int fd, const void* buf, size_t count, - off64_t offset)); -UNIFYCR_DECL(posix_fadvise, int, (int fd, off_t offset, off_t len, int advice)); -UNIFYCR_DECL(lseek, off_t, (int fd, off_t offset, int whence)); -UNIFYCR_DECL(lseek64, off64_t, (int fd, off64_t offset, int whence)); -UNIFYCR_DECL(ftruncate, int, (int fd, off_t length)); -UNIFYCR_DECL(fsync, int, (int fd)); -UNIFYCR_DECL(fdatasync, int, (int fd)); -UNIFYCR_DECL(flock, int, (int fd, int operation)); -UNIFYCR_DECL(mmap, void*, (void* addr, size_t length, int prot, int flags, - int fd, off_t offset)); -UNIFYCR_DECL(mmap64, void*, (void* addr, size_t length, int prot, int flags, - int fd, off64_t offset)); -UNIFYCR_DECL(munmap, int, (void* addr, size_t length)); -UNIFYCR_DECL(msync, int, (void* addr, size_t length, int flags)); -UNIFYCR_DECL(__fxstat, int, (int vers, int fd, struct stat* buf)); -UNIFYCR_DECL(close, int, (int fd)); -UNIFYCR_DECL(lio_listio, int, (int mode, struct aiocb* const aiocb_list[], - int nitems, struct sigevent* sevp)); - -/* read count bytes info buf from file starting at offset pos, - * returns number of bytes actually read in retcount, - * retcount will be less than count only if an error occurs - * or end of file is reached */ -int unifycr_fd_read(int fd, off_t pos, void* buf, size_t count, - size_t* retcount); - -/* write count bytes from buf into file starting at offset pos, - * allocates new bytes and updates file size as necessary, - * fills any gaps with zeros */ -int unifycr_fd_write(int fd, off_t pos, const void* buf, size_t count); -int unifycr_fd_logreadlist(read_req_t* read_req, int count); - -#include "unifycr-dirops.h" - -#endif /* UNIFYCR_SYSIO_H */ diff --git a/client/src/unifycr_client.c b/client/src/unifycr_client.c deleted file mode 100644 index a1f6668d5..000000000 --- a/client/src/unifycr_client.c +++ /dev/null @@ -1,372 +0,0 @@ - -/******************************************************************************************* - * unifycr_client.c - Implements the RPC handlers for the - * application-client, shared-memory interface. - ********************************************************************************************/ - -#include "unifycr-internal.h" -#include "unifycr_client.h" -#include "unifycr_clientcalls_rpc.h" -#include "unifycr_log.h" - -/* add margo and argobots */ -#include -#include - -/* declaration should be moved to header file */ -int unifycr_sync_to_del(unifycr_mount_in_t* in); - -static int set_global_file_meta(unifycr_metaset_in_t* in, - unifycr_file_attr_t* f_meta) -{ - in->fid = f_meta->fid; - in->gfid = f_meta->gfid; - in->filename = f_meta->filename; - - /* TODO: unifycr_metaset_in_t is missing struct stat info - * in->file_attr = f_meta->file_attr; */ - - return UNIFYCR_SUCCESS; -} - -static int get_global_file_meta(int gfid, unifycr_metaget_out_t* out, - unifycr_file_attr_t* file_meta) -{ - memset(file_meta, 0, sizeof(unifycr_file_attr_t)); - - file_meta->gfid = gfid; - strcpy(file_meta->filename, out->filename); - file_meta->file_attr.st_size = out->st_size; - - return UNIFYCR_SUCCESS; -} - -/* invokes the mount rpc function by calling unifycr_sync_to_del */ -int32_t unifycr_client_mount_rpc_invoke(unifycr_client_rpc_context_t** - unifycr_rpc_context) -{ - hg_handle_t handle; - unifycr_mount_in_t in; - unifycr_mount_out_t out; - hg_return_t hret; - - LOGDBG("invoking the mount rpc function in client"); - - /* fill in input struct */ - hret = margo_create((*unifycr_rpc_context)->mid, - (*unifycr_rpc_context)->svr_addr, - (*unifycr_rpc_context)->unifycr_mount_rpc_id, &handle); - assert(hret == HG_SUCCESS); - - /* fill in input struct by calling unifycr_sync_to_del */ - unifycr_sync_to_del(&in); - hret = margo_forward(handle, &in); - assert(hret == HG_SUCCESS); - free((void*)in.external_spill_dir); - - /* decode response */ - hret = margo_get_output(handle, &out); - assert(hret == HG_SUCCESS); - - unifycr_key_slice_range = out.max_recs_per_slice; - LOGDBG("set unifycr_key_slice_range=%zu", unifycr_key_slice_range); - - margo_free_output(handle, &out); - margo_destroy(handle); - return out.ret; -} - -/* function invokes the unmount rpc - * TODO: Need secondary function to do actual cleanup */ -int32_t unifycr_client_unmount_rpc_invoke(unifycr_client_rpc_context_t** - unifycr_rpc_context) -{ - hg_handle_t handle; - unifycr_unmount_in_t in; - unifycr_unmount_out_t out; - hg_return_t hret; - int ret; - - LOGDBG("invoking the unmount rpc function in client"); - - hret = margo_create((*unifycr_rpc_context)->mid, - (*unifycr_rpc_context)->svr_addr, - (*unifycr_rpc_context)->unifycr_unmount_rpc_id, - &handle); - assert(hret == HG_SUCCESS); - - /* fill in input struct */ - in.app_id = app_id; - in.local_rank_idx = local_rank_idx; - - hret = margo_forward(handle, &in); - assert(hret == HG_SUCCESS); - - /* decode response */ - hret = margo_get_output(handle, &out); - assert(hret == HG_SUCCESS); - ret = out.ret; - - margo_free_output(handle, &out); - margo_destroy(handle); - return ret; -} - -/* invokes the client metaset rpc function by calling set_global_file_meta */ -int32_t unifycr_client_metaset_rpc_invoke(unifycr_client_rpc_context_t** - unifycr_rpc_context, - unifycr_file_attr_t* f_meta) -{ - hg_handle_t handle; - unifycr_metaset_in_t in; - unifycr_metaset_out_t out; - hg_return_t hret; - - LOGDBG("invoking the metaset rpc function in client"); - - /* fill in input struct */ - hret = margo_create((*unifycr_rpc_context)->mid, - (*unifycr_rpc_context)->svr_addr, - (*unifycr_rpc_context)->unifycr_metaset_rpc_id, - &handle); - assert(hret == HG_SUCCESS); - - /* fill in input struct by calling unifycr_sync_to_del */ - set_global_file_meta(&in, f_meta); - - hret = margo_forward(handle, &in); - assert(hret == HG_SUCCESS); - - /* decode response */ - hret = margo_get_output(handle, &out); - assert(hret == HG_SUCCESS); - - LOGDBG("Got response ret: %d", (int)out.ret); - - margo_free_output(handle, &out); - margo_destroy(handle); - return out.ret; -} - -/* invokes the client metaget rpc function by calling get_global_file_meta */ -int32_t unifycr_client_metaget_rpc_invoke(unifycr_client_rpc_context_t** - unifycr_rpc_context, - int32_t gfid, - unifycr_file_attr_t* file_meta) -{ - hg_handle_t handle; - unifycr_metaget_in_t in; - unifycr_metaget_out_t out; - hg_return_t hret; - - LOGDBG("invoking the metaget rpc function in client"); - - /* fill in input struct */ - hret = margo_create((*unifycr_rpc_context)->mid, - (*unifycr_rpc_context)->svr_addr, - (*unifycr_rpc_context)->unifycr_metaget_rpc_id, - &handle); - assert(hret == HG_SUCCESS); - - in.gfid = gfid; - hret = margo_forward(handle, &in); - assert(hret == HG_SUCCESS); - - /* decode response */ - hret = margo_get_output(handle, &out); - assert(hret == HG_SUCCESS); - - LOGDBG("Got metaget response ret: %d", (int)out.ret); - if (out.ret == ACK_SUCCESS) { - /* fill in results */ - get_global_file_meta(gfid, &out, file_meta); - } - - margo_free_output(handle, &out); - margo_destroy(handle); - return out.ret; -} - -/* invokes the client fsync rpc function */ -int32_t unifycr_client_fsync_rpc_invoke(unifycr_client_rpc_context_t** - unifycr_rpc_context, - int32_t app_id, - int32_t local_rank_idx, - int32_t gfid) -{ - hg_handle_t handle; - unifycr_fsync_in_t in; - unifycr_fsync_out_t out; - hg_return_t hret; - - LOGDBG("invoking the fsync rpc function in client"); - - /* fill in input struct */ - hret = margo_create((*unifycr_rpc_context)->mid, - (*unifycr_rpc_context)->svr_addr, - (*unifycr_rpc_context)->unifycr_fsync_rpc_id, - &handle); - assert(hret == HG_SUCCESS); - - /* fill in input struct */ - in.app_id = app_id; - in.local_rank_idx = local_rank_idx; - in.gfid = gfid; - - hret = margo_forward(handle, &in); - assert(hret == HG_SUCCESS); - - /* decode response */ - hret = margo_get_output(handle, &out); - assert(hret == HG_SUCCESS); - - LOGDBG("Got response ret: %d", (int)out.ret); - - margo_free_output(handle, &out); - margo_destroy(handle); - return out.ret; -} - -/* invokes the client filesize rpc function */ -uint32_t unifycr_client_filesize_rpc_invoke(unifycr_client_rpc_context_t** - unifycr_rpc_context, - int32_t app_id, - int32_t local_rank_idx, - int32_t gfid, - hg_size_t* outsize) -{ - printf("invoking the filesize rpc function in client\n"); - - /* get handle to rpc function */ - hg_handle_t handle; - hg_return_t hret = margo_create((*unifycr_rpc_context)->mid, - (*unifycr_rpc_context)->svr_addr, - (*unifycr_rpc_context)->unifycr_filesize_rpc_id, - &handle); - assert(hret == HG_SUCCESS); - assert(hret == HG_SUCCESS); - - /* fill in input struct */ - unifycr_filesize_in_t in; - in.app_id = app_id; - in.local_rank_idx = local_rank_idx; - in.gfid = gfid; - - /* call rpc function */ - hret = margo_forward(handle, &in); - assert(hret == HG_SUCCESS); - printf("Forwarded\n"); - - /* decode response */ - unifycr_filesize_out_t out; - hret = margo_get_output(handle, &out); - assert(hret == HG_SUCCESS); - - /* save output from function */ - printf("Got response in read, ret: %d\n", out.ret); - uint32_t ret = (uint32_t) out.ret; - *outsize = (hg_size_t) out.filesize; - - /* free resources */ - margo_free_output(handle, &out); - margo_destroy(handle); - - return ret; -} - -/* invokes the client read rpc function */ -int32_t unifycr_client_read_rpc_invoke(unifycr_client_rpc_context_t** - unifycr_rpc_context, - int32_t app_id, - int32_t local_rank_idx, - int32_t gfid, - hg_size_t offset, - hg_size_t length) -{ - hg_handle_t handle; - unifycr_read_in_t in; - unifycr_read_out_t out; - hg_return_t hret; - - LOGDBG("invoking the read rpc function in client"); - - /* fill in input struct */ - hret = margo_create((*unifycr_rpc_context)->mid, - (*unifycr_rpc_context)->svr_addr, - (*unifycr_rpc_context)->unifycr_read_rpc_id, - &handle); - assert(hret == HG_SUCCESS); - assert(hret == HG_SUCCESS); - - /* fill in input struct */ - in.app_id = app_id; - in.local_rank_idx = local_rank_idx; - in.gfid = gfid; - in.offset = offset; - in.length = length; - - hret = margo_forward(handle, &in); - assert(hret == HG_SUCCESS); - LOGDBG("Forwarded"); - - /* decode response */ - hret = margo_get_output(handle, &out); - assert(hret == HG_SUCCESS); - - LOGDBG("Got response in read, ret: %d", (int)out.ret); - - margo_free_output(handle, &out); - margo_destroy(handle); - return out.ret; -} - -/* invokes the client mread rpc function */ -int32_t unifycr_client_mread_rpc_invoke(unifycr_client_rpc_context_t** - unifycr_rpc_context, - int32_t app_id, - int32_t local_rank_idx, - int32_t gfid, - int32_t read_count, - hg_size_t size, - void* buffer) -{ - hg_handle_t handle; - unifycr_mread_in_t in; - unifycr_mread_out_t out; - hg_return_t hret; - - LOGDBG("invoking the read rpc function in client"); - - /* fill in input struct */ - hret = margo_create((*unifycr_rpc_context)->mid, - (*unifycr_rpc_context)->svr_addr, - (*unifycr_rpc_context)->unifycr_mread_rpc_id, - &handle); - assert(hret == HG_SUCCESS); - hret = margo_bulk_create((*unifycr_rpc_context)->mid, 1, &buffer, &size, - HG_BULK_READ_ONLY, &in.bulk_handle); - assert(hret == HG_SUCCESS); - - /* fill in input struct */ - in.app_id = app_id; - in.local_rank_idx = local_rank_idx; - in.gfid = gfid; - in.read_count = read_count; - in.bulk_size = size; - - hret = margo_forward(handle, &in); - assert(hret == HG_SUCCESS); - LOGDBG("Forwarded"); - - /* decode response */ - hret = margo_get_output(handle, &out); - assert(hret == HG_SUCCESS); - - LOGDBG("Got response ret: %d", (int)out.ret); - - margo_bulk_free(in.bulk_handle); - margo_free_output(handle, &out); - margo_destroy(handle); - return out.ret; -} - diff --git a/client/src/unifycr_client.h b/client/src/unifycr_client.h deleted file mode 100644 index 8f47926a3..000000000 --- a/client/src/unifycr_client.h +++ /dev/null @@ -1,85 +0,0 @@ -#ifndef __UNIFYCR_CLIENT_H -#define __UNIFYCR_CLIENT_H - -/******************************************************************************* - * unifycr_client.h - * - * Declarations for the unifycr client interface. - * - * ******************************************************************************/ - -#include "unifycr-internal.h" - -#include -#include - -typedef struct ClientRpcContext { - margo_instance_id mid; - hg_context_t* hg_context; - hg_class_t* hg_class; - hg_addr_t svr_addr; - hg_id_t unifycr_filesize_rpc_id; - hg_id_t unifycr_read_rpc_id; - hg_id_t unifycr_mread_rpc_id; - hg_id_t unifycr_mount_rpc_id; - hg_id_t unifycr_unmount_rpc_id; - hg_id_t unifycr_metaget_rpc_id; - hg_id_t unifycr_metaset_rpc_id; - hg_id_t unifycr_fsync_rpc_id; -} unifycr_client_rpc_context_t; - -/* global rpc context (probably should find a better spot for this) */ -extern unifycr_client_rpc_context_t* unifycr_rpc_context; - -/* -int unifycr_client_rpc_init(char* svr_addr_str, - unifycr_client_rpc_context_t** - unifycr_rpc_context); -*/ - -int32_t unifycr_client_mount_rpc_invoke(unifycr_client_rpc_context_t** - unifycr_rpc_context); - -int32_t unifycr_client_unmount_rpc_invoke(unifycr_client_rpc_context_t** - unifycr_rpc_context); - -int32_t unifycr_client_metaset_rpc_invoke(unifycr_client_rpc_context_t** - unifycr_rpc_context, - unifycr_file_attr_t* f_meta); - -int32_t unifycr_client_metaget_rpc_invoke( - unifycr_client_rpc_context_t** unifycr_rpc_context, - int32_t gfid, - unifycr_file_attr_t* f_meta); - -int32_t unifycr_client_fsync_rpc_invoke(unifycr_client_rpc_context_t** - unifycr_rpc_context, - int32_t app_id, - int32_t local_rank_idx, - int32_t gfid); - -uint32_t unifycr_client_filesize_rpc_invoke(unifycr_client_rpc_context_t** - unifycr_rpc_context, - int32_t app_id, - int32_t local_rank_idx, - int32_t gfid, - hg_size_t* filesize); - -int32_t unifycr_client_read_rpc_invoke(unifycr_client_rpc_context_t** - unifycr_rpc_context, - int32_t app_id, - int32_t local_rank_idx, - int32_t gfid, - hg_size_t offset, - hg_size_t length); - -int32_t unifycr_client_mread_rpc_invoke(unifycr_client_rpc_context_t** - unifycr_rpc_context, - int32_t app_id, - int32_t local_rank_idx, - int32_t gfid, - int32_t read_count, - hg_size_t size, - void* buffer); - -#endif diff --git a/client/src/unifycr_clientcalls_rpc.h b/client/src/unifycr_clientcalls_rpc.h deleted file mode 100644 index 24402727c..000000000 --- a/client/src/unifycr_clientcalls_rpc.h +++ /dev/null @@ -1,121 +0,0 @@ -#ifndef __UNIFYCR_CLIENTCALLS_RPC_H -#define __UNIFYCR_CLIENTCALLS_RPC_H - -/******************************************************************************* - * unifycr_clientcalls_rpc.h - * Declarations for the RPC shared-memory interfaces to the UCR server. - ********************************************************************************/ - -#include -#include -#include -#include -#include -#include - -/* called by client to register with the server, client provides a - * structure of values on input, some of which specify global - * values across all clients in the app_id, and some of which are - * specific to the client process, - * - * server creates a structure for the given app_id (if needed), - * and then fills in a set of values for the particular client, - * - * server attaches to client shared memory regions, opens files - * holding spill over data, and launchers request manager for - * client */ -MERCURY_GEN_PROC(unifycr_mount_out_t, - ((hg_size_t)(max_recs_per_slice)) - ((int32_t)(ret))) -MERCURY_GEN_PROC(unifycr_mount_in_t, - ((int32_t)(app_id)) - ((int32_t)(local_rank_idx)) - ((int32_t)(dbg_rank)) - ((int32_t)(num_procs_per_node)) - ((hg_size_t)(req_buf_sz)) - ((hg_size_t)(recv_buf_sz)) - ((hg_size_t)(superblock_sz)) - ((hg_size_t)(meta_offset)) - ((hg_size_t)(meta_size)) - ((hg_size_t)(fmeta_offset)) - ((hg_size_t)(fmeta_size)) - ((hg_size_t)(data_offset)) - ((hg_size_t)(data_size)) - ((hg_const_string_t)(external_spill_dir))) -DECLARE_MARGO_RPC_HANDLER(unifycr_mount_rpc) - -MERCURY_GEN_PROC(unifycr_unmount_out_t, ((int32_t)(ret))) -MERCURY_GEN_PROC(unifycr_unmount_in_t, - ((int32_t)(app_id)) - ((int32_t)(local_rank_idx))) -DECLARE_MARGO_RPC_HANDLER(unifycr_unmount_rpc) - -/* given a global file id and a file name, - * record key/value entry for this file */ -MERCURY_GEN_PROC(unifycr_metaset_out_t, ((int32_t)(ret))) -MERCURY_GEN_PROC(unifycr_metaset_in_t, - ((int32_t)(fid)) - ((int32_t)(gfid)) - ((hg_const_string_t)(filename))) -DECLARE_MARGO_RPC_HANDLER(unifycr_metaset_rpc) - -/* returns file meta data including file size and file name - * given a global file id */ -MERCURY_GEN_PROC(unifycr_metaget_out_t, - ((hg_size_t)(st_size)) - ((int32_t)(ret)) - ((hg_const_string_t)(filename))) -MERCURY_GEN_PROC(unifycr_metaget_in_t, - ((int32_t)(gfid))) -DECLARE_MARGO_RPC_HANDLER(unifycr_metaget_rpc) - -/* given app_id, client_id, and a global file id as input, - * read extent location metadata from client shared memory - * and insert corresponding key/value pairs into global index */ -MERCURY_GEN_PROC(unifycr_fsync_out_t, ((int32_t)(ret))) -MERCURY_GEN_PROC(unifycr_fsync_in_t, - ((int32_t)(app_id)) - ((int32_t)(local_rank_idx)) - ((int32_t)(gfid))) -DECLARE_MARGO_RPC_HANDLER(unifycr_fsync_rpc) - -/* given an app_id, client_id, global file id, - * return filesize for given file */ -MERCURY_GEN_PROC(unifycr_filesize_out_t, - ((int32_t)(ret)) - ((hg_size_t)(filesize))) -MERCURY_GEN_PROC(unifycr_filesize_in_t, - ((int32_t)(app_id)) - ((int32_t)(local_rank_idx)) - ((int32_t)(gfid))) -DECLARE_MARGO_RPC_HANDLER(unifycr_filesize_rpc) - -/* given an app_id, client_id, global file id, an offset, and a length, - * initiate read operation to lookup and return data, - * client synchronizes with server again later when data is available - * to be copied into user buffers */ -MERCURY_GEN_PROC(unifycr_read_out_t, ((int32_t)(ret))) -MERCURY_GEN_PROC(unifycr_read_in_t, - ((int32_t)(app_id)) - ((int32_t)(local_rank_idx)) - ((int32_t)(gfid)) - ((hg_size_t)(offset)) - ((hg_size_t)(length))) -DECLARE_MARGO_RPC_HANDLER(unifycr_read_rpc) - -/* given an app_id, client_id, global file id, and a count - * of read requests, follow by list of offset/length tuples - * initiate read requests for data, - * client synchronizes with server again later when data is available - * to be copied into user buffers */ -MERCURY_GEN_PROC(unifycr_mread_out_t, ((int32_t)(ret))) -MERCURY_GEN_PROC(unifycr_mread_in_t, - ((int32_t)(app_id)) - ((int32_t)(local_rank_idx)) - ((int32_t)(gfid)) - ((int32_t)(read_count)) - ((hg_size_t)(bulk_size)) - ((hg_bulk_t)(bulk_handle))) -DECLARE_MARGO_RPC_HANDLER(unifycr_mread_rpc) - -#endif diff --git a/client/src/unifycr-dirops.c b/client/src/unifyfs-dirops.c similarity index 56% rename from client/src/unifycr-dirops.c rename to client/src/unifyfs-dirops.c index 06cc37e3f..8325152f5 100644 --- a/client/src/unifycr-dirops.c +++ b/client/src/unifyfs-dirops.c @@ -7,20 +7,20 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ #include -#include "unifycr-sysio.h" +#include "unifyfs-sysio.h" /* given a file id corresponding to a directory, * allocate and initialize a directory stream */ -static inline unifycr_dirstream_t* unifycr_dirstream_alloc(int fid) +static inline unifyfs_dirstream_t* unifyfs_dirstream_alloc(int fid) { /* allocate a file descriptor for this stream */ - int fd = unifycr_stack_pop(unifycr_fd_stack); + int fd = unifyfs_stack_pop(unifyfs_fd_stack); if (fd < 0) { /* exhausted our file descriptors */ errno = EMFILE; @@ -28,29 +28,29 @@ static inline unifycr_dirstream_t* unifycr_dirstream_alloc(int fid) } /* allocate a directory stream id */ - int dirid = unifycr_stack_pop(unifycr_dirstream_stack); + int dirid = unifyfs_stack_pop(unifyfs_dirstream_stack); if (dirid < 0) { /* exhausted our directory streams, * return our file descriptor and set errno */ - unifycr_stack_push(unifycr_fd_stack, fd); + unifyfs_stack_push(unifyfs_fd_stack, fd); errno = EMFILE; return NULL; } /* get file descriptor for this fd */ - unifycr_fd_t* filedesc = unifycr_get_filedesc_from_fd(fd); + unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(fd); filedesc->fid = fid; filedesc->pos = 0; filedesc->read = 0; filedesc->write = 0; /* get pointer to file stream structure */ - unifycr_dirstream_t* dirp = &(unifycr_dirstreams[dirid]); + unifyfs_dirstream_t* dirp = &(unifyfs_dirstreams[dirid]); /* initialize fields in structure */ memset((void*) dirp, 0, sizeof(*dirp)); - /* record index into unifycr_dirstreams */ + /* record index into unifyfs_dirstreams */ dirp->dirid = dirid; /* record local file id corresponding to directory */ @@ -65,33 +65,33 @@ static inline unifycr_dirstream_t* unifycr_dirstream_alloc(int fid) return dirp; } -/* release resources allocated in unifycr_dirstream_alloc */ -static inline int unifycr_dirstream_free(unifycr_dirstream_t* dirp) +/* release resources allocated in unifyfs_dirstream_alloc */ +static inline int unifyfs_dirstream_free(unifyfs_dirstream_t* dirp) { /* reinit file descriptor to indicate that it's no longer in use, * not really necessary, but should help find bugs */ - unifycr_fd_init(dirp->fd); + unifyfs_fd_init(dirp->fd); /* return file descriptor to the free stack */ - unifycr_stack_push(unifycr_fd_stack, dirp->fd); + unifyfs_stack_push(unifyfs_fd_stack, dirp->fd); /* reinit dir stream to indicate that it's no longer in use, * not really necessary, but should help find bugs */ - unifycr_dirstream_init(dirp->dirid); + unifyfs_dirstream_init(dirp->dirid); /* return our index to directory stream stack */ - unifycr_stack_push(unifycr_dirstream_stack, dirp->dirid); + unifyfs_stack_push(unifyfs_dirstream_stack, dirp->dirid); - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } -DIR* UNIFYCR_WRAP(opendir)(const char* name) +DIR* UNIFYFS_WRAP(opendir)(const char* name) { /* call real opendir and return early if this is * not one of our paths */ - if (!unifycr_intercept_path(name)) { + if (!unifyfs_intercept_path(name)) { MAP_OR_FAIL(opendir); - return UNIFYCR_REAL(opendir)(name); + return UNIFYFS_REAL(opendir)(name); } /* @@ -99,32 +99,35 @@ DIR* UNIFYCR_WRAP(opendir)(const char* name) * if valid, populate the local file meta cache accordingly. */ - int fid = unifycr_get_fid_from_path(name); - int gfid = unifycr_generate_gfid(name); + int fid = unifyfs_get_fid_from_path(name); + int gfid = unifyfs_generate_gfid(name); - unifycr_file_attr_t gfattr = { 0, }; - int ret = unifycr_get_global_file_meta(fid, gfid, &gfattr); - if (ret != UNIFYCR_SUCCESS) { + unifyfs_file_attr_t gfattr = { 0, }; + int ret = unifyfs_get_global_file_meta(fid, gfid, &gfattr); + if (ret != UNIFYFS_SUCCESS) { errno = ENOENT; return NULL; } - struct stat* sb = &gfattr.file_attr; - if (!S_ISDIR(sb->st_mode)) { + struct stat sb = { 0, }; + + unifyfs_file_attr_to_stat(&gfattr, &sb); + + if (!S_ISDIR(sb.st_mode)) { errno = ENOTDIR; return NULL; } - unifycr_filemeta_t* meta = NULL; + unifyfs_filemeta_t* meta = NULL; if (fid >= 0) { - meta = unifycr_get_meta_from_fid(fid); + meta = unifyfs_get_meta_from_fid(fid); /* * FIXME: We found an inconsistent status between local cache and * global metadb. is it safe to invalidate the local entry and * re-populate with the global data? */ - if (!meta->is_dir) { + if (!unifyfs_fid_is_dir(fid)) { errno = EIO; return NULL; } @@ -132,135 +135,135 @@ DIR* UNIFYCR_WRAP(opendir)(const char* name) /* * FIXME: also, is it safe to oeverride this local data? */ - meta->size = sb->st_size; - meta->chunks = sb->st_blocks; + meta->size = sb.st_size; + meta->chunks = sb.st_blocks; meta->log_size = 0; /* no need of local storage for dir operations */ } else { - fid = unifycr_fid_create_file(name); + fid = unifyfs_fid_create_file(name); if (fid < 0) { errno = EIO; return NULL; } - meta = unifycr_get_meta_from_fid(fid); - meta->is_dir = 1; - meta->size = sb->st_size; - meta->chunks = sb->st_blocks; + meta = unifyfs_get_meta_from_fid(fid); + meta->mode = (meta->mode & ~S_IFREG) | S_IFDIR; /* set as directory */ + meta->size = sb.st_size; + meta->chunks = sb.st_blocks; meta->log_size = 0; } - unifycr_dirstream_t* dirp = unifycr_dirstream_alloc(fid); + unifyfs_dirstream_t* dirp = unifyfs_dirstream_alloc(fid); return (DIR*) dirp; } -DIR* UNIFYCR_WRAP(fdopendir)(int fd) +DIR* UNIFYFS_WRAP(fdopendir)(int fd) { - if (unifycr_intercept_fd(&fd)) { + if (unifyfs_intercept_fd(&fd)) { fprintf(stderr, "Function not yet supported @ %s:%d\n", __FILE__, __LINE__); errno = ENOSYS; return NULL; } else { MAP_OR_FAIL(fdopendir); - DIR* ret = UNIFYCR_REAL(fdopendir)(fd); + DIR* ret = UNIFYFS_REAL(fdopendir)(fd); return ret; } } -int UNIFYCR_WRAP(closedir)(DIR* dirp) +int UNIFYFS_WRAP(closedir)(DIR* dirp) { - if (unifycr_intercept_dirstream(dirp)) { - unifycr_dirstream_t* d = (unifycr_dirstream_t*) dirp; - unifycr_dirstream_free(d); + if (unifyfs_intercept_dirstream(dirp)) { + unifyfs_dirstream_t* d = (unifyfs_dirstream_t*) dirp; + unifyfs_dirstream_free(d); return 0; } else { MAP_OR_FAIL(closedir); - int ret = UNIFYCR_REAL(closedir)(dirp); + int ret = UNIFYFS_REAL(closedir)(dirp); return ret; } } -struct dirent* UNIFYCR_WRAP(readdir)(DIR* dirp) +struct dirent* UNIFYFS_WRAP(readdir)(DIR* dirp) { - if (unifycr_intercept_dirstream(dirp)) { + if (unifyfs_intercept_dirstream(dirp)) { fprintf(stderr, "Function not yet supported @ %s:%d\n", __FILE__, __LINE__); errno = ENOSYS; return NULL; } else { MAP_OR_FAIL(readdir); - struct dirent* d = UNIFYCR_REAL(readdir)(dirp); + struct dirent* d = UNIFYFS_REAL(readdir)(dirp); return d; } } -void UNIFYCR_WRAP(rewinddir)(DIR* dirp) +void UNIFYFS_WRAP(rewinddir)(DIR* dirp) { - if (unifycr_intercept_dirstream(dirp)) { - unifycr_dirstream_t* _dirp = (unifycr_dirstream_t*) dirp; + if (unifyfs_intercept_dirstream(dirp)) { + unifyfs_dirstream_t* _dirp = (unifyfs_dirstream_t*) dirp; /* TODO: update the pos in the file descriptor (fd) via lseek */ _dirp->pos = 0; } else { MAP_OR_FAIL(rewinddir); - UNIFYCR_REAL(rewinddir)(dirp); + UNIFYFS_REAL(rewinddir)(dirp); return; } } -int UNIFYCR_WRAP(dirfd)(DIR* dirp) +int UNIFYFS_WRAP(dirfd)(DIR* dirp) { - if (unifycr_intercept_dirstream(dirp)) { - unifycr_dirstream_t* d = (unifycr_dirstream_t*) dirp; + if (unifyfs_intercept_dirstream(dirp)) { + unifyfs_dirstream_t* d = (unifyfs_dirstream_t*) dirp; int fd = d->fd; return fd; } else { MAP_OR_FAIL(dirfd); - int fd = UNIFYCR_REAL(dirfd)(dirp); + int fd = UNIFYFS_REAL(dirfd)(dirp); return fd; } } -long UNIFYCR_WRAP(telldir)(DIR* dirp) +long UNIFYFS_WRAP(telldir)(DIR* dirp) { - if (unifycr_intercept_dirstream(dirp)) { - unifycr_dirstream_t* d = (unifycr_dirstream_t*) dirp; + if (unifyfs_intercept_dirstream(dirp)) { + unifyfs_dirstream_t* d = (unifyfs_dirstream_t*) dirp; return d->pos; } else { MAP_OR_FAIL(telldir); - long ret = UNIFYCR_REAL(telldir)(dirp); + long ret = UNIFYFS_REAL(telldir)(dirp); return ret; } } -int UNIFYCR_WRAP(scandir)(const char* path, struct dirent** namelist, +int UNIFYFS_WRAP(scandir)(const char* path, struct dirent** namelist, int (*filter)(const struct dirent*), int (*compar)(const struct dirent**, const struct dirent**)) { - if (unifycr_intercept_path(path)) { + if (unifyfs_intercept_path(path)) { fprintf(stderr, "Function not yet supported @ %s:%d\n", __FILE__, __LINE__); errno = ENOSYS; return -1; } else { MAP_OR_FAIL(scandir); - long ret = UNIFYCR_REAL(scandir)(path, namelist, filter, compar); + long ret = UNIFYFS_REAL(scandir)(path, namelist, filter, compar); return ret; } } -void UNIFYCR_WRAP(seekdir)(DIR* dirp, long loc) +void UNIFYFS_WRAP(seekdir)(DIR* dirp, long loc) { - if (unifycr_intercept_dirstream(dirp)) { + if (unifyfs_intercept_dirstream(dirp)) { fprintf(stderr, "Function not yet supported @ %s:%d\n", __FILE__, __LINE__); errno = ENOSYS; } else { MAP_OR_FAIL(seekdir); - UNIFYCR_REAL(seekdir)(dirp, loc); + UNIFYFS_REAL(seekdir)(dirp, loc); return; } } diff --git a/client/src/unifycr-dirops.h b/client/src/unifyfs-dirops.h similarity index 53% rename from client/src/unifycr-dirops.h rename to client/src/unifyfs-dirops.h index 60680db7f..7694ed492 100644 --- a/client/src/unifycr-dirops.h +++ b/client/src/unifyfs-dirops.h @@ -7,12 +7,12 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ -#ifndef __UNIFYCR_DIROPS_H -#define __UNIFYCR_DIROPS_H +#ifndef __UNIFYFS_DIROPS_H +#define __UNIFYFS_DIROPS_H #include @@ -38,18 +38,18 @@ * seekdir(3) */ -UNIFYCR_DECL(opendir, DIR*, (const char* name)); -UNIFYCR_DECL(fdopendir, DIR*, (int fd)); -UNIFYCR_DECL(closedir, int, (DIR* dirp)); -UNIFYCR_DECL(readdir, struct dirent*, (DIR* dirp)); -UNIFYCR_DECL(rewinddir, void, (DIR* dirp)); -UNIFYCR_DECL(dirfd, int, (DIR* dirp)); -UNIFYCR_DECL(telldir, long, (DIR* dirp)); -UNIFYCR_DECL(scandir, int, (const char* dirp, struct dirent** namelist, +UNIFYFS_DECL(opendir, DIR*, (const char* name)); +UNIFYFS_DECL(fdopendir, DIR*, (int fd)); +UNIFYFS_DECL(closedir, int, (DIR* dirp)); +UNIFYFS_DECL(readdir, struct dirent*, (DIR* dirp)); +UNIFYFS_DECL(rewinddir, void, (DIR* dirp)); +UNIFYFS_DECL(dirfd, int, (DIR* dirp)); +UNIFYFS_DECL(telldir, long, (DIR* dirp)); +UNIFYFS_DECL(scandir, int, (const char* dirp, struct dirent** namelist, int (*filter)(const struct dirent*), int (*compar)(const struct dirent**, const struct dirent**))); -UNIFYCR_DECL(seekdir, void, (DIR* dirp, long loc)); +UNIFYFS_DECL(seekdir, void, (DIR* dirp, long loc)); -#endif /* __UNIFYCR_DIROPS_H */ +#endif /* __UNIFYFS_DIROPS_H */ diff --git a/client/src/unifycr-fixed.c b/client/src/unifyfs-fixed.c similarity index 67% rename from client/src/unifycr-fixed.c rename to client/src/unifyfs-fixed.c index 97f3a4935..88b4c72a3 100644 --- a/client/src/unifycr-fixed.c +++ b/client/src/unifyfs-fixed.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -40,25 +40,39 @@ * Please also read this file LICENSE.CRUISE */ -#include "unifycr-fixed.h" -#include "unifycr_log.h" +#include "unifyfs-fixed.h" +#include "unifyfs_log.h" + +static inline +unifyfs_chunkmeta_t* filemeta_get_chunkmeta(const unifyfs_filemeta_t* meta, + int cid) +{ + unifyfs_chunkmeta_t* chunkmeta = NULL; + uint64_t limit = 0; + + if (unifyfs_use_memfs) { + limit += unifyfs_max_chunks; + } + + if (unifyfs_use_spillover) { + limit += unifyfs_spillover_max_chunks; + } + + if (meta && (cid >= 0 && cid < limit)) { + chunkmeta = &unifyfs_chunkmetas[meta->chunkmeta_idx + cid]; + } + + return chunkmeta; +} /* given a file id and logical chunk id, return pointer to meta data * for specified chunk, return NULL if not found */ -unifycr_chunkmeta_t* unifycr_get_chunkmeta(int fid, int cid) +static inline unifyfs_chunkmeta_t* unifyfs_get_chunkmeta(int fid, int cid) { /* lookup file meta data for specified file id */ - unifycr_filemeta_t* meta = unifycr_get_meta_from_fid(fid); - if (meta != NULL) { - /* now lookup chunk meta data for specified chunk id */ - if (cid >= 0 && cid < unifycr_max_chunks) { - unifycr_chunkmeta_t* chunk_meta = &(meta->chunk_meta[cid]); - return chunk_meta; - } - } + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - /* failed to find file or chunk id is out of range */ - return (unifycr_chunkmeta_t*)NULL; + return filemeta_get_chunkmeta(meta, cid); } /* --------------------------------------- @@ -67,21 +81,19 @@ unifycr_chunkmeta_t* unifycr_get_chunkmeta(int fid, int cid) /* given a logical chunk id and an offset within that chunk, return the pointer * to the memory location corresponding to that location */ -static inline void* unifycr_compute_chunk_buf( - const unifycr_filemeta_t* meta, - int logical_id, - off_t logical_offset) +static inline void* unifyfs_compute_chunk_buf(const unifyfs_filemeta_t* meta, + int cid, off_t offset) { /* get pointer to chunk meta */ - const unifycr_chunkmeta_t* chunk_meta = &(meta->chunk_meta[logical_id]); + const unifyfs_chunkmeta_t* chunk_meta = filemeta_get_chunkmeta(meta, cid); /* identify physical chunk id */ int physical_id = chunk_meta->id; /* compute the start of the chunk */ char* start = NULL; - if (physical_id < unifycr_max_chunks) { - start = unifycr_chunks + ((long)physical_id << unifycr_chunk_bits); + if (physical_id < unifyfs_max_chunks) { + start = unifyfs_chunks + ((long)physical_id << unifyfs_chunk_bits); } else { /* chunk is in spill over */ LOGERR("wrong chunk ID"); @@ -89,69 +101,67 @@ static inline void* unifycr_compute_chunk_buf( } /* now add offset */ - char* buf = start + logical_offset; + char* buf = start + offset; return (void*)buf; } /* given a chunk id and an offset within that chunk, return the offset * in the spillover file corresponding to that location */ -static inline off_t unifycr_compute_spill_offset( - const unifycr_filemeta_t* meta, - int logical_id, - off_t logical_offset) +static inline off_t unifyfs_compute_spill_offset(const unifyfs_filemeta_t* meta, + int cid, off_t offset) { /* get pointer to chunk meta */ - const unifycr_chunkmeta_t* chunk_meta = &(meta->chunk_meta[logical_id]); + const unifyfs_chunkmeta_t* chunk_meta = filemeta_get_chunkmeta(meta, cid); /* identify physical chunk id */ int physical_id = chunk_meta->id; /* compute start of chunk in spill over device */ off_t start = 0; - if (physical_id < unifycr_max_chunks) { + if (physical_id < unifyfs_max_chunks) { LOGERR("wrong spill-chunk ID"); return -1; } else { /* compute buffer loc within spillover device chunk */ - /* account for the unifycr_max_chunks added to identify location when + /* account for the unifyfs_max_chunks added to identify location when * grabbing this chunk */ - start = ((long)(physical_id - unifycr_max_chunks) << unifycr_chunk_bits); + start = ((long)(physical_id - unifyfs_max_chunks) << unifyfs_chunk_bits); } - off_t buf = start + logical_offset; + off_t buf = start + offset; return buf; } /* allocate a new chunk for the specified file and logical chunk id */ -static int unifycr_chunk_alloc(int fid, unifycr_filemeta_t* meta, int chunk_id) +static int unifyfs_chunk_alloc(int fid, unifyfs_filemeta_t* meta, int chunk_id) { /* get pointer to chunk meta data */ - unifycr_chunkmeta_t* chunk_meta = &(meta->chunk_meta[chunk_id]); + unifyfs_chunkmeta_t* chunk_meta = filemeta_get_chunkmeta(meta, chunk_id); /* allocate a chunk and record its location */ - if (unifycr_use_memfs) { + if (unifyfs_use_memfs) { /* allocate a new chunk from memory */ - unifycr_stack_lock(); - int id = unifycr_stack_pop(free_chunk_stack); - unifycr_stack_unlock(); + unifyfs_stack_lock(); + int id = unifyfs_stack_pop(free_chunk_stack); + unifyfs_stack_unlock(); /* if we got one return, otherwise try spill over */ if (id >= 0) { /* got a chunk from memory */ chunk_meta->location = CHUNK_LOCATION_MEMFS; chunk_meta->id = id; - } else if (unifycr_use_spillover) { + } else if (unifyfs_use_spillover) { /* shm segment out of space, grab a block from spill-over device */ LOGDBG("getting blocks from spill-over device"); /* TODO: missing lock calls? */ - /* add unifycr_max_chunks to identify chunk location */ - unifycr_stack_lock(); - id = unifycr_stack_pop(free_spillchunk_stack) + unifycr_max_chunks; - unifycr_stack_unlock(); - if (id < unifycr_max_chunks) { + /* add unifyfs_max_chunks to identify chunk location */ + unifyfs_stack_lock(); + id = unifyfs_stack_pop(free_spillchunk_stack) + unifyfs_max_chunks; + unifyfs_stack_unlock(); + if (id < unifyfs_max_chunks) { LOGERR("spill-over device out of space (%d)", id); - return UNIFYCR_ERROR_NOSPC; + return UNIFYFS_ERROR_NOSPC; } /* got one from spill over */ @@ -160,22 +170,22 @@ static int unifycr_chunk_alloc(int fid, unifycr_filemeta_t* meta, int chunk_id) } else { /* spill over isn't available, so we're out of space */ LOGERR("memfs out of space (%d)", id); - return UNIFYCR_ERROR_NOSPC; + return UNIFYFS_ERROR_NOSPC; } - } else if (unifycr_use_spillover) { + } else if (unifyfs_use_spillover) { /* memory file system is not enabled, but spill over is */ /* shm segment out of space, grab a block from spill-over device */ LOGDBG("getting blocks from spill-over device"); /* TODO: missing lock calls? */ - /* add unifycr_max_chunks to identify chunk location */ - unifycr_stack_lock(); - int id = unifycr_stack_pop(free_spillchunk_stack) + unifycr_max_chunks; - unifycr_stack_unlock(); - if (id < unifycr_max_chunks) { + /* add unifyfs_max_chunks to identify chunk location */ + unifyfs_stack_lock(); + int id = unifyfs_stack_pop(free_spillchunk_stack) + unifyfs_max_chunks; + unifyfs_stack_unlock(); + if (id < unifyfs_max_chunks) { LOGERR("spill-over device out of space (%d)", id); - return UNIFYCR_ERROR_NOSPC; + return UNIFYFS_ERROR_NOSPC; } /* got one from spill over */ @@ -184,16 +194,16 @@ static int unifycr_chunk_alloc(int fid, unifycr_filemeta_t* meta, int chunk_id) } else { /* don't know how to allocate chunk */ chunk_meta->location = CHUNK_LOCATION_NULL; - return UNIFYCR_ERROR_IO; + return UNIFYFS_ERROR_IO; } - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } -static int unifycr_chunk_free(int fid, unifycr_filemeta_t* meta, int chunk_id) +static int unifyfs_chunk_free(int fid, unifyfs_filemeta_t* meta, int chunk_id) { /* get pointer to chunk meta data */ - unifycr_chunkmeta_t* chunk_meta = &(meta->chunk_meta[chunk_id]); + unifyfs_chunkmeta_t* chunk_meta = filemeta_get_chunkmeta(meta, chunk_id); /* get physical id of chunk */ int id = chunk_meta->id; @@ -201,67 +211,64 @@ static int unifycr_chunk_free(int fid, unifycr_filemeta_t* meta, int chunk_id) /* determine location of chunk */ if (chunk_meta->location == CHUNK_LOCATION_MEMFS) { - unifycr_stack_lock(); - unifycr_stack_push(free_chunk_stack, id); - unifycr_stack_unlock(); + unifyfs_stack_lock(); + unifyfs_stack_push(free_chunk_stack, id); + unifyfs_stack_unlock(); } else if (chunk_meta->location == CHUNK_LOCATION_SPILLOVER) { /* TODO: free spill over chunk */ } else { /* unkwown chunk location */ LOGERR("unknown chunk location %d", chunk_meta->location); - return UNIFYCR_ERROR_IO; + return UNIFYFS_ERROR_IO; } /* update location of chunk */ chunk_meta->location = CHUNK_LOCATION_NULL; - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /* read data from specified chunk id, chunk offset, and count into user buffer, * count should fit within chunk starting from specified offset */ -static int unifycr_chunk_read( - unifycr_filemeta_t* meta, /* pointer to file meta data */ +static int unifyfs_chunk_read( + unifyfs_filemeta_t* meta, /* pointer to file meta data */ int chunk_id, /* logical chunk id to read data from */ off_t chunk_offset, /* logical offset within chunk to read from */ void* buf, /* buffer to store data to */ size_t count) /* number of bytes to read */ { /* get chunk meta data */ - unifycr_chunkmeta_t* chunk_meta = &(meta->chunk_meta[chunk_id]); + unifyfs_chunkmeta_t* chunk_meta = filemeta_get_chunkmeta(meta, chunk_id); /* determine location of chunk */ if (chunk_meta->location == CHUNK_LOCATION_MEMFS) { /* just need a memcpy to read data */ - void* chunk_buf = unifycr_compute_chunk_buf( + void* chunk_buf = unifyfs_compute_chunk_buf( meta, chunk_id, chunk_offset); memcpy(buf, chunk_buf, count); } else if (chunk_meta->location == CHUNK_LOCATION_SPILLOVER) { /* spill over to a file, so read from file descriptor */ //MAP_OR_FAIL(pread); - off_t spill_offset = unifycr_compute_spill_offset(meta, chunk_id, chunk_offset); - ssize_t rc = pread(unifycr_spilloverblock, buf, count, spill_offset); + off_t spill_offset = unifyfs_compute_spill_offset(meta, chunk_id, chunk_offset); + ssize_t rc = pread(unifyfs_spilloverblock, buf, count, spill_offset); if (rc < 0) { - return unifycr_errno_map_to_err(rc); + return unifyfs_errno_map_to_err(rc); } } else { /* unknown chunk type */ LOGERR("unknown chunk type"); - return UNIFYCR_ERROR_IO; + return UNIFYFS_ERROR_IO; } /* assume read was successful if we get to here */ - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } -/* - * given an index, split it into multiple indices whose range is equal or smaller - * than slice_range size - * @param cur_idx: the index to split +/* given an index, split it into multiple indices whose range is equal or + * smaller than slice_range size @param cur_idx: the index to split * @param slice_range: the slice size of the key-value store - * @return index_set: the set of split indices - * */ -int unifycr_split_index(unifycr_index_t* cur_idx, index_set_t* index_set, + * @return index_set: the set of split indices */ +int unifyfs_split_index(unifyfs_index_t* cur_idx, index_set_t* index_set, long slice_range) { @@ -329,91 +336,93 @@ int unifycr_split_index(unifycr_index_t* cur_idx, index_set_t* index_set, /* read data from specified chunk id, chunk offset, and count into user buffer, * count should fit within chunk starting from specified offset */ -static int unifycr_logio_chunk_write( +static int unifyfs_logio_chunk_write( int fid, /* local file id */ long pos, /* write offset inside the file */ - unifycr_filemeta_t* meta, /* pointer to file meta data */ + unifyfs_filemeta_t* meta, /* pointer to file meta data */ int chunk_id, /* logical chunk id to write to */ off_t chunk_offset, /* logical offset within chunk to write to */ const void* buf, /* buffer holding data to be written */ size_t count) /* number of bytes to write */ { /* get chunk meta data */ - unifycr_chunkmeta_t* chunk_meta = &(meta->chunk_meta[chunk_id]); + unifyfs_chunkmeta_t* chunk_meta = filemeta_get_chunkmeta(meta, chunk_id); if (chunk_meta->location != CHUNK_LOCATION_MEMFS && chunk_meta->location != CHUNK_LOCATION_SPILLOVER) { /* unknown chunk type */ LOGERR("unknown chunk type"); - return UNIFYCR_ERROR_IO; + return UNIFYFS_ERROR_IO; } /* determine location of chunk */ off_t log_offset = 0; if (chunk_meta->location == CHUNK_LOCATION_MEMFS) { /* just need a memcpy to write data */ - char* chunk_buf = unifycr_compute_chunk_buf( + char* chunk_buf = unifyfs_compute_chunk_buf( meta, chunk_id, chunk_offset); memcpy(chunk_buf, buf, count); - log_offset = chunk_buf - unifycr_chunks; + log_offset = chunk_buf - unifyfs_chunks; } else if (chunk_meta->location == CHUNK_LOCATION_SPILLOVER) { /* spill over to a file, so write to file descriptor */ //MAP_OR_FAIL(pwrite); - off_t spill_offset = unifycr_compute_spill_offset(meta, chunk_id, chunk_offset); - ssize_t rc = __real_pwrite(unifycr_spilloverblock, buf, count, spill_offset); + off_t spill_offset = unifyfs_compute_spill_offset(meta, chunk_id, chunk_offset); + ssize_t rc = __real_pwrite(unifyfs_spilloverblock, buf, count, spill_offset); if (rc < 0) { LOGERR("pwrite failed: errno=%d (%s)", errno, strerror(errno)); } - log_offset = spill_offset + unifycr_max_chunks * (1 << unifycr_chunk_bits); + log_offset = spill_offset + unifyfs_max_chunks * (1 << unifyfs_chunk_bits); } /* find the corresponding file attr entry and update attr*/ - unifycr_file_attr_t tmp_meta_entry; + unifyfs_file_attr_t tmp_meta_entry; tmp_meta_entry.fid = fid; - unifycr_file_attr_t* ptr_meta_entry - = (unifycr_file_attr_t*)bsearch(&tmp_meta_entry, - unifycr_fattrs.meta_entry, - *unifycr_fattrs.ptr_num_entries, - sizeof(unifycr_file_attr_t), + unifyfs_file_attr_t* ptr_meta_entry + = (unifyfs_file_attr_t*)bsearch(&tmp_meta_entry, + unifyfs_fattrs.meta_entry, + *unifyfs_fattrs.ptr_num_entries, + sizeof(unifyfs_file_attr_t), compare_fattr); if (ptr_meta_entry != NULL) { - ptr_meta_entry->file_attr.st_size = pos + count; + ptr_meta_entry->size = pos + count; } /* define an new index entry for this write operation */ - unifycr_index_t cur_idx; + unifyfs_index_t cur_idx; cur_idx.fid = ptr_meta_entry->gfid; cur_idx.file_pos = pos; cur_idx.mem_pos = log_offset; cur_idx.length = count; - /* split the write requests larger than unifycr_key_slice_range into - * the ones smaller than unifycr_key_slice_range + /* split the write requests larger than unifyfs_key_slice_range into + * the ones smaller than unifyfs_key_slice_range * */ - unifycr_split_index(&cur_idx, &tmp_index_set, - unifycr_key_slice_range); + index_set_t tmp_index_set; + memset(&tmp_index_set, 0, sizeof(tmp_index_set)); + unifyfs_split_index(&cur_idx, &tmp_index_set, + unifyfs_key_slice_range); /* lookup number of existing index entries */ - off_t num_entries = *(unifycr_indices.ptr_num_entries); + off_t num_entries = *(unifyfs_indices.ptr_num_entries); /* number of new entries we may add */ off_t tmp_entries = (off_t) tmp_index_set.count; /* check whether there is room to add new entries */ - if (num_entries + tmp_entries < unifycr_max_index_entries) { + if (num_entries + tmp_entries < unifyfs_max_index_entries) { /* get pointer to index array */ - unifycr_index_t* idxs = unifycr_indices.index_entry; + unifyfs_index_t* idxs = unifyfs_indices.index_entry; /* coalesce contiguous indices */ int i = 0; if (num_entries > 0) { /* pointer to last element in index array */ - unifycr_index_t* prev_idx = &idxs[num_entries - 1]; + unifyfs_index_t* prev_idx = &idxs[num_entries - 1]; /* pointer to first element in temp list */ - unifycr_index_t* next_idx = &tmp_index_set.idxes[0]; + unifyfs_index_t* next_idx = &tmp_index_set.idxes[0]; /* offset of last byte for last index in list */ off_t prev_offset = prev_idx->file_pos + prev_idx->length; @@ -424,8 +433,8 @@ static int unifycr_logio_chunk_write( prev_offset == next_idx->file_pos) { /* got contiguous bytes in the same file, * check if both index values fall in the same slice */ - off_t prev_slice = prev_idx->file_pos / unifycr_key_slice_range; - off_t next_slice = next_idx->file_pos / unifycr_key_slice_range; + off_t prev_slice = prev_idx->file_pos / unifyfs_key_slice_range; + off_t next_slice = next_idx->file_pos / unifyfs_key_slice_range; if (prev_slice == next_slice) { /* index values also are in same slice, * so append first index in temp list to @@ -439,7 +448,7 @@ static int unifycr_logio_chunk_write( } /* pointer to temp index list */ - unifycr_index_t* newidxs = tmp_index_set.idxes; + unifyfs_index_t* newidxs = tmp_index_set.idxes; /* copy remaining items in temp index list to index list */ while (i < tmp_index_set.count) { @@ -455,7 +464,7 @@ static int unifycr_logio_chunk_write( } /* update number of entries in index array */ - (*unifycr_indices.ptr_num_entries) = num_entries; + (*unifyfs_indices.ptr_num_entries) = num_entries; } else { /* TODO: no room to write additional index metadata entries, * swap out existing metadata buffer to disk*/ @@ -463,34 +472,34 @@ static int unifycr_logio_chunk_write( } /* assume read was successful if we get to here */ - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /* read data from specified chunk id, chunk offset, and count into user buffer, * count should fit within chunk starting from specified offset */ -static int unifycr_chunk_write( - unifycr_filemeta_t* meta, /* pointer to file meta data */ +static int unifyfs_chunk_write( + unifyfs_filemeta_t* meta, /* pointer to file meta data */ int chunk_id, /* logical chunk id to write to */ off_t chunk_offset, /* logical offset within chunk to write to */ const void* buf, /* buffer holding data to be written */ size_t count) /* number of bytes to write */ { /* get chunk meta data */ - unifycr_chunkmeta_t* chunk_meta = &(meta->chunk_meta[chunk_id]); + unifyfs_chunkmeta_t* chunk_meta = filemeta_get_chunkmeta(meta, chunk_id); /* determine location of chunk */ if (chunk_meta->location == CHUNK_LOCATION_MEMFS) { /* just need a memcpy to write data */ - void* chunk_buf = unifycr_compute_chunk_buf( + void* chunk_buf = unifyfs_compute_chunk_buf( meta, chunk_id, chunk_offset); memcpy(chunk_buf, buf, count); // _intel_fast_memcpy(chunk_buf, buf, count); -// unifycr_memcpy(chunk_buf, buf, count); +// unifyfs_memcpy(chunk_buf, buf, count); } else if (chunk_meta->location == CHUNK_LOCATION_SPILLOVER) { /* spill over to a file, so write to file descriptor */ //MAP_OR_FAIL(pwrite); - off_t spill_offset = unifycr_compute_spill_offset(meta, chunk_id, chunk_offset); - ssize_t rc = pwrite(unifycr_spilloverblock, buf, count, spill_offset); + off_t spill_offset = unifyfs_compute_spill_offset(meta, chunk_id, chunk_offset); + ssize_t rc = pwrite(unifyfs_spilloverblock, buf, count, spill_offset); if (rc < 0) { LOGERR("pwrite failed: errno=%d (%s)", errno, strerror(errno)); } @@ -499,11 +508,11 @@ static int unifycr_chunk_write( } else { /* unknown chunk type */ LOGERR("unknown chunk type"); - return UNIFYCR_ERROR_IO; + return UNIFYFS_ERROR_IO; } /* assume read was successful if we get to here */ - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /* --------------------------------------- @@ -511,91 +520,91 @@ static int unifycr_chunk_write( * --------------------------------------- */ /* if length is greater than reserved space, reserve space up to length */ -int unifycr_fid_store_fixed_extend(int fid, unifycr_filemeta_t* meta, +int unifyfs_fid_store_fixed_extend(int fid, unifyfs_filemeta_t* meta, off_t length) { /* determine whether we need to allocate more chunks */ - off_t maxsize = meta->chunks << unifycr_chunk_bits; + off_t maxsize = meta->chunks << unifyfs_chunk_bits; if (length > maxsize) { /* compute number of additional bytes we need */ off_t additional = length - maxsize; while (additional > 0) { /* check that we don't overrun max number of chunks for file */ - if (meta->chunks == unifycr_max_chunks + unifycr_spillover_max_chunks) { - return UNIFYCR_ERROR_NOSPC; + if (meta->chunks == unifyfs_max_chunks + unifyfs_spillover_max_chunks) { + return UNIFYFS_ERROR_NOSPC; } /* allocate a new chunk */ - int rc = unifycr_chunk_alloc(fid, meta, meta->chunks); - if (rc != UNIFYCR_SUCCESS) { + int rc = unifyfs_chunk_alloc(fid, meta, meta->chunks); + if (rc != UNIFYFS_SUCCESS) { LOGERR("failed to allocate chunk"); - return UNIFYCR_ERROR_NOSPC; + return UNIFYFS_ERROR_NOSPC; } /* increase chunk count and subtract bytes from the number we need */ meta->chunks++; - additional -= unifycr_chunk_size; + additional -= unifyfs_chunk_size; } } - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /* if length is shorter than reserved space, give back space down to length */ -int unifycr_fid_store_fixed_shrink(int fid, unifycr_filemeta_t* meta, +int unifyfs_fid_store_fixed_shrink(int fid, unifyfs_filemeta_t* meta, off_t length) { /* determine the number of chunks to leave after truncating */ off_t num_chunks = 0; if (length > 0) { - num_chunks = (length >> unifycr_chunk_bits) + 1; + num_chunks = (length >> unifyfs_chunk_bits) + 1; } /* clear off any extra chunks */ while (meta->chunks > num_chunks) { meta->chunks--; - unifycr_chunk_free(fid, meta, meta->chunks); + unifyfs_chunk_free(fid, meta, meta->chunks); } - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /* read data from file stored as fixed-size chunks */ -int unifycr_fid_store_fixed_read(int fid, unifycr_filemeta_t* meta, off_t pos, +int unifyfs_fid_store_fixed_read(int fid, unifyfs_filemeta_t* meta, off_t pos, void* buf, size_t count) { int rc; /* get pointer to position within first chunk */ - int chunk_id = pos >> unifycr_chunk_bits; - off_t chunk_offset = pos & unifycr_chunk_mask; + int chunk_id = pos >> unifyfs_chunk_bits; + off_t chunk_offset = pos & unifyfs_chunk_mask; /* determine how many bytes remain in the current chunk */ - size_t remaining = unifycr_chunk_size - chunk_offset; + size_t remaining = unifyfs_chunk_size - chunk_offset; if (count <= remaining) { /* all bytes for this read fit within the current chunk */ - rc = unifycr_chunk_read(meta, chunk_id, chunk_offset, buf, count); + rc = unifyfs_chunk_read(meta, chunk_id, chunk_offset, buf, count); } else { /* read what's left of current chunk */ char* ptr = (char*) buf; - rc = unifycr_chunk_read(meta, chunk_id, + rc = unifyfs_chunk_read(meta, chunk_id, chunk_offset, (void*)ptr, remaining); ptr += remaining; /* read from the next chunk */ size_t processed = remaining; - while (processed < count && rc == UNIFYCR_SUCCESS) { + while (processed < count && rc == UNIFYFS_SUCCESS) { /* get pointer to start of next chunk */ chunk_id++; /* compute size to read from this chunk */ size_t num = count - processed; - if (num > unifycr_chunk_size) { - num = unifycr_chunk_size; + if (num > unifyfs_chunk_size) { + num = unifyfs_chunk_size; } /* read data */ - rc = unifycr_chunk_read(meta, chunk_id, 0, (void*)ptr, num); + rc = unifyfs_chunk_read(meta, chunk_id, 0, (void*)ptr, num); ptr += num; /* update number of bytes written */ @@ -607,7 +616,7 @@ int unifycr_fid_store_fixed_read(int fid, unifycr_filemeta_t* meta, off_t pos, } /* write data to file stored as fixed-size chunks */ -int unifycr_fid_store_fixed_write(int fid, unifycr_filemeta_t* meta, off_t pos, +int unifyfs_fid_store_fixed_write(int fid, unifyfs_filemeta_t* meta, off_t pos, const void* buf, size_t count) { int rc; @@ -617,38 +626,38 @@ int unifycr_fid_store_fixed_write(int fid, unifycr_filemeta_t* meta, off_t pos, off_t chunk_offset; if (meta->storage == FILE_STORAGE_FIXED_CHUNK) { - chunk_id = pos >> unifycr_chunk_bits; - chunk_offset = pos & unifycr_chunk_mask; + chunk_id = pos >> unifyfs_chunk_bits; + chunk_offset = pos & unifyfs_chunk_mask; } else if (meta->storage == FILE_STORAGE_LOGIO) { - chunk_id = meta->size >> unifycr_chunk_bits; - chunk_offset = meta->size & unifycr_chunk_mask; + chunk_id = meta->size >> unifyfs_chunk_bits; + chunk_offset = meta->size & unifyfs_chunk_mask; } else { - return UNIFYCR_ERROR_IO; + return UNIFYFS_ERROR_IO; } /* determine how many bytes remain in the current chunk */ - size_t remaining = unifycr_chunk_size - chunk_offset; + size_t remaining = unifyfs_chunk_size - chunk_offset; if (count <= remaining) { /* all bytes for this write fit within the current chunk */ if (meta->storage == FILE_STORAGE_FIXED_CHUNK) { - rc = unifycr_chunk_write(meta, chunk_id, chunk_offset, buf, count); + rc = unifyfs_chunk_write(meta, chunk_id, chunk_offset, buf, count); } else if (meta->storage == FILE_STORAGE_LOGIO) { - rc = unifycr_logio_chunk_write(fid, pos, meta, chunk_id, chunk_offset, + rc = unifyfs_logio_chunk_write(fid, pos, meta, chunk_id, chunk_offset, buf, count); } else { - return UNIFYCR_ERROR_IO; + return UNIFYFS_ERROR_IO; } } else { /* otherwise, fill up the remainder of the current chunk */ char* ptr = (char*) buf; if (meta->storage == FILE_STORAGE_FIXED_CHUNK) { - rc = unifycr_chunk_write(meta, chunk_id, + rc = unifyfs_chunk_write(meta, chunk_id, chunk_offset, (void*)ptr, remaining); } else if (meta->storage == FILE_STORAGE_LOGIO) { - rc = unifycr_logio_chunk_write(fid, pos, meta, chunk_id, + rc = unifyfs_logio_chunk_write(fid, pos, meta, chunk_id, chunk_offset, (void*)ptr, remaining); } else { - return UNIFYCR_ERROR_IO; + return UNIFYFS_ERROR_IO; } ptr += remaining; @@ -657,24 +666,24 @@ int unifycr_fid_store_fixed_write(int fid, unifycr_filemeta_t* meta, off_t pos, /* then write the rest of the bytes starting from beginning * of chunks */ size_t processed = remaining; - while (processed < count && rc == UNIFYCR_SUCCESS) { + while (processed < count && rc == UNIFYFS_SUCCESS) { /* get pointer to start of next chunk */ chunk_id++; /* compute size to write to this chunk */ size_t num = count - processed; - if (num > unifycr_chunk_size) { - num = unifycr_chunk_size; + if (num > unifyfs_chunk_size) { + num = unifyfs_chunk_size; } /* write data */ if (meta->storage == FILE_STORAGE_FIXED_CHUNK) { - rc = unifycr_chunk_write(meta, chunk_id, 0, (void*)ptr, num); - } else if (meta->storage == FILE_STORAGE_LOGIO) - rc = unifycr_logio_chunk_write(fid, pos, meta, chunk_id, 0, + rc = unifyfs_chunk_write(meta, chunk_id, 0, (void*)ptr, num); + } else if (meta->storage == FILE_STORAGE_LOGIO) { + rc = unifyfs_logio_chunk_write(fid, pos, meta, chunk_id, 0, (void*)ptr, num); - else { - return UNIFYCR_ERROR_IO; + } else { + return UNIFYFS_ERROR_IO; } ptr += num; pos += num; diff --git a/client/src/unifycr-fixed.h b/client/src/unifyfs-fixed.h similarity index 77% rename from client/src/unifycr-fixed.h rename to client/src/unifyfs-fixed.h index 27c2a751b..dbd46b4e9 100644 --- a/client/src/unifycr-fixed.h +++ b/client/src/unifyfs-fixed.h @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -40,45 +40,45 @@ * Please also read this file LICENSE.CRUISE */ -#ifndef UNIFYCR_FIXED_H -#define UNIFYCR_FIXED_H +#ifndef UNIFYFS_FIXED_H +#define UNIFYFS_FIXED_H -#include "unifycr-internal.h" +#include "unifyfs-internal.h" /* if length is greater than reserved space, * reserve space up to length */ -int unifycr_fid_store_fixed_extend( +int unifyfs_fid_store_fixed_extend( int fid, /* file id to reserve space for */ - unifycr_filemeta_t* meta, /* meta data for file */ + unifyfs_filemeta_t* meta, /* meta data for file */ off_t length /* number of bytes to reserve for file */ ); /* if length is shorter than reserved space, * give back space down to length */ -int unifycr_fid_store_fixed_shrink( +int unifyfs_fid_store_fixed_shrink( int fid, /* file id to free space for */ - unifycr_filemeta_t* meta, /* meta data for file */ + unifyfs_filemeta_t* meta, /* meta data for file */ off_t length /* number of bytes to reserve for file */ ); /* read data from file stored as fixed-size chunks, - * returns UNIFYCR error code */ -int unifycr_fid_store_fixed_read( + * returns UNIFYFS error code */ +int unifyfs_fid_store_fixed_read( int fid, /* file id to read from */ - unifycr_filemeta_t* meta, /* meta data for file */ + unifyfs_filemeta_t* meta, /* meta data for file */ off_t pos, /* position within file to read from */ void* buf, /* user buffer to store data in */ size_t count /* number of bytes to read */ ); /* write data to file stored as fixed-size chunks, - * returns UNIFYCR error code */ -int unifycr_fid_store_fixed_write( + * returns UNIFYFS error code */ +int unifyfs_fid_store_fixed_write( int fid, /* file id to write to */ - unifycr_filemeta_t* meta, /* meta data for file */ + unifyfs_filemeta_t* meta, /* meta data for file */ off_t pos, /* position within file to write to */ const void* buf, /* user buffer holding data */ size_t count /* number of bytes to write */ ); -#endif /* UNIFYCR_FIXED_H */ +#endif /* UNIFYFS_FIXED_H */ diff --git a/client/src/unifycr-internal.h b/client/src/unifyfs-internal.h similarity index 65% rename from client/src/unifycr-internal.h rename to client/src/unifyfs-internal.h index 5e2a3c8fa..f31504f18 100644 --- a/client/src/unifycr-internal.h +++ b/client/src/unifyfs-internal.h @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -40,8 +40,8 @@ * Please also read this file LICENSE.CRUISE */ -#ifndef UNIFYCR_INTERNAL_H -#define UNIFYCR_INTERNAL_H +#ifndef UNIFYFS_INTERNAL_H +#define UNIFYFS_INTERNAL_H #include "config.h" @@ -56,6 +56,8 @@ * Common includes * ------------------------------- */ + +// system headers #include #include #include @@ -87,15 +89,20 @@ #include #include +// common headers +#include "unifyfs_configurator.h" +#include "unifyfs_const.h" +#include "unifyfs_keyval.h" +#include "unifyfs_log.h" +#include "unifyfs_meta.h" +#include "unifyfs_shm.h" + +// client headers +#include "unifyfs.h" +#include "unifyfs-stack.h" #include "utlist.h" #include "uthash.h" -#include "unifycr.h" -#include "unifycr_configurator.h" -#include "unifycr_meta.h" -#include "unifycr-stack.h" - - /* ------------------------------- * Defines and types * ------------------------------- @@ -103,32 +110,32 @@ /* define a macro to capture function name, file name, and line number * along with user-defined string */ -#define UNIFYCR_UNSUPPORTED(fmt, args...) \ - unifycr_unsupported(__func__, __FILE__, __LINE__, fmt, ##args) +#define UNIFYFS_UNSUPPORTED(fmt, args...) \ + unifyfs_unsupported(__func__, __FILE__, __LINE__, fmt, ##args) -#ifdef UNIFYCR_GOTCHA +#ifdef UNIFYFS_GOTCHA /* gotcha fills in address of original/real function * and we need to declare function prototype for each * wrapper */ -#define UNIFYCR_DECL(name,ret,args) \ +#define UNIFYFS_DECL(name,ret,args) \ extern ret(*__real_ ## name)args; \ ret __wrap_ ## name args; /* define each DECL function in a .c file */ -#define UNIFYCR_DEF(name,ret,args) \ +#define UNIFYFS_DEF(name,ret,args) \ ret(*__real_ ## name)args = NULL; /* we define our wrapper function as __wrap_ instead of */ -#define UNIFYCR_WRAP(name) __wrap_ ## name +#define UNIFYFS_WRAP(name) __wrap_ ## name /* gotcha maps the call to __real_() */ -#define UNIFYCR_REAL(name) __real_ ## name +#define UNIFYFS_REAL(name) __real_ ## name /* no need to look up the address of the real function (gotcha does that) */ #define MAP_OR_FAIL(func) -#elif UNIFYCR_PRELOAD +#elif UNIFYFS_PRELOAD /* =================================================================== * Using LD_PRELOAD to intercept @@ -143,14 +150,14 @@ /* define a static variable called __real_open to record address of * real open call and initialize it to NULL */ -#define UNIFYCR_DECL(name,ret,args) \ +#define UNIFYFS_DECL(name,ret,args) \ static ret (*__real_ ## name)args = NULL; /* our open wrapper assumes the name of open() */ -#define UNIFYCR_WRAP(name) name +#define UNIFYFS_WRAP(name) name /* the address of the real open call is stored in __real_open variable */ -#define UNIFYCR_REAL(name) __real_ ## name +#define UNIFYFS_REAL(name) __real_ ## name /* if __real_open is still NULL, call dlsym to lookup address of real * function and record it */ @@ -158,8 +165,8 @@ if (!(__real_ ## func)) \ { \ __real_ ## func = dlsym(RTLD_NEXT, #func); \ - if(!(__real_ ## func)) { \ - fprintf(stderr, "UNIFYCR failed to map symbol: %s\n", #func); \ + if (!(__real_ ## func)) { \ + fprintf(stderr, "UNIFYFS failed to map symbol: %s\n", #func); \ exit(1); \ } \ } @@ -176,15 +183,15 @@ * just declare the existence of __real_open so the compiler knows the * prototype of this function (linker will provide it), also need to * declare prototype for __wrap_open */ -#define UNIFYCR_DECL(name,ret,args) \ +#define UNIFYFS_DECL(name,ret,args) \ extern ret __real_ ## name args; \ ret __wrap_ ## name args; /* we define our wrapper function as __wrap_open instead of open */ -#define UNIFYCR_WRAP(name) __wrap_ ## name +#define UNIFYFS_WRAP(name) __wrap_ ## name /* the linker maps the open call to __real_open() */ -#define UNIFYCR_REAL(name) __real_ ## name +#define UNIFYFS_REAL(name) __real_ ## name /* no need to look up the address of the real function */ #define MAP_OR_FAIL(func) @@ -197,22 +204,22 @@ typedef struct { off_t pos; /* current file pointer */ int read; /* whether file is opened for read */ int write; /* whether file is opened for write */ -} unifycr_fd_t; +} unifyfs_fd_t; -enum unifycr_stream_orientation { - UNIFYCR_STREAM_ORIENTATION_NULL = 0, - UNIFYCR_STREAM_ORIENTATION_BYTE, - UNIFYCR_STREAM_ORIENTATION_WIDE, +enum unifyfs_stream_orientation { + UNIFYFS_STREAM_ORIENTATION_NULL = 0, + UNIFYFS_STREAM_ORIENTATION_BYTE, + UNIFYFS_STREAM_ORIENTATION_WIDE, }; /* structure to represent FILE* streams */ typedef struct { - int sid; /* index within unifycr_streams */ + int sid; /* index within unifyfs_streams */ int err; /* stream error indicator flag */ int eof; /* stream end-of-file indicator flag */ int fd; /* file descriptor associated with stream */ int append; /* whether file is opened in append mode */ - int orient; /* stream orientation, UNIFYCR_STREAM_ORIENTATION_{NULL,BYTE,WIDE} */ + int orient; /* stream orientation, UNIFYFS_STREAM_ORIENTATION_{NULL,BYTE,WIDE} */ void* buf; /* pointer to buffer */ int buffree; /* whether we need to free buffer */ @@ -228,15 +235,15 @@ typedef struct { unsigned char* _p; /* pointer to character in buffer */ size_t _r; /* number of bytes left at pointer */ -} unifycr_stream_t; +} unifyfs_stream_t; /* structure to represent DIR* streams */ typedef struct { - int dirid; /* index within unifycr_dirstreams */ + int dirid; /* index within unifyfs_dirstreams */ int fid; /* local file id of directory for this stream */ int fd; /* file descriptor associated with stream */ off_t pos; /* position within directory stream */ -} unifycr_dirstream_t; +} unifyfs_dirstream_t; enum flock_enum { UNLOCKED, @@ -255,22 +262,27 @@ enum flock_enum { #define CHUNK_LOCATION_SPILLOVER 2 typedef struct { - int location; /* CHUNK_LOCATION specifies how chunk is stored */ + int location; /* CHUNK_LOCATION type */ off_t id; /* physical id of chunk in its respective storage */ -} unifycr_chunkmeta_t; +} unifyfs_chunkmeta_t; typedef struct { - off_t size; /* current file size */ - off_t log_size; /* real size of the file for logio*/ - int is_dir; /* is this file a directory */ - pthread_spinlock_t fspinlock; /* file lock variable */ - enum flock_enum flock_status; /* file lock status */ + off_t size; /* current file size */ + off_t log_size; /* real size of the file for logio*/ + pthread_spinlock_t fspinlock; /* file lock variable */ + enum flock_enum flock_status; /* file lock status */ - int storage; /* FILE_STORAGE specifies file data management */ + int storage; /* FILE_STORAGE type */ + + int needs_sync; /* have unsynced writes */ off_t chunks; /* number of chunks allocated to file */ - unifycr_chunkmeta_t* chunk_meta; /* meta data for chunks */ -} unifycr_filemeta_t; + off_t chunkmeta_idx; /* starting index in unifyfs_chunkmeta */ + int is_laminated; /* Is this file laminated */ + uint32_t mode; /* st_mode bits. This has file + * permission info and will tell you if this + * is a regular file or directory. */ +} unifyfs_filemeta_t; /* struct used to map a full path to its local file id, * an array of these is kept and a simple linear search @@ -280,10 +292,10 @@ typedef struct { int in_use; /* full path and name of file */ - const char filename[UNIFYCR_MAX_FILENAME]; -} unifycr_filename_t; + const char filename[UNIFYFS_MAX_FILENAME]; +} unifyfs_filename_t; -/*unifycr structures*/ +/*unifyfs structures*/ typedef struct { int fid; int errcode; @@ -294,32 +306,28 @@ typedef struct { typedef struct { size_t* ptr_num_entries; - unifycr_index_t* index_entry; -} unifycr_index_buf_t; + unifyfs_index_t* index_entry; +} unifyfs_index_buf_t; typedef struct { size_t* ptr_num_entries; - unifycr_file_attr_t* meta_entry; -} unifycr_fattr_buf_t; + unifyfs_file_attr_t* meta_entry; +} unifyfs_fattr_buf_t; typedef struct { - unifycr_index_t idxes[UNIFYCR_MAX_SPLIT_CNT]; + unifyfs_index_t idxes[UNIFYFS_MAX_SPLIT_CNT]; int count; } index_set_t; typedef struct { - read_req_t read_reqs[UNIFYCR_MAX_READ_CNT]; + read_req_t read_reqs[UNIFYFS_MAX_READ_CNT]; int count; } read_req_set_t; -read_req_set_t read_req_set; -index_set_t tmp_index_set; - -extern unifycr_index_buf_t unifycr_indices; -extern unsigned long unifycr_max_index_entries; -extern long unifycr_spillover_max_chunks; +extern unifyfs_index_buf_t unifyfs_indices; +extern unsigned long unifyfs_max_index_entries; +extern long unifyfs_spillover_max_chunks; -extern int* local_rank_lst; extern int local_rank_cnt; extern int local_rank_idx; extern int local_del_cnt; @@ -328,17 +336,17 @@ extern struct pollfd cmd_fd; extern void* shm_req_buf; extern void* shm_recv_buf; extern char cmd_buf[CMD_BUF_SIZE]; -extern unifycr_fattr_buf_t unifycr_fattrs; +extern unifyfs_fattr_buf_t unifyfs_fattrs; extern int app_id; -extern size_t unifycr_key_slice_range; +extern size_t unifyfs_key_slice_range; /* ------------------------------- * Global varaible declarations * ------------------------------- */ -/*definition for unifycr*/ -#define UNIFYCR_CLI_TIME_OUT 5000 +/*definition for unifyfs*/ +#define UNIFYFS_CLI_TIME_OUT 5000 typedef enum { ACK_SUCCESS, @@ -346,131 +354,135 @@ typedef enum { } ack_status_t; /* keep track of what we've initialized */ -extern int unifycr_initialized; +extern int unifyfs_initialized; /* list of file name structures of fixed length, * used to map a full path to its local file id, * an array of these is kept and a simple linear search * is used to find a match */ -extern unifycr_filename_t* unifycr_filelist; +extern unifyfs_filename_t* unifyfs_filelist; /* mount directory */ -extern char* unifycr_mount_prefix; -extern size_t unifycr_mount_prefixlen; +extern char* unifyfs_mount_prefix; +extern size_t unifyfs_mount_prefixlen; /* array of file descriptors */ -extern unifycr_fd_t unifycr_fds[UNIFYCR_MAX_FILEDESCS]; -extern rlim_t unifycr_fd_limit; +extern unifyfs_fd_t unifyfs_fds[UNIFYFS_MAX_FILEDESCS]; +extern rlim_t unifyfs_fd_limit; /* array of file streams */ -extern unifycr_stream_t unifycr_streams[UNIFYCR_MAX_FILEDESCS]; +extern unifyfs_stream_t unifyfs_streams[UNIFYFS_MAX_FILEDESCS]; /* array of directory streams */ -extern unifycr_dirstream_t unifycr_dirstreams[UNIFYCR_MAX_FILEDESCS]; +extern unifyfs_dirstream_t unifyfs_dirstreams[UNIFYFS_MAX_FILEDESCS]; /* stack of free file descriptor values, - * each is an index into unifycr_fds array */ -extern void* unifycr_fd_stack; + * each is an index into unifyfs_fds array */ +extern void* unifyfs_fd_stack; /* stack of free streams, - * each is an index into unifycr_streams array */ -extern void* unifycr_stream_stack; + * each is an index into unifyfs_streams array */ +extern void* unifyfs_stream_stack; /* stack of directory streams, - * each is an index into unifycr_dirstreams array */ -extern void* unifycr_dirstream_stack; + * each is an index into unifyfs_dirstreams array */ +extern void* unifyfs_dirstream_stack; -extern int unifycr_use_memfs; -extern int unifycr_use_spillover; +extern int unifyfs_use_memfs; +extern int unifyfs_use_spillover; -extern int unifycr_max_files; /* maximum number of files to store */ +extern int unifyfs_max_files; /* maximum number of files to store */ extern size_t -unifycr_chunk_mem; /* number of bytes in memory to be used for chunk storage */ -extern int unifycr_chunk_bits; /* we set chunk size = 2^unifycr_chunk_bits */ -extern off_t unifycr_chunk_size; /* chunk size in bytes */ +unifyfs_chunk_mem; /* number of bytes in memory to be used for chunk storage */ +extern int unifyfs_chunk_bits; /* we set chunk size = 2^unifyfs_chunk_bits */ +extern off_t unifyfs_chunk_size; /* chunk size in bytes */ extern off_t -unifycr_chunk_mask; /* mask applied to logical offset to determine physical offset within chunk */ +unifyfs_chunk_mask; /* mask applied to logical offset to determine physical offset within chunk */ extern long -unifycr_max_chunks; /* maximum number of chunks that fit in memory */ +unifyfs_max_chunks; /* maximum number of chunks that fit in memory */ extern void* free_chunk_stack; extern void* free_spillchunk_stack; -extern char* unifycr_chunks; -int unifycr_spilloverblock; +extern char* unifyfs_chunks; +extern unifyfs_chunkmeta_t* unifyfs_chunkmetas; +int unifyfs_spilloverblock; /* ------------------------------- * Common functions * ------------------------------- */ /* single function to route all unsupported wrapper calls through */ -int unifycr_unsupported(const char* fn_name, const char* file, int line, +int unifyfs_unsupported(const char* fn_name, const char* file, int line, const char* fmt, ...); /* returns 1 if two input parameters will overflow their type when * added together */ -int unifycr_would_overflow_offt(off_t a, off_t b); +int unifyfs_would_overflow_offt(off_t a, off_t b); /* returns 1 if two input parameters will overflow their type when * added together */ -int unifycr_would_overflow_long(long a, long b); +int unifyfs_would_overflow_long(long a, long b); /* given an input mode, mask it with umask and return, can specify * an input mode==0 to specify all read/write bits */ -mode_t unifycr_getmode(mode_t perms); +mode_t unifyfs_getmode(mode_t perms); -int unifycr_stack_lock(); +int unifyfs_stack_lock(); -int unifycr_stack_unlock(); +int unifyfs_stack_unlock(); /* sets flag if the path is a special path */ -int unifycr_intercept_path(const char* path); +int unifyfs_intercept_path(const char* path); /* given an fd, return 1 if we should intercept this file, 0 otherwise, * convert fd to new fd value if needed */ -int unifycr_intercept_fd(int* fd); +int unifyfs_intercept_fd(int* fd); /* given a FILE*, returns 1 if we should intercept this file, * 0 otherwise */ -int unifycr_intercept_stream(FILE* stream); +int unifyfs_intercept_stream(FILE* stream); /* given a DIR*, returns 1 if we should intercept this directory, * 0 otherwise */ -int unifycr_intercept_dirstream(DIR* dirp); +int unifyfs_intercept_dirstream(DIR* dirp); /* given a path, return the file id */ -int unifycr_get_fid_from_path(const char* path); +int unifyfs_get_fid_from_path(const char* path); /* given a file descriptor, return the file id */ -int unifycr_get_fid_from_fd(int fd); +int unifyfs_get_fid_from_fd(int fd); /* initialze file descriptor structure corresponding to fd value */ -int unifycr_fd_init(int fd); +int unifyfs_fd_init(int fd); /* initialze file stream structure corresponding to id value */ -int unifycr_stream_init(int sid); +int unifyfs_stream_init(int sid); /* initialze directory stream descriptor structure * corresponding to id value */ -int unifycr_dirstream_init(int dirid); +int unifyfs_dirstream_init(int dirid); /* return address of file descriptor structure or NULL if fd is out * of range */ -unifycr_fd_t* unifycr_get_filedesc_from_fd(int fd); +unifyfs_fd_t* unifyfs_get_filedesc_from_fd(int fd); /* given a file id, return a pointer to the meta data, * otherwise return NULL */ -unifycr_filemeta_t* unifycr_get_meta_from_fid(int fid); +unifyfs_filemeta_t* unifyfs_get_meta_from_fid(int fid); + +/* Given a fid, return the path. */ +const char* unifyfs_path_from_fid(int fid); -/* given an UNIFYCR error code, return corresponding errno code */ -int unifycr_err_map_to_errno(int rc); +/* given an UNIFYFS error code, return corresponding errno code */ +int unifyfs_err_map_to_errno(int rc); -/* given an errno error code, return corresponding UnifyCR error code */ -int unifycr_errno_map_to_err(int rc); +/* given an errno error code, return corresponding UnifyFS error code */ +int unifyfs_errno_map_to_err(int rc); /* checks to see if fid is a directory * returns 1 for yes * returns 0 for no */ -int unifycr_fid_is_dir(int fid); +int unifyfs_fid_is_dir(int fid); /* checks to see if a directory is empty * assumes that check for is_dir has already been made @@ -478,82 +490,81 @@ int unifycr_fid_is_dir(int fid); * e.g. ../dirname will not work * returns 1 for yes it is empty * returns 0 for no */ -int unifycr_fid_is_dir_empty(const char* path); +int unifyfs_fid_is_dir_empty(const char* path); /* return current size of given file id */ -off_t unifycr_fid_size(int fid); +off_t unifyfs_fid_size(int fid); /* fill in limited amount of stat information for global file id */ -int unifycr_gfid_stat(int gfid, struct stat* buf); +int unifyfs_gfid_stat(int gfid, struct stat* buf); /* fill in limited amount of stat information */ -int unifycr_fid_stat(int fid, struct stat* buf); +int unifyfs_fid_stat(int fid, struct stat* buf); /* allocate a file id slot for a new file * return the fid or -1 on error */ -int unifycr_fid_alloc(); +int unifyfs_fid_alloc(); /* return the file id back to the free pool */ -int unifycr_fid_free(int fid); +int unifyfs_fid_free(int fid); /* add a new file and initialize metadata * returns the new fid, or negative value on error */ -int unifycr_fid_create_file(const char* path); +int unifyfs_fid_create_file(const char* path); /* add a new directory and initialize metadata * returns the new fid, or a negative value on error */ -int unifycr_fid_create_directory(const char* path); +int unifyfs_fid_create_directory(const char* path); /* read count bytes from file starting from pos and store into buf, * all bytes are assumed to exist, so checks on file size should be * done before calling this routine */ -int unifycr_fid_read(int fid, off_t pos, void* buf, size_t count); +int unifyfs_fid_read(int fid, off_t pos, void* buf, size_t count); /* write count bytes from buf into file starting at offset pos, * all bytes are assumed to be allocated to file, so file should * be extended before calling this routine */ -int unifycr_fid_write(int fid, off_t pos, const void* buf, size_t count); +int unifyfs_fid_write(int fid, off_t pos, const void* buf, size_t count); /* given a file id, write zero bytes to region of specified offset * and length, assumes space is already reserved */ -int unifycr_fid_write_zero(int fid, off_t pos, off_t count); +int unifyfs_fid_write_zero(int fid, off_t pos, off_t count); /* increase size of file if length is greater than current size, * and allocate additional chunks as needed to reserve space for * length bytes */ -int unifycr_fid_extend(int fid, off_t length); +int unifyfs_fid_extend(int fid, off_t length); /* truncate file id to given length, frees resources if length is * less than size and allocates and zero-fills new bytes if length * is more than size */ -int unifycr_fid_truncate(int fid, off_t length); +int unifyfs_fid_truncate(int fid, off_t length); /* opens a new file id with specified path, access flags, and permissions, * fills outfid with file id and outpos with position for current file pointer, - * returns UNIFYCR error code */ -int unifycr_fid_open(const char* path, int flags, mode_t mode, int* outfid, + * returns UNIFYFS error code */ +int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, off_t* outpos); -int unifycr_fid_close(int fid); +int unifyfs_fid_close(int fid); /* delete a file id and return file its resources to free pools */ -int unifycr_fid_unlink(int fid); +int unifyfs_fid_unlink(int fid); -/* functions used in UnifyCR */ +/* functions used in UnifyFS */ -int unifycr_generate_gfid(const char* path); +int unifyfs_generate_gfid(const char* path); -int unifycr_set_global_file_meta(const char* path, int fid, int gfid, - struct stat* sb); +int unifyfs_set_global_file_meta(int fid, int gfid); -int unifycr_get_global_file_meta(int fid, int gfid, - unifycr_file_attr_t* gfattr); +int unifyfs_get_global_file_meta(int fid, int gfid, + unifyfs_file_attr_t* gfattr); // These require types/structures defined above -#include "unifycr-fixed.h" -#include "unifycr-stdio.h" -#include "unifycr-sysio.h" -#include "unifycr-dirops.h" +#include "unifyfs-fixed.h" +#include "unifyfs-stdio.h" +#include "unifyfs-sysio.h" +#include "unifyfs-dirops.h" -#endif /* UNIFYCR_INTERNAL_H */ +#endif /* UNIFYFS_INTERNAL_H */ diff --git a/client/src/unifycr-stack.c b/client/src/unifyfs-stack.c similarity index 79% rename from client/src/unifycr-stack.c rename to client/src/unifyfs-stack.c index 08ed19285..8d40b36c9 100644 --- a/client/src/unifycr-stack.c +++ b/client/src/unifyfs-stack.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -48,24 +48,24 @@ * last records index within entries that points to item one past * the item at the top of the stack */ -#include "unifycr-stack.h" +#include "unifyfs-stack.h" /* returns number of bytes needed to represent stack data structure */ -size_t unifycr_stack_bytes(int size) +size_t unifyfs_stack_bytes(int size) { - size_t bytes = sizeof(unifycr_stack) + size * sizeof(int); + size_t bytes = sizeof(unifyfs_stack) + size * sizeof(int); return bytes; } /* intializes stack to record all entries as being free */ -void unifycr_stack_init(void* start, int size) +void unifyfs_stack_init(void* start, int size) { - unifycr_stack* stack = (unifycr_stack*) start; + unifyfs_stack* stack = (unifyfs_stack*) start; stack->size = size; stack->last = size; int i; - int* entries = (int*)((char*)start + sizeof(unifycr_stack)); + int* entries = (int*)((char*)start + sizeof(unifyfs_stack)); for (i = 0; i < size; i++) { /* flip the order so low numbers are at the top * to make debugging easier */ @@ -74,9 +74,9 @@ void unifycr_stack_init(void* start, int size) } /* pops one entry from stack and returns its value */ -int unifycr_stack_pop(void* start) +int unifyfs_stack_pop(void* start) { - unifycr_stack* stack = (unifycr_stack*) start; + unifyfs_stack* stack = (unifyfs_stack*) start; /* check that the stack isn't empty */ if (stack->last > 0) { @@ -88,7 +88,7 @@ int unifycr_stack_pop(void* start) int idx = stack->last; /* return the value of this item */ - int* entries = (int*)((char*)start + sizeof(unifycr_stack)); + int* entries = (int*)((char*)start + sizeof(unifyfs_stack)); int value = entries[idx]; return value; } else { @@ -98,9 +98,9 @@ int unifycr_stack_pop(void* start) } /* pushes item onto free stack */ -void unifycr_stack_push(void* start, int value) +void unifyfs_stack_push(void* start, int value) { - unifycr_stack* stack = (unifycr_stack*) start; + unifyfs_stack* stack = (unifyfs_stack*) start; /* check that we have space to push item onto stack */ if (stack->last < stack->size) { @@ -108,7 +108,7 @@ void unifycr_stack_push(void* start, int value) int idx = stack->last; /* place item on stack */ - int* entries = (int*)((char*)start + sizeof(unifycr_stack)); + int* entries = (int*)((char*)start + sizeof(unifyfs_stack)); entries[idx] = value; /* increment last pointer to point to next item */ diff --git a/client/src/unifycr-stack.h b/client/src/unifyfs-stack.h similarity index 70% rename from client/src/unifycr-stack.h rename to client/src/unifyfs-stack.h index b12013d63..3dd475e19 100644 --- a/client/src/unifycr-stack.h +++ b/client/src/unifyfs-stack.h @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -22,9 +22,9 @@ * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror * LLNL-CODE-728877. All rights reserved. * - * This file is part of UnifyCR. - * For details, see https://github.com/llnl/unifycr - * Please read https://github.com/llnl/unifycr/LICENSE for full license text. + * This file is part of UnifyFS. + * For details, see https://github.com/llnl/unifyfs + * Please read https://github.com/llnl/unifyfs/LICENSE for full license text. */ /* @@ -35,13 +35,13 @@ * Kathryn Mohror * Adam Moody * All rights reserved. - * This file is part of UNIFYCR. - * For details, see https://github.com/hpc/unifycr - * Please also read this file LICENSE.UNIFYCR + * This file is part of UNIFYFS. + * For details, see https://github.com/hpc/unifyfs + * Please also read this file LICENSE.UNIFYFS */ -#ifndef UNIFYCR_STACK_H -#define UNIFYCR_STACK_H +#ifndef UNIFYFS_STACK_H +#define UNIFYFS_STACK_H /* implements a fixed-size stack which stores integer values in range * of 0 to size-1, entire structure stored in an int array of size+2 @@ -58,18 +58,18 @@ typedef struct { int size; int last; -} unifycr_stack; +} unifyfs_stack; /* returns number of bytes needed to represent stack data structure */ -size_t unifycr_stack_bytes(int size); +size_t unifyfs_stack_bytes(int size); /* intializes stack to record all entries as being free */ -void unifycr_stack_init(void* start, int size); +void unifyfs_stack_init(void* start, int size); /* pops one entry from stack and returns its value */ -int unifycr_stack_pop(void* start); +int unifyfs_stack_pop(void* start); /* pushes item onto free stack */ -void unifycr_stack_push(void* start, int value); +void unifyfs_stack_push(void* start, int value); -#endif /* UNIFYCR_STACK_H */ +#endif /* UNIFYFS_STACK_H */ diff --git a/client/src/unifycr-stdio.c b/client/src/unifyfs-stdio.c similarity index 79% rename from client/src/unifycr-stdio.c rename to client/src/unifyfs-stdio.c index 5b813564f..0c3787023 100644 --- a/client/src/unifycr-stdio.c +++ b/client/src/unifyfs-stdio.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -40,10 +40,10 @@ * Please also read this file LICENSE.CRUISE */ -#include "unifycr-stdio.h" -#include "unifycr-sysio.h" +#include "unifyfs-stdio.h" +#include "unifyfs-sysio.h" -static int unifycr_fpos_enabled = 1; /* whether we can use fgetpos/fsetpos */ +static int unifyfs_fpos_enabled = 1; /* whether we can use fgetpos/fsetpos */ /* --------------------------------------- * POSIX wrappers: file streams @@ -75,24 +75,24 @@ static int unifycr_fpos_enabled = 1; /* whether we can use fgetpos/fsetpos */ /* http://www.open-std.org/jtc1/sc22/wg14/www/docs/n1124.pdf 7.19 */ -/* TODO: add "unifycr_unsupported" call to report details of unsupported fns */ +/* TODO: add "unifyfs_unsupported" call to report details of unsupported fns */ /* given a stream, return file name or NULL if invalid */ -static const char* unifycr_stream_name(FILE* fp) +static const char* unifyfs_stream_name(FILE* fp) { - /* convert to unifycr_stream_t pointer */ - unifycr_stream_t* s = (unifycr_stream_t*) fp; + /* convert to unifyfs_stream_t pointer */ + unifyfs_stream_t* s = (unifyfs_stream_t*) fp; /* get name of file */ const char* name = NULL; - int fid = unifycr_get_fid_from_fd(s->fd); + int fid = unifyfs_get_fid_from_fd(s->fd); if (fid >= 0) { - name = unifycr_filelist[fid].filename; + name = unifyfs_filelist[fid].filename; } return name; } -int unifycr_unsupported_stream( +int unifyfs_unsupported_stream( FILE* fp, const char* wrap_fn, const char* wrap_file, @@ -100,14 +100,14 @@ int unifycr_unsupported_stream( const char* format, ...) { - /* convert to unifycr_stream_t pointer */ - unifycr_stream_t* s = (unifycr_stream_t*) fp; + /* convert to unifyfs_stream_t pointer */ + unifyfs_stream_t* s = (unifyfs_stream_t*) fp; /* get name of file */ - const char* name = unifycr_stream_name(fp); + const char* name = unifyfs_stream_name(fp); /* get file position */ - unifycr_fd_t* filedesc = unifycr_get_filedesc_from_fd(s->fd); + unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(s->fd); off_t pos = filedesc->pos; /* determine length of string to hold formatted args */ @@ -132,7 +132,7 @@ int unifycr_unsupported_stream( /* print message */ va_list args; va_start(args, format); - int rc = unifycr_unsupported( + int rc = unifyfs_unsupported( wrap_fn, wrap_file, wrap_line, "file %s pos %lu msg %s", name, (unsigned long) pos, str ); @@ -144,15 +144,15 @@ int unifycr_unsupported_stream( return rc; } -int unifycr_stream_set_pointers(unifycr_stream_t* s) +int unifyfs_stream_set_pointers(unifyfs_stream_t* s) { /* get pointer to file descriptor structure */ - unifycr_fd_t* filedesc = unifycr_get_filedesc_from_fd(s->fd); + unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(s->fd); if (filedesc == NULL) { /* ERROR: invalid file descriptor */ s->err = 1; errno = EBADF; - return UNIFYCR_ERROR_BADF; + return UNIFYFS_ERROR_BADF; } /* if we have anything on the push back buffer, that must be @@ -161,7 +161,7 @@ int unifycr_stream_set_pointers(unifycr_stream_t* s) if (s->ubuflen > 0) { s->_p = s->ubuf + s->ubufsize - s->ubuflen; s->_r = s->ubuflen; - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /* check that current falls within buffer */ @@ -181,7 +181,7 @@ int unifycr_stream_set_pointers(unifycr_stream_t* s) s->_r = stream_remaining; } - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /* TODO: support other modes as listed in @@ -189,9 +189,9 @@ int unifycr_stream_set_pointers(unifycr_stream_t* s) /* given a mode like "r", "wb+", or "a+" return flags read, write, * append, and plus to indicate which were set, - * returns UNIFYCR_ERROR_INVAL if invalid character is found + * returns UNIFYFS_ERROR_INVAL if invalid character is found */ -static int unifycr_fopen_parse_mode( +static int unifyfs_fopen_parse_mode( const char* mode, int* read, int* write, @@ -204,15 +204,15 @@ static int unifycr_fopen_parse_mode( *append = 0; *plus = 0; - /* ensure that user specifed an input mode */ + /* ensure that user specified an input mode */ if (mode == NULL) { - return UNIFYCR_ERROR_INVAL; + return UNIFYFS_ERROR_INVAL; } /* get number of characters in mode */ size_t len = strlen(mode); if (len <= 0 || len > 3) { - return UNIFYCR_ERROR_INVAL; + return UNIFYFS_ERROR_INVAL; } /* first character must either be r, w, or a */ @@ -228,7 +228,7 @@ static int unifycr_fopen_parse_mode( *append = 1; break; default: - return UNIFYCR_ERROR_INVAL; + return UNIFYFS_ERROR_INVAL; } /* optional second character may either be + or b */ @@ -243,7 +243,7 @@ static int unifycr_fopen_parse_mode( char third = mode[2]; if (third != 'b') { /* third character something other than + or b */ - return UNIFYCR_ERROR_INVAL; + return UNIFYFS_ERROR_INVAL; } } } else if (second == 'b') { @@ -254,23 +254,24 @@ static int unifycr_fopen_parse_mode( *plus = 1; } else { /* third character something other than + or b */ - return UNIFYCR_ERROR_INVAL; + return UNIFYFS_ERROR_INVAL; } } } else { /* second character something other than + or b */ - return UNIFYCR_ERROR_INVAL; + return UNIFYFS_ERROR_INVAL; } } - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } -/* calls unifycr_fid_open to open specified file in mode according to - * fopen mode semantics, initializes outstream and returns - * UNIFYCR_SUCCESS if successful, returns some other UNIFYCR error - * otherwise */ -static int unifycr_fopen( +/* + * Calls unifyfs_fid_open() to open specified file in mode according to + * fopen mode semantics, initializes outstream and returns UNIFYFS_SUCCESS if + * successful. Returns some other UNIFYFS/errno error otherwise. + */ +static int unifyfs_fopen( const char* path, const char* mode, FILE** outstream) @@ -280,39 +281,39 @@ static int unifycr_fopen( /* parse the fopen mode string */ int read, write, append, plus; - int parse_rc = unifycr_fopen_parse_mode(mode, &read, &write, &append, &plus); - if (parse_rc != UNIFYCR_SUCCESS) { + int parse_rc = unifyfs_fopen_parse_mode(mode, &read, &write, &append, &plus); + if (parse_rc != UNIFYFS_SUCCESS) { return parse_rc; } /* TODO: get real permissions via umask */ /* assume default permissions */ - mode_t perms = unifycr_getmode(0); + mode_t perms = unifyfs_getmode(0); int open_rc = -1; int fid; off_t pos; if (read) { - /* read shall fail if file does not already exist, unifycr_fid_open - * returns UNIFYCR_ERROR_NOENT if file does not exist w/o O_CREAT + /* read shall fail if file does not already exist, unifyfs_fid_open + * returns UNIFYFS_ERROR_NOENT if file does not exist w/o O_CREAT */ if (plus) { /* r+ ==> open file for update (reading and writing) */ - open_rc = unifycr_fid_open(path, O_RDWR, perms, &fid, &pos); + open_rc = unifyfs_fid_open(path, O_RDWR, perms, &fid, &pos); } else { /* r ==> open file for reading */ - open_rc = unifycr_fid_open(path, O_RDONLY, perms, &fid, &pos); + open_rc = unifyfs_fid_open(path, O_RDONLY, perms, &fid, &pos); } } else if (write) { if (plus) { /* w+ ==> truncate to zero length or create file for update * (read/write) */ - open_rc = unifycr_fid_open(path, O_RDWR | O_CREAT | O_TRUNC, + open_rc = unifyfs_fid_open(path, O_RDWR | O_CREAT | O_TRUNC, perms, &fid, &pos); } else { /* w ==> truncate to zero length or create file for writing */ - open_rc = unifycr_fid_open(path, O_WRONLY | O_CREAT | O_TRUNC, + open_rc = unifyfs_fid_open(path, O_WRONLY | O_CREAT | O_TRUNC, perms, &fid, &pos); } } else if (append) { @@ -320,48 +321,48 @@ static int unifycr_fopen( if (plus) { /* a+ ==> append, open or create file for update, initial file * position for reading should be at start */ - open_rc = unifycr_fid_open(path, O_RDWR | O_CREAT, + open_rc = unifyfs_fid_open(path, O_RDWR | O_CREAT, perms, &fid, &pos); } else { /* a ==> append, open or create file for writing, at end of file */ - open_rc = unifycr_fid_open(path, O_WRONLY | O_CREAT | O_APPEND, + open_rc = unifyfs_fid_open(path, O_WRONLY | O_CREAT | O_APPEND, perms, &fid, &pos); } } /* check the open return code */ - if (open_rc != UNIFYCR_SUCCESS) { + if (open_rc != UNIFYFS_SUCCESS) { return open_rc; } /* allocate a stream for this file */ - int sid = unifycr_stack_pop(unifycr_stream_stack); + int sid = unifyfs_stack_pop(unifyfs_stream_stack); if (sid < 0) { /* TODO: would like to return EMFILE to indicate * process has hit file stream limit, not the OS */ /* exhausted our file streams */ - return UNIFYCR_ERROR_NFILE; + return UNIFYFS_ERROR_NFILE; } /* get stream structure corresponding to stream id */ - unifycr_stream_t* s = &(unifycr_streams[sid]); + unifyfs_stream_t* s = &(unifyfs_streams[sid]); /* allocate a file descriptor for this file */ - int fd = unifycr_stack_pop(unifycr_fd_stack); + int fd = unifyfs_stack_pop(unifyfs_fd_stack); if (fd < 0) { /* TODO: would like to return EMFILE to indicate * process has hit file descriptor limit, not the OS */ /* put back our stream id */ - unifycr_stack_push(unifycr_stream_stack, sid); + unifyfs_stack_push(unifyfs_stream_stack, sid); /* exhausted our file descriptors */ - return UNIFYCR_ERROR_NFILE; + return UNIFYFS_ERROR_NFILE; } /* set file pointer and read/write mode in file descriptor */ - unifycr_fd_t* filedesc = unifycr_get_filedesc_from_fd(fd); + unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(fd); filedesc->fid = fid; filedesc->pos = pos; filedesc->read = read || plus; @@ -379,7 +380,7 @@ static int unifycr_fopen( s->append = append; /* set orientation to NULL */ - s->orient = UNIFYCR_STREAM_ORIENTATION_NULL; + s->orient = UNIFYFS_STREAM_ORIENTATION_NULL; /* default to fully buffered, set buffer to NULL to indicate * setvbuf has not been called */ @@ -401,39 +402,39 @@ static int unifycr_fopen( /* set return parameter and return */ *outstream = (FILE*)s; - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /* associate buffer with stream, allocates a buffer of specified size * if buf is NULL, otherwise uses buffer passed by caller, also sets - * stream to fully/line/unbuffered, returns UNIFYCR error codes */ -static int unifycr_setvbuf( + * stream to fully/line/unbuffered, returns UNIFYFS error codes */ +static int unifyfs_setvbuf( FILE* stream, char* buf, int type, size_t size) { /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; /* TODO: check that stream is valid */ /* check whether we've already associated a buffer */ if (s->buf != NULL) { /* ERROR: stream already has buffer */ - return UNIFYCR_ERROR_BADF; + return UNIFYFS_ERROR_BADF; } /* check that the type argument is valid */ if (type != _IOFBF && type != _IOLBF && type != _IONBF) { /* ERROR: invalid type argument */ - return UNIFYCR_ERROR_INVAL; + return UNIFYFS_ERROR_INVAL; } /* check that size is valid */ if (size <= 0) { /* ERROR: invalid size argument */ - return UNIFYCR_ERROR_INVAL; + return UNIFYFS_ERROR_INVAL; } /* associate buffer with stream */ @@ -442,7 +443,7 @@ static int unifycr_setvbuf( s->buf = malloc(size); if (s->buf == NULL) { /* ERROR: no memory */ - return UNIFYCR_ERROR_NOMEM; + return UNIFYFS_ERROR_NOMEM; } /* remember that we need to free the buffer at the end */ s->buffree = 1; @@ -460,25 +461,25 @@ static int unifycr_setvbuf( s->buflen = 0; s->bufdirty = 0; - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } -/* calls unifycr_fd_write to flush stream if it is dirty, - * returns UNIFYCR error codes, sets stream error indicator and errno +/* calls unifyfs_fd_write to flush stream if it is dirty, + * returns UNIFYFS error codes, sets stream error indicator and errno * upon error */ -static int unifycr_stream_flush(FILE* stream) +static int unifyfs_stream_flush(FILE* stream) { /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; /* TODO: check that stream is valid */ /* if buffer is dirty, write data to file */ if (s->buf != NULL && s->bufdirty) { - int write_rc = unifycr_fd_write(s->fd, s->bufpos, s->buf, s->buflen); - if (write_rc != UNIFYCR_SUCCESS) { + int write_rc = unifyfs_fd_write(s->fd, s->bufpos, s->buf, s->buflen); + if (write_rc != UNIFYFS_SUCCESS) { s->err = 1; - errno = unifycr_err_map_to_errno(write_rc); + errno = unifyfs_err_map_to_errno(write_rc); return write_rc; } @@ -486,57 +487,57 @@ static int unifycr_stream_flush(FILE* stream) s->bufdirty = 0; } - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /* reads count bytes from stream into buf, sets stream EOF and error * indicators as appropriate, sets errno if error, updates file - * position, returns number of bytes read in retcount, returns UNIFYCR + * position, returns number of bytes read in retcount, returns UNIFYFS * error codes*/ -static int unifycr_stream_read( +static int unifyfs_stream_read( FILE* stream, void* buf, size_t count, size_t* retcount) { /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; /* clear pointers, will force a reset when refill is called */ s->_p = NULL; s->_r = 0; /* get pointer to file descriptor structure */ - unifycr_fd_t* filedesc = unifycr_get_filedesc_from_fd(s->fd); + unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(s->fd); if (filedesc == NULL) { /* ERROR: invalid file descriptor */ s->err = 1; errno = EBADF; - return UNIFYCR_ERROR_BADF; + return UNIFYFS_ERROR_BADF; } /* bail with error if stream not open for reading */ - if (! filedesc->read) { + if (!filedesc->read) { s->err = 1; errno = EBADF; - return UNIFYCR_ERROR_BADF; + return UNIFYFS_ERROR_BADF; } /* associate buffer with stream if we need to */ if (s->buf == NULL) { - int setvbuf_rc = unifycr_setvbuf(stream, NULL, s->buftype, - UNIFYCR_STREAM_BUFSIZE); - if (setvbuf_rc != UNIFYCR_SUCCESS) { + int setvbuf_rc = unifyfs_setvbuf(stream, NULL, s->buftype, + UNIFYFS_STREAM_BUFSIZE); + if (setvbuf_rc != UNIFYFS_SUCCESS) { /* ERROR: failed to associate buffer */ s->err = 1; - errno = unifycr_err_map_to_errno(setvbuf_rc); + errno = unifyfs_err_map_to_errno(setvbuf_rc); return setvbuf_rc; } } /* don't attempt read if end-of-file indicator is set */ if (s->eof) { - return UNIFYCR_FAILURE; + return UNIFYFS_FAILURE; } /* track our current position in the file and number of bytes @@ -545,10 +546,10 @@ static int unifycr_stream_read( size_t remaining = count; /* check that current + count doesn't overflow */ - if (unifycr_would_overflow_offt(current, (off_t) count)) { + if (unifyfs_would_overflow_offt(current, (off_t) count)) { s->err = 1; errno = EOVERFLOW; - return UNIFYCR_ERROR_OVERFLOW; + return UNIFYFS_ERROR_OVERFLOW; } /* take bytes from push back buffer if they exist */ @@ -585,19 +586,19 @@ static int unifycr_stream_read( /* current is outside the range of our buffer */ /* flush buffer if needed before read */ - int flush_rc = unifycr_stream_flush(stream); - if (flush_rc != UNIFYCR_SUCCESS) { + int flush_rc = unifyfs_stream_flush(stream); + if (flush_rc != UNIFYFS_SUCCESS) { /* ERROR: flush sets error indicator and errno */ return flush_rc; } /* read data from file into buffer */ size_t bufcount; - int read_rc = unifycr_fd_read(s->fd, current, s->buf, s->bufsize, &bufcount); - if (read_rc != UNIFYCR_SUCCESS) { + int read_rc = unifyfs_fd_read(s->fd, current, s->buf, s->bufsize, &bufcount); + if (read_rc != UNIFYFS_SUCCESS) { /* ERROR: read error, set error indicator and errno */ s->err = 1; - errno = unifycr_err_map_to_errno(read_rc); + errno = unifyfs_err_map_to_errno(read_rc); return read_rc; } @@ -644,19 +645,19 @@ static int unifycr_stream_read( } /* return success */ - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /* writes count bytes from buf to stream, sets stream EOF and error * indicators as appropriate, sets errno if error, updates file - * position, return UNIFYCR error codes */ -static int unifycr_stream_write( + * position, return UNIFYFS error codes */ +static int unifyfs_stream_write( FILE* stream, const void* buf, size_t count) { /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; /* clear pointers, will force a reset when refill is called */ s->_p = NULL; @@ -665,19 +666,19 @@ static int unifycr_stream_write( /* TODO: check that stream is valid */ /* get pointer to file descriptor structure */ - unifycr_fd_t* filedesc = unifycr_get_filedesc_from_fd(s->fd); + unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(s->fd); if (filedesc == NULL) { /* ERROR: invalid file descriptor */ s->err = 1; errno = EBADF; - return UNIFYCR_ERROR_BADF; + return UNIFYFS_ERROR_BADF; } /* bail with error if stream not open for writing */ - if (! filedesc->write) { + if (!filedesc->write) { s->err = 1; errno = EBADF; - return UNIFYCR_ERROR_BADF; + return UNIFYFS_ERROR_BADF; } /* TODO: Don't know what to do with push back bytes if write @@ -687,13 +688,13 @@ static int unifycr_stream_write( off_t current; if (s->append) { /* if in append mode, always write to end of file */ - int fid = unifycr_get_fid_from_fd(s->fd); + int fid = unifyfs_get_fid_from_fd(s->fd); if (fid < 0) { s->err = 1; errno = EBADF; - return UNIFYCR_ERROR_BADF; + return UNIFYFS_ERROR_BADF; } - current = unifycr_fid_size(fid); + current = unifyfs_fid_size(fid); /* like a seek, we discard push back bytes */ s->ubuflen = 0; @@ -712,20 +713,20 @@ static int unifycr_stream_write( } /* check that current + count doesn't overflow */ - if (unifycr_would_overflow_offt(current, (off_t) count)) { + if (unifyfs_would_overflow_offt(current, (off_t) count)) { s->err = 1; errno = EFBIG; - return UNIFYCR_ERROR_FBIG; + return UNIFYFS_ERROR_FBIG; } /* associate buffer with stream if we need to */ if (s->buf == NULL) { - int setvbuf_rc = unifycr_setvbuf(stream, NULL, s->buftype, - UNIFYCR_STREAM_BUFSIZE); - if (setvbuf_rc != UNIFYCR_SUCCESS) { + int setvbuf_rc = unifyfs_setvbuf(stream, NULL, s->buftype, + UNIFYFS_STREAM_BUFSIZE); + if (setvbuf_rc != UNIFYFS_SUCCESS) { /* ERROR: failed to associate buffer */ s->err = 1; - errno = unifycr_err_map_to_errno(setvbuf_rc); + errno = unifyfs_err_map_to_errno(setvbuf_rc); return setvbuf_rc; } } @@ -733,18 +734,18 @@ static int unifycr_stream_write( /* if unbuffered, write data directly to file */ if (s->buftype == _IONBF) { /* write data directly to file */ - int write_rc = unifycr_fd_write(s->fd, current, buf, count); - if (write_rc != UNIFYCR_SUCCESS) { + int write_rc = unifyfs_fd_write(s->fd, current, buf, count); + if (write_rc != UNIFYFS_SUCCESS) { /* ERROR: write error, set error indicator and errno */ s->err = 1; - errno = unifycr_err_map_to_errno(write_rc); + errno = unifyfs_err_map_to_errno(write_rc); return write_rc; } /* update file position */ filedesc->pos = current + (off_t) count; - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /* TODO: if count is large enough, write directly to file @@ -754,7 +755,7 @@ static int unifycr_stream_write( size_t remaining = count; while (remaining > 0) { /* if buffer is clean, set start of buffer to current */ - if (! s->bufdirty) { + if (!s->bufdirty) { s->bufpos = current; s->buflen = 0; } @@ -787,7 +788,7 @@ static int unifycr_stream_write( /* ERROR: write error, set error indicator and errno */ s->err = 1; errno = ENOMEM; - return UNIFYCR_ERROR_NOMEM; + return UNIFYFS_ERROR_NOMEM; } } else { /* fully buffered, write until we hit the buffer limit */ @@ -812,8 +813,8 @@ static int unifycr_stream_write( /* if we've filled the buffer, flush it */ if (need_flush) { /* flush stream */ - int flush_rc = unifycr_stream_flush(stream); - if (flush_rc != UNIFYCR_SUCCESS) { + int flush_rc = unifyfs_stream_flush(stream); + if (flush_rc != UNIFYFS_SUCCESS) { /* ERROR: flush sets error indicator and errno */ return flush_rc; } @@ -832,23 +833,23 @@ static int unifycr_stream_write( /* update file position */ filedesc->pos = current; - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /* fseek, fseeko, rewind, and fsetpos all call this function, sets error * indicator and errno if necessary, returns -1 on error, returns * 0 for success */ -static int unifycr_fseek(FILE* stream, off_t offset, int whence) +static int unifyfs_fseek(FILE* stream, off_t offset, int whence) { /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; /* clear pointers, will force a reset when refill is called */ s->_p = NULL; s->_r = 0; /* get pointer to file descriptor structure */ - unifycr_fd_t* filedesc = unifycr_get_filedesc_from_fd(s->fd); + unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(s->fd); if (filedesc == NULL) { /* ERROR: invalid file descriptor */ s->err = 1; @@ -857,14 +858,14 @@ static int unifycr_fseek(FILE* stream, off_t offset, int whence) } /* flush stream if we need to */ - int flush_rc = unifycr_stream_flush(stream); - if (flush_rc != UNIFYCR_SUCCESS) { + int flush_rc = unifyfs_stream_flush(stream); + if (flush_rc != UNIFYFS_SUCCESS) { /* ERROR: flush sets error indicator and errno */ return -1; } /* get the file id for this file descriptor */ - int fid = unifycr_get_fid_from_fd(s->fd); + int fid = unifyfs_get_fid_from_fd(s->fd); if (fid < 0) { /* couldn't find file id, so assume we're at the end, * feof defines to errors */ @@ -885,7 +886,7 @@ static int unifycr_fseek(FILE* stream, off_t offset, int whence) break; case SEEK_CUR: /* seek to current position + offset */ - if (unifycr_would_overflow_offt(current_pos, offset)) { + if (unifyfs_would_overflow_offt(current_pos, offset)) { s->err = 1; errno = EOVERFLOW; return -1; @@ -894,8 +895,8 @@ static int unifycr_fseek(FILE* stream, off_t offset, int whence) break; case SEEK_END: /* seek to EOF + offset */ - filesize = unifycr_fid_size(fid); - if (unifycr_would_overflow_offt(filesize, offset)) { + filesize = unifyfs_fid_size(fid); + if (unifyfs_would_overflow_offt(filesize, offset)) { s->err = 1; errno = EOVERFLOW; return -1; @@ -924,82 +925,82 @@ static int unifycr_fseek(FILE* stream, off_t offset, int whence) return 0; } -FILE* UNIFYCR_WRAP(fopen)(const char* path, const char* mode) +FILE* UNIFYFS_WRAP(fopen)(const char* path, const char* mode) { /* check whether we should intercept this path */ - if (unifycr_intercept_path(path)) { + if (unifyfs_intercept_path(path)) { FILE* stream; - int rc = unifycr_fopen(path, mode, &stream); - if (rc != UNIFYCR_SUCCESS) { - errno = unifycr_err_map_to_errno(rc); + int rc = unifyfs_fopen(path, mode, &stream); + if (rc != UNIFYFS_SUCCESS) { + errno = unifyfs_err_map_to_errno(rc); return NULL; } return stream; } else { MAP_OR_FAIL(fopen); - FILE* ret = UNIFYCR_REAL(fopen)(path, mode); + FILE* ret = UNIFYFS_REAL(fopen)(path, mode); return ret; } } -FILE* UNIFYCR_WRAP(freopen)(const char* path, const char* mode, FILE* stream) +FILE* UNIFYFS_WRAP(freopen)(const char* path, const char* mode, FILE* stream) { /* check whether we should intercept this path */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* return file descriptor associated with stream */ - unifycr_unsupported_stream(stream, __func__, __FILE__, __LINE__, "new file %s", + unifyfs_unsupported_stream(stream, __func__, __FILE__, __LINE__, "new file %s", path); /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; s->err = 1; errno = EIO; return NULL; } else { MAP_OR_FAIL(freopen); - FILE* ret = UNIFYCR_REAL(freopen)(path, mode, stream); + FILE* ret = UNIFYFS_REAL(freopen)(path, mode, stream); return ret; } } -int UNIFYCR_WRAP(setvbuf)(FILE* stream, char* buf, int type, size_t size) +int UNIFYFS_WRAP(setvbuf)(FILE* stream, char* buf, int type, size_t size) { /* check whether we should intercept this path */ - if (unifycr_intercept_stream(stream)) { - int rc = unifycr_setvbuf(stream, buf, type, size); - if (rc != UNIFYCR_SUCCESS) { - errno = unifycr_err_map_to_errno(rc); + if (unifyfs_intercept_stream(stream)) { + int rc = unifyfs_setvbuf(stream, buf, type, size); + if (rc != UNIFYFS_SUCCESS) { + errno = unifyfs_err_map_to_errno(rc); return 1; } return 0; } else { MAP_OR_FAIL(setvbuf); - int ret = UNIFYCR_REAL(setvbuf)(stream, buf, type, size); + int ret = UNIFYFS_REAL(setvbuf)(stream, buf, type, size); return ret; } } -void UNIFYCR_WRAP(setbuf)(FILE* stream, char* buf) +void UNIFYFS_WRAP(setbuf)(FILE* stream, char* buf) { /* check whether we should intercept this path */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { if (buf != NULL) { - unifycr_setvbuf(stream, buf, _IOFBF, BUFSIZ); + unifyfs_setvbuf(stream, buf, _IOFBF, BUFSIZ); } else { - unifycr_setvbuf(stream, buf, _IONBF, BUFSIZ); + unifyfs_setvbuf(stream, buf, _IONBF, BUFSIZ); } return; } else { MAP_OR_FAIL(setbuf); - UNIFYCR_REAL(setbuf)(stream, buf); + UNIFYFS_REAL(setbuf)(stream, buf); return; } } -int UNIFYCR_WRAP(ungetc)(int c, FILE* stream) +int UNIFYFS_WRAP(ungetc)(int c, FILE* stream) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* operation shall fail and input stream left unchanged */ if (c == EOF) { return EOF; @@ -1009,10 +1010,10 @@ int UNIFYCR_WRAP(ungetc)(int c, FILE* stream) unsigned char uc = (unsigned char) c; /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; /* get filedescriptor and check that stream is valid */ - unifycr_fd_t* filedesc = unifycr_get_filedesc_from_fd(s->fd); + unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(s->fd); if (filedesc == NULL) { return EOF; } @@ -1078,21 +1079,21 @@ int UNIFYCR_WRAP(ungetc)(int c, FILE* stream) return (int) uc; } else { MAP_OR_FAIL(ungetc); - int ret = UNIFYCR_REAL(ungetc)(c, stream); + int ret = UNIFYFS_REAL(ungetc)(c, stream); return ret; } } -int UNIFYCR_WRAP(fgetc)(FILE* stream) +int UNIFYFS_WRAP(fgetc)(FILE* stream) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* read next character from file */ unsigned char charbuf; size_t count = 1; size_t retcount; - int read_rc = unifycr_stream_read(stream, &charbuf, count, &retcount); - if (read_rc != UNIFYCR_SUCCESS || retcount == 0) { + int read_rc = unifyfs_stream_read(stream, &charbuf, count, &retcount); + if (read_rc != UNIFYFS_SUCCESS || retcount == 0) { /* stream read sets error indicator, EOF indicator, * and errno for us */ return EOF; @@ -1102,20 +1103,20 @@ int UNIFYCR_WRAP(fgetc)(FILE* stream) return (int) charbuf; } else { MAP_OR_FAIL(fgetc); - int ret = UNIFYCR_REAL(fgetc)(stream); + int ret = UNIFYFS_REAL(fgetc)(stream); return ret; } } -int UNIFYCR_WRAP(fputc)(int c, FILE* stream) +int UNIFYFS_WRAP(fputc)(int c, FILE* stream) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* write data to file */ unsigned char charbuf = (unsigned char) c; size_t count = 1; - int write_rc = unifycr_stream_write(stream, &charbuf, count); - if (write_rc != UNIFYCR_SUCCESS) { + int write_rc = unifyfs_stream_write(stream, &charbuf, count); + if (write_rc != UNIFYFS_SUCCESS) { /* stream write sets error indicator, EOF indicator, * and errno for us */ return EOF; @@ -1125,21 +1126,21 @@ int UNIFYCR_WRAP(fputc)(int c, FILE* stream) return (int) charbuf; } else { MAP_OR_FAIL(fputc); - int ret = UNIFYCR_REAL(fputc)(c, stream); + int ret = UNIFYFS_REAL(fputc)(c, stream); return ret; } } -int UNIFYCR_WRAP(getc)(FILE* stream) +int UNIFYFS_WRAP(getc)(FILE* stream) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* read next character from file */ unsigned char charbuf; size_t count = 1; size_t retcount; - int read_rc = unifycr_stream_read(stream, &charbuf, count, &retcount); - if (read_rc != UNIFYCR_SUCCESS || retcount == 0) { + int read_rc = unifyfs_stream_read(stream, &charbuf, count, &retcount); + if (read_rc != UNIFYFS_SUCCESS || retcount == 0) { /* stream read sets error indicator, EOF indicator, * and errno for us */ return EOF; @@ -1149,20 +1150,20 @@ int UNIFYCR_WRAP(getc)(FILE* stream) return (int) charbuf; } else { MAP_OR_FAIL(getc); - int ret = UNIFYCR_REAL(getc)(stream); + int ret = UNIFYFS_REAL(getc)(stream); return ret; } } -int UNIFYCR_WRAP(putc)(int c, FILE* stream) +int UNIFYFS_WRAP(putc)(int c, FILE* stream) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* write data to file */ unsigned char charbuf = (unsigned char) c; size_t count = 1; - int write_rc = unifycr_stream_write(stream, &charbuf, count); - if (write_rc != UNIFYCR_SUCCESS) { + int write_rc = unifyfs_stream_write(stream, &charbuf, count); + if (write_rc != UNIFYFS_SUCCESS) { /* stream write sets error indicator, EOF indicator, * and errno for us */ return EOF; @@ -1172,21 +1173,21 @@ int UNIFYCR_WRAP(putc)(int c, FILE* stream) return (int) charbuf; } else { MAP_OR_FAIL(putc); - int ret = UNIFYCR_REAL(putc)(c, stream); + int ret = UNIFYFS_REAL(putc)(c, stream); return ret; } } -char* UNIFYCR_WRAP(fgets)(char* s, int n, FILE* stream) +char* UNIFYFS_WRAP(fgets)(char* s, int n, FILE* stream) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* TODO: this isn't the most efficient algorithm, but it works, * would be faster to read a large block of characters then * scan for newline */ /* lookup stream */ - unifycr_stream_t* stm = (unifycr_stream_t*) stream; + unifyfs_stream_t* stm = (unifyfs_stream_t*) stream; /* TODO: check that stream is valid */ @@ -1205,11 +1206,11 @@ char* UNIFYCR_WRAP(fgets)(char* s, int n, FILE* stream) /* read the next character from the file */ char charbuf; size_t retcount; - int read_rc = unifycr_stream_read(stream, &charbuf, 1, &retcount); - if (read_rc != UNIFYCR_SUCCESS) { + int read_rc = unifyfs_stream_read(stream, &charbuf, 1, &retcount); + if (read_rc != UNIFYFS_SUCCESS) { /* if error flag is not set, we must have hit EOF, * terminate string before returning */ - if (! stm->err) { + if (!stm->err) { s[limit] = '\0'; } @@ -1234,23 +1235,23 @@ char* UNIFYCR_WRAP(fgets)(char* s, int n, FILE* stream) return s; } else { MAP_OR_FAIL(fgets); - char* ret = UNIFYCR_REAL(fgets)(s, n, stream); + char* ret = UNIFYFS_REAL(fgets)(s, n, stream); return ret; } } -int UNIFYCR_WRAP(fputs)(const char* s, FILE* stream) +int UNIFYFS_WRAP(fputs)(const char* s, FILE* stream) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* TODO: check that s is not NULL */ /* get length of string, less the NUL-terminating byte */ size_t count = strlen(s); /* write data to file */ - int write_rc = unifycr_stream_write(stream, (const void*)s, count); - if (write_rc != UNIFYCR_SUCCESS) { + int write_rc = unifyfs_stream_write(stream, (const void*)s, count); + if (write_rc != UNIFYFS_SUCCESS) { /* stream write sets error indicator, EOF indicator, * and errno for us */ return EOF; @@ -1260,15 +1261,15 @@ int UNIFYCR_WRAP(fputs)(const char* s, FILE* stream) return 0; } else { MAP_OR_FAIL(fputs); - int ret = UNIFYCR_REAL(fputs)(s, stream); + int ret = UNIFYFS_REAL(fputs)(s, stream); return ret; } } -size_t UNIFYCR_WRAP(fread)(void* ptr, size_t size, size_t nitems, FILE* stream) +size_t UNIFYFS_WRAP(fread)(void* ptr, size_t size, size_t nitems, FILE* stream) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* must return 0 and do nothing if size or nitems is zero */ if (size == 0 || nitems == 0) { return 0; @@ -1280,8 +1281,8 @@ size_t UNIFYCR_WRAP(fread)(void* ptr, size_t size, size_t nitems, FILE* stream) /* read next character from file */ size_t retcount; - int read_rc = unifycr_stream_read(stream, ptr, count, &retcount); - if (read_rc != UNIFYCR_SUCCESS) { + int read_rc = unifyfs_stream_read(stream, ptr, count, &retcount); + if (read_rc != UNIFYFS_SUCCESS) { /* stream read sets error indicator, EOF indicator, * and errno for us */ return 0; @@ -1297,16 +1298,16 @@ size_t UNIFYCR_WRAP(fread)(void* ptr, size_t size, size_t nitems, FILE* stream) } } else { MAP_OR_FAIL(fread); - size_t ret = UNIFYCR_REAL(fread)(ptr, size, nitems, stream); + size_t ret = UNIFYFS_REAL(fread)(ptr, size, nitems, stream); return ret; } } -size_t UNIFYCR_WRAP(fwrite)(const void* ptr, size_t size, size_t nitems, +size_t UNIFYFS_WRAP(fwrite)(const void* ptr, size_t size, size_t nitems, FILE* stream) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* must return 0 and do nothing if size or nitems is zero */ if (size == 0 || nitems == 0) { return 0; @@ -1317,8 +1318,8 @@ size_t UNIFYCR_WRAP(fwrite)(const void* ptr, size_t size, size_t nitems, size_t count = size * nitems; /* write data to file */ - int write_rc = unifycr_stream_write(stream, ptr, count); - if (write_rc != UNIFYCR_SUCCESS) { + int write_rc = unifyfs_stream_write(stream, ptr, count); + if (write_rc != UNIFYFS_SUCCESS) { /* stream write sets error indicator, EOF indicator, * and errno for us */ return 0; @@ -1328,17 +1329,17 @@ size_t UNIFYCR_WRAP(fwrite)(const void* ptr, size_t size, size_t nitems, return nitems; } else { MAP_OR_FAIL(fwrite); - size_t ret = UNIFYCR_REAL(fwrite)(ptr, size, nitems, stream); + size_t ret = UNIFYFS_REAL(fwrite)(ptr, size, nitems, stream); return ret; } } -int UNIFYCR_WRAP(vfprintf)(FILE* stream, const char* format, va_list ap) +int UNIFYFS_WRAP(vfprintf)(FILE* stream, const char* format, va_list ap) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; /* TODO: check that stream is active */ @@ -1370,8 +1371,8 @@ int UNIFYCR_WRAP(vfprintf)(FILE* stream, const char* format, va_list ap) } /* write data to file */ - int write_rc = unifycr_stream_write(stream, str, chars); - if (write_rc != UNIFYCR_SUCCESS) { + int write_rc = unifyfs_stream_write(stream, str, chars); + if (write_rc != UNIFYFS_SUCCESS) { /* stream write sets error indicator, EOF indicator, * and errno for us */ return -1; @@ -1386,115 +1387,115 @@ int UNIFYCR_WRAP(vfprintf)(FILE* stream, const char* format, va_list ap) va_list ap2; va_copy(ap2, ap); MAP_OR_FAIL(vfprintf); - int ret = UNIFYCR_REAL(vfprintf)(stream, format, ap2); + int ret = UNIFYFS_REAL(vfprintf)(stream, format, ap2); va_end(ap2); return ret; } } -int UNIFYCR_WRAP(fprintf)(FILE* stream, const char* format, ...) +int UNIFYFS_WRAP(fprintf)(FILE* stream, const char* format, ...) { va_list args; int ret; /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* delegate work to vfprintf */ va_start(args, format); - ret = UNIFYCR_WRAP(vfprintf)(stream, format, args); + ret = UNIFYFS_WRAP(vfprintf)(stream, format, args); va_end(args); return ret; } else { va_start(args, format); MAP_OR_FAIL(vfprintf); - ret = UNIFYCR_REAL(vfprintf)(stream, format, args); + ret = UNIFYFS_REAL(vfprintf)(stream, format, args); va_end(args); return ret; } } /* need to declare this before calling it */ -static int __svfscanf(unifycr_stream_t* fp, const char* fmt0, va_list ap); +static int __svfscanf(unifyfs_stream_t* fp, const char* fmt0, va_list ap); -int UNIFYCR_WRAP(vfscanf)(FILE* stream, const char* format, va_list ap) +int UNIFYFS_WRAP(vfscanf)(FILE* stream, const char* format, va_list ap) { va_list args; int ret; /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { va_copy(args, ap); - ret = __svfscanf((unifycr_stream_t*)stream, format, args); + ret = __svfscanf((unifyfs_stream_t*)stream, format, args); va_end(args); return ret; } else { va_copy(args, ap); MAP_OR_FAIL(vfscanf); - ret = UNIFYCR_REAL(vfscanf)(stream, format, args); + ret = UNIFYFS_REAL(vfscanf)(stream, format, args); va_end(args); return ret; } } -int UNIFYCR_WRAP(fscanf)(FILE* stream, const char* format, ...) +int UNIFYFS_WRAP(fscanf)(FILE* stream, const char* format, ...) { va_list args; int ret; /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* delegate work to vfscanf */ va_start(args, format); - ret = UNIFYCR_WRAP(vfscanf)(stream, format, args); + ret = UNIFYFS_WRAP(vfscanf)(stream, format, args); va_end(args); return ret; } else { va_start(args, format); MAP_OR_FAIL(vfscanf); - ret = UNIFYCR_REAL(vfscanf)(stream, format, args); + ret = UNIFYFS_REAL(vfscanf)(stream, format, args); va_end(args); return ret; } } /* TODO: return error if new position overflows long */ -int UNIFYCR_WRAP(fseek)(FILE* stream, long offset, int whence) +int UNIFYFS_WRAP(fseek)(FILE* stream, long offset, int whence) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { off_t offset_offt = (off_t) offset; - int rc = unifycr_fseek(stream, offset_offt, whence); + int rc = unifyfs_fseek(stream, offset_offt, whence); return rc; } else { MAP_OR_FAIL(fseek); - int ret = UNIFYCR_REAL(fseek)(stream, offset, whence); + int ret = UNIFYFS_REAL(fseek)(stream, offset, whence); return ret; } } -int UNIFYCR_WRAP(fseeko)(FILE* stream, off_t offset, int whence) +int UNIFYFS_WRAP(fseeko)(FILE* stream, off_t offset, int whence) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { - int rc = unifycr_fseek(stream, offset, whence); + if (unifyfs_intercept_stream(stream)) { + int rc = unifyfs_fseek(stream, offset, whence); return rc; } else { MAP_OR_FAIL(fseeko); - int ret = UNIFYCR_REAL(fseeko)(stream, offset, whence); + int ret = UNIFYFS_REAL(fseeko)(stream, offset, whence); return ret; } } /* TODO: set EOVERFLOW if position overflows long */ -long UNIFYCR_WRAP(ftell)(FILE* stream) +long UNIFYFS_WRAP(ftell)(FILE* stream) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; /* get pointer to file descriptor structure */ - unifycr_fd_t* filedesc = unifycr_get_filedesc_from_fd(s->fd); + unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(s->fd); if (filedesc == NULL) { /* ERROR: invalid file descriptor */ s->err = 1; @@ -1507,20 +1508,20 @@ long UNIFYCR_WRAP(ftell)(FILE* stream) return (long)current_pos; } else { MAP_OR_FAIL(ftell); - long ret = UNIFYCR_REAL(ftell)(stream); + long ret = UNIFYFS_REAL(ftell)(stream); return ret; } } -off_t UNIFYCR_WRAP(ftello)(FILE* stream) +off_t UNIFYFS_WRAP(ftello)(FILE* stream) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; /* get pointer to file descriptor structure */ - unifycr_fd_t* filedesc = unifycr_get_filedesc_from_fd(s->fd); + unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(s->fd); if (filedesc == NULL) { /* ERROR: invalid file descriptor */ s->err = 1; @@ -1533,27 +1534,27 @@ off_t UNIFYCR_WRAP(ftello)(FILE* stream) return current_pos; } else { MAP_OR_FAIL(ftello); - off_t ret = UNIFYCR_REAL(ftello)(stream); + off_t ret = UNIFYFS_REAL(ftello)(stream); return ret; } } /* equivalent to fseek(stream, 0L, SEEK_SET) except shall also clear * error indicator */ -void UNIFYCR_WRAP(rewind)(FILE* stream) +void UNIFYFS_WRAP(rewind)(FILE* stream) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; /* TODO: check that stream is active */ /* seek to front of file */ - int rc = unifycr_fseek(stream, (off_t) 0L, SEEK_SET); + int rc = unifyfs_fseek(stream, (off_t) 0L, SEEK_SET); /* set errno */ - errno = unifycr_err_map_to_errno(rc); + errno = unifyfs_err_map_to_errno(rc); /* clear error indicator if seek successful */ if (rc == 0) { @@ -1563,32 +1564,32 @@ void UNIFYCR_WRAP(rewind)(FILE* stream) return; } else { MAP_OR_FAIL(rewind); - UNIFYCR_REAL(rewind)(stream); + UNIFYFS_REAL(rewind)(stream); return; } } -struct unifycr_fpos_t { +struct unifyfs_fpos_t { off_t pos; }; -int UNIFYCR_WRAP(fgetpos)(FILE* stream, fpos_t* pos) +int UNIFYFS_WRAP(fgetpos)(FILE* stream, fpos_t* pos) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* check that we can store a pointer in fpos_t */ - if (! unifycr_fpos_enabled) { + if (!unifyfs_fpos_enabled) { errno = EOVERFLOW; return 1; } /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; /* TODO: check that stream is active */ /* get file descriptor for stream */ - unifycr_fd_t* filedesc = unifycr_get_filedesc_from_fd(s->fd); + unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(s->fd); if (filedesc == NULL) { /* ERROR: invalid file descriptor */ errno = EBADF; @@ -1596,7 +1597,7 @@ int UNIFYCR_WRAP(fgetpos)(FILE* stream, fpos_t* pos) } /* allocate memory to hold state of stream */ - struct unifycr_fpos_t* state = malloc(sizeof(struct unifycr_fpos_t)); + struct unifyfs_fpos_t* state = malloc(sizeof(struct unifyfs_fpos_t)); if (state == NULL) { errno = ENOMEM; return 1; @@ -1612,22 +1613,22 @@ int UNIFYCR_WRAP(fgetpos)(FILE* stream, fpos_t* pos) return 0; } else { MAP_OR_FAIL(fgetpos); - int ret = UNIFYCR_REAL(fgetpos)(stream, pos); + int ret = UNIFYFS_REAL(fgetpos)(stream, pos); return ret; } } -int UNIFYCR_WRAP(fsetpos)(FILE* stream, const fpos_t* pos) +int UNIFYFS_WRAP(fsetpos)(FILE* stream, const fpos_t* pos) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; /* TODO: check that stream is active */ /* check that we can store a pointer in fpos_t */ - if (! unifycr_fpos_enabled) { + if (!unifyfs_fpos_enabled) { s->err = 1; errno = EOVERFLOW; return -1; @@ -1636,10 +1637,10 @@ int UNIFYCR_WRAP(fsetpos)(FILE* stream, const fpos_t* pos) /* get pointer to state from pos input parameter, * assumes pos is a pointer to void*, deference to get value * of void*, which we then cast to a state pointer, ugh */ - struct unifycr_fpos_t* state = (struct unifycr_fpos_t*) *(void**)pos; + struct unifyfs_fpos_t* state = (struct unifyfs_fpos_t*) *(void**)pos; /* semantics of fsetpos seem to match a seek */ - int seek_rc = unifycr_fseek(stream, state->pos, SEEK_SET); + int seek_rc = unifyfs_fseek(stream, state->pos, SEEK_SET); if (seek_rc != 0) { return seek_rc; } @@ -1650,30 +1651,30 @@ int UNIFYCR_WRAP(fsetpos)(FILE* stream, const fpos_t* pos) return 0; } else { MAP_OR_FAIL(fsetpos); - int ret = UNIFYCR_REAL(fsetpos)(stream, pos); + int ret = UNIFYFS_REAL(fsetpos)(stream, pos); return ret; } } -int UNIFYCR_WRAP(fflush)(FILE* stream) +int UNIFYFS_WRAP(fflush)(FILE* stream) { /* if stream is NULL, flush output on all streams */ if (stream == NULL) { /* first, have real library flush all of its streams, * important to do this in this order since it may set errno, - * which could override our setting for unifycr streams */ + * which could override our setting for unifyfs streams */ MAP_OR_FAIL(fflush); - int ret = UNIFYCR_REAL(fflush)(NULL); + int ret = UNIFYFS_REAL(fflush)(NULL); - /* flush each active unifycr stream */ + /* flush each active unifyfs stream */ int i; - for (i = 0; i < UNIFYCR_MAX_FILEDESCS; i++) { + for (i = 0; i < UNIFYFS_MAX_FILEDESCS; i++) { /* get stream and check whether it's active */ - unifycr_stream_t* s = &(unifycr_streams[i]); + unifyfs_stream_t* s = &(unifyfs_streams[i]); if (s->fd >= 0) { /* attempt to flush stream */ - int flush_rc = unifycr_stream_flush((FILE*)s); - if (flush_rc != UNIFYCR_SUCCESS) { + int flush_rc = unifyfs_stream_flush((FILE*)s); + if (flush_rc != UNIFYFS_SUCCESS) { /* ERROR: flush sets error indicator and errno */ ret = EOF; } @@ -1684,12 +1685,12 @@ int UNIFYCR_WRAP(fflush)(FILE* stream) } /* otherwise, check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* TODO: check that stream is active */ /* flush output on stream */ - int rc = unifycr_stream_flush(stream); + int rc = unifyfs_stream_flush(stream); - if (rc != UNIFYCR_SUCCESS) { + if (rc != UNIFYFS_SUCCESS) { /* ERROR: flush sets error indicator and errno */ return EOF; } @@ -1697,19 +1698,19 @@ int UNIFYCR_WRAP(fflush)(FILE* stream) return 0; } else { MAP_OR_FAIL(fflush); - int ret = UNIFYCR_REAL(fflush)(stream); + int ret = UNIFYFS_REAL(fflush)(stream); return ret; } } /* return non-zero if and only if end-of-file indicator is set * for stream */ -int UNIFYCR_WRAP(feof)(FILE* stream) +int UNIFYFS_WRAP(feof)(FILE* stream) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; /* TODO: ensure stream is active */ @@ -1717,17 +1718,17 @@ int UNIFYCR_WRAP(feof)(FILE* stream) return ret; } else { MAP_OR_FAIL(feof); - int ret = UNIFYCR_REAL(feof)(stream); + int ret = UNIFYFS_REAL(feof)(stream); return ret; } } -int UNIFYCR_WRAP(ferror)(FILE* stream) +int UNIFYFS_WRAP(ferror)(FILE* stream) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* lookup stream and file descriptor */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; /* TODO: ensure stream is active */ @@ -1735,17 +1736,17 @@ int UNIFYCR_WRAP(ferror)(FILE* stream) return ret; } else { MAP_OR_FAIL(ferror); - int ret = UNIFYCR_REAL(ferror)(stream); + int ret = UNIFYFS_REAL(ferror)(stream); return ret; } } -void UNIFYCR_WRAP(clearerr)(FILE* stream) +void UNIFYFS_WRAP(clearerr)(FILE* stream) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; /* TODO: ensure stream is active */ @@ -1755,18 +1756,19 @@ void UNIFYCR_WRAP(clearerr)(FILE* stream) return; } else { MAP_OR_FAIL(clearerr); - UNIFYCR_REAL(clearerr)(stream); + UNIFYFS_REAL(clearerr)(stream); return; } } -int UNIFYCR_WRAP(fileno)(FILE* stream) +int UNIFYFS_WRAP(fileno)(FILE* stream) { + int ret; /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; /* TODO: check that stream is valid */ int fd = s->fd; @@ -1775,32 +1777,34 @@ int UNIFYCR_WRAP(fileno)(FILE* stream) return -1; } - /* return file descriptor associated with stream */ - return fd; + /* return file descriptor associated with stream but don't conflict + * with active system fds that range from 0 - (fd_limit) */ + ret = fd + unifyfs_fd_limit; + return ret; } else { MAP_OR_FAIL(fileno); - int ret = UNIFYCR_REAL(fileno)(stream); + ret = UNIFYFS_REAL(fileno)(stream); return ret; } } -int UNIFYCR_WRAP(fclose)(FILE* stream) +int UNIFYFS_WRAP(fclose)(FILE* stream) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; /* get the file id for this file descriptor */ - int fid = unifycr_get_fid_from_fd(s->fd); + int fid = unifyfs_get_fid_from_fd(s->fd); if (fid < 0) { errno = EBADF; return EOF; } /* flush stream */ - int flush_rc = unifycr_stream_flush(stream); - if (flush_rc != UNIFYCR_SUCCESS) { + int flush_rc = unifyfs_stream_flush(stream); + if (flush_rc != UNIFYFS_SUCCESS) { /* ERROR: flush sets error indicator and errno */ return EOF; } @@ -1819,46 +1823,46 @@ int UNIFYCR_WRAP(fclose)(FILE* stream) } /* close the file */ - int close_rc = unifycr_fid_close(fid); - if (close_rc != UNIFYCR_SUCCESS) { - errno = unifycr_err_map_to_errno(close_rc); + int close_rc = unifyfs_fid_close(fid); + if (close_rc != UNIFYFS_SUCCESS) { + errno = unifyfs_err_map_to_errno(close_rc); return EOF; } /* reinitialize file descriptor to indicate that * it is no longer associated with a file, * not technically needed but may help catch bugs */ - unifycr_fd_init(s->fd); + unifyfs_fd_init(s->fd); /* add file descriptor back to free stack */ - unifycr_stack_push(unifycr_fd_stack, s->fd); + unifyfs_stack_push(unifyfs_fd_stack, s->fd); /* set file descriptor to -1 to indicate stream is invalid */ - unifycr_stream_init(s->sid); + unifyfs_stream_init(s->sid); /* add stream back to free stack */ - unifycr_stack_push(unifycr_stream_stack, s->sid); + unifyfs_stack_push(unifyfs_stream_stack, s->sid); /* currently a no-op */ return 0; } else { MAP_OR_FAIL(fclose); - int ret = UNIFYCR_REAL(fclose)(stream); + int ret = UNIFYFS_REAL(fclose)(stream); return ret; } } -int UNIFYCR_WRAP(fwprintf)(FILE* stream, const wchar_t* format, ...) +int UNIFYFS_WRAP(fwprintf)(FILE* stream, const wchar_t* format, ...) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* return file descriptor associated with stream */ - unifycr_unsupported_stream(stream, __func__, __FILE__, __LINE__, "%s", format); + unifyfs_unsupported_stream(stream, __func__, __FILE__, __LINE__, "%s", format); /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; s->err = 1; errno = EIO; return -1; @@ -1866,21 +1870,21 @@ int UNIFYCR_WRAP(fwprintf)(FILE* stream, const wchar_t* format, ...) va_list args; va_start(args, format); MAP_OR_FAIL(vfwprintf); - int ret = UNIFYCR_REAL(vfwprintf)(stream, format, args); + int ret = UNIFYFS_REAL(vfwprintf)(stream, format, args); va_end(args); return ret; } } -int UNIFYCR_WRAP(fwscanf)(FILE* stream, const wchar_t* format, ...) +int UNIFYFS_WRAP(fwscanf)(FILE* stream, const wchar_t* format, ...) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* return file descriptor associated with stream */ - unifycr_unsupported_stream(stream, __func__, __FILE__, __LINE__, "%s", format); + unifyfs_unsupported_stream(stream, __func__, __FILE__, __LINE__, "%s", format); /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; s->err = 1; errno = EIO; return EOF; @@ -1888,21 +1892,21 @@ int UNIFYCR_WRAP(fwscanf)(FILE* stream, const wchar_t* format, ...) va_list args; va_start(args, format); MAP_OR_FAIL(vfwscanf); - int ret = UNIFYCR_REAL(vfwscanf)(stream, format, args); + int ret = UNIFYFS_REAL(vfwscanf)(stream, format, args); va_end(args); return ret; } } -int UNIFYCR_WRAP(vfwprintf)(FILE* stream, const wchar_t* format, va_list arg) +int UNIFYFS_WRAP(vfwprintf)(FILE* stream, const wchar_t* format, va_list arg) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* return file descriptor associated with stream */ - unifycr_unsupported_stream(stream, __func__, __FILE__, __LINE__, "%s", format); + unifyfs_unsupported_stream(stream, __func__, __FILE__, __LINE__, "%s", format); /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; s->err = 1; errno = EIO; return -1; @@ -1910,21 +1914,21 @@ int UNIFYCR_WRAP(vfwprintf)(FILE* stream, const wchar_t* format, va_list arg) va_list args; va_copy(args, arg); MAP_OR_FAIL(vfwprintf); - int ret = UNIFYCR_REAL(vfwprintf)(stream, format, args); + int ret = UNIFYFS_REAL(vfwprintf)(stream, format, args); va_end(args); return ret; } } -int UNIFYCR_WRAP(vfwscanf)(FILE* stream, const wchar_t* format, va_list arg) +int UNIFYFS_WRAP(vfwscanf)(FILE* stream, const wchar_t* format, va_list arg) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* return file descriptor associated with stream */ - unifycr_unsupported_stream(stream, __func__, __FILE__, __LINE__, "%s", format); + unifyfs_unsupported_stream(stream, __func__, __FILE__, __LINE__, "%s", format); /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; s->err = 1; errno = EIO; return EOF; @@ -1932,160 +1936,160 @@ int UNIFYCR_WRAP(vfwscanf)(FILE* stream, const wchar_t* format, va_list arg) va_list args; va_copy(args, arg); MAP_OR_FAIL(vfwscanf); - int ret = UNIFYCR_REAL(vfwscanf)(stream, format, args); + int ret = UNIFYFS_REAL(vfwscanf)(stream, format, args); va_end(args); return ret; } } -wint_t UNIFYCR_WRAP(fgetwc)(FILE* stream) +wint_t UNIFYFS_WRAP(fgetwc)(FILE* stream) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* return file descriptor associated with stream */ - unifycr_unsupported_stream(stream, __func__, __FILE__, __LINE__, ""); + unifyfs_unsupported_stream(stream, __func__, __FILE__, __LINE__, ""); /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; s->err = 1; errno = EIO; return WEOF; } else { MAP_OR_FAIL(fgetwc); - wint_t ret = UNIFYCR_REAL(fgetwc)(stream); + wint_t ret = UNIFYFS_REAL(fgetwc)(stream); return ret; } } -wchar_t* UNIFYCR_WRAP(fgetws)(wchar_t* s, int n, FILE* stream) +wchar_t* UNIFYFS_WRAP(fgetws)(wchar_t* s, int n, FILE* stream) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* return file descriptor associated with stream */ - unifycr_unsupported_stream(stream, __func__, __FILE__, __LINE__, ""); + unifyfs_unsupported_stream(stream, __func__, __FILE__, __LINE__, ""); /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; s->err = 1; errno = EIO; return NULL; } else { MAP_OR_FAIL(fgetws); - wchar_t* ret = UNIFYCR_REAL(fgetws)(s, n, stream); + wchar_t* ret = UNIFYFS_REAL(fgetws)(s, n, stream); return ret; } } -wint_t UNIFYCR_WRAP(fputwc)(wchar_t wc, FILE* stream) +wint_t UNIFYFS_WRAP(fputwc)(wchar_t wc, FILE* stream) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* return file descriptor associated with stream */ - unifycr_unsupported_stream(stream, __func__, __FILE__, __LINE__, ""); + unifyfs_unsupported_stream(stream, __func__, __FILE__, __LINE__, ""); /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; s->err = 1; errno = EIO; return WEOF; } else { MAP_OR_FAIL(fputwc); - wint_t ret = UNIFYCR_REAL(fputwc)(wc, stream); + wint_t ret = UNIFYFS_REAL(fputwc)(wc, stream); return ret; } } -int UNIFYCR_WRAP(fputws)(const wchar_t* s, FILE* stream) +int UNIFYFS_WRAP(fputws)(const wchar_t* s, FILE* stream) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* return file descriptor associated with stream */ - unifycr_unsupported_stream(stream, __func__, __FILE__, __LINE__, ""); + unifyfs_unsupported_stream(stream, __func__, __FILE__, __LINE__, ""); /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; s->err = 1; errno = EIO; return -1; } else { MAP_OR_FAIL(fputws); - int ret = UNIFYCR_REAL(fputws)(s, stream); + int ret = UNIFYFS_REAL(fputws)(s, stream); return ret; } } -int UNIFYCR_WRAP(fwide)(FILE* stream, int mode) +int UNIFYFS_WRAP(fwide)(FILE* stream, int mode) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* return file descriptor associated with stream */ - unifycr_unsupported_stream(stream, __func__, __FILE__, __LINE__, ""); + unifyfs_unsupported_stream(stream, __func__, __FILE__, __LINE__, ""); /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; s->err = 1; errno = EIO; return 0; } else { MAP_OR_FAIL(fwide); - int ret = UNIFYCR_REAL(fwide)(stream, mode); + int ret = UNIFYFS_REAL(fwide)(stream, mode); return ret; } } -wint_t UNIFYCR_WRAP(getwc)(FILE* stream) +wint_t UNIFYFS_WRAP(getwc)(FILE* stream) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* return file descriptor associated with stream */ - unifycr_unsupported_stream(stream, __func__, __FILE__, __LINE__, ""); + unifyfs_unsupported_stream(stream, __func__, __FILE__, __LINE__, ""); /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; s->err = 1; errno = EIO; return WEOF; } else { MAP_OR_FAIL(getwc); - wint_t ret = UNIFYCR_REAL(getwc)(stream); + wint_t ret = UNIFYFS_REAL(getwc)(stream); return ret; } } -wint_t UNIFYCR_WRAP(putwc)(wchar_t c, FILE* stream) +wint_t UNIFYFS_WRAP(putwc)(wchar_t c, FILE* stream) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* return file descriptor associated with stream */ - unifycr_unsupported_stream(stream, __func__, __FILE__, __LINE__, ""); + unifyfs_unsupported_stream(stream, __func__, __FILE__, __LINE__, ""); /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; s->err = 1; errno = EIO; return WEOF; } else { MAP_OR_FAIL(putwc); - wint_t ret = UNIFYCR_REAL(putwc)(c, stream); + wint_t ret = UNIFYFS_REAL(putwc)(c, stream); return ret; } } -wint_t UNIFYCR_WRAP(ungetwc)(wint_t c, FILE* stream) +wint_t UNIFYFS_WRAP(ungetwc)(wint_t c, FILE* stream) { /* check whether we should intercept this stream */ - if (unifycr_intercept_stream(stream)) { + if (unifyfs_intercept_stream(stream)) { /* return file descriptor associated with stream */ - unifycr_unsupported_stream(stream, __func__, __FILE__, __LINE__, ""); + unifyfs_unsupported_stream(stream, __func__, __FILE__, __LINE__, ""); /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; s->err = 1; errno = EIO; return WEOF; } else { MAP_OR_FAIL(ungetwc); - wint_t ret = UNIFYCR_REAL(ungetwc)(c, stream); + wint_t ret = UNIFYFS_REAL(ungetwc)(c, stream); return ret; } } @@ -2139,8 +2143,8 @@ wint_t UNIFYCR_WRAP(ungetwc)(wint_t c, FILE* stream) #include #include #include -#include "unifycr-stdio.h" -#include "unifycr-internal.h" +#include "unifyfs-stdio.h" +#include "unifyfs-internal.h" #define BUF 513 /* Maximum length of numeric string. */ @@ -2184,7 +2188,7 @@ wint_t UNIFYCR_WRAP(ungetwc)(wint_t c, FILE* stream) static const u_char* __sccl(char*, const u_char*); #ifndef NO_FLOATING_POINT -static int parsefloat(unifycr_stream_t*, char*, char*); +static int parsefloat(unifyfs_stream_t*, char*, char*); #endif /* @@ -2195,15 +2199,15 @@ static int parsefloat(unifycr_stream_t*, char*, char*); * NULL pointer. */ static const int suppress; -#define SUPPRESS_PTR ((void *)&suppress) +#define SUPPRESS_PTR ((void*)&suppress) -static int __srefill(unifycr_stream_t* stream) +static int __srefill(unifyfs_stream_t* stream) { /* lookup stream */ - unifycr_stream_t* s = (unifycr_stream_t*) stream; + unifyfs_stream_t* s = (unifyfs_stream_t*) stream; /* get pointer to file descriptor structure */ - unifycr_fd_t* filedesc = unifycr_get_filedesc_from_fd(s->fd); + unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(s->fd); if (filedesc == NULL) { /* ERROR: invalid file descriptor */ s->err = 1; @@ -2212,7 +2216,7 @@ static int __srefill(unifycr_stream_t* stream) } /* bail with error if stream not open for reading */ - if (! filedesc->read) { + if (!filedesc->read) { s->err = 1; errno = EBADF; return 1; @@ -2220,12 +2224,12 @@ static int __srefill(unifycr_stream_t* stream) /* associate buffer with stream if we need to */ if (s->buf == NULL) { - int setvbuf_rc = unifycr_setvbuf((FILE*)stream, NULL, s->buftype, - UNIFYCR_STREAM_BUFSIZE); - if (setvbuf_rc != UNIFYCR_SUCCESS) { + int setvbuf_rc = unifyfs_setvbuf((FILE*)stream, NULL, s->buftype, + UNIFYFS_STREAM_BUFSIZE); + if (setvbuf_rc != UNIFYFS_SUCCESS) { /* ERROR: failed to associate buffer */ s->err = 1; - errno = unifycr_err_map_to_errno(setvbuf_rc); + errno = unifyfs_err_map_to_errno(setvbuf_rc); return 1; } } @@ -2247,19 +2251,19 @@ static int __srefill(unifycr_stream_t* stream) /* current is outside the range of our buffer */ /* flush buffer if needed before read */ - int flush_rc = unifycr_stream_flush((FILE*)stream); - if (flush_rc != UNIFYCR_SUCCESS) { + int flush_rc = unifyfs_stream_flush((FILE*)stream); + if (flush_rc != UNIFYFS_SUCCESS) { /* ERROR: flush sets error indicator and errno */ return 1; } /* read data from file into buffer */ size_t bufcount; - int read_rc = unifycr_fd_read(s->fd, current, s->buf, s->bufsize, &bufcount); - if (read_rc != UNIFYCR_SUCCESS) { + int read_rc = unifyfs_fd_read(s->fd, current, s->buf, s->bufsize, &bufcount); + if (read_rc != UNIFYFS_SUCCESS) { /* ERROR: read error, set error indicator and errno */ s->err = 1; - errno = unifycr_err_map_to_errno(read_rc); + errno = unifyfs_err_map_to_errno(read_rc); return 1; } @@ -2289,7 +2293,7 @@ static int __srefill(unifycr_stream_t* stream) */ static __inline int -convert_char(unifycr_stream_t* fp, char* p, int width) +convert_char(unifyfs_stream_t* fp, char* p, int width) { int n; @@ -2325,7 +2329,7 @@ convert_char(unifycr_stream_t* fp, char* p, int width) } static __inline int -convert_ccl(unifycr_stream_t* fp, char* p, int width, const char* ccltab) +convert_ccl(unifyfs_stream_t* fp, char* p, int width, const char* ccltab) { char* p0; int n; @@ -2366,7 +2370,7 @@ convert_ccl(unifycr_stream_t* fp, char* p, int width, const char* ccltab) } static __inline int -convert_string(unifycr_stream_t* fp, char* p, int width) +convert_string(unifyfs_stream_t* fp, char* p, int width) { char* p0; int n; @@ -2408,7 +2412,7 @@ convert_string(unifycr_stream_t* fp, char* p, int width) * otherwise. */ static __inline int -parseint(unifycr_stream_t* fp, char* __restrict buf, int width, int base, +parseint(unifyfs_stream_t* fp, char* __restrict buf, int width, int base, int flags) { /* `basefix' is used to avoid `if' tests */ @@ -2554,7 +2558,7 @@ parseint(unifycr_stream_t* fp, char* __restrict buf, int width, int base, * __svfscanf - non-MT-safe version of __vfscanf */ static int -__svfscanf(unifycr_stream_t* fp, const char* fmt0, va_list ap) +__svfscanf(unifyfs_stream_t* fp, const char* fmt0, va_list ap) { #define GETARG(type) ((flags & SUPPRESS) ? SUPPRESS_PTR : va_arg(ap, type)) const u_char* fmt = (const u_char*)fmt0; @@ -2801,7 +2805,7 @@ __svfscanf(unifycr_stream_t* fp, const char* fmt0, va_list ap) width = 1; } if (flags & LONG) { - unifycr_unsupported_stream((FILE*)fp, __func__, __FILE__, + unifyfs_unsupported_stream((FILE*)fp, __func__, __FILE__, __LINE__, "%s", fmt0); } else { nr = convert_char(fp, GETARG(char*), width); @@ -2817,7 +2821,7 @@ __svfscanf(unifycr_stream_t* fp, const char* fmt0, va_list ap) width = (size_t)~0; /* `infinity' */ } if (flags & LONG) { - unifycr_unsupported_stream((FILE*)fp, __func__, __FILE__, + unifyfs_unsupported_stream((FILE*)fp, __func__, __FILE__, __LINE__, "%s", fmt0); } else { nr = convert_ccl(fp, GETARG(char*), width, @@ -2838,7 +2842,7 @@ __svfscanf(unifycr_stream_t* fp, const char* fmt0, va_list ap) width = (size_t)~0; } if (flags & LONG) { - unifycr_unsupported_stream((FILE*)fp, __func__, __FILE__, + unifyfs_unsupported_stream((FILE*)fp, __func__, __FILE__, __LINE__, "%s", fmt0); } else { nr = convert_string(fp, GETARG(char*), width); @@ -2874,10 +2878,10 @@ __svfscanf(unifycr_stream_t* fp, const char* fmt0, va_list ap) } else { res = strtoumax(buf, (char**)NULL, base); } - if (flags & POINTER) + if (flags & POINTER) { *va_arg(ap, void**) = (void*)(uintptr_t)res; - else if (flags & SHORTSHORT) { + } else if (flags & SHORTSHORT) { *va_arg(ap, char*) = res; } else if (flags & SHORT) { *va_arg(ap, short*) = res; @@ -3031,8 +3035,8 @@ const u_char* fmt; #ifndef NO_FLOATING_POINT static int -//parsefloat(unifycr_stream_t *fp, char *buf, char *end, locale_t locale) -parsefloat(unifycr_stream_t* fp, char* buf, char* end) +//parsefloat(unifyfs_stream_t *fp, char *buf, char *end, locale_t locale) +parsefloat(unifyfs_stream_t* fp, char* buf, char* end) { char* commit, *p; int infnanpos = 0, decptpos = 0; diff --git a/client/src/unifyfs-stdio.h b/client/src/unifyfs-stdio.h new file mode 100644 index 000000000..447ea0634 --- /dev/null +++ b/client/src/unifyfs-stdio.h @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2017, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +/* + * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * Copyright (c) 2017, Florida State University. Contributions from + * the Computer Architecture and Systems Research Laboratory (CASTL) + * at the Department of Computer Science. + * + * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror + * LLNL-CODE-728877. All rights reserved. + * + * This file is part of UnifyFS. + * For details, see https://github.com/llnl/unifyfs + * Please read https://github.com/llnl/unifyfs/LICENSE for full license text. + */ + +/* + * Copyright (c) 2013, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * code Written by + * Raghunath Rajachandrasekar + * Kathryn Mohror + * Adam Moody + * All rights reserved. + * This file is part of UNIFYFS. + * For details, see https://github.com/hpc/unifyfs + * Please also read this file LICENSE.UNIFYFS + */ + +#ifndef UNIFYFS_STDIO_H +#define UNIFYFS_STDIO_H + +#include "unifyfs-internal.h" + +/* TODO: declare the wrapper functions we define in unifyfs-stdio.c + * so other routines can call them */ +UNIFYFS_DECL(fclose, int, (FILE* stream)); +UNIFYFS_DECL(fflush, int, (FILE* stream)); +UNIFYFS_DECL(fopen, FILE*, (const char* path, const char* mode)); +UNIFYFS_DECL(freopen, FILE*, (const char* path, const char* mode, + FILE* stream)); +UNIFYFS_DECL(setbuf, void, (FILE* stream, char* buf)); +UNIFYFS_DECL(setvbuf, int, (FILE* stream, char* buf, int type, size_t size)); + +UNIFYFS_DECL(fprintf, int, (FILE* stream, const char* format, ...)); +UNIFYFS_DECL(fscanf, int, (FILE* stream, const char* format, ...)); +UNIFYFS_DECL(vfprintf, int, (FILE* stream, const char* format, va_list ap)); +UNIFYFS_DECL(vfscanf, int, (FILE* stream, const char* format, va_list ap)); + +UNIFYFS_DECL(fgetc, int, (FILE* stream)); +UNIFYFS_DECL(fgets, char*, (char* s, int n, FILE* stream)); +UNIFYFS_DECL(fputc, int, (int c, FILE* stream)); +UNIFYFS_DECL(fputs, int, (const char* s, FILE* stream)); +UNIFYFS_DECL(getc, int, (FILE* stream)); +UNIFYFS_DECL(putc, int, (int c, FILE* stream)); +UNIFYFS_DECL(ungetc, int, (int c, FILE* stream)); + +UNIFYFS_DECL(fread, size_t, (void* ptr, size_t size, size_t nitems, + FILE* stream)); +UNIFYFS_DECL(fwrite, size_t, (const void* ptr, size_t size, size_t nitems, + FILE* stream)); + +UNIFYFS_DECL(fgetpos, int, (FILE* stream, fpos_t* pos)); +UNIFYFS_DECL(fseek, int, (FILE* stream, long offset, int whence)); +UNIFYFS_DECL(fsetpos, int, (FILE* stream, const fpos_t* pos)); +UNIFYFS_DECL(ftell, long, (FILE* stream)); +UNIFYFS_DECL(rewind, void, (FILE* stream)); + +UNIFYFS_DECL(clearerr, void, (FILE* stream)); +UNIFYFS_DECL(feof, int, (FILE* stream)); +UNIFYFS_DECL(ferror, int, (FILE* stream)); + +UNIFYFS_DECL(fseeko, int, (FILE* stream, off_t offset, int whence)); +UNIFYFS_DECL(ftello, off_t, (FILE* stream)); +UNIFYFS_DECL(fileno, int, (FILE* stream)); + +/* http://www.open-std.org/jtc1/sc22/wg14/www/docs/n1124.pdf 7.24 */ + +UNIFYFS_DECL(fwprintf, int, (FILE* stream, const wchar_t* format, ...)); +UNIFYFS_DECL(fwscanf, int, (FILE* stream, const wchar_t* format, ...)); +UNIFYFS_DECL(vfwprintf, int, (FILE* stream, const wchar_t* format, + va_list arg)); +UNIFYFS_DECL(vfwscanf, int, (FILE* stream, const wchar_t* format, + va_list arg)); +UNIFYFS_DECL(fgetwc, wint_t, (FILE* stream)); +UNIFYFS_DECL(fgetws, wchar_t*, (wchar_t* s, int n, FILE* stream)); +UNIFYFS_DECL(fputwc, wint_t, (wchar_t wc, FILE* stream)); +UNIFYFS_DECL(fputws, int, (const wchar_t* s, FILE* stream)); +UNIFYFS_DECL(fwide, int, (FILE* stream, int mode)); +UNIFYFS_DECL(getwc, wint_t, (FILE* stream)); +UNIFYFS_DECL(putwc, wint_t, (wchar_t c, FILE* stream)); +UNIFYFS_DECL(ungetwc, wint_t, (wint_t c, FILE* stream)); + +UNIFYFS_DECL(chmod, int, (const char* path, mode_t mode)); +UNIFYFS_DECL(fchmod, int, (int fd, mode_t mode)); + +#endif /* UNIFYFS_STDIO_H */ diff --git a/client/src/unifycr-sysio.c b/client/src/unifyfs-sysio.c similarity index 64% rename from client/src/unifycr-sysio.c rename to client/src/unifyfs-sysio.c index c5b04f35c..0dd46c36a 100644 --- a/client/src/unifycr-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -40,30 +40,29 @@ * Please also read this file LICENSE.CRUISE */ -#include "unifycr-sysio.h" -#include "unifycr-internal.h" -#include "unifycr_client.h" +#include "unifyfs-internal.h" +#include "unifyfs-sysio.h" +#include "margo_client.h" #include "ucr_read_builder.h" -#include "unifycr_log.h" /* ------------------- * define external variables * --------------------*/ -extern int unifycr_spilloverblock; -extern int unifycr_use_spillover; +extern int unifyfs_spilloverblock; +extern int unifyfs_use_spillover; /* --------------------------------------- * POSIX wrappers: paths * --------------------------------------- */ -int UNIFYCR_WRAP(access)(const char* path, int mode) +int UNIFYFS_WRAP(access)(const char* path, int mode) { /* determine whether we should intercept this path */ - if (unifycr_intercept_path(path)) { + if (unifyfs_intercept_path(path)) { /* check if path exists */ - if (unifycr_get_fid_from_path(path) < 0) { - LOGDBG("access: unifycr_get_id_from path failed, returning -1, %s", + if (unifyfs_get_fid_from_path(path) < 0) { + LOGDBG("access: unifyfs_get_id_from path failed, returning -1, %s", path); errno = ENOENT; return -1; @@ -75,13 +74,13 @@ int UNIFYCR_WRAP(access)(const char* path, int mode) } else { LOGDBG("access: calling MAP_OR_FAIL, %s", path); MAP_OR_FAIL(access); - int ret = UNIFYCR_REAL(access)(path, mode); + int ret = UNIFYFS_REAL(access)(path, mode); LOGDBG("access: returning __real_access %d, %s", ret, path); return ret; } } -int UNIFYCR_WRAP(mkdir)(const char* path, mode_t mode) +int UNIFYFS_WRAP(mkdir)(const char* path, mode_t mode) { /* Support for directories is very limited at this time * mkdir simply puts an entry into the filelist for the @@ -89,19 +88,19 @@ int UNIFYCR_WRAP(mkdir)(const char* path, mode_t mode) * It doesn't check to see if parent directory exists */ /* determine whether we should intercept this path */ - if (unifycr_intercept_path(path)) { + if (unifyfs_intercept_path(path)) { /* check if it already exists */ - if (unifycr_get_fid_from_path(path) >= 0) { + if (unifyfs_get_fid_from_path(path) >= 0) { errno = EEXIST; return -1; } /* add directory to file list */ - int ret = unifycr_fid_create_directory(path); - if (ret != UNIFYCR_SUCCESS) { + int ret = unifyfs_fid_create_directory(path); + if (ret != UNIFYFS_SUCCESS) { /* failed to create the directory, * set errno and return */ - errno = unifycr_err_map_to_errno(ret); + errno = unifyfs_err_map_to_errno(ret); return -1; } @@ -109,46 +108,46 @@ int UNIFYCR_WRAP(mkdir)(const char* path, mode_t mode) return 0; } else { MAP_OR_FAIL(mkdir); - int ret = UNIFYCR_REAL(mkdir)(path, mode); + int ret = UNIFYFS_REAL(mkdir)(path, mode); return ret; } } -int UNIFYCR_WRAP(rmdir)(const char* path) +int UNIFYFS_WRAP(rmdir)(const char* path) { /* determine whether we should intercept this path */ - if (unifycr_intercept_path(path)) { + if (unifyfs_intercept_path(path)) { /* check if the mount point itself is being deleted */ - if (!strcmp(path, unifycr_mount_prefix)) { + if (!strcmp(path, unifyfs_mount_prefix)) { errno = EBUSY; return -1; } /* check if path exists */ - int fid = unifycr_get_fid_from_path(path); + int fid = unifyfs_get_fid_from_path(path); if (fid < 0) { errno = ENOENT; return -1; } /* is it a directory? */ - if (!unifycr_fid_is_dir(fid)) { + if (!unifyfs_fid_is_dir(fid)) { errno = ENOTDIR; return -1; } /* is it empty? */ - if (!unifycr_fid_is_dir_empty(path)) { + if (!unifyfs_fid_is_dir_empty(path)) { errno = ENOTEMPTY; return -1; } /* remove the directory from the file list */ - int ret = unifycr_fid_unlink(fid); - if (ret != UNIFYCR_SUCCESS) { + int ret = unifyfs_fid_unlink(fid); + if (ret != UNIFYFS_SUCCESS) { /* failed to remove the directory, * set errno and return */ - errno = unifycr_err_map_to_errno(ret); + errno = unifyfs_err_map_to_errno(ret); return -1; } @@ -156,30 +155,30 @@ int UNIFYCR_WRAP(rmdir)(const char* path) return 0; } else { MAP_OR_FAIL(rmdir); - int ret = UNIFYCR_REAL(rmdir)(path); + int ret = UNIFYFS_REAL(rmdir)(path); return ret; } } -int UNIFYCR_WRAP(rename)(const char* oldpath, const char* newpath) +int UNIFYFS_WRAP(rename)(const char* oldpath, const char* newpath) { /* TODO: allow oldpath / newpath to split across memfs and normal * linux fs, which means we'll need to do a read / write */ /* check whether the old path is in our file system */ - if (unifycr_intercept_path(oldpath)) { + if (unifyfs_intercept_path(oldpath)) { /* for now, we can only rename within our file system */ - if (!unifycr_intercept_path(newpath)) { + if (!unifyfs_intercept_path(newpath)) { /* ERROR: can't yet rename across file systems */ errno = EXDEV; return -1; } /* verify that we really have a file by the old name */ - int fid = unifycr_get_fid_from_path(oldpath); + int fid = unifyfs_get_fid_from_path(oldpath); if (fid < 0) { /* ERROR: oldname does not exist */ - LOGDBG("Couldn't find entry for %s in UNIFYCR", oldpath); + LOGDBG("Couldn't find entry for %s in UNIFYFS", oldpath); errno = ENOENT; return -1; } @@ -187,7 +186,7 @@ int UNIFYCR_WRAP(rename)(const char* oldpath, const char* newpath) /* check that new name is within bounds */ size_t newpathlen = strlen(newpath) + 1; - if (newpathlen > UNIFYCR_MAX_FILENAME) { + if (newpathlen > UNIFYFS_MAX_FILENAME) { errno = ENAMETOOLONG; return -1; } @@ -195,10 +194,10 @@ int UNIFYCR_WRAP(rename)(const char* oldpath, const char* newpath) /* TODO: rename should replace existing file atomically */ /* verify that we don't already have a file by the new name */ - int newfid = unifycr_get_fid_from_path(newpath); + int newfid = unifyfs_get_fid_from_path(newpath); if (newfid >= 0) { /* something exists in newpath, need to delete it */ - int ret = UNIFYCR_WRAP(unlink)(newpath); + int ret = UNIFYFS_WRAP(unlink)(newpath); if (ret == -1) { /* failed to unlink */ errno = EBUSY; @@ -208,14 +207,14 @@ int UNIFYCR_WRAP(rename)(const char* oldpath, const char* newpath) /* finally overwrite the old name with the new name */ LOGDBG("Changing %s to %s", - (char*)&unifycr_filelist[fid].filename, newpath); - strcpy((void*)&unifycr_filelist[fid].filename, newpath); + (char*)&unifyfs_filelist[fid].filename, newpath); + strcpy((void*)&unifyfs_filelist[fid].filename, newpath); /* success */ return 0; } else { /* for now, we can only rename within our file system */ - if (unifycr_intercept_path(newpath)) { + if (unifyfs_intercept_path(newpath)) { /* ERROR: can't yet rename across file systems */ errno = EXDEV; return -1; @@ -223,28 +222,28 @@ int UNIFYCR_WRAP(rename)(const char* oldpath, const char* newpath) /* both files are normal linux files, delegate to system call */ MAP_OR_FAIL(rename); - int ret = UNIFYCR_REAL(rename)(oldpath, newpath); + int ret = UNIFYFS_REAL(rename)(oldpath, newpath); return ret; } } -int UNIFYCR_WRAP(truncate)(const char* path, off_t length) +int UNIFYFS_WRAP(truncate)(const char* path, off_t length) { /* determine whether we should intercept this path or not */ - if (unifycr_intercept_path(path)) { + if (unifyfs_intercept_path(path)) { /* lookup the fid for the path */ - int fid = unifycr_get_fid_from_path(path); + int fid = unifyfs_get_fid_from_path(path); if (fid < 0) { /* ERROR: file does not exist */ - LOGDBG("Couldn't find entry for %s in UNIFYCR", path); + LOGDBG("Couldn't find entry for %s in UNIFYFS", path); errno = ENOENT; return -1; } /* truncate the file */ - int rc = unifycr_fid_truncate(fid, length); - if (rc != UNIFYCR_SUCCESS) { - LOGDBG("unifycr_fid_truncate failed for %s in UNIFYCR", path); + int rc = unifyfs_fid_truncate(fid, length); + if (rc != UNIFYFS_SUCCESS) { + LOGDBG("unifyfs_fid_truncate failed for %s in UNIFYFS", path); errno = EIO; return -1; } @@ -253,36 +252,36 @@ int UNIFYCR_WRAP(truncate)(const char* path, off_t length) return 0; } else { MAP_OR_FAIL(truncate); - int ret = UNIFYCR_REAL(truncate)(path, length); + int ret = UNIFYFS_REAL(truncate)(path, length); return ret; } } -int UNIFYCR_WRAP(unlink)(const char* path) +int UNIFYFS_WRAP(unlink)(const char* path) { /* determine whether we should intercept this path or not */ - if (unifycr_intercept_path(path)) { + if (unifyfs_intercept_path(path)) { /* get file id for path name */ - int fid = unifycr_get_fid_from_path(path); + int fid = unifyfs_get_fid_from_path(path); if (fid < 0) { /* ERROR: file does not exist */ - LOGDBG("Couldn't find entry for %s in UNIFYCR", path); + LOGDBG("Couldn't find entry for %s in UNIFYFS", path); errno = ENOENT; return -1; } /* check that it's not a directory */ - if (unifycr_fid_is_dir(fid)) { + if (unifyfs_fid_is_dir(fid)) { /* ERROR: is a directory */ - LOGDBG("Attempting to unlink a directory %s in UNIFYCR", path); + LOGDBG("Attempting to unlink a directory %s in UNIFYFS", path); errno = EISDIR; return -1; } /* delete the file */ - int ret = unifycr_fid_unlink(fid); - if (ret != UNIFYCR_SUCCESS) { - errno = unifycr_err_map_to_errno(ret); + int ret = unifyfs_fid_unlink(fid); + if (ret != UNIFYFS_SUCCESS) { + errno = unifyfs_err_map_to_errno(ret); return -1; } @@ -290,38 +289,38 @@ int UNIFYCR_WRAP(unlink)(const char* path) return 0; } else { MAP_OR_FAIL(unlink); - int ret = UNIFYCR_REAL(unlink)(path); + int ret = UNIFYFS_REAL(unlink)(path); return ret; } } -int UNIFYCR_WRAP(remove)(const char* path) +int UNIFYFS_WRAP(remove)(const char* path) { /* determine whether we should intercept this path or not */ - if (unifycr_intercept_path(path)) { + if (unifyfs_intercept_path(path)) { /* get file id for path name */ - int fid = unifycr_get_fid_from_path(path); + int fid = unifyfs_get_fid_from_path(path); if (fid < 0) { /* ERROR: file does not exist */ - LOGDBG("Couldn't find entry for %s in UNIFYCR", path); + LOGDBG("Couldn't find entry for %s in UNIFYFS", path); errno = ENOENT; return -1; } /* check that it's not a directory */ - if (unifycr_fid_is_dir(fid)) { + if (unifyfs_fid_is_dir(fid)) { /* TODO: shall be equivalent to rmdir(path) */ /* ERROR: is a directory */ - LOGDBG("Attempting to remove a directory %s in UNIFYCR", path); + LOGDBG("Attempting to remove a directory %s in UNIFYFS", path); errno = EISDIR; return -1; } /* shall be equivalent to unlink(path) */ /* delete the file */ - int ret = unifycr_fid_unlink(fid); - if (ret != UNIFYCR_SUCCESS) { - errno = unifycr_err_map_to_errno(ret); + int ret = unifyfs_fid_unlink(fid); + if (ret != UNIFYFS_SUCCESS) { + errno = unifyfs_err_map_to_errno(ret); return -1; } @@ -329,16 +328,16 @@ int UNIFYCR_WRAP(remove)(const char* path) return 0; } else { MAP_OR_FAIL(remove); - int ret = UNIFYCR_REAL(remove)(path); + int ret = UNIFYFS_REAL(remove)(path); return ret; } } -int UNIFYCR_WRAP(stat)(const char* path, struct stat* buf) +int UNIFYFS_WRAP(stat)(const char* path, struct stat* buf) { LOGDBG("stat was called for %s", path); - if (unifycr_intercept_path(path)) { + if (unifyfs_intercept_path(path)) { /* check that caller gave us a buffer to write to */ if (!buf) { errno = EFAULT; @@ -346,12 +345,12 @@ int UNIFYCR_WRAP(stat)(const char* path, struct stat* buf) } /* get global file id for path */ - int gfid = unifycr_generate_gfid(path); + int gfid = unifyfs_generate_gfid(path); /* lookup stat info for global file id */ - int ret = unifycr_gfid_stat(gfid, buf); - if (ret != UNIFYCR_SUCCESS) { - errno = unifycr_err_map_to_errno(ret); + int ret = unifyfs_gfid_stat(gfid, buf); + if (ret != UNIFYFS_SUCCESS) { + errno = unifyfs_err_map_to_errno(ret); return -1; } @@ -359,34 +358,34 @@ int UNIFYCR_WRAP(stat)(const char* path, struct stat* buf) return 0; } else { MAP_OR_FAIL(stat); - int ret = UNIFYCR_REAL(stat)(path, buf); + int ret = UNIFYFS_REAL(stat)(path, buf); return ret; } } #if 0 -int UNIFYCR_WRAP(fstat)(int fd, struct stat* buf) +int UNIFYFS_WRAP(fstat)(int fd, struct stat* buf) { LOGDBG("fstat was called for fd: %d", fd); /* check whether we should intercept this file descriptor */ - if (unifycr_intercept_fd(&fd)) { + if (unifyfs_intercept_fd(&fd)) { if (!buf) { errno = EFAULT; return -1; } /* get the file id for this file descriptor */ - int fid = unifycr_get_fid_from_fd(fd); + int fid = unifyfs_get_fid_from_fd(fd); if (fid < 0) { errno = EBADF; return -1; } /* lookup stat info for this file id */ - int ret = unifycr_fid_stat(fid, buf); + int ret = unifyfs_fid_stat(fid, buf); if (ret < 0) { - errno = unifycr_err_map_to_errno(ret); + errno = unifyfs_err_map_to_errno(ret); return -1; } @@ -394,7 +393,7 @@ int UNIFYCR_WRAP(fstat)(int fd, struct stat* buf) return 0; } else { MAP_OR_FAIL(fstat); - int ret = UNIFYCR_REAL(fstat)(fd, buf); + int ret = UNIFYFS_REAL(fstat)(fd, buf); return ret; } } @@ -409,11 +408,11 @@ int UNIFYCR_WRAP(fstat)(int fd, struct stat* buf) * instead of using the absolute value 3. */ -int UNIFYCR_WRAP(__xstat)(int vers, const char* path, struct stat* buf) +int UNIFYFS_WRAP(__xstat)(int vers, const char* path, struct stat* buf) { LOGDBG("xstat was called for %s", path); - if (unifycr_intercept_path(path)) { + if (unifyfs_intercept_path(path)) { if (vers != _STAT_VER) { errno = EINVAL; return -1; @@ -425,12 +424,12 @@ int UNIFYCR_WRAP(__xstat)(int vers, const char* path, struct stat* buf) } /* get global file id for path */ - int gfid = unifycr_generate_gfid(path); + int gfid = unifyfs_generate_gfid(path); /* lookup stat info for global file id */ - int ret = unifycr_gfid_stat(gfid, buf); - if (ret != UNIFYCR_SUCCESS) { - errno = unifycr_err_map_to_errno(ret); + int ret = unifyfs_gfid_stat(gfid, buf); + if (ret != UNIFYFS_SUCCESS) { + errno = unifyfs_err_map_to_errno(ret); return -1; } @@ -438,16 +437,16 @@ int UNIFYCR_WRAP(__xstat)(int vers, const char* path, struct stat* buf) return 0; } else { MAP_OR_FAIL(__xstat); - int ret = UNIFYCR_REAL(__xstat)(vers, path, buf); + int ret = UNIFYFS_REAL(__xstat)(vers, path, buf); return ret; } } -int UNIFYCR_WRAP(__lxstat)(int vers, const char* path, struct stat* buf) +int UNIFYFS_WRAP(__lxstat)(int vers, const char* path, struct stat* buf) { LOGDBG("lxstat was called for %s", path); - if (unifycr_intercept_path(path)) { + if (unifyfs_intercept_path(path)) { if (vers != _STAT_VER) { errno = EINVAL; return -1; @@ -459,12 +458,12 @@ int UNIFYCR_WRAP(__lxstat)(int vers, const char* path, struct stat* buf) } /* get global file id for path */ - int gfid = unifycr_generate_gfid(path); + int gfid = unifyfs_generate_gfid(path); /* lookup stat info for global file id */ - int ret = unifycr_gfid_stat(gfid, buf); - if (ret != UNIFYCR_SUCCESS) { - errno = unifycr_err_map_to_errno(ret); + int ret = unifyfs_gfid_stat(gfid, buf); + if (ret != UNIFYFS_SUCCESS) { + errno = unifyfs_err_map_to_errno(ret); return -1; } @@ -472,17 +471,17 @@ int UNIFYCR_WRAP(__lxstat)(int vers, const char* path, struct stat* buf) return 0; } else { MAP_OR_FAIL(__lxstat); - int ret = UNIFYCR_REAL(__lxstat)(vers, path, buf); + int ret = UNIFYFS_REAL(__lxstat)(vers, path, buf); return ret; } } -int UNIFYCR_WRAP(__fxstat)(int vers, int fd, struct stat* buf) +int UNIFYFS_WRAP(__fxstat)(int vers, int fd, struct stat* buf) { LOGDBG("fxstat was called for fd %d", fd); /* check whether we should intercept this file descriptor */ - if (unifycr_intercept_fd(&fd)) { + if (unifyfs_intercept_fd(&fd)) { if (vers != _STAT_VER) { errno = EINVAL; return -1; @@ -494,16 +493,16 @@ int UNIFYCR_WRAP(__fxstat)(int vers, int fd, struct stat* buf) } /* get the file id for this file descriptor */ - int fid = unifycr_get_fid_from_fd(fd); + int fid = unifyfs_get_fid_from_fd(fd); if (fid < 0) { errno = EBADF; return -1; } /* lookup stat info for this file id */ - int ret = unifycr_fid_stat(fid, buf); - if (ret != UNIFYCR_SUCCESS) { - errno = unifycr_err_map_to_errno(ret); + int ret = unifyfs_fid_stat(fid, buf); + if (ret != UNIFYFS_SUCCESS) { + errno = unifyfs_err_map_to_errno(ret); return -1; } @@ -511,7 +510,7 @@ int UNIFYCR_WRAP(__fxstat)(int vers, int fd, struct stat* buf) return 0; } else { MAP_OR_FAIL(__fxstat); - int ret = UNIFYCR_REAL(__fxstat)(vers, fd, buf); + int ret = UNIFYFS_REAL(__fxstat)(vers, fd, buf); return ret; } } @@ -524,38 +523,38 @@ int UNIFYCR_WRAP(__fxstat)(int vers, int fd, struct stat* buf) * returns number of bytes actually read in retcount, * retcount will be less than count only if an error occurs * or end of file is reached */ -int unifycr_fd_read(int fd, off_t pos, void* buf, size_t count, +int unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count, size_t* retcount) { /* get the file id for this file descriptor */ - int fid = unifycr_get_fid_from_fd(fd); + int fid = unifyfs_get_fid_from_fd(fd); if (fid < 0) { - return UNIFYCR_ERROR_BADF; + return UNIFYFS_ERROR_BADF; } /* it's an error to read from a directory */ - if (unifycr_fid_is_dir(fid)) { + if (unifyfs_fid_is_dir(fid)) { /* TODO: note that read/pread can return this, but not fread */ - return UNIFYCR_ERROR_ISDIR; + return UNIFYFS_ERROR_ISDIR; } /* check that file descriptor is open for read */ - unifycr_fd_t* filedesc = unifycr_get_filedesc_from_fd(fd); + unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(fd); if (!filedesc->read) { - return UNIFYCR_ERROR_BADF; + return UNIFYFS_ERROR_BADF; } /* TODO: is it safe to assume that off_t is bigger than size_t? */ /* check that we don't overflow the file length */ - if (unifycr_would_overflow_offt(pos, (off_t) count)) { - return UNIFYCR_ERROR_OVERFLOW; + if (unifyfs_would_overflow_offt(pos, (off_t) count)) { + return UNIFYFS_ERROR_OVERFLOW; } /* TODO: check that file is open for reading */ /* check that we don't try to read past the end of the file */ off_t lastread = pos + (off_t) count; - off_t filesize = unifycr_fid_size(fid); + off_t filesize = unifyfs_fid_size(fid); if (filesize < lastread) { /* adjust count so we don't read past end of file */ if (filesize > pos) { @@ -572,83 +571,82 @@ int unifycr_fd_read(int fd, off_t pos, void* buf, size_t count, /* if we don't read any bytes, return success */ if (count == 0) { - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /* read data from file */ - int read_rc = unifycr_fid_read(fid, pos, buf, count); + int read_rc = unifyfs_fid_read(fid, pos, buf, count); return read_rc; } /* write count bytes from buf into file starting at offset pos, * allocates new bytes and updates file size as necessary, * fills any gaps with zeros */ -int unifycr_fd_write(int fd, off_t pos, const void* buf, size_t count) +int unifyfs_fd_write(int fd, off_t pos, const void* buf, size_t count) { /* get the file id for this file descriptor */ - int fid = unifycr_get_fid_from_fd(fd); + int fid = unifyfs_get_fid_from_fd(fd); if (fid < 0) { - return UNIFYCR_ERROR_BADF; + return UNIFYFS_ERROR_BADF; } /* it's an error to write to a directory */ - if (unifycr_fid_is_dir(fid)) { - return UNIFYCR_ERROR_INVAL; + if (unifyfs_fid_is_dir(fid)) { + return UNIFYFS_ERROR_INVAL; } /* check that file descriptor is open for write */ - unifycr_fd_t* filedesc = unifycr_get_filedesc_from_fd(fd); + unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(fd); if (!filedesc->write) { - return UNIFYCR_ERROR_BADF; + return UNIFYFS_ERROR_BADF; } /* TODO: is it safe to assume that off_t is bigger than size_t? */ /* check that our write won't overflow the length */ - if (unifycr_would_overflow_offt(pos, (off_t) count)) { + if (unifyfs_would_overflow_offt(pos, (off_t) count)) { /* TODO: want to return EFBIG here for streams */ - return UNIFYCR_ERROR_OVERFLOW; + return UNIFYFS_ERROR_OVERFLOW; } /* TODO: check that file is open for writing */ /* get current file size before extending the file */ - off_t filesize = unifycr_fid_size(fid); + off_t filesize = unifyfs_fid_size(fid); /* compute new position based on storage type */ off_t newpos; - unifycr_filemeta_t* meta = unifycr_get_meta_from_fid(fid); + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); if (meta->storage == FILE_STORAGE_FIXED_CHUNK) { /* extend file size and allocate chunks if needed */ newpos = pos + (off_t) count; - int extend_rc = unifycr_fid_extend(fid, newpos); - if (extend_rc != UNIFYCR_SUCCESS) { + int extend_rc = unifyfs_fid_extend(fid, newpos); + if (extend_rc != UNIFYFS_SUCCESS) { return extend_rc; } /* fill any new bytes between old size and pos with zero values */ if (filesize < pos) { off_t gap_size = pos - filesize; - int zero_rc = unifycr_fid_write_zero(fid, filesize, gap_size); - if (zero_rc != UNIFYCR_SUCCESS) { + int zero_rc = unifyfs_fid_write_zero(fid, filesize, gap_size); + if (zero_rc != UNIFYFS_SUCCESS) { return zero_rc; } } } else if (meta->storage == FILE_STORAGE_LOGIO) { newpos = filesize + (off_t)count; - int extend_rc = unifycr_fid_extend(fid, newpos); - if (extend_rc != UNIFYCR_SUCCESS) { + int extend_rc = unifyfs_fid_extend(fid, newpos); + if (extend_rc != UNIFYFS_SUCCESS) { return extend_rc; } } else { - return UNIFYCR_ERROR_IO; + return UNIFYFS_ERROR_IO; } /* finally write specified data to file */ - int write_rc = unifycr_fid_write(fid, pos, buf, count); - - if (meta->storage == FILE_STORAGE_LOGIO) { - unifycr_filemeta_t* meta = unifycr_get_meta_from_fid(fid); - if (write_rc == 0) { + int write_rc = unifyfs_fid_write(fid, pos, buf, count); + if (write_rc == 0) { + meta->needs_sync = 1; + if (meta->storage == FILE_STORAGE_LOGIO) { meta->size = newpos; meta->log_size = pos + count; } @@ -656,25 +654,25 @@ int unifycr_fd_write(int fd, off_t pos, const void* buf, size_t count) return write_rc; } -int UNIFYCR_WRAP(creat)(const char* path, mode_t mode) +int UNIFYFS_WRAP(creat)(const char* path, mode_t mode) { /* equivalent to open(path, O_WRONLY|O_CREAT|O_TRUNC, mode) */ /* check whether we should intercept this path */ - if (unifycr_intercept_path(path)) { + if (unifyfs_intercept_path(path)) { /* TODO: handle relative paths using current working directory */ /* create the file */ int fid; off_t pos; - int rc = unifycr_fid_open(path, O_WRONLY | O_CREAT | O_TRUNC, mode, &fid, &pos); - if (rc != UNIFYCR_SUCCESS) { - errno = unifycr_err_map_to_errno(rc); + int rc = unifyfs_fid_open(path, O_WRONLY | O_CREAT | O_TRUNC, mode, &fid, &pos); + if (rc != UNIFYFS_SUCCESS) { + errno = unifyfs_err_map_to_errno(rc); return -1; } /* allocate a free file descriptor value */ - int fd = unifycr_stack_pop(unifycr_fd_stack); + int fd = unifyfs_stack_pop(unifyfs_fd_stack); if (fd < 0) { /* ran out of file descriptors */ errno = EMFILE; @@ -682,38 +680,40 @@ int UNIFYCR_WRAP(creat)(const char* path, mode_t mode) } /* set file id and file pointer, flags include O_WRONLY */ - unifycr_fd_t* filedesc = unifycr_get_filedesc_from_fd(fd); + unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(fd); filedesc->fid = fid; filedesc->pos = pos; filedesc->read = 0; filedesc->write = 1; - LOGDBG("UNIFYCR_open generated fd %d for file %s", fd, path); + LOGDBG("UNIFYFS_open generated fd %d for file %s", fd, path); /* don't conflict with active system fds that range from 0 - (fd_limit) */ - int ret = fd + unifycr_fd_limit; + int ret = fd + unifyfs_fd_limit; return ret; } else { MAP_OR_FAIL(creat); - int ret = UNIFYCR_REAL(creat)(path, mode); - return ret ; + int ret = UNIFYFS_REAL(creat)(path, mode); + return ret; } } -int UNIFYCR_WRAP(creat64)(const char* path, mode_t mode) +int UNIFYFS_WRAP(creat64)(const char* path, mode_t mode) { /* check whether we should intercept this path */ - if (unifycr_intercept_path(path)) { + if (unifyfs_intercept_path(path)) { /* ERROR: fn not yet supported */ - fprintf(stderr, "Function not yet supported @ %s:%d\n", __FILE__, __LINE__); + fprintf(stderr, "Function not yet supported @ %s:%d\n", + __FILE__, __LINE__); + errno = ENOTSUP; return -1; } else { MAP_OR_FAIL(creat64); - int ret = UNIFYCR_REAL(creat64)(path, mode); + int ret = UNIFYFS_REAL(creat64)(path, mode); return ret; } } -int UNIFYCR_WRAP(open)(const char* path, int flags, ...) +int UNIFYFS_WRAP(open)(const char* path, int flags, ...) { /* if O_CREAT is set, we should also have some mode flags */ int mode = 0; @@ -726,20 +726,20 @@ int UNIFYCR_WRAP(open)(const char* path, int flags, ...) /* determine whether we should intercept this path */ int ret; - if (unifycr_intercept_path(path)) { + if (unifyfs_intercept_path(path)) { /* TODO: handle relative paths using current working directory */ /* create the file */ int fid; off_t pos; - int rc = unifycr_fid_open(path, flags, mode, &fid, &pos); - if (rc != UNIFYCR_SUCCESS) { - errno = unifycr_err_map_to_errno(rc); + int rc = unifyfs_fid_open(path, flags, mode, &fid, &pos); + if (rc != UNIFYFS_SUCCESS) { + errno = unifyfs_err_map_to_errno(rc); return -1; } /* allocate a free file descriptor value */ - int fd = unifycr_stack_pop(unifycr_fd_stack); + int fd = unifyfs_stack_pop(unifyfs_fd_stack); if (fd < 0) { /* ran out of file descriptors */ errno = EMFILE; @@ -747,30 +747,30 @@ int UNIFYCR_WRAP(open)(const char* path, int flags, ...) } /* set file id and file pointer */ - unifycr_fd_t* filedesc = unifycr_get_filedesc_from_fd(fd); + unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(fd); filedesc->fid = fid; filedesc->pos = pos; filedesc->read = ((flags & O_RDONLY) == O_RDONLY) || ((flags & O_RDWR) == O_RDWR); filedesc->write = ((flags & O_WRONLY) == O_WRONLY) || ((flags & O_RDWR) == O_RDWR); - LOGDBG("UNIFYCR_open generated fd %d for file %s", fd, path); + LOGDBG("UNIFYFS_open generated fd %d for file %s", fd, path); /* don't conflict with active system fds that range from 0 - (fd_limit) */ - ret = fd + unifycr_fd_limit; + ret = fd + unifyfs_fd_limit; return ret; } else { MAP_OR_FAIL(open); if (flags & O_CREAT) { - ret = UNIFYCR_REAL(open)(path, flags, mode); + ret = UNIFYFS_REAL(open)(path, flags, mode); } else { - ret = UNIFYCR_REAL(open)(path, flags); + ret = UNIFYFS_REAL(open)(path, flags); } return ret; } } -int UNIFYCR_WRAP(open64)(const char* path, int flags, ...) +int UNIFYFS_WRAP(open64)(const char* path, int flags, ...) { /* if O_CREAT is set, we should also have some mode flags */ int mode = 0; @@ -783,26 +783,26 @@ int UNIFYCR_WRAP(open64)(const char* path, int flags, ...) /* check whether we should intercept this path */ int ret; - if (unifycr_intercept_path(path)) { + if (unifyfs_intercept_path(path)) { /* Call open wrapper with LARGEFILE flag set*/ if (flags & O_CREAT) { - ret = UNIFYCR_WRAP(open)(path, flags | O_LARGEFILE, mode); + ret = UNIFYFS_WRAP(open)(path, flags | O_LARGEFILE, mode); } else { - ret = UNIFYCR_WRAP(open)(path, flags | O_LARGEFILE); + ret = UNIFYFS_WRAP(open)(path, flags | O_LARGEFILE); } } else { MAP_OR_FAIL(open64); if (flags & O_CREAT) { - ret = UNIFYCR_REAL(open64)(path, flags, mode); + ret = UNIFYFS_REAL(open64)(path, flags, mode); } else { - ret = UNIFYCR_REAL(open64)(path, flags); + ret = UNIFYFS_REAL(open64)(path, flags); } } return ret; } -int UNIFYCR_WRAP(__open_2)(const char* path, int flags, ...) +int UNIFYFS_WRAP(__open_2)(const char* path, int flags, ...) { int ret; @@ -818,35 +818,35 @@ int UNIFYCR_WRAP(__open_2)(const char* path, int flags, ...) } /* check whether we should intercept this path */ - if (unifycr_intercept_path(path)) { + if (unifyfs_intercept_path(path)) { LOGDBG("__open_2 was intercepted for path %s", path); /* Call open wrapper */ if (flags & O_CREAT) { - ret = UNIFYCR_WRAP(open)(path, flags, mode); + ret = UNIFYFS_WRAP(open)(path, flags, mode); } else { - ret = UNIFYCR_WRAP(open)(path, flags); + ret = UNIFYFS_WRAP(open)(path, flags); } } else { MAP_OR_FAIL(open); if (flags & O_CREAT) { - ret = UNIFYCR_REAL(open)(path, flags, mode); + ret = UNIFYFS_REAL(open)(path, flags, mode); } else { - ret = UNIFYCR_REAL(open)(path, flags); + ret = UNIFYFS_REAL(open)(path, flags); } } return ret; } -off_t UNIFYCR_WRAP(lseek)(int fd, off_t offset, int whence) +off_t UNIFYFS_WRAP(lseek)(int fd, off_t offset, int whence) { /* check whether we should intercept this file descriptor */ - if (unifycr_intercept_fd(&fd)) { + if (unifyfs_intercept_fd(&fd)) { /* TODO: check that fd is actually in use */ /* get the file id for this file descriptor */ - int fid = unifycr_get_fid_from_fd(fd); + int fid = unifyfs_get_fid_from_fd(fd); if (fid < 0) { /* bad file descriptor */ errno = EBADF; @@ -854,7 +854,7 @@ off_t UNIFYCR_WRAP(lseek)(int fd, off_t offset, int whence) } /* lookup meta to get file size */ - unifycr_filemeta_t* meta = unifycr_get_meta_from_fid(fid); + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); if (meta == NULL) { /* bad file descriptor */ errno = EBADF; @@ -862,7 +862,7 @@ off_t UNIFYCR_WRAP(lseek)(int fd, off_t offset, int whence) } /* get file descriptor for fd */ - unifycr_fd_t* filedesc = unifycr_get_filedesc_from_fd(fd); + unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(fd); /* get current file position */ off_t current_pos = filedesc->pos; @@ -895,42 +895,42 @@ off_t UNIFYCR_WRAP(lseek)(int fd, off_t offset, int whence) return current_pos; } else { MAP_OR_FAIL(lseek); - off_t ret = UNIFYCR_REAL(lseek)(fd, offset, whence); + off_t ret = UNIFYFS_REAL(lseek)(fd, offset, whence); return ret; } } -off64_t UNIFYCR_WRAP(lseek64)(int fd, off64_t offset, int whence) +off64_t UNIFYFS_WRAP(lseek64)(int fd, off64_t offset, int whence) { /* check whether we should intercept this file descriptor */ int origfd = fd; - if (unifycr_intercept_fd(&fd)) { + if (unifyfs_intercept_fd(&fd)) { if (sizeof(off_t) == sizeof(off64_t)) { /* off_t and off64_t are the same size, * delegate to lseek warpper */ - off64_t ret = (off64_t)UNIFYCR_WRAP(lseek)( + off64_t ret = (off64_t)UNIFYFS_WRAP(lseek)( origfd, (off_t) offset, whence); return ret; } else { /* ERROR: fn not yet supported */ fprintf(stderr, "Function not yet supported @ %s:%d\n", __FILE__, __LINE__); - errno = EBADF; + errno = ENOTSUP; return (off64_t)(-1); } } else { MAP_OR_FAIL(lseek64); - off64_t ret = UNIFYCR_REAL(lseek64)(fd, offset, whence); + off64_t ret = UNIFYFS_REAL(lseek64)(fd, offset, whence); return ret; } } -int UNIFYCR_WRAP(posix_fadvise)(int fd, off_t offset, off_t len, int advice) +int UNIFYFS_WRAP(posix_fadvise)(int fd, off_t offset, off_t len, int advice) { /* check whether we should intercept this file descriptor */ - if (unifycr_intercept_fd(&fd)) { + if (unifyfs_intercept_fd(&fd)) { /* check that the file descriptor is valid */ - int fid = unifycr_get_fid_from_fd(fd); + int fid = unifyfs_get_fid_from_fd(fd); if (fid < 0) { errno = EBADF; return errno; @@ -955,8 +955,10 @@ int UNIFYCR_WRAP(posix_fadvise)(int fd, off_t offset, off_t len, int advice) */ /* ERROR: fn not yet supported */ - fprintf(stderr, "Function not yet supported @ %s:%d\n", __FILE__, __LINE__); - break; + fprintf(stderr, "Function not yet supported @ %s:%d\n", + __FILE__, __LINE__); + errno = ENOTSUP; + return errno; default: /* this function returns the errno itself, not -1 */ errno = EINVAL; @@ -967,17 +969,17 @@ int UNIFYCR_WRAP(posix_fadvise)(int fd, off_t offset, off_t len, int advice) return 0; } else { MAP_OR_FAIL(posix_fadvise); - int ret = UNIFYCR_REAL(posix_fadvise)(fd, offset, len, advice); + int ret = UNIFYFS_REAL(posix_fadvise)(fd, offset, len, advice); return ret; } } -ssize_t UNIFYCR_WRAP(read)(int fd, void* buf, size_t count) +ssize_t UNIFYFS_WRAP(read)(int fd, void* buf, size_t count) { /* check whether we should intercept this file descriptor */ - if (unifycr_intercept_fd(&fd)) { + if (unifyfs_intercept_fd(&fd)) { /* get file id */ - int fid = unifycr_get_fid_from_fd(fd); + int fid = unifyfs_get_fid_from_fd(fd); if (fid < 0) { /* ERROR: invalid file descriptor */ errno = EBADF; @@ -985,14 +987,15 @@ ssize_t UNIFYCR_WRAP(read)(int fd, void* buf, size_t count) } /* get pointer to file descriptor structure */ - unifycr_fd_t* filedesc = unifycr_get_filedesc_from_fd(fd); + unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(fd); if (filedesc == NULL) { /* ERROR: invalid file descriptor */ errno = EBADF; return (ssize_t)(-1); } - unifycr_filemeta_t* meta = unifycr_get_meta_from_fid(fid); +#if 0 // THIS IS BROKEN UNTIL WE HAVE GLOBAL SIZE + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); if (meta == NULL) { /* ERROR: invalid file descriptor */ errno = EBADF; @@ -1003,6 +1006,7 @@ ssize_t UNIFYCR_WRAP(read)(int fd, void* buf, size_t count) if (filedesc->pos >= meta->size) { return 0; /* EOF */ } +#endif /* assume we'll succeed in read */ size_t retcount = count; @@ -1011,22 +1015,26 @@ ssize_t UNIFYCR_WRAP(read)(int fd, void* buf, size_t count) tmp_req.fid = fid; tmp_req.offset = (size_t) filedesc->pos; tmp_req.length = count; - tmp_req.errcode = UNIFYCR_SUCCESS; + tmp_req.errcode = UNIFYFS_SUCCESS; tmp_req.buf = buf; /* * this returns error code, which is zero for successful cases. */ - int ret = unifycr_fd_logreadlist(&tmp_req, 1); + int ret = unifyfs_fd_logreadlist(&tmp_req, 1); - if (ret != UNIFYCR_SUCCESS) { - /* error reading data */ - errno = EIO; - retcount = -1; - } else if (tmp_req.errcode != UNIFYCR_SUCCESS) { - /* error reading data */ - errno = EIO; - retcount = -1; + /* + * FIXME: when we can get the global file size correctly, the following + * should be rewritten. currently, we cannot detect EOF reliably. + */ + if (ret != UNIFYFS_SUCCESS) { + if (tmp_req.errcode != UNIFYFS_SUCCESS) { + /* error reading data */ + errno = EIO; + retcount = -1; + } else { + retcount = 0; /* possible EOF */ + } } else { /* success, update position */ filedesc->pos += (off_t) retcount; @@ -1036,22 +1044,22 @@ ssize_t UNIFYCR_WRAP(read)(int fd, void* buf, size_t count) return (ssize_t) retcount; } else { MAP_OR_FAIL(read); - ssize_t ret = UNIFYCR_REAL(read)(fd, buf, count); + ssize_t ret = UNIFYFS_REAL(read)(fd, buf, count); return ret; } } /* TODO: find right place to msync spillover mapping */ -ssize_t UNIFYCR_WRAP(write)(int fd, const void* buf, size_t count) +ssize_t UNIFYFS_WRAP(write)(int fd, const void* buf, size_t count) { ssize_t ret; LOGDBG("write was called for fd %d", fd); /* check whether we should intercept this file descriptor */ - if (unifycr_intercept_fd(&fd)) { + if (unifyfs_intercept_fd(&fd)) { /* get pointer to file descriptor structure */ - unifycr_fd_t* filedesc = unifycr_get_filedesc_from_fd(fd); + unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(fd); if (filedesc == NULL) { /* ERROR: invalid file descriptor */ errno = EBADF; @@ -1059,9 +1067,9 @@ ssize_t UNIFYCR_WRAP(write)(int fd, const void* buf, size_t count) } /* write data to file */ - int write_rc = unifycr_fd_write(fd, filedesc->pos, buf, count); - if (write_rc != UNIFYCR_SUCCESS) { - errno = unifycr_err_map_to_errno(write_rc); + int write_rc = unifyfs_fd_write(fd, filedesc->pos, buf, count); + if (write_rc != UNIFYFS_SUCCESS) { + errno = unifyfs_err_map_to_errno(write_rc); return (ssize_t)(-1); } ret = count; @@ -1070,97 +1078,169 @@ ssize_t UNIFYCR_WRAP(write)(int fd, const void* buf, size_t count) filedesc->pos += ret; } else { MAP_OR_FAIL(write); - ret = UNIFYCR_REAL(write)(fd, buf, count); + ret = UNIFYFS_REAL(write)(fd, buf, count); } return ret; } -ssize_t UNIFYCR_WRAP(readv)(int fd, const struct iovec* iov, int iovcnt) +ssize_t UNIFYFS_WRAP(readv)(int fd, const struct iovec* iov, int iovcnt) { + ssize_t ret; + /* check whether we should intercept this file descriptor */ - if (unifycr_intercept_fd(&fd)) { - /* ERROR: fn not yet supported */ - fprintf(stderr, "Function not yet supported @ %s:%d\n", __FILE__, __LINE__); - errno = EBADF; - return -1; + if (unifyfs_intercept_fd(&fd)) { + ssize_t rret; + int i; + ret = 0; + for (i = 0; i < iovcnt; i++) { + rret = UNIFYFS_WRAP(read)(fd, (void*)iov[i].iov_base, + iov[i].iov_len); + if (-1 == rret) { + return -1; + } else if (0 == rret) { + return ret; + } else { + ret += rret; + } + } + return ret; } else { MAP_OR_FAIL(readv); - ssize_t ret = UNIFYCR_REAL(readv)(fd, iov, iovcnt); + ret = UNIFYFS_REAL(readv)(fd, iov, iovcnt); return ret; } } -ssize_t UNIFYCR_WRAP(writev)(int fd, const struct iovec* iov, int iovcnt) +ssize_t UNIFYFS_WRAP(writev)(int fd, const struct iovec* iov, int iovcnt) { + ssize_t ret; + /* check whether we should intercept this file descriptor */ - if (unifycr_intercept_fd(&fd)) { - /* ERROR: fn not yet supported */ - fprintf(stderr, "Function not yet supported @ %s:%d\n", __FILE__, __LINE__); - errno = EBADF; - return -1; + if (unifyfs_intercept_fd(&fd)) { + ssize_t wret; + int i; + ret = 0; + for (i = 0; i < iovcnt; i++) { + wret = UNIFYFS_WRAP(write)(fd, (const void*)iov[i].iov_base, + iov[i].iov_len); + if (-1 == wret) { + return -1; + } else { + ret += wret; + if ((size_t)wret != iov[i].iov_len) { + return ret; + } + } + } + return ret; } else { MAP_OR_FAIL(writev); - ssize_t ret = UNIFYCR_REAL(writev)(fd, iov, iovcnt); + ret = UNIFYFS_REAL(writev)(fd, iov, iovcnt); return ret; } } -int UNIFYCR_WRAP(lio_listio)(int mode, struct aiocb* const aiocb_list[], +int UNIFYFS_WRAP(lio_listio)(int mode, struct aiocb* const aiocb_list[], int nitems, struct sigevent* sevp) { - /* TODO: missing pass through for non-intercepted lio_listio */ + /* TODO - support for LIO_NOWAIT mode */ - /* TODO: require all elements in list to refer to unify files? */ + read_req_t* reqs = calloc(nitems, sizeof(read_req_t)); + if (NULL == reqs) { + errno = ENOMEM; // EAGAIN? + return -1; + } - read_req_t* reqs = malloc(nitems * sizeof(read_req_t)); + int ret = 0; + int reqcnt = 0; + int i, fd, fid, ndx, rc; + struct aiocb* cbp; - int i; for (i = 0; i < nitems; i++) { - if (aiocb_list[i]->aio_lio_opcode != LIO_READ) { - //does not support write operation currently - free(reqs); - errno = EIO; - return -1; + cbp = aiocb_list[i]; + fd = cbp->aio_fildes; + switch (cbp->aio_lio_opcode) { + case LIO_WRITE: { + ssize_t wret; + wret = UNIFYFS_WRAP(pwrite)(fd, (const void*)cbp->aio_buf, + cbp->aio_nbytes, cbp->aio_offset); + if (-1 == wret) { + AIOCB_ERROR_CODE(cbp) = errno; + } else { + AIOCB_ERROR_CODE(cbp) = 0; + AIOCB_RETURN_VAL(cbp) = wret; + } + break; } - - int fd = aiocb_list[i]->aio_fildes; - if (unifycr_intercept_fd(&fd)) { - /* get local file id for this request */ - int fid = unifycr_get_fid_from_fd(fd); - if (fid < 0) { - /* TODO: need to set error code on each element */ - errno = EIO; - free(reqs); - return -1; + case LIO_READ: { + if (unifyfs_intercept_fd(&fd)) { + /* get local file id for this request */ + fid = unifyfs_get_fid_from_fd(fd); + if (fid < 0) { + AIOCB_ERROR_CODE(cbp) = EINVAL; + } else { + reqs[reqcnt].fid = fid; + reqs[reqcnt].offset = (size_t)(cbp->aio_offset); + reqs[reqcnt].length = cbp->aio_nbytes; + reqs[reqcnt].errcode = EINPROGRESS; + reqs[reqcnt].buf = (char*)(cbp->aio_buf); + reqcnt++; + } + } else { + ssize_t rret; + rret = UNIFYFS_WRAP(pread)(fd, (void*)cbp->aio_buf, + cbp->aio_nbytes, cbp->aio_offset); + if (-1 == rret) { + AIOCB_ERROR_CODE(cbp) = errno; + } else { + AIOCB_ERROR_CODE(cbp) = 0; + AIOCB_RETURN_VAL(cbp) = rret; + } } - - reqs[i].fid = fid; - reqs[i].offset = (size_t) aiocb_list[i]->aio_offset; - reqs[i].length = aiocb_list[i]->aio_nbytes; - reqs[i].errcode = UNIFYCR_SUCCESS; - reqs[i].buf = (char*)aiocb_list[i]->aio_buf; + break; + } + default: // LIO_NOP + break; } } - int ret = unifycr_fd_logreadlist(reqs, nitems); - - for (i = 0; i < nitems; i++) { - /* TODO: update fields to record error status - * see /usr/include/aio.h, - * update __error_code and __return_val fields? */ + if (reqcnt) { + rc = unifyfs_fd_logreadlist(reqs, reqcnt); + if (rc != UNIFYFS_SUCCESS) { + /* error reading data */ + ret = -1; + } + /* update aiocb fields to record error status and return value */ + ndx = 0; + for (i = 0; i < reqcnt; i++) { + char* buf = reqs[i].buf; + for (; ndx < nitems; ndx++) { + cbp = aiocb_list[ndx]; + if ((char*)(cbp->aio_buf) == buf) { + AIOCB_ERROR_CODE(cbp) = reqs[i].errcode; + if (0 == reqs[i].errcode) { + AIOCB_RETURN_VAL(cbp) = reqs[i].length; + } + break; // continue outer loop + } + } + } } free(reqs); + if (-1 == ret) { + errno = EIO; + } return ret; } /* order by file id then by file position */ static int compare_index_entry(const void* a, const void* b) { - const unifycr_index_t* ptr_a = a; - const unifycr_index_t* ptr_b = b; + const unifyfs_index_t* ptr_a = a; + const unifyfs_index_t* ptr_b = b; if (ptr_a->fid != ptr_b->fid) { if (ptr_a->fid < ptr_b->fid) { @@ -1204,7 +1284,7 @@ static int compare_read_req(const void* a, const void* b) /* returns index into read_req of item whose offset is * just below offset of target item (if one exists) */ -static int unifycr_locate_req(read_req_t* read_reqs, int count, +static int unifyfs_locate_req(read_req_t* read_reqs, int count, read_req_t* match_req) { /* if list is empty, indicate that there is valid starting request */ @@ -1280,7 +1360,7 @@ static int unifycr_locate_req(read_req_t* read_reqs, int count, * @param slice_range: the slice size of the key-value store * @return out_set: the set of split read requests * */ -static int unifycr_split_read_requests(read_req_t* req, +static int unifyfs_split_read_requests(read_req_t* req, read_req_set_t* out_set, size_t slice_range) { @@ -1288,7 +1368,7 @@ static int unifycr_split_read_requests(read_req_t* req, size_t req_start = req->offset; size_t req_end = req->offset + req->length - 1; - /* compute offset of first and lasy byte of slice + /* compute offset of first and last byte of slice * that contains first byte of request */ size_t slice_start = (req->offset / slice_range) * slice_range; size_t slice_end = slice_start + slice_range - 1; @@ -1334,7 +1414,7 @@ static int unifycr_split_read_requests(read_req_t* req, out_set->read_reqs[count].fid = req->fid; out_set->read_reqs[count].offset = slice_start; out_set->read_reqs[count].length = slice_range; - out_set->read_reqs[count].errcode = UNIFYCR_SUCCESS; + out_set->read_reqs[count].errcode = UNIFYFS_SUCCESS; count++; } while (1); @@ -1342,7 +1422,7 @@ static int unifycr_split_read_requests(read_req_t* req, out_set->read_reqs[count].fid = req->fid; out_set->read_reqs[count].offset = slice_start; out_set->read_reqs[count].length = req_end - slice_start + 1; - out_set->read_reqs[count].errcode = UNIFYCR_SUCCESS; + out_set->read_reqs[count].errcode = UNIFYFS_SUCCESS; count++; } @@ -1353,9 +1433,9 @@ static int unifycr_split_read_requests(read_req_t* req, } /* - * coalesce read requests refering to contiguous data within a given + * coalesce read requests referring to contiguous data within a given * file id, and split read requests whose size is larger than - * unifycr_key_slice_range into more requests that are smaller + * unifyfs_key_slice_range into more requests that are smaller * * Note: a series of read requests that have overlapping spans * will prevent merging of contiguous ranges, this should still @@ -1365,12 +1445,12 @@ static int unifycr_split_read_requests(read_req_t* req, * @param count: number of read requests * @param tmp_set: a temporary read requests buffer * to hold the intermediate result - * @param unifycr_key_slice_range: slice size of distributed + * @param unifyfs_key_slice_range: slice size of distributed * key-value store * @return out_set: the coalesced read requests * * */ -static int unifycr_coalesce_read_reqs(read_req_t* read_req, int count, +static int unifyfs_coalesce_read_reqs(read_req_t* read_req, int count, size_t slice_range, read_req_set_t* out_set) { @@ -1388,7 +1468,7 @@ static int unifycr_coalesce_read_reqs(read_req_t* read_req, int count, /* split this read request into parts based on slice range * store resulting requests in tmp_set */ - unifycr_split_read_requests(&read_req[i], &tmp_set, slice_range); + unifyfs_split_read_requests(&read_req[i], &tmp_set, slice_range); /* look to merge last item in output set with first item * in split requests */ @@ -1430,13 +1510,13 @@ static int unifycr_coalesce_read_reqs(read_req_t* read_req, int count, /* * match the received read_requests with the * client's read requests - * @param read_req: a list of read requests + * @param read_reqs: a list of read requests * @param count: number of read requests * @param match_req: received read request to match * @return error code * * */ -static int unifycr_match_received_ack(read_req_t* read_req, int count, +static int unifyfs_match_received_ack(read_req_t* read_reqs, int count, read_req_t* match_req) { /* given fid, offset, and length of match_req that holds read reply, @@ -1451,20 +1531,20 @@ static int unifycr_match_received_ack(read_req_t* read_req, int count, match_end.offset += match_end.length - 1; /* find index of read request that contains our first byte */ - int start_pos = unifycr_locate_req(read_req, count, &match_start); + int start_pos = unifyfs_locate_req(read_reqs, count, &match_start); /* find index of read request that contains our last byte */ - int end_pos = unifycr_locate_req(read_req, count, &match_end); + int end_pos = unifyfs_locate_req(read_reqs, count, &match_end); /* could not find a valid read request in read_req array */ if (start_pos == -1) { - return UNIFYCR_FAILURE; + return UNIFYFS_FAILURE; } /* s: start of match_req, e: end of match_req */ if (start_pos == 0) { - if (compare_read_req(&match_start, &read_req[0]) < 0) { + if (compare_read_req(&match_start, &read_reqs[0]) < 0) { /* starting offset in read reply comes before lowest * offset in read requests, consider this to be an error * @@ -1472,15 +1552,15 @@ static int unifycr_match_received_ack(read_req_t* read_req, int count, * s * * */ - return UNIFYCR_FAILURE; + return UNIFYFS_FAILURE; } } /* create read request corresponding to first byte of first read request */ - read_req_t first_start = read_req[start_pos]; + read_req_t first_start = read_reqs[start_pos]; /* create read request corresponding to last byte of first read request */ - read_req_t first_end = read_req[start_pos]; + read_req_t first_end = read_reqs[start_pos]; first_end.offset += first_end.length - 1; /* check whether read reply is fully contained by first read request */ @@ -1495,7 +1575,7 @@ static int unifycr_match_received_ack(read_req_t* read_req, int count, * */ /* copy data to user buffer if no error */ - if (match_req->errcode == UNIFYCR_SUCCESS) { + if (match_req->errcode == UNIFYFS_SUCCESS) { /* compute buffer location to copy data */ size_t offset = (size_t)(match_start.offset - first_start.offset); char* buf = first_start.buf + offset; @@ -1503,20 +1583,20 @@ static int unifycr_match_received_ack(read_req_t* read_req, int count, /* copy data to user buffer */ memcpy(buf, match_req->buf, match_req->length); - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } else { /* hit an error during read, so record this fact * in user's original read request */ - read_req[start_pos].errcode = match_req->errcode; - return UNIFYCR_FAILURE; + read_reqs[start_pos].errcode = match_req->errcode; + return UNIFYFS_FAILURE; } } /* define read request for offset of first byte in last read request */ - read_req_t last_start = read_req[end_pos]; + read_req_t last_start = read_reqs[end_pos]; /* define read request for offset of last byte in last read request */ - read_req_t last_end = read_req[end_pos]; + read_req_t last_end = read_reqs[end_pos]; last_end.offset += last_end.length - 1; /* determine whether read reply is contained in a range of read requests */ @@ -1534,16 +1614,15 @@ static int unifycr_match_received_ack(read_req_t* read_req, int count, * define a contiguous set of bytes */ int i; for (i = start_pos + 1; i <= end_pos; i++) { - if (read_req[i - 1].offset + read_req[i - 1].length != read_req[i].offset) { - /* read requests are noncontiguous, so we returned data - * covering a hole in the original array of read requests, error */ - return UNIFYCR_FAILURE; + if ((read_reqs[i - 1].offset + read_reqs[i - 1].length) + != read_reqs[i].offset) { + /* read requests are noncontiguous, error */ + return UNIFYFS_FAILURE; } } /* read requests are contiguous, fill all buffers in middle */ - - if (match_req->errcode == UNIFYCR_SUCCESS) { + if (match_req->errcode == UNIFYFS_SUCCESS) { /* get pointer to start of read reply data */ char* ptr = match_req->buf; @@ -1561,8 +1640,8 @@ static int unifycr_match_received_ack(read_req_t* read_req, int count, /* copy data for middle read requests */ for (i = start_pos + 1; i < end_pos; i++) { - memcpy(read_req[i].buf, ptr, read_req[i].length); - ptr += read_req[i].length; + memcpy(read_reqs[i].buf, ptr, read_reqs[i].length); + ptr += read_reqs[i].length; } /* compute bytes for last read request */ @@ -1572,41 +1651,41 @@ static int unifycr_match_received_ack(read_req_t* read_req, int count, memcpy(last_start.buf, ptr, length); ptr += length; - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } else { /* hit an error during read, update errcode in user's * original read request from start to end inclusive */ for (i = start_pos; i <= end_pos; i++) { - read_req[i].errcode = match_req->errcode; + read_reqs[i].errcode = match_req->errcode; } - return UNIFYCR_FAILURE; + return UNIFYFS_FAILURE; } } /* could not find a matching read request, return an error */ - return UNIFYCR_FAILURE; + return UNIFYFS_FAILURE; } /* notify our delegator that the shared memory buffer * is now clear and ready to hold more read data */ -static int delegator_signal() +static void delegator_signal(void) { - int rc = UNIFYCR_SUCCESS; + LOGDBG("receive buffer now empty"); - /* set shm flag to 0 to signal delegator we're done */ - volatile int* ptr_flag = (volatile int*)shm_recv_buf; - *ptr_flag = 0; + /* set shm flag to signal delegator we're done */ + shm_header_t* hdr = (shm_header_t*)shm_recv_buf; + hdr->state = SHMEM_REGION_EMPTY; /* TODO: MEM_FLUSH */ - - return rc; } /* wait for delegator to inform us that shared memory buffer * is filled with read data */ -static int delegator_wait() +static int delegator_wait(void) { -#if 0 + int rc = (int)UNIFYFS_SUCCESS; + +#if defined(UNIFYFS_USE_DOMAIN_SOCKET) /* wait for signal on socket */ cmd_fd.events = POLLIN | POLLPRI; cmd_fd.revents = 0; @@ -1616,7 +1695,7 @@ static int delegator_wait() if (rc == 0) { if (cmd_fd.revents != 0) { if (cmd_fd.revents == POLLIN) { - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } else { printf("poll returned %d; error: %s\n", rc, strerror(errno)); } @@ -1635,78 +1714,77 @@ static int delegator_wait() shm_wait_tm.tv_nsec = SHM_WAIT_INTERVAL; /* get pointer to flag in shared memory */ - volatile int* ptr_flag = (volatile int*)shm_recv_buf; - - /* TODO: MEM_FETCH */ + shm_header_t* hdr = (shm_header_t*)shm_recv_buf; /* wait for server to set flag to non-zero */ - while (*ptr_flag == 0) { + int max_sleep = 5000000; // 5s + volatile int* vip = (volatile int*)&(hdr->state); + while (*vip == SHMEM_REGION_EMPTY) { /* not there yet, sleep for a while */ nanosleep(&shm_wait_tm, NULL); /* TODO: MEM_FETCH */ + max_sleep--; + if (0 == max_sleep) { + LOGERR("timed out waiting for non-empty"); + rc = (int)UNIFYFS_ERROR_SHMEM; + break; + } } - return UNIFYCR_SUCCESS; + return rc; } /* copy read data from shared memory buffer to user buffers from read * calls, sets done=1 on return when delegator informs us it has no * more data */ -static int process_read_data(read_req_t* read_req, int count, int* done) +static int process_read_data(read_req_t* read_reqs, int count, int* done) { /* assume we'll succeed */ - int rc = UNIFYCR_SUCCESS; + int rc = UNIFYFS_SUCCESS; /* get pointer to start of shared memory buffer */ - char* shmptr = (char*)shm_recv_buf; - - /* extract pointer to flag */ - int* ptr_flag = (int*)shmptr; - shmptr += sizeof(int); - - /* extract pointer to size */ - int* ptr_size = (int*)shmptr; - shmptr += sizeof(int); + shm_header_t* shm_hdr = (shm_header_t*)shm_recv_buf; + char* shmptr = ((char*)shm_hdr) + sizeof(shm_header_t); - /* extract number of read replies in buffer */ - int* ptr_num = (int*)shmptr; - shmptr += sizeof(int); - int num = *ptr_num; + size_t num = shm_hdr->meta_cnt; + if (0 == num) { + LOGDBG("no read responses available"); + return rc; + } /* process each of our replies */ - int j; - for (j = 0; j < num; j++) { + size_t i; + for (i = 0; i < num; i++) { /* get pointer to current read reply header */ shm_meta_t* msg = (shm_meta_t*)shmptr; shmptr += sizeof(shm_meta_t); /* define request object */ read_req_t req; - req.fid = msg->src_fid; + req.fid = msg->gfid; req.offset = msg->offset; req.length = msg->length; req.errcode = msg->errcode; + LOGDBG("read reply: gfid=%d offset=%zu size=%zu", + req.fid, req.offset, req.length); + /* get pointer to data */ req.buf = shmptr; shmptr += msg->length; /* process this read reply, identify which application read * request this reply goes to and copy data to user buffer */ - int tmp_rc = unifycr_match_received_ack(read_req, count, &req); - if (tmp_rc != UNIFYCR_SUCCESS) { - rc = UNIFYCR_FAILURE; + int tmp_rc = unifyfs_match_received_ack(read_reqs, count, &req); + if (tmp_rc != UNIFYFS_SUCCESS) { + rc = UNIFYFS_FAILURE; } } /* set done flag if there is no more data */ - if (*ptr_flag == 2) { + if (shm_hdr->state == SHMEM_REGION_DATA_COMPLETE) { *done = 1; - }; - - /* zero out size and count in recv buffer */ - *ptr_size = 0; - *ptr_num = 0; + } return rc; } @@ -1714,16 +1792,16 @@ static int process_read_data(read_req_t* read_req, int count, int* done) /* * get data for a list of read requests from the * delegator - * @param read_req: a list of read requests + * @param read_reqs: a list of read requests * @param count: number of read requests * @return error code * * */ -int unifycr_fd_logreadlist(read_req_t* read_req, int count) +int unifyfs_fd_logreadlist(read_req_t* read_reqs, int count) { int i; int tot_sz = 0; - int rc = UNIFYCR_SUCCESS; + int rc = UNIFYFS_SUCCESS; int num = 0; int* ptr_size = NULL; int* ptr_num = NULL; @@ -1732,12 +1810,12 @@ int unifycr_fd_logreadlist(read_req_t* read_req, int count) /* Adjust length for fitting the EOF. */ for (i = 0; i < count; i++) { /* get pointer to read request */ - read_req_t* req = &read_req[i]; + read_req_t* req = &read_reqs[i]; /* get metadata for this file */ - unifycr_filemeta_t* meta = unifycr_get_meta_from_fid(req->fid); + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(req->fid); if (meta == NULL) { - return UNIFYCR_ERROR_BADF; + return UNIFYFS_ERROR_BADF; } /* compute last byte of read request */ @@ -1756,40 +1834,40 @@ int unifycr_fd_logreadlist(read_req_t* read_req, int count) * */ /* convert local fid to global fid */ - unifycr_file_attr_t tmp_meta_entry; - unifycr_file_attr_t* ptr_meta_entry; + unifyfs_file_attr_t tmp_meta_entry; + unifyfs_file_attr_t* ptr_meta_entry; for (i = 0; i < count; i++) { /* look for global meta data for this local file id */ - tmp_meta_entry.fid = read_req[i].fid; + tmp_meta_entry.fid = read_reqs[i].fid; ptr_meta_entry = - (unifycr_file_attr_t*) bsearch(&tmp_meta_entry, - unifycr_fattrs.meta_entry, - *unifycr_fattrs.ptr_num_entries, - sizeof(unifycr_file_attr_t), + (unifyfs_file_attr_t*) bsearch(&tmp_meta_entry, + unifyfs_fattrs.meta_entry, + *unifyfs_fattrs.ptr_num_entries, + sizeof(unifyfs_file_attr_t), compare_fattr); /* replace local file id with global file id in request */ if (ptr_meta_entry != NULL) { - read_req[i].fid = ptr_meta_entry->gfid; + read_reqs[i].fid = ptr_meta_entry->gfid; } else { /* failed to find gfid for this request */ - return UNIFYCR_ERROR_BADF; + return UNIFYFS_ERROR_BADF; } } /* order read request by increasing file id, then increasing offset */ - qsort(read_req, count, sizeof(read_req_t), compare_read_req); + qsort(read_reqs, count, sizeof(read_req_t), compare_read_req); /* coalesce the contiguous read requests */ - unifycr_coalesce_read_reqs(read_req, count, - unifycr_key_slice_range, + read_req_set_t read_req_set; + unifyfs_coalesce_read_reqs(read_reqs, count, + unifyfs_key_slice_range, &read_req_set); - shm_meta_t* tmp_sh_meta; - /* prepare our shared memory buffer for delegator */ delegator_signal(); + int read_rc; if (read_req_set.count > 1) { /* got multiple read requests, * build up a flat buffer to include them all */ @@ -1797,32 +1875,30 @@ int unifycr_fd_logreadlist(read_req_t* read_req, int count) flatcc_builder_init(&builder); /* create request vector */ - unifycr_Extent_vec_start(&builder); + unifyfs_Extent_vec_start(&builder); /* fill in values for each request entry */ for (i = 0; i < read_req_set.count; i++) { - unifycr_Extent_vec_push_create(&builder, + unifyfs_Extent_vec_push_create(&builder, read_req_set.read_reqs[i].fid, read_req_set.read_reqs[i].offset, read_req_set.read_reqs[i].length); } /* complete the array */ - unifycr_Extent_vec_ref_t extents = unifycr_Extent_vec_end(&builder); - unifycr_ReadRequest_create_as_root(&builder, extents); - //unifycr_ReadRequest_end_as_root(&builder); + unifyfs_Extent_vec_ref_t extents = unifyfs_Extent_vec_end(&builder); + unifyfs_ReadRequest_create_as_root(&builder, extents); + //unifyfs_ReadRequest_end_as_root(&builder); /* allocate our buffer to be sent */ size_t size = 0; void* buffer = flatcc_builder_finalize_buffer(&builder, &size); assert(buffer); - LOGDBG("mread: n_reqs:%zu, flatcc buffer (%p) sz:%zu", + LOGDBG("mread: n_reqs:%d, flatcc buffer (%p) sz:%zu", read_req_set.count, buffer, size); /* invoke read rpc here */ - unifycr_client_mread_rpc_invoke(&unifycr_rpc_context, app_id, - local_rank_idx, ptr_meta_entry->gfid, - read_req_set.count, size, buffer); + read_rc = invoke_client_mread_rpc(read_req_set.count, size, buffer); flatcc_builder_clear(&builder); free(buffer); @@ -1832,8 +1908,12 @@ int unifycr_fd_logreadlist(read_req_t* read_req, int count) size_t offset = read_req_set.read_reqs[0].offset; size_t length = read_req_set.read_reqs[0].length; LOGDBG("read: offset:%zu, len:%zu", offset, length); - unifycr_client_read_rpc_invoke(&unifycr_rpc_context, app_id, - local_rank_idx, gfid, offset, length); + read_rc = invoke_client_read_rpc(gfid, offset, length); + } + + /* bail out with error if we failed to even start the read */ + if (read_rc != UNIFYFS_SUCCESS) { + return read_rc; } /* @@ -1843,44 +1923,40 @@ int unifycr_fd_logreadlist(read_req_t* read_req, int count) int done = 0; while (!done) { - delegator_wait(); - - int tmp_rc = process_read_data(read_req, count, &done); - if (tmp_rc != UNIFYCR_SUCCESS) { - rc = UNIFYCR_FAILURE; + int tmp_rc = delegator_wait(); + if (tmp_rc != UNIFYFS_SUCCESS) { + rc = UNIFYFS_FAILURE; + done = 1; + } else { + tmp_rc = process_read_data(read_reqs, count, &done); + if (tmp_rc != UNIFYFS_SUCCESS) { + rc = UNIFYFS_FAILURE; + } + delegator_signal(); } - - delegator_signal(); } return rc; } -ssize_t UNIFYCR_WRAP(pread)(int fd, void* buf, size_t count, off_t offset) +ssize_t UNIFYFS_WRAP(pread)(int fd, void* buf, size_t count, off_t offset) { /* equivalent to read(), except that it shall read from a given * position in the file without changing the file pointer */ /* check whether we should intercept this file descriptor */ - if (unifycr_intercept_fd(&fd)) { + if (unifyfs_intercept_fd(&fd)) { /* get file id */ - int fid = unifycr_get_fid_from_fd(fd); + int fid = unifyfs_get_fid_from_fd(fd); if (fid < 0) { /* ERROR: invalid file descriptor */ errno = EBADF; return (ssize_t)(-1); } - /* get file descriptor structure */ - unifycr_fd_t* filedesc = unifycr_get_filedesc_from_fd(fd); - if (filedesc == NULL) { - /* ERROR: invalid file descriptor */ - errno = EBADF; - return (ssize_t)(-1); - } - +#if 0 // THIS IS BROKEN UNTIL WE HAVE GLOBAL SIZE /* get pointer to file descriptor structure */ - unifycr_filemeta_t* meta = unifycr_get_meta_from_fid(fid); + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); if (meta == NULL) { /* ERROR: invalid file descriptor */ errno = EBADF; @@ -1891,6 +1967,7 @@ ssize_t UNIFYCR_WRAP(pread)(int fd, void* buf, size_t count, off_t offset) if (offset >= meta->size) { return 0; } +#endif /* assume we'll succeed in read */ size_t retcount = count; @@ -1899,16 +1976,15 @@ ssize_t UNIFYCR_WRAP(pread)(int fd, void* buf, size_t count, off_t offset) tmp_req.fid = fid; tmp_req.offset = offset; tmp_req.length = count; - tmp_req.errcode = UNIFYCR_SUCCESS; + tmp_req.errcode = UNIFYFS_SUCCESS; tmp_req.buf = buf; - int ret = unifycr_fd_logreadlist(&tmp_req, 1); - - if (ret != UNIFYCR_SUCCESS) { + int ret = unifyfs_fd_logreadlist(&tmp_req, 1); + if (ret != UNIFYFS_SUCCESS) { /* error reading data */ errno = EIO; retcount = -1; - } else if (tmp_req.errcode != UNIFYCR_SUCCESS) { + } else if (tmp_req.errcode != UNIFYFS_SUCCESS) { /* error reading data */ errno = EIO; retcount = -1; @@ -1918,35 +1994,32 @@ ssize_t UNIFYCR_WRAP(pread)(int fd, void* buf, size_t count, off_t offset) return (ssize_t) retcount; } else { MAP_OR_FAIL(pread); - ssize_t ret = UNIFYCR_REAL(pread)(fd, buf, count, offset); + ssize_t ret = UNIFYFS_REAL(pread)(fd, buf, count, offset); return ret; } } -ssize_t UNIFYCR_WRAP(pread64)(int fd, void* buf, size_t count, off64_t offset) +ssize_t UNIFYFS_WRAP(pread64)(int fd, void* buf, size_t count, off64_t offset) { /* check whether we should intercept this file descriptor */ - if (unifycr_intercept_fd(&fd)) { - /* ERROR: fn not yet supported */ - fprintf(stderr, "Function not yet supported @ %s:%d\n", __FILE__, __LINE__); - errno = EBADF; - return -1; + if (unifyfs_intercept_fd(&fd)) { + return UNIFYFS_WRAP(pread)(fd, buf, count, (off_t)offset); } else { MAP_OR_FAIL(pread64); - ssize_t ret = UNIFYCR_REAL(pread64)(fd, buf, count, offset); + ssize_t ret = UNIFYFS_REAL(pread64)(fd, buf, count, offset); return ret; } } -ssize_t UNIFYCR_WRAP(pwrite)(int fd, const void* buf, size_t count, +ssize_t UNIFYFS_WRAP(pwrite)(int fd, const void* buf, size_t count, off_t offset) { /* equivalent to write(), except that it writes into a given * position without changing the file pointer */ /* check whether we should intercept this file descriptor */ - if (unifycr_intercept_fd(&fd)) { + if (unifyfs_intercept_fd(&fd)) { /* get pointer to file descriptor structure */ - unifycr_fd_t* filedesc = unifycr_get_filedesc_from_fd(fd); + unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(fd); if (filedesc == NULL) { /* ERROR: invalid file descriptor */ errno = EBADF; @@ -1954,9 +2027,9 @@ ssize_t UNIFYCR_WRAP(pwrite)(int fd, const void* buf, size_t count, } /* write data to file */ - int write_rc = unifycr_fd_write(fd, offset, buf, count); - if (write_rc != UNIFYCR_SUCCESS) { - errno = unifycr_err_map_to_errno(write_rc); + int write_rc = unifyfs_fd_write(fd, offset, buf, count); + if (write_rc != UNIFYFS_SUCCESS) { + errno = unifyfs_err_map_to_errno(write_rc); return (ssize_t)(-1); } @@ -1964,31 +2037,30 @@ ssize_t UNIFYCR_WRAP(pwrite)(int fd, const void* buf, size_t count, return (ssize_t) count; } else { MAP_OR_FAIL(pwrite); - ssize_t ret = UNIFYCR_REAL(pwrite)(fd, buf, count, offset); + ssize_t ret = UNIFYFS_REAL(pwrite)(fd, buf, count, offset); return ret; } } -ssize_t UNIFYCR_WRAP(pwrite64)(int fd, const void* buf, size_t count, +ssize_t UNIFYFS_WRAP(pwrite64)(int fd, const void* buf, size_t count, off64_t offset) { /* check whether we should intercept this file descriptor */ - if (unifycr_intercept_fd(&fd)) { - /* ERROR: fn not yet supported */ - return -1; + if (unifyfs_intercept_fd(&fd)) { + return UNIFYFS_WRAP(pwrite)(fd, buf, count, (off_t)offset); } else { MAP_OR_FAIL(pwrite64); - ssize_t ret = UNIFYCR_REAL(pwrite64)(fd, buf, count, offset); + ssize_t ret = UNIFYFS_REAL(pwrite64)(fd, buf, count, offset); return ret; } } -int UNIFYCR_WRAP(ftruncate)(int fd, off_t length) +int UNIFYFS_WRAP(ftruncate)(int fd, off_t length) { /* check whether we should intercept this file descriptor */ - if (unifycr_intercept_fd(&fd)) { + if (unifyfs_intercept_fd(&fd)) { /* get the file id for this file descriptor */ - int fid = unifycr_get_fid_from_fd(fd); + int fid = unifyfs_get_fid_from_fd(fd); if (fid < 0) { /* ERROR: invalid file descriptor */ errno = EBADF; @@ -1996,15 +2068,15 @@ int UNIFYCR_WRAP(ftruncate)(int fd, off_t length) } /* check that file descriptor is open for write */ - unifycr_fd_t* filedesc = unifycr_get_filedesc_from_fd(fd); + unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(fd); if (!filedesc->write) { errno = EBADF; return -1; } /* truncate the file */ - int rc = unifycr_fid_truncate(fid, length); - if (rc != UNIFYCR_SUCCESS) { + int rc = unifyfs_fid_truncate(fid, length); + if (rc != UNIFYFS_SUCCESS) { errno = EIO; return -1; } @@ -2012,7 +2084,7 @@ int UNIFYCR_WRAP(ftruncate)(int fd, off_t length) return 0; } else { MAP_OR_FAIL(ftruncate); - int ret = UNIFYCR_REAL(ftruncate)(fd, length); + int ret = UNIFYFS_REAL(ftruncate)(fd, length); return ret; } } @@ -2021,15 +2093,15 @@ int UNIFYCR_WRAP(ftruncate)(int fd, off_t length) * TODO: maybe move this somewhere else */ uint32_t get_gfid(int fid) { - unifycr_file_attr_t target; + unifyfs_file_attr_t target; target.fid = fid; - const void* entries = unifycr_fattrs.meta_entry; - size_t num = *unifycr_fattrs.ptr_num_entries; - size_t size = sizeof(unifycr_file_attr_t); + const void* entries = unifyfs_fattrs.meta_entry; + size_t num = *unifyfs_fattrs.ptr_num_entries; + size_t size = sizeof(unifyfs_file_attr_t); - unifycr_file_attr_t* entry = - (unifycr_file_attr_t*) bsearch(&target, entries, num, size, + unifyfs_file_attr_t* entry = + (unifyfs_file_attr_t*) bsearch(&target, entries, num, size, compare_fattr); uint32_t gfid; @@ -2042,20 +2114,25 @@ uint32_t get_gfid(int fid) return gfid; } -int UNIFYCR_WRAP(fsync)(int fd) +int UNIFYFS_WRAP(fsync)(int fd) { /* check whether we should intercept this file descriptor */ - if (unifycr_intercept_fd(&fd)) { + if (unifyfs_intercept_fd(&fd)) { /* get the file id for this file descriptor */ - int fid = unifycr_get_fid_from_fd(fd); + int fid = unifyfs_get_fid_from_fd(fd); if (fid < 0) { errno = EBADF; return -1; } + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + if (!meta->needs_sync) { + return 0; + } + /* if using spill over, fsync spillover data to disk */ - if (unifycr_use_spillover) { - int ret = __real_fsync(unifycr_spilloverblock); + if (unifyfs_use_spillover) { + int ret = __real_fsync(unifyfs_spilloverblock); if (ret != 0) { /* error, need to set errno appropriately, * we called the real fsync which should @@ -2065,43 +2142,43 @@ int UNIFYCR_WRAP(fsync)(int fd) } /* if using LOGIO, call fsync rpc */ - unifycr_filemeta_t* meta = unifycr_get_meta_from_fid(fid); if (meta->storage == FILE_STORAGE_LOGIO) { /* invoke fsync rpc to register index metadata with server */ - uint32_t gfid = get_gfid(fid); - unifycr_client_fsync_rpc_invoke(&unifycr_rpc_context, - app_id, - local_rank_idx, - gfid); + int gfid = get_gfid(fid); + invoke_client_fsync_rpc(gfid); } + + meta->needs_sync = 0; + return 0; } else { MAP_OR_FAIL(fsync); - int ret = UNIFYCR_REAL(fsync)(fd); + int ret = UNIFYFS_REAL(fsync)(fd); return ret; } } -int UNIFYCR_WRAP(fdatasync)(int fd) +int UNIFYFS_WRAP(fdatasync)(int fd) { /* check whether we should intercept this file descriptor */ - if (unifycr_intercept_fd(&fd)) { + if (unifyfs_intercept_fd(&fd)) { /* ERROR: fn not yet supported */ - fprintf(stderr, "Function not yet supported @ %s:%d\n", __FILE__, __LINE__); - errno = EBADF; + fprintf(stderr, "Function not yet supported @ %s:%d\n", + __FILE__, __LINE__); + errno = ENOTSUP; return -1; } else { MAP_OR_FAIL(fdatasync); - int ret = UNIFYCR_REAL(fdatasync)(fd); + int ret = UNIFYFS_REAL(fdatasync)(fd); return ret; } } -int UNIFYCR_WRAP(flock)(int fd, int operation) +int UNIFYFS_WRAP(flock)(int fd, int operation) { int ret; /* check whether we should intercept this file descriptor */ - if (unifycr_intercept_fd(&fd)) { + if (unifyfs_intercept_fd(&fd)) { // KMM I removed the locking code because it was causing // hangs /* @@ -2136,17 +2213,17 @@ int UNIFYCR_WRAP(flock)(int fd, int operation) return 0; } else { MAP_OR_FAIL(flock); - ret = UNIFYCR_REAL(flock)(fd, operation); + ret = UNIFYFS_REAL(flock)(fd, operation); return ret; } } /* TODO: handle different flags */ -void* UNIFYCR_WRAP(mmap)(void* addr, size_t length, int prot, int flags, +void* UNIFYFS_WRAP(mmap)(void* addr, size_t length, int prot, int flags, int fd, off_t offset) { /* check whether we should intercept this file descriptor */ - if (unifycr_intercept_fd(&fd)) { + if (unifyfs_intercept_fd(&fd)) { /* for now, tell user that we can't support mmap, * we'll need to track assigned memory region so that * we can identify our files on msync and munmap */ @@ -2155,8 +2232,9 @@ void* UNIFYCR_WRAP(mmap)(void* addr, size_t length, int prot, int flags, errno = ENODEV; return MAP_FAILED; +#if 0 // TODO - mmap support /* get the file id for this file descriptor */ - int fid = unifycr_get_fid_from_fd(fd); + int fid = unifyfs_get_fid_from_fd(fd); if (fid < 0) { errno = EBADF; return MAP_FAILED; @@ -2187,7 +2265,7 @@ void* UNIFYCR_WRAP(mmap)(void* addr, size_t length, int prot, int flags, /* check that we don't copy past the end of the file */ off_t last_byte = offset + length; - off_t file_size = unifycr_fid_size(fid); + off_t file_size = unifyfs_fid_size(fid); if (last_byte > file_size) { /* trying to copy past the end of the file, so * adjust the total amount to be copied */ @@ -2195,81 +2273,79 @@ void* UNIFYCR_WRAP(mmap)(void* addr, size_t length, int prot, int flags, } /* read data from file */ - int rc = unifycr_fid_read(fid, offset, addr, length); - if (rc != UNIFYCR_SUCCESS) { + int rc = unifyfs_fid_read(fid, offset, addr, length); + if (rc != UNIFYFS_SUCCESS) { /* TODO: need to free memory in this case? */ errno = ENOMEM; return MAP_FAILED; } return addr; +#endif } else { MAP_OR_FAIL(mmap); - void* ret = UNIFYCR_REAL(mmap)(addr, length, prot, flags, fd, offset); + void* ret = UNIFYFS_REAL(mmap)(addr, length, prot, flags, fd, offset); return ret; } } -int UNIFYCR_WRAP(munmap)(void* addr, size_t length) +int UNIFYFS_WRAP(munmap)(void* addr, size_t length) { -#if 0 +#if 0 // TODO - mmap support fprintf(stderr, "Function not yet supported @ %s:%d\n", __FILE__, __LINE__); errno = EINVAL; return -1; #endif MAP_OR_FAIL(munmap); - int ret = UNIFYCR_REAL(munmap)(addr, length); + int ret = UNIFYFS_REAL(munmap)(addr, length); return ret; } -int UNIFYCR_WRAP(msync)(void* addr, size_t length, int flags) +int UNIFYFS_WRAP(msync)(void* addr, size_t length, int flags) { +#if 0 // TODO - mmap support /* TODO: need to keep track of all the mmaps that are linked to * a given file before this function can be implemented */ -#if 0 fprintf(stderr, "Function not yet supported @ %s:%d\n", __FILE__, __LINE__); errno = EINVAL; return -1; #endif MAP_OR_FAIL(msync); - int ret = UNIFYCR_REAL(msync)(addr, length, flags); + int ret = UNIFYFS_REAL(msync)(addr, length, flags); return ret; } -void* UNIFYCR_WRAP(mmap64)(void* addr, size_t length, int prot, int flags, +void* UNIFYFS_WRAP(mmap64)(void* addr, size_t length, int prot, int flags, int fd, off64_t offset) { /* check whether we should intercept this file descriptor */ - if (unifycr_intercept_fd(&fd)) { - /* ERROR: fn not yet supported */ - fprintf(stderr, "Function not yet supported @ %s:%d\n", __FILE__, __LINE__); - errno = ENOSYS; - return MAP_FAILED; + if (unifyfs_intercept_fd(&fd)) { + return UNIFYFS_WRAP(mmap)(addr, length, prot, flags, fd, (off_t)offset); } else { MAP_OR_FAIL(mmap64); - void* ret = UNIFYCR_REAL(mmap64)(addr, length, prot, flags, fd, offset); + void* ret = UNIFYFS_REAL(mmap64)(addr, length, prot, flags, fd, offset); return ret; } } -int UNIFYCR_WRAP(close)(int fd) +int UNIFYFS_WRAP(close)(int fd) { /* check whether we should intercept this file descriptor */ int origfd = fd; - if (unifycr_intercept_fd(&fd)) { + if (unifyfs_intercept_fd(&fd)) { LOGDBG("closing fd %d", fd); /* TODO: what to do if underlying file has been deleted? */ /* check that fd is actually in use */ - int fid = unifycr_get_fid_from_fd(fd); + int fid = unifyfs_get_fid_from_fd(fd); if (fid < 0) { errno = EBADF; return -1; } /* get file descriptor for this file */ - unifycr_fd_t* filedesc = unifycr_get_filedesc_from_fd(fd); + unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(fd); if (filedesc == NULL) { errno = EBADF; return -1; @@ -2277,12 +2353,12 @@ int UNIFYCR_WRAP(close)(int fd) /* if file was opened for writing, fsync it */ if (filedesc->write) { - UNIFYCR_WRAP(fsync)(origfd); + UNIFYFS_WRAP(fsync)(origfd); } /* close the file id */ - int close_rc = unifycr_fid_close(fid); - if (close_rc != UNIFYCR_SUCCESS) { + int close_rc = unifyfs_fid_close(fid); + if (close_rc != UNIFYFS_SUCCESS) { errno = EIO; return -1; } @@ -2290,15 +2366,127 @@ int UNIFYCR_WRAP(close)(int fd) /* reinitialize file descriptor to indicate that * it is no longer associated with a file, * not technically needed but may help catch bugs */ - unifycr_fd_init(fd); + unifyfs_fd_init(fd); /* add file descriptor back to free stack */ - unifycr_stack_push(unifycr_fd_stack, fd); + unifyfs_stack_push(unifyfs_fd_stack, fd); return 0; } else { MAP_OR_FAIL(close); - int ret = UNIFYCR_REAL(close)(fd); + int ret = UNIFYFS_REAL(close)(fd); + return ret; + } +} + +/* Helper function used by fchmod() and chmod() */ +static int __chmod(int fid, mode_t mode) +{ + int gfid; + unifyfs_filemeta_t* meta; + const char* path; + int ret; + + path = unifyfs_path_from_fid(fid); + + meta = unifyfs_get_meta_from_fid(fid); + if (!meta) { + LOGDBG("chmod: %s no metadata info", path); + errno = ENOENT; + return -1; + } + + /* Once a file is laminated, you can't modify it in any way */ + if (meta->is_laminated) { + LOGDBG("chmod: %s is already laminated", path); + errno = EROFS; + return -1; + } + + gfid = unifyfs_generate_gfid(path); + + /* + * If the chmod clears all the existing write bits, then it's a laminate. + * + * meta->mode & 0222 Was at least one write bit set before? + * ((meta->mode & 0222) & mode) == 0 Will all the write bits be cleared? + */ + if ((meta->mode & 0222) && + (((meta->mode & 0222) & mode) == 0)) { + + /* + * We're laminating. Calculate the file size so we can cache it + * (both locally and on the server). + */ + ret = invoke_client_filesize_rpc(gfid, &meta->size); + if (ret) { + LOGERR("chmod: couldn't get the global file size on laminate"); + errno = EIO; + return -1; + } + meta->is_laminated = 1; + } + + /* Clear out our old permission bits, and set the new ones in */ + meta->mode = meta->mode & ~0777; + meta->mode = meta->mode | mode; + + ret = unifyfs_set_global_file_meta(fid, gfid); + if (ret) { + LOGERR("chmod: can't set global meta entry for %s (fid:%d)", + path, fid); + errno = EIO; + return -1; + } + return 0; +} + +int UNIFYFS_WRAP(fchmod)(int fd, mode_t mode) +{ + /* check whether we should intercept this file descriptor */ + int origfd = fd; + if (unifyfs_intercept_fd(&fd)) { + + /* TODO: what to do if underlying file has been deleted? */ + + /* check that fd is actually in use */ + int fid = unifyfs_get_fid_from_fd(fd); + if (fid < 0) { + errno = EBADF; + return -1; + } + LOGDBG("fchmod: setting fd %d to %o", fd, mode); + return __chmod(fid, mode); + + } else { + MAP_OR_FAIL(fchmod); + int ret = UNIFYFS_REAL(fchmod)(fd, mode); + return ret; + } +} + + +int UNIFYFS_WRAP(chmod)(const char* path, mode_t mode) +{ + int fid, gfid; + int ret; + unifyfs_filemeta_t* meta; + /* determine whether we should intercept this path */ + if (unifyfs_intercept_path(path)) { + /* check if path exists */ + fid = unifyfs_get_fid_from_path(path); + if (fid < 0) { + LOGDBG("chmod: unifyfs_get_id_from path failed, returning -1, %s", + path); + errno = ENOENT; + return -1; + } + + LOGDBG("chmod: setting %s to %o", path, mode); + return __chmod(fid, mode); + } else { + MAP_OR_FAIL(chmod); + int ret = UNIFYFS_REAL(chmod)(path, mode); return ret; } } diff --git a/client/src/unifyfs-sysio.h b/client/src/unifyfs-sysio.h new file mode 100644 index 000000000..d829874f8 --- /dev/null +++ b/client/src/unifyfs-sysio.h @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2017, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +/* + * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * Copyright (c) 2017, Florida State University. Contributions from + * the Computer Architecture and Systems Research Laboratory (CASTL) + * at the Department of Computer Science. + * + * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror + * LLNL-CODE-728877. All rights reserved. + * + * This file is part of UnifyFS. + * For details, see https://github.com/llnl/unifyfs + * Please read https://github.com/llnl/unifyfs/LICENSE for full license text. + */ + +/* + * Copyright (c) 2013, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * code Written by + * Raghunath Rajachandrasekar + * Kathryn Mohror + * Adam Moody + * All rights reserved. + * This file is part of UNIFYFS. + * For details, see https://github.com/hpc/unifyfs + * Please also read this file LICENSE.UNIFYFS + */ + +#ifndef UNIFYFS_SYSIO_H +#define UNIFYFS_SYSIO_H + +#include "unifyfs-internal.h" + +#define AIOCB_ERROR_CODE(cbp) (cbp->__error_code) +#define AIOCB_RETURN_VAL(cbp) (cbp->__return_value) + +/* --------------------------------------- + * POSIX wrappers: paths + * --------------------------------------- */ + +UNIFYFS_DECL(access, int, (const char* pathname, int mode)); +UNIFYFS_DECL(mkdir, int, (const char* path, mode_t mode)); +UNIFYFS_DECL(rmdir, int, (const char* path)); +UNIFYFS_DECL(unlink, int, (const char* path)); +UNIFYFS_DECL(remove, int, (const char* path)); +UNIFYFS_DECL(rename, int, (const char* oldpath, const char* newpath)); +UNIFYFS_DECL(truncate, int, (const char* path, off_t length)); +UNIFYFS_DECL(stat, int, (const char* path, struct stat* buf)); +//UNIFYFS_DECL(fstat, int, (int fd, struct stat* buf)); +UNIFYFS_DECL(__xstat, int, (int vers, const char* path, struct stat* buf)); +UNIFYFS_DECL(__lxstat, int, (int vers, const char* path, struct stat* buf)); +UNIFYFS_DECL(__fxstat, int, (int vers, int fd, struct stat* buf)); + +/* --------------------------------------- + * POSIX wrappers: file descriptors + * --------------------------------------- */ + +UNIFYFS_DECL(creat, int, (const char* path, mode_t mode)); +UNIFYFS_DECL(creat64, int, (const char* path, mode_t mode)); +UNIFYFS_DECL(open, int, (const char* path, int flags, ...)); +UNIFYFS_DECL(open64, int, (const char* path, int flags, ...)); +UNIFYFS_DECL(__open_2, int, (const char* path, int flags, ...)); +UNIFYFS_DECL(read, ssize_t, (int fd, void* buf, size_t count)); +UNIFYFS_DECL(write, ssize_t, (int fd, const void* buf, size_t count)); +UNIFYFS_DECL(readv, ssize_t, (int fd, const struct iovec* iov, int iovcnt)); +UNIFYFS_DECL(writev, ssize_t, (int fd, const struct iovec* iov, int iovcnt)); +UNIFYFS_DECL(pread, ssize_t, (int fd, void* buf, size_t count, off_t offset)); +UNIFYFS_DECL(pread64, ssize_t, (int fd, void* buf, size_t count, + off64_t offset)); +UNIFYFS_DECL(pwrite, ssize_t, (int fd, const void* buf, size_t count, + off_t offset)); +UNIFYFS_DECL(pwrite64, ssize_t, (int fd, const void* buf, size_t count, + off64_t offset)); +UNIFYFS_DECL(posix_fadvise, int, (int fd, off_t offset, off_t len, int advice)); +UNIFYFS_DECL(lseek, off_t, (int fd, off_t offset, int whence)); +UNIFYFS_DECL(lseek64, off64_t, (int fd, off64_t offset, int whence)); +UNIFYFS_DECL(ftruncate, int, (int fd, off_t length)); +UNIFYFS_DECL(fsync, int, (int fd)); +UNIFYFS_DECL(fdatasync, int, (int fd)); +UNIFYFS_DECL(flock, int, (int fd, int operation)); +UNIFYFS_DECL(mmap, void*, (void* addr, size_t length, int prot, int flags, + int fd, off_t offset)); +UNIFYFS_DECL(mmap64, void*, (void* addr, size_t length, int prot, int flags, + int fd, off64_t offset)); +UNIFYFS_DECL(munmap, int, (void* addr, size_t length)); +UNIFYFS_DECL(msync, int, (void* addr, size_t length, int flags)); +UNIFYFS_DECL(__fxstat, int, (int vers, int fd, struct stat* buf)); +UNIFYFS_DECL(close, int, (int fd)); +UNIFYFS_DECL(lio_listio, int, (int mode, struct aiocb* const aiocb_list[], + int nitems, struct sigevent* sevp)); + +/* read count bytes info buf from file starting at offset pos, + * returns number of bytes actually read in retcount, + * retcount will be less than count only if an error occurs + * or end of file is reached */ +int unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count, + size_t* retcount); + +/* write count bytes from buf into file starting at offset pos, + * allocates new bytes and updates file size as necessary, + * fills any gaps with zeros */ +int unifyfs_fd_write(int fd, off_t pos, const void* buf, size_t count); +int unifyfs_fd_logreadlist(read_req_t* read_req, int count); + +#include "unifyfs-dirops.h" + +#endif /* UNIFYFS_SYSIO_H */ diff --git a/client/src/unifycr.c b/client/src/unifyfs.c similarity index 52% rename from client/src/unifycr.c rename to client/src/unifyfs.c index e79c81e15..1626916d7 100644 --- a/client/src/unifycr.c +++ b/client/src/unifyfs.c @@ -1,15 +1,15 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2019, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2019, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -40,12 +40,9 @@ * Please also read this file LICENSE.CRUISE */ -#include "unifycr-internal.h" -#include "unifycr-fixed.h" -#include "unifycr_meta.h" -#include "unifycr_pmix.h" -#include "unifycr_shm.h" -#include "unifycr_runstate.h" +#include "unifyfs-internal.h" +#include "unifyfs-fixed.h" +#include "unifyfs_runstate.h" #include #include @@ -55,39 +52,33 @@ #include #endif -#ifdef UNIFYCR_GOTCHA +#ifdef UNIFYFS_GOTCHA #include "gotcha/gotcha_types.h" #include "gotcha/gotcha.h" -#include "gotcha_map_unifycr_list.h" +#include "gotcha_map_unifyfs_list.h" #endif -#include "unifycr_client.h" -#include "unifycr_clientcalls_rpc.h" - -#include "unifycr_log.h" - -/* global rpc context (probably should find a better spot for this) */ -unifycr_client_rpc_context_t* unifycr_rpc_context = NULL; +#include "unifyfs_client_rpcs.h" +#include "unifyfs_server_rpcs.h" +#include "unifyfs_rpc_util.h" +#include "margo_client.h" -static int unifycr_fpos_enabled = 1; /* whether we can use fgetpos/fsetpos */ +/* avoid duplicate mounts (for now) */ +static int unifyfs_mounted = -1; -/* - * unifycr variable: - * */ +static int unifyfs_fpos_enabled = 1; /* whether we can use fgetpos/fsetpos */ -unifycr_cfg_t client_cfg; +unifyfs_cfg_t client_cfg; -unifycr_index_buf_t unifycr_indices; -unifycr_fattr_buf_t unifycr_fattrs; -static size_t -unifycr_index_buf_size; /* size of metadata log for log-structured io*/ -static size_t unifycr_fattr_buf_size; -unsigned long -unifycr_max_index_entries; /*max number of metadata entries for log-structured write*/ -unsigned int unifycr_max_fattr_entries; -int unifycr_spillmetablock; +unifyfs_index_buf_t unifyfs_indices; +unifyfs_fattr_buf_t unifyfs_fattrs; +static size_t unifyfs_index_buf_size; /* size of metadata log */ +static size_t unifyfs_fattr_buf_size; +unsigned long unifyfs_max_index_entries; /* max metadata log entries */ +unsigned int unifyfs_max_fattr_entries; +int unifyfs_spillmetablock; -int* local_rank_lst; +int global_rank_cnt; /* count of world ranks */ int local_rank_cnt; int local_rank_idx; @@ -98,59 +89,60 @@ struct pollfd cmd_fd; /* shared memory buffer to transfer read requests * from client to server */ static char shm_req_name[GEN_STR_LEN] = {0}; -static size_t shm_req_size = UNIFYCR_SHMEM_REQ_SIZE; -void* shm_req_buf; +static size_t shm_req_size = UNIFYFS_SHMEM_REQ_SIZE; +void* shm_req_buf; /* shared memory buffer to transfer read replies * from server to client */ static char shm_recv_name[GEN_STR_LEN] = {0}; -static size_t shm_recv_size = UNIFYCR_SHMEM_RECV_SIZE; -void* shm_recv_buf; +static size_t shm_recv_size = UNIFYFS_SHMEM_RECV_SIZE; +void* shm_recv_buf; char cmd_buf[CMD_BUF_SIZE] = {0}; +int client_rank; int app_id; -size_t unifycr_key_slice_range; +size_t unifyfs_key_slice_range; /* whether chunks should be allocated to * store file contents in memory */ -int unifycr_use_memfs = 1; +int unifyfs_use_memfs = 1; /* whether chunks should be allocated to * store file contents on spill over device */ -int unifycr_use_spillover = 1; +int unifyfs_use_spillover = 1; -static int unifycr_use_single_shm = 0; -static int unifycr_page_size = 0; +static int unifyfs_use_single_shm = 0; +static int unifyfs_page_size = 0; -static off_t unifycr_max_offt; -static off_t unifycr_min_offt; -static off_t unifycr_max_long; -static off_t unifycr_min_long; +static off_t unifyfs_max_offt; +static off_t unifyfs_min_offt; +static off_t unifyfs_max_long; +static off_t unifyfs_min_long; /* TODO: moved these to fixed file */ -int unifycr_max_files; /* maximum number of files to store */ -size_t unifycr_chunk_mem; /* number of bytes in memory to be used for chunk storage */ -int unifycr_chunk_bits; /* we set chunk size = 2^unifycr_chunk_bits */ -off_t unifycr_chunk_size; /* chunk size in bytes */ -off_t unifycr_chunk_mask; /* mask applied to logical offset to determine physical offset within chunk */ -long unifycr_max_chunks; /* maximum number of chunks that fit in memory */ +int unifyfs_max_files; /* maximum number of files to store */ +size_t unifyfs_chunk_mem; /* number of bytes in memory to be used for chunk storage */ +int unifyfs_chunk_bits; /* we set chunk size = 2^unifyfs_chunk_bits */ +off_t unifyfs_chunk_size; /* chunk size in bytes */ +off_t unifyfs_chunk_mask; /* mask applied to logical offset to determine physical offset within chunk */ +long unifyfs_max_chunks; /* maximum number of chunks that fit in memory */ /* number of bytes in spillover to be used for chunk storage */ -static size_t unifycr_spillover_size; +static size_t unifyfs_spillover_size; /* maximum number of chunks that fit in spillover storage */ -long unifycr_spillover_max_chunks; +long unifyfs_spillover_max_chunks; #ifdef HAVE_LIBNUMA -static char unifycr_numa_policy[10]; -static int unifycr_numa_bank = -1; +static char unifyfs_numa_policy[10]; +static int unifyfs_numa_bank = -1; #endif -extern pthread_mutex_t unifycr_stack_mutex; +extern pthread_mutex_t unifyfs_stack_mutex; /* keep track of what we've initialized */ -int unifycr_initialized = 0; +int unifyfs_initialized = 0; /* shared memory for superblock */ static char shm_super_name[GEN_STR_LEN] = {0}; @@ -161,67 +153,68 @@ void* shm_super_buf; static void* free_fid_stack; void* free_chunk_stack; void* free_spillchunk_stack; -unifycr_filename_t* unifycr_filelist; -static unifycr_filemeta_t* unifycr_filemetas; -static unifycr_chunkmeta_t* unifycr_chunkmetas; +unifyfs_filename_t* unifyfs_filelist; +static unifyfs_filemeta_t* unifyfs_filemetas; -char* unifycr_chunks; -int unifycr_spilloverblock = 0; -int unifycr_spillmetablock = 0; /*used for log-structured i/o*/ +unifyfs_chunkmeta_t* unifyfs_chunkmetas; + +char* unifyfs_chunks; +int unifyfs_spilloverblock = 0; +int unifyfs_spillmetablock = 0; /*used for log-structured i/o*/ /* array of file descriptors */ -unifycr_fd_t unifycr_fds[UNIFYCR_MAX_FILEDESCS]; -rlim_t unifycr_fd_limit; +unifyfs_fd_t unifyfs_fds[UNIFYFS_MAX_FILEDESCS]; +rlim_t unifyfs_fd_limit; /* array of file streams */ -unifycr_stream_t unifycr_streams[UNIFYCR_MAX_FILEDESCS]; +unifyfs_stream_t unifyfs_streams[UNIFYFS_MAX_FILEDESCS]; /* * TODO: the number of open directories clearly won't exceed the number of file - * descriptors. however, the current UNIFYCR_MAX_FILEDESCS value of 256 will + * descriptors. however, the current UNIFYFS_MAX_FILEDESCS value of 256 will * quickly run out. if this value is fixed to be reasonably larger, then we * would need a way to dynamically allocate the dirstreams instead of the * following fixed size array. */ /* array of DIR* streams to be used */ -unifycr_dirstream_t unifycr_dirstreams[UNIFYCR_MAX_FILEDESCS]; +unifyfs_dirstream_t unifyfs_dirstreams[UNIFYFS_MAX_FILEDESCS]; /* stack to track free file descriptor values, - * each is an index into unifycr_fds array */ -void* unifycr_fd_stack; + * each is an index into unifyfs_fds array */ +void* unifyfs_fd_stack; /* stack to track free file streams, - * each is an index into unifycr_streams array */ -void* unifycr_stream_stack; + * each is an index into unifyfs_streams array */ +void* unifyfs_stream_stack; /* stack to track free directory streams, - * each is an index into unifycr_dirstreams array */ -void* unifycr_dirstream_stack; + * each is an index into unifyfs_dirstreams array */ +void* unifyfs_dirstream_stack; /* mount point information */ -char* unifycr_mount_prefix; -size_t unifycr_mount_prefixlen = 0; -static key_t unifycr_mount_shmget_key = 0; +char* unifyfs_mount_prefix; +size_t unifyfs_mount_prefixlen = 0; +static key_t unifyfs_mount_shmget_key = 0; /* mutex to lock stack operations */ -pthread_mutex_t unifycr_stack_mutex = PTHREAD_MUTEX_INITIALIZER; +pthread_mutex_t unifyfs_stack_mutex = PTHREAD_MUTEX_INITIALIZER; /* path of external storage's mount point*/ -char external_data_dir[UNIFYCR_MAX_FILENAME] = {0}; -char external_meta_dir[UNIFYCR_MAX_FILENAME] = {0}; +char external_data_dir[UNIFYFS_MAX_FILENAME] = {0}; +char external_meta_dir[UNIFYFS_MAX_FILENAME] = {0}; /* single function to route all unsupported wrapper calls through */ -int unifycr_vunsupported( +int unifyfs_vunsupported( const char* fn_name, const char* file, int line, const char* fmt, va_list args) { - /* print a message about where in the UNIFYCR code we are */ - printf("UNIFYCR UNSUPPORTED: %s() at %s:%d: ", fn_name, file, line); + /* print a message about where in the UNIFYFS code we are */ + printf("UNIFYFS UNSUPPORTED: %s() at %s:%d: ", fn_name, file, line); /* print string with more info about call, e.g., param values */ va_list args2; @@ -231,11 +224,11 @@ int unifycr_vunsupported( /* TODO: optionally abort */ - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /* single function to route all unsupported wrapper calls through */ -int unifycr_unsupported( +int unifyfs_unsupported( const char* fn_name, const char* file, int line, @@ -245,97 +238,97 @@ int unifycr_unsupported( /* print string with more info about call, e.g., param values */ va_list args; va_start(args, fmt); - int rc = unifycr_vunsupported(fn_name, file, line, fmt, args); + int rc = unifyfs_vunsupported(fn_name, file, line, fmt, args); va_end(args); return rc; } -/* given an UNIFYCR error code, return corresponding errno code */ -int unifycr_err_map_to_errno(int rc) +/* given an UNIFYFS error code, return corresponding errno code */ +int unifyfs_err_map_to_errno(int rc) { - unifycr_error_e ec = (unifycr_error_e)rc; + unifyfs_error_e ec = (unifyfs_error_e)rc; switch (ec) { - case UNIFYCR_SUCCESS: + case UNIFYFS_SUCCESS: return 0; - case UNIFYCR_ERROR_BADF: + case UNIFYFS_ERROR_BADF: return EBADF; - case UNIFYCR_ERROR_EXIST: + case UNIFYFS_ERROR_EXIST: return EEXIST; - case UNIFYCR_ERROR_FBIG: + case UNIFYFS_ERROR_FBIG: return EFBIG; - case UNIFYCR_ERROR_INVAL: + case UNIFYFS_ERROR_INVAL: return EINVAL; - case UNIFYCR_ERROR_ISDIR: + case UNIFYFS_ERROR_ISDIR: return EISDIR; - case UNIFYCR_ERROR_NAMETOOLONG: + case UNIFYFS_ERROR_NAMETOOLONG: return ENAMETOOLONG; - case UNIFYCR_ERROR_NFILE: + case UNIFYFS_ERROR_NFILE: return ENFILE; - case UNIFYCR_ERROR_NOENT: + case UNIFYFS_ERROR_NOENT: return ENOENT; - case UNIFYCR_ERROR_NOMEM: + case UNIFYFS_ERROR_NOMEM: return ENOMEM; - case UNIFYCR_ERROR_NOSPC: + case UNIFYFS_ERROR_NOSPC: return ENOSPC; - case UNIFYCR_ERROR_NOTDIR: + case UNIFYFS_ERROR_NOTDIR: return ENOTDIR; - case UNIFYCR_ERROR_OVERFLOW: + case UNIFYFS_ERROR_OVERFLOW: return EOVERFLOW; default: break; } - return EIO; + return ec; } -/* given an errno error code, return corresponding UnifyCR error code */ -int unifycr_errno_map_to_err(int rc) +/* given an errno error code, return corresponding UnifyFS error code */ +int unifyfs_errno_map_to_err(int rc) { switch (rc) { case 0: - return (int)UNIFYCR_SUCCESS; + return (int)UNIFYFS_SUCCESS; case EBADF: - return (int)UNIFYCR_ERROR_BADF; + return (int)UNIFYFS_ERROR_BADF; case EEXIST: - return (int)UNIFYCR_ERROR_EXIST; + return (int)UNIFYFS_ERROR_EXIST; case EFBIG: - return (int)UNIFYCR_ERROR_FBIG; + return (int)UNIFYFS_ERROR_FBIG; case EINVAL: - return (int)UNIFYCR_ERROR_INVAL; + return (int)UNIFYFS_ERROR_INVAL; case EIO: - return (int)UNIFYCR_ERROR_IO; + return (int)UNIFYFS_ERROR_IO; case EISDIR: - return (int)UNIFYCR_ERROR_ISDIR; + return (int)UNIFYFS_ERROR_ISDIR; case ENAMETOOLONG: - return (int)UNIFYCR_ERROR_NAMETOOLONG; + return (int)UNIFYFS_ERROR_NAMETOOLONG; case ENFILE: - return (int)UNIFYCR_ERROR_NFILE; + return (int)UNIFYFS_ERROR_NFILE; case ENOENT: - return (int)UNIFYCR_ERROR_NOENT; + return (int)UNIFYFS_ERROR_NOENT; case ENOMEM: - return (int)UNIFYCR_ERROR_NOMEM; + return (int)UNIFYFS_ERROR_NOMEM; case ENOSPC: - return (int)UNIFYCR_ERROR_NOSPC; + return (int)UNIFYFS_ERROR_NOSPC; case ENOTDIR: - return (int)UNIFYCR_ERROR_NOTDIR; + return (int)UNIFYFS_ERROR_NOTDIR; case EOVERFLOW: - return (int)UNIFYCR_ERROR_OVERFLOW; + return (int)UNIFYFS_ERROR_OVERFLOW; default: break; } - return (int)UNIFYCR_FAILURE; + return (int)UNIFYFS_FAILURE; } /* returns 1 if two input parameters will overflow their type when * added together */ -inline int unifycr_would_overflow_offt(off_t a, off_t b) +inline int unifyfs_would_overflow_offt(off_t a, off_t b) { /* if both parameters are positive, they could overflow when * added together */ if (a > 0 && b > 0) { /* if the distance between a and max is greater than or equal to * b, then we could add a and b and still not exceed max */ - if (unifycr_max_offt - a >= b) { + if (unifyfs_max_offt - a >= b) { return 0; } return 1; @@ -346,7 +339,7 @@ inline int unifycr_would_overflow_offt(off_t a, off_t b) if (a < 0 && b < 0) { /* if the distance between min and a is less than or equal to * b, then we could add a and b and still not exceed min */ - if (unifycr_min_offt - a <= b) { + if (unifyfs_min_offt - a <= b) { return 0; } return 1; @@ -360,14 +353,14 @@ inline int unifycr_would_overflow_offt(off_t a, off_t b) /* returns 1 if two input parameters will overflow their type when * added together */ -inline int unifycr_would_overflow_long(long a, long b) +inline int unifyfs_would_overflow_long(long a, long b) { /* if both parameters are positive, they could overflow when * added together */ if (a > 0 && b > 0) { /* if the distance between a and max is greater than or equal to * b, then we could add a and b and still not exceed max */ - if (unifycr_max_long - a >= b) { + if (unifyfs_max_long - a >= b) { return 0; } return 1; @@ -378,7 +371,7 @@ inline int unifycr_would_overflow_long(long a, long b) if (a < 0 && b < 0) { /* if the distance between min and a is less than or equal to * b, then we could add a and b and still not exceed min */ - if (unifycr_min_long - a <= b) { + if (unifyfs_min_long - a <= b) { return 0; } return 1; @@ -392,7 +385,7 @@ inline int unifycr_would_overflow_long(long a, long b) /* given an input mode, mask it with umask and return, can specify * an input mode==0 to specify all read/write bits */ -mode_t unifycr_getmode(mode_t perms) +mode_t unifyfs_getmode(mode_t perms) { /* perms == 0 is shorthand for all read and write bits */ if (perms == 0) { @@ -409,33 +402,33 @@ mode_t unifycr_getmode(mode_t perms) } /* lock access to shared data structures in superblock */ -inline int unifycr_stack_lock() +inline int unifyfs_stack_lock() { - if (unifycr_use_single_shm) { - return pthread_mutex_lock(&unifycr_stack_mutex); + if (unifyfs_use_single_shm) { + return pthread_mutex_lock(&unifyfs_stack_mutex); } return 0; } /* unlock access to shared data structures in superblock */ -inline int unifycr_stack_unlock() +inline int unifyfs_stack_unlock() { - if (unifycr_use_single_shm) { - return pthread_mutex_unlock(&unifycr_stack_mutex); + if (unifyfs_use_single_shm) { + return pthread_mutex_unlock(&unifyfs_stack_mutex); } return 0; } /* sets flag if the path is a special path */ -inline int unifycr_intercept_path(const char* path) +inline int unifyfs_intercept_path(const char* path) { /* don't intecept anything until we're initialized */ - if (!unifycr_initialized) { + if (!unifyfs_initialized) { return 0; } /* if the path starts with our mount point, intercept it */ - if (strncmp(path, unifycr_mount_prefix, unifycr_mount_prefixlen) == 0) { + if (strncmp(path, unifyfs_mount_prefix, unifyfs_mount_prefixlen) == 0) { return 1; } return 0; @@ -443,16 +436,16 @@ inline int unifycr_intercept_path(const char* path) /* given an fd, return 1 if we should intercept this file, 0 otherwise, * convert fd to new fd value if needed */ -inline int unifycr_intercept_fd(int* fd) +inline int unifyfs_intercept_fd(int* fd) { int oldfd = *fd; /* don't intecept anything until we're initialized */ - if (!unifycr_initialized) { + if (!unifyfs_initialized) { return 0; } - if (oldfd < unifycr_fd_limit) { + if (oldfd < unifyfs_fd_limit) { /* this fd is a real system fd, so leave it as is */ return 0; } else if (oldfd < 0) { @@ -461,7 +454,7 @@ inline int unifycr_intercept_fd(int* fd) } else { /* this is an fd we generated and returned to the user, * so intercept the call and shift the fd */ - int newfd = oldfd - unifycr_fd_limit; + int newfd = oldfd - unifyfs_fd_limit; *fd = newfd; LOGDBG("Changing fd from exposed %d to internal %d", oldfd, newfd); return 1; @@ -470,18 +463,18 @@ inline int unifycr_intercept_fd(int* fd) /* given an fd, return 1 if we should intercept this file, 0 otherwise, * convert fd to new fd value if needed */ -inline int unifycr_intercept_stream(FILE* stream) +inline int unifyfs_intercept_stream(FILE* stream) { /* don't intecept anything until we're initialized */ - if (!unifycr_initialized) { + if (!unifyfs_initialized) { return 0; } /* check whether this pointer lies within range of our * file stream array */ - unifycr_stream_t* ptr = (unifycr_stream_t*) stream; - unifycr_stream_t* start = &(unifycr_streams[0]); - unifycr_stream_t* end = &(unifycr_streams[UNIFYCR_MAX_FILEDESCS]); + unifyfs_stream_t* ptr = (unifyfs_stream_t*) stream; + unifyfs_stream_t* start = &(unifyfs_streams[0]); + unifyfs_stream_t* end = &(unifyfs_streams[UNIFYFS_MAX_FILEDESCS]); if (ptr >= start && ptr < end) { return 1; } @@ -491,18 +484,18 @@ inline int unifycr_intercept_stream(FILE* stream) /* given an directory stream, return 1 if we should intercept this * fdirecotry, 0 otherwise */ -inline int unifycr_intercept_dirstream(DIR* dirp) +inline int unifyfs_intercept_dirstream(DIR* dirp) { /* don't intecept anything until we're initialized */ - if (!unifycr_initialized) { + if (!unifyfs_initialized) { return 0; } /* check whether this pointer lies within range of our * directory stream array */ - unifycr_dirstream_t* ptr = (unifycr_dirstream_t*) dirp; - unifycr_dirstream_t* start = &(unifycr_dirstreams[0]); - unifycr_dirstream_t* end = &(unifycr_dirstreams[UNIFYCR_MAX_FILEDESCS]); + unifyfs_dirstream_t* ptr = (unifyfs_dirstream_t*) dirp; + unifyfs_dirstream_t* start = &(unifyfs_dirstreams[0]); + unifyfs_dirstream_t* end = &(unifyfs_dirstreams[UNIFYFS_MAX_FILEDESCS]); if (ptr >= start && ptr < end) { return 1; } @@ -511,14 +504,14 @@ inline int unifycr_intercept_dirstream(DIR* dirp) } /* given a path, return the file id */ -inline int unifycr_get_fid_from_path(const char* path) +inline int unifyfs_get_fid_from_path(const char* path) { int i = 0; - while (i < unifycr_max_files) { - if (unifycr_filelist[i].in_use && - strcmp((void*)&unifycr_filelist[i].filename, path) == 0) { - LOGDBG("File found: unifycr_filelist[%d].filename = %s", - i, (char*)&unifycr_filelist[i].filename); + while (i < unifyfs_max_files) { + if (unifyfs_filelist[i].in_use && + strcmp((void*)&unifyfs_filelist[i].filename, path) == 0) { + LOGDBG("File found: unifyfs_filelist[%d].filename = %s", + i, (char*)&unifyfs_filelist[i].filename); return i; } i++; @@ -529,10 +522,10 @@ inline int unifycr_get_fid_from_path(const char* path) } /* initialize file descriptor structure for given fd value */ -int unifycr_fd_init(int fd) +int unifyfs_fd_init(int fd) { /* get pointer to file descriptor struct for this fd value */ - unifycr_fd_t* filedesc = &(unifycr_fds[fd]); + unifyfs_fd_t* filedesc = &(unifyfs_fds[fd]); /* set fid to -1 to indicate fd is not active, * set file position to max value, @@ -542,14 +535,14 @@ int unifycr_fd_init(int fd) filedesc->read = 0; filedesc->write = 0; - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /* initialize file streams structure for given sid value */ -int unifycr_stream_init(int sid) +int unifyfs_stream_init(int sid) { /* get pointer to file stream struct for this id value */ - unifycr_stream_t* s = &(unifycr_streams[sid]); + unifyfs_stream_t* s = &(unifyfs_streams[sid]); /* record our id so when given a pointer to the stream * struct we can easily recover our id value */ @@ -558,14 +551,14 @@ int unifycr_stream_init(int sid) /* set fd to -1 to indicate stream is not active */ s->fd = -1; - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /* initialize directory streams structure for given dirid value */ -int unifycr_dirstream_init(int dirid) +int unifyfs_dirstream_init(int dirid) { /* get pointer to directory stream struct for this id value */ - unifycr_dirstream_t* dirp = &(unifycr_dirstreams[dirid]); + unifyfs_dirstream_t* dirp = &(unifyfs_dirstreams[dirid]); /* initialize fields in structure */ memset((void*) dirp, 0, sizeof(*dirp)); @@ -577,29 +570,29 @@ int unifycr_dirstream_init(int dirid) /* set fid to -1 to indicate stream is not active */ dirp->fid = -1; - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /* given a file descriptor, return the file id */ -inline int unifycr_get_fid_from_fd(int fd) +inline int unifyfs_get_fid_from_fd(int fd) { /* check that file descriptor is within range */ - if (fd < 0 || fd >= UNIFYCR_MAX_FILEDESCS) { + if (fd < 0 || fd >= UNIFYFS_MAX_FILEDESCS) { return -1; } /* get local file id that file descriptor is assocated with, * will be -1 if not active */ - int fid = unifycr_fds[fd].fid; + int fid = unifyfs_fds[fd].fid; return fid; } /* return address of file descriptor structure or NULL if fd is out * of range */ -inline unifycr_fd_t* unifycr_get_filedesc_from_fd(int fd) +inline unifyfs_fd_t* unifyfs_get_filedesc_from_fd(int fd) { - if (fd >= 0 && fd < UNIFYCR_MAX_FILEDESCS) { - unifycr_fd_t* filedesc = &(unifycr_fds[fd]); + if (fd >= 0 && fd < UNIFYFS_MAX_FILEDESCS) { + unifyfs_fd_t* filedesc = &(unifyfs_fds[fd]); return filedesc; } return NULL; @@ -607,12 +600,12 @@ inline unifycr_fd_t* unifycr_get_filedesc_from_fd(int fd) /* given a file id, return a pointer to the meta data, * otherwise return NULL */ -unifycr_filemeta_t* unifycr_get_meta_from_fid(int fid) +unifyfs_filemeta_t* unifyfs_get_meta_from_fid(int fid) { /* check that the file id is within range of our array */ - if (fid >= 0 && fid < unifycr_max_files) { + if (fid >= 0 && fid < unifyfs_max_files) { /* get a pointer to the file meta data structure */ - unifycr_filemeta_t* meta = &unifycr_filemetas[fid]; + unifyfs_filemeta_t* meta = &unifyfs_filemetas[fid]; return meta; } return NULL; @@ -623,27 +616,27 @@ unifycr_filemeta_t* unifycr_get_meta_from_fid(int fid) * --------------------------------------- */ /* allocate and initialize data management resource for file */ -static int unifycr_fid_store_alloc(int fid) +static int unifyfs_fid_store_alloc(int fid) { /* get meta data for this file */ - unifycr_filemeta_t* meta = unifycr_get_meta_from_fid(fid); + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); /* indicate that we're using LOGIO to store data for this file */ meta->storage = FILE_STORAGE_LOGIO; - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /* free data management resource for file */ -static int unifycr_fid_store_free(int fid) +static int unifyfs_fid_store_free(int fid) { /* get meta data for this file */ - unifycr_filemeta_t* meta = unifycr_get_meta_from_fid(fid); + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); /* set storage type back to NULL */ meta->storage = FILE_STORAGE_NULL; - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /* --------------------------------------- @@ -653,13 +646,11 @@ static int unifycr_fid_store_free(int fid) /* checks to see if fid is a directory * returns 1 for yes * returns 0 for no */ -int unifycr_fid_is_dir(int fid) +int unifyfs_fid_is_dir(int fid) { - unifycr_filemeta_t* meta = unifycr_get_meta_from_fid(fid); - if (meta) { - /* found a file with that id, return value of directory flag */ - int rc = meta->is_dir; - return rc; + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + if (meta && meta->mode & S_IFDIR) { + return 1; } else { /* if it doesn't exist, then it's not a directory? */ return 0; @@ -671,7 +662,7 @@ int unifycr_fid_is_dir(int fid) * @param path: file path * return: gfid */ -int unifycr_generate_gfid(const char* path) +int unifyfs_generate_gfid(const char* path) { unsigned char digested[16] = { 0, }; unsigned long len = strlen(path); @@ -682,45 +673,55 @@ int unifycr_generate_gfid(const char* path) return abs(ival[0]); } -static int unifycr_gfid_from_fid(const int fid) +static int unifyfs_gfid_from_fid(const int fid) { /* check that local file id is in range */ - if (fid < 0 || fid >= unifycr_max_files) { + if (fid < 0 || fid >= unifyfs_max_files) { return -EINVAL; } /* lookup file name structure for this local file id */ - unifycr_filename_t* fname = &unifycr_filelist[fid]; + unifyfs_filename_t* fname = &unifyfs_filelist[fid]; /* generate global file id from path if file is valid */ if (fname->in_use) { - int gfid = unifycr_generate_gfid(fname->filename); + int gfid = unifyfs_generate_gfid(fname->filename); return gfid; } return -EINVAL; } +/* Given a fid, return the path. */ +const char* unifyfs_path_from_fid(int fid) +{ + unifyfs_filename_t* fname = &unifyfs_filelist[fid]; + if (fname->in_use) { + return fname->filename; + } + return NULL; +} + /* checks to see if a directory is empty * assumes that check for is_dir has already been made * only checks for full path matches, does not check relative paths, * e.g. ../dirname will not work * returns 1 for yes it is empty * returns 0 for no */ -int unifycr_fid_is_dir_empty(const char* path) +int unifyfs_fid_is_dir_empty(const char* path) { int i = 0; - while (i < unifycr_max_files) { + while (i < unifyfs_max_files) { /* only check this element if it's active */ - if (unifycr_filelist[i].in_use) { + if (unifyfs_filelist[i].in_use) { /* if the file starts with the path, it is inside of that directory * also check to make sure that it's not the directory entry itself */ - char* strptr = strstr(path, unifycr_filelist[i].filename); - if (strptr == unifycr_filelist[i].filename && - strcmp(path, unifycr_filelist[i].filename) != 0) { + char* strptr = strstr(path, unifyfs_filelist[i].filename); + if (strptr == unifyfs_filelist[i].filename && + strcmp(path, unifyfs_filelist[i].filename) != 0) { /* found a child item in path */ - LOGDBG("File found: unifycr_filelist[%d].filename = %s", - i, (char*)&unifycr_filelist[i].filename); + LOGDBG("File found: unifyfs_filelist[%d].filename = %s", + i, (char*)&unifyfs_filelist[i].filename); return 0; } } @@ -734,10 +735,10 @@ int unifycr_fid_is_dir_empty(const char* path) } /* return current size of given file id */ -off_t unifycr_fid_size(int fid) +off_t unifyfs_fid_size(int fid) { /* get meta data for this file */ - unifycr_filemeta_t* meta = unifycr_get_meta_from_fid(fid); + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); return meta->size; } @@ -745,11 +746,11 @@ off_t unifycr_fid_size(int fid) * insert file attribute to attributed shared memory buffer, * keep entries ordered by file id */ -static int ins_file_meta(unifycr_fattr_buf_t* ptr_f_meta_log, - unifycr_file_attr_t* ins_fattr) +static int ins_file_meta(unifyfs_fattr_buf_t* ptr_f_meta_log, + unifyfs_file_attr_t* ins_fattr) { /* get pointer to start of stat structures in shared memory buffer */ - unifycr_file_attr_t* meta_entry = ptr_f_meta_log->meta_entry; + unifyfs_file_attr_t* meta_entry = ptr_f_meta_log->meta_entry; /* get number of active entries currently in the buffer */ int meta_cnt = *(ptr_f_meta_log->ptr_num_entries); @@ -783,44 +784,63 @@ static int ins_file_meta(unifycr_fattr_buf_t* ptr_f_meta_log, return 0; } -int unifycr_set_global_file_meta(const char* path, int fid, int gfid, - struct stat* sb) +unifyfs_filemeta_t* meta; +int unifyfs_set_global_file_meta(int fid, int gfid) { int ret = 0; - unifycr_file_attr_t new_fmeta = { 0, }; + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + unifyfs_file_attr_t new_fmeta = {0}; + struct timespec tp = {0}; - memset((void*) &new_fmeta, 0, sizeof(new_fmeta)); + const char* path = unifyfs_path_from_fid(fid); sprintf(new_fmeta.filename, "%s", path); new_fmeta.fid = fid; new_fmeta.gfid = gfid; - new_fmeta.file_attr = *sb; - ret = unifycr_client_metaset_rpc_invoke(&unifycr_rpc_context, - &new_fmeta); - //ret = set_global_file_meta(&new_fmeta); + clock_gettime(CLOCK_REALTIME, &tp); + new_fmeta.atime = tp; + new_fmeta.mtime = tp; + new_fmeta.ctime = tp; + + new_fmeta.mode = meta->mode; + new_fmeta.is_laminated = meta->is_laminated; + + if (meta->is_laminated) { + /* + * If is_laminated is set, we're either laminating for the first time, + * in which case meta->size will have already been calculated and + * filled in with the global file size, or, the file was already + * laminated. In either case, we write meta->size to the server. + */ + new_fmeta.size = meta->size; + } else { + new_fmeta.size = 0; + } + + new_fmeta.uid = getuid(); + new_fmeta.gid = getgid(); + + ret = invoke_client_metaset_rpc(&new_fmeta); if (ret < 0) { return ret; } - ins_file_meta(&unifycr_fattrs, &new_fmeta); + ins_file_meta(&unifyfs_fattrs, &new_fmeta); return 0; } -int unifycr_get_global_file_meta(int fid, int gfid, unifycr_file_attr_t* gfattr) +int unifyfs_get_global_file_meta(int fid, int gfid, unifyfs_file_attr_t* gfattr) { - int rc = 0; - if (!gfattr) { return -EINVAL; } - unifycr_file_attr_t fmeta; - int ret = unifycr_client_metaget_rpc_invoke( - &unifycr_rpc_context, gfid, &fmeta); - if (ret == UNIFYCR_SUCCESS) { + unifyfs_file_attr_t fmeta; + int ret = invoke_client_metaget_rpc(gfid, &fmeta); + if (ret == UNIFYFS_SUCCESS) { *gfattr = fmeta; gfattr->fid = fid; } @@ -829,92 +849,85 @@ int unifycr_get_global_file_meta(int fid, int gfid, unifycr_file_attr_t* gfattr) } /* fill in limited amount of stat information for global file id */ -int unifycr_gfid_stat(int gfid, struct stat* buf) +int unifyfs_gfid_stat(int gfid, struct stat* buf) { /* check that we have an output buffer to write to */ if (!buf) { - return UNIFYCR_ERROR_INVAL; + return UNIFYFS_ERROR_INVAL; } /* zero out user's stat buffer */ memset(buf, 0, sizeof(struct stat)); /* lookup stat data for global file id */ - unifycr_file_attr_t fattr; - int ret = unifycr_client_metaget_rpc_invoke( - &unifycr_rpc_context, gfid, &fattr); - if (ret != UNIFYCR_SUCCESS) { - return UNIFYCR_ERROR_IO; + unifyfs_file_attr_t fattr; + int ret = invoke_client_metaget_rpc(gfid, &fattr); + if (ret != UNIFYFS_SUCCESS) { + return ret; } - /* execute rpc to get file size */ - size_t filesize; - ret = unifycr_client_filesize_rpc_invoke(&unifycr_rpc_context, - app_id, local_rank_idx, gfid, &filesize); - if (ret != UNIFYCR_SUCCESS) { - return UNIFYCR_ERROR_IO; + /* It was decided that non-laminated files return a file size of 0 */ + if (!fattr.is_laminated) { + fattr.size = 0; } /* copy stat structure */ - *buf = fattr.file_attr; + unifyfs_file_attr_to_stat(&fattr, buf); - /* set the file size */ - buf->st_size = filesize; - - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /* fill in limited amount of stat information */ -int unifycr_fid_stat(int fid, struct stat* buf) +int unifyfs_fid_stat(int fid, struct stat* buf) { /* check that fid is defined */ - unifycr_filemeta_t* meta = unifycr_get_meta_from_fid(fid); + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); if (meta == NULL) { - return UNIFYCR_ERROR_IO; + return UNIFYFS_ERROR_IO; } /* get global file id corresponding to local file id */ - int gfid = unifycr_gfid_from_fid(fid); + int gfid = unifyfs_gfid_from_fid(fid); /* lookup stat info for global file id */ - int ret = unifycr_gfid_stat(gfid, buf); - if (ret != UNIFYCR_SUCCESS) { - return UNIFYCR_ERROR_IO; + int ret = unifyfs_gfid_stat(gfid, buf); + if (ret != UNIFYFS_SUCCESS) { + return UNIFYFS_ERROR_IO; } - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /* allocate a file id slot for a new file * return the fid or -1 on error */ -int unifycr_fid_alloc() +int unifyfs_fid_alloc() { - unifycr_stack_lock(); - int fid = unifycr_stack_pop(free_fid_stack); - unifycr_stack_unlock(); - LOGDBG("unifycr_stack_pop() gave %d", fid); + unifyfs_stack_lock(); + int fid = unifyfs_stack_pop(free_fid_stack); + unifyfs_stack_unlock(); + LOGDBG("unifyfs_stack_pop() gave %d", fid); if (fid < 0) { /* need to create a new file, but we can't */ - LOGERR("unifycr_stack_pop() failed (%d)", fid); + LOGERR("unifyfs_stack_pop() failed (%d)", fid); return -1; } return fid; } /* return the file id back to the free pool */ -int unifycr_fid_free(int fid) +int unifyfs_fid_free(int fid) { - unifycr_stack_lock(); - unifycr_stack_push(free_fid_stack, fid); - unifycr_stack_unlock(); - return UNIFYCR_SUCCESS; + unifyfs_stack_lock(); + unifyfs_stack_push(free_fid_stack, fid); + unifyfs_stack_unlock(); + return UNIFYFS_SUCCESS; } /* add a new file and initialize metadata * returns the new fid, or negative value on error */ -int unifycr_fid_create_file(const char* path) +int unifyfs_fid_create_file(const char* path) { - int fid = unifycr_fid_alloc(); + int fid = unifyfs_fid_alloc(); if (fid < 0) { /* was there an error? if so, return it */ errno = ENOSPC; @@ -922,25 +935,27 @@ int unifycr_fid_create_file(const char* path) } /* mark this slot as in use and copy the filename */ - unifycr_filelist[fid].in_use = 1; + unifyfs_filelist[fid].in_use = 1; /* TODO: check path length to see if it is < 128 bytes * and return appropriate error if it is greater */ /* copy file name into slot */ - strcpy((void*)&unifycr_filelist[fid].filename, path); - LOGDBG("Filename %s got unifycr fd %d", - unifycr_filelist[fid].filename, fid); + strcpy((void*)&unifyfs_filelist[fid].filename, path); + LOGDBG("Filename %s got unifyfs fd %d", + unifyfs_filelist[fid].filename, fid); /* initialize meta data */ - unifycr_filemeta_t* meta = unifycr_get_meta_from_fid(fid); + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); meta->size = 0; meta->chunks = 0; - meta->is_dir = 0; meta->log_size = 0; meta->storage = FILE_STORAGE_NULL; + meta->needs_sync = 0; meta->flock_status = UNLOCKED; + meta->is_laminated = 0; + meta->mode = UNIFYFS_STAT_DEFAULT_FILE_MODE; /* PTHREAD_PROCESS_SHARED allows Process-Shared Synchronization*/ pthread_spin_init(&meta->fspinlock, PTHREAD_PROCESS_SHARED); @@ -948,41 +963,7 @@ int unifycr_fid_create_file(const char* path) return fid; } -/* - * TODO: we need to generate proper mode for each entry. - */ -static const mode_t unifycr_default_file_mode = S_IFREG | 0644; -static const mode_t unifycr_default_dir_mode = S_IFDIR | 0755; - -static inline void unifycr_init_file_attr(struct stat* sb, int gfid) -{ - if (sb) { - memset((void*) sb, 0, sizeof(*sb)); - - sb->st_ino = gfid; - sb->st_size = 0; - sb->st_blocks = 1; - sb->st_blksize = 4096; - sb->st_atime = sb->st_mtime = sb->st_ctime = time(NULL); - sb->st_mode = unifycr_default_file_mode; - } -} - -static inline void unifycr_init_dir_attr(struct stat* sb, int gfid) -{ - if (sb) { - memset((void*) sb, 0, sizeof(*sb)); - - sb->st_ino = gfid; - sb->st_size = 0; - sb->st_blocks = 1; - sb->st_blksize = 4096; - sb->st_atime = sb->st_mtime = sb->st_ctime = time(NULL); - sb->st_mode = unifycr_default_dir_mode; - } -} - -int unifycr_fid_create_directory(const char* path) +int unifyfs_fid_create_directory(const char* path) { int ret = 0; int fid = 0; @@ -991,22 +972,22 @@ int unifycr_fid_create_directory(const char* path) int found_local = 0; size_t pathlen = strlen(path) + 1; struct stat sb = { 0, }; - unifycr_file_attr_t gfattr = { 0, }; - unifycr_filemeta_t* meta = NULL; + unifyfs_file_attr_t gfattr = { 0, }; + unifyfs_filemeta_t* meta = NULL; - if (pathlen > UNIFYCR_MAX_FILENAME) { - return (int) UNIFYCR_ERROR_NAMETOOLONG; + if (pathlen > UNIFYFS_MAX_FILENAME) { + return (int) UNIFYFS_ERROR_NAMETOOLONG; } - fid = unifycr_get_fid_from_path(path); - gfid = unifycr_generate_gfid(path); + fid = unifyfs_get_fid_from_path(path); + gfid = unifyfs_generate_gfid(path); found_global = - (unifycr_get_global_file_meta(fid, gfid, &gfattr) == UNIFYCR_SUCCESS); + (unifyfs_get_global_file_meta(fid, gfid, &gfattr) == UNIFYFS_SUCCESS); found_local = (fid >= 0); if (found_local && found_global) { - return (int) UNIFYCR_ERROR_EXIST; + return (int) UNIFYFS_ERROR_EXIST; } if (found_local && !found_global) { @@ -1024,58 +1005,58 @@ int unifycr_fid_create_directory(const char* path) * we currently return EIO, and this needs to be addressed according to * a consistency model this fs intance assumes. */ - return (int) UNIFYCR_ERROR_IO; + return (int) UNIFYFS_ERROR_IO; } if (!found_local && found_global) { /* populate the local cache, then return EEXIST */ - return (int) UNIFYCR_ERROR_EXIST; + return (int) UNIFYFS_ERROR_EXIST; } /* now, we need to create a new directory. */ - fid = unifycr_fid_create_file(path); + fid = unifyfs_fid_create_file(path); if (fid < 0) { - return (int) UNIFYCR_ERROR_IO; /* FIXME: ENOSPC or EIO? */ + return (int) UNIFYFS_ERROR_IO; /* FIXME: ENOSPC or EIO? */ } - meta = unifycr_get_meta_from_fid(fid); - meta->is_dir = 1; + meta = unifyfs_get_meta_from_fid(fid); - unifycr_init_dir_attr(&sb, gfid); + /* Set as directory */ + meta->mode = (meta->mode & ~S_IFREG) | S_IFDIR; - ret = unifycr_set_global_file_meta(path, fid, gfid, &sb); + ret = unifyfs_set_global_file_meta(fid, gfid); if (ret) { LOGERR("Failed to populate the global meta entry for %s (fid:%d)", path, fid); - return (int) UNIFYCR_ERROR_IO; + return (int) UNIFYFS_ERROR_IO; } - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /* read count bytes from file starting from pos and store into buf, * all bytes are assumed to exist, so checks on file size should be * done before calling this routine */ -int unifycr_fid_read(int fid, off_t pos, void* buf, size_t count) +int unifyfs_fid_read(int fid, off_t pos, void* buf, size_t count) { int rc; /* short-circuit a 0-byte read */ if (count == 0) { - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /* get meta for this file id */ - unifycr_filemeta_t* meta = unifycr_get_meta_from_fid(fid); + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); /* determine storage type to read file data */ if (meta->storage == FILE_STORAGE_FIXED_CHUNK) { /* file stored in fixed-size chunks */ - rc = unifycr_fid_store_fixed_read(fid, meta, pos, buf, count); + rc = unifyfs_fid_store_fixed_read(fid, meta, pos, buf, count); } else { /* unknown storage type */ - rc = (int)UNIFYCR_ERROR_IO; + rc = (int)UNIFYFS_ERROR_IO; } return rc; @@ -1084,26 +1065,26 @@ int unifycr_fid_read(int fid, off_t pos, void* buf, size_t count) /* write count bytes from buf into file starting at offset pos, * all bytes are assumed to be allocated to file, so file should * be extended before calling this routine */ -int unifycr_fid_write(int fid, off_t pos, const void* buf, size_t count) +int unifyfs_fid_write(int fid, off_t pos, const void* buf, size_t count) { int rc; /* short-circuit a 0-byte write */ if (count == 0) { - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /* get meta for this file id */ - unifycr_filemeta_t* meta = unifycr_get_meta_from_fid(fid); + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); /* determine storage type to write file data */ if (meta->storage == FILE_STORAGE_FIXED_CHUNK || meta->storage == FILE_STORAGE_LOGIO) { /* file stored in fixed-size chunks */ - rc = unifycr_fid_store_fixed_write(fid, meta, pos, buf, count); + rc = unifyfs_fid_store_fixed_write(fid, meta, pos, buf, count); } else { /* unknown storage type */ - rc = (int)UNIFYCR_ERROR_IO; + rc = (int)UNIFYFS_ERROR_IO; } return rc; @@ -1111,15 +1092,15 @@ int unifycr_fid_write(int fid, off_t pos, const void* buf, size_t count) /* given a file id, write zero bytes to region of specified offset * and length, assumes space is already reserved */ -int unifycr_fid_write_zero(int fid, off_t pos, off_t count) +int unifyfs_fid_write_zero(int fid, off_t pos, off_t count) { - int rc = UNIFYCR_SUCCESS; + int rc = UNIFYFS_SUCCESS; /* allocate an aligned chunk of memory */ size_t buf_size = 1024 * 1024; void* buf = (void*) malloc(buf_size); if (buf == NULL) { - return (int)UNIFYCR_ERROR_IO; + return (int)UNIFYFS_ERROR_IO; } /* set values in this buffer to zero */ @@ -1137,9 +1118,9 @@ int unifycr_fid_write_zero(int fid, off_t pos, off_t count) } /* write data to file */ - int write_rc = unifycr_fid_write(fid, curpos, buf, num); - if (write_rc != UNIFYCR_SUCCESS) { - rc = (int)UNIFYCR_ERROR_IO; + int write_rc = unifyfs_fid_write(fid, curpos, buf, num); + if (write_rc != UNIFYFS_SUCCESS) { + rc = (int)UNIFYFS_ERROR_IO; break; } @@ -1157,21 +1138,21 @@ int unifycr_fid_write_zero(int fid, off_t pos, off_t count) /* increase size of file if length is greater than current size, * and allocate additional chunks as needed to reserve space for * length bytes */ -int unifycr_fid_extend(int fid, off_t length) +int unifyfs_fid_extend(int fid, off_t length) { int rc; /* get meta data for this file */ - unifycr_filemeta_t* meta = unifycr_get_meta_from_fid(fid); + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); /* determine file storage type */ if (meta->storage == FILE_STORAGE_FIXED_CHUNK || meta->storage == FILE_STORAGE_LOGIO) { /* file stored in fixed-size chunks */ - rc = unifycr_fid_store_fixed_extend(fid, meta, length); + rc = unifyfs_fid_store_fixed_extend(fid, meta, length); } else { /* unknown storage type */ - rc = (int)UNIFYCR_ERROR_IO; + rc = (int)UNIFYFS_ERROR_IO; } /* TODO: move this statement elsewhere */ @@ -1186,20 +1167,20 @@ int unifycr_fid_extend(int fid, off_t length) } /* if length is less than reserved space, give back space down to length */ -int unifycr_fid_shrink(int fid, off_t length) +int unifyfs_fid_shrink(int fid, off_t length) { int rc; /* get meta data for this file */ - unifycr_filemeta_t* meta = unifycr_get_meta_from_fid(fid); + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); /* determine file storage type */ if (meta->storage == FILE_STORAGE_FIXED_CHUNK) { /* file stored in fixed-size chunks */ - rc = unifycr_fid_store_fixed_shrink(fid, meta, length); + rc = unifyfs_fid_store_fixed_shrink(fid, meta, length); } else { /* unknown storage type */ - rc = (int)UNIFYCR_ERROR_IO; + rc = (int)UNIFYFS_ERROR_IO; } return rc; @@ -1208,10 +1189,10 @@ int unifycr_fid_shrink(int fid, off_t length) /* truncate file id to given length, frees resources if length is * less than size and allocates and zero-fills new bytes if length * is more than size */ -int unifycr_fid_truncate(int fid, off_t length) +int unifyfs_fid_truncate(int fid, off_t length) { /* get meta data for this file */ - unifycr_filemeta_t* meta = unifycr_get_meta_from_fid(fid); + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); /* get current size of file */ off_t size = meta->size; @@ -1220,29 +1201,29 @@ int unifycr_fid_truncate(int fid, off_t length) * allocate new space and zero fill it if bigger */ if (length < size) { /* determine the number of chunks to leave after truncating */ - int shrink_rc = unifycr_fid_shrink(fid, length); - if (shrink_rc != UNIFYCR_SUCCESS) { + int shrink_rc = unifyfs_fid_shrink(fid, length); + if (shrink_rc != UNIFYFS_SUCCESS) { return shrink_rc; } } else if (length > size) { /* file size has been extended, allocate space */ - int extend_rc = unifycr_fid_extend(fid, length); - if (extend_rc != UNIFYCR_SUCCESS) { - return (int)UNIFYCR_ERROR_NOSPC; + int extend_rc = unifyfs_fid_extend(fid, length); + if (extend_rc != UNIFYFS_SUCCESS) { + return (int)UNIFYFS_ERROR_NOSPC; } /* write zero values to new bytes */ off_t gap_size = length - size; - int zero_rc = unifycr_fid_write_zero(fid, size, gap_size); - if (zero_rc != UNIFYCR_SUCCESS) { - return (int)UNIFYCR_ERROR_IO; + int zero_rc = unifyfs_fid_write_zero(fid, size, gap_size); + if (zero_rc != UNIFYFS_SUCCESS) { + return (int)UNIFYFS_ERROR_IO; } } /* set the new size */ meta->size = length; - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /* @@ -1250,7 +1231,7 @@ int unifycr_fid_truncate(int fid, off_t length) * @param path: file path * return: error code, gfid * */ -static int unifycr_get_global_fid(const char* path, int* gfid) +static int unifyfs_get_global_fid(const char* path, int* gfid) { MD5_CTX ctx; @@ -1262,14 +1243,14 @@ static int unifycr_get_global_fid(const char* path, int* gfid) MD5_Final(md, &ctx); *gfid = *((int*)md); - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /* opens a new file id with specified path, access flags, and permissions, * fills outfid with file id and outpos with position for current file pointer, - * returns UNIFYCR error code + * returns UNIFYFS error code */ -int unifycr_fid_open(const char* path, int flags, mode_t mode, int* outfid, +int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, off_t* outpos) { /* check that path is short enough */ @@ -1280,10 +1261,10 @@ int unifycr_fid_open(const char* path, int flags, mode_t mode, int* outfid, int found_global = 0; int found_local = 0; off_t pos = 0; /* set the pointer to the start of the file */ - unifycr_file_attr_t gfattr = { 0, }; + unifyfs_file_attr_t gfattr = { 0, }; - if (pathlen > UNIFYCR_MAX_FILENAME) { - return (int) UNIFYCR_ERROR_NAMETOOLONG; + if (pathlen > UNIFYFS_MAX_FILENAME) { + return (int) UNIFYFS_ERROR_NAMETOOLONG; } /* check whether this file already exists */ @@ -1294,82 +1275,93 @@ int unifycr_fid_open(const char* path, int flags, mode_t mode, int* outfid, * the broadcast for cache invalidation has not been implemented, yet. */ - gfid = unifycr_generate_gfid(path); - fid = unifycr_get_fid_from_path(path); + gfid = unifyfs_generate_gfid(path); + fid = unifyfs_get_fid_from_path(path); - LOGDBG("unifycr_get_fid_from_path() gave %d (gfid = %d)", fid, gfid); + LOGDBG("unifyfs_get_fid_from_path() gave %d (gfid = %d)", fid, gfid); found_global = - (unifycr_get_global_file_meta(fid, gfid, &gfattr) == UNIFYCR_SUCCESS); + (unifyfs_get_global_file_meta(fid, gfid, &gfattr) == UNIFYFS_SUCCESS); found_local = (fid >= 0); + /* + * Catch any case where we could potentially want to write to a laminated + * file. + */ + if (gfattr.is_laminated && + ((flags & (O_CREAT | O_TRUNC | O_APPEND | O_WRONLY)) || + (mode & 0222))) { + LOGDBG("Can't open with a writable flag on laminated file."); + return EROFS; + } + /* possibly, the file still exists in our local cache but globally * unlinked. Invalidate the entry * - * FIXME: unifycr_fid_unlink() always returns success. + * FIXME: unifyfs_fid_unlink() always returns success. */ if (found_local && !found_global) { LOGDBG("file found locally, but seems to be deleted globally. " "invalidating the local cache."); + return EROFS; - unifycr_fid_unlink(fid); + unifyfs_fid_unlink(fid); - return (int) UNIFYCR_ERROR_NOENT; + return (int) UNIFYFS_ERROR_NOENT; } /* for all other three cases below, we need to open the file and allocate a * file descriptor for the client. */ - if (!found_local && found_global) { /* file has possibly been created by another process. We need to * create a local meta cache and also initialize the local storage * space. */ - unifycr_filemeta_t* meta = NULL; + unifyfs_filemeta_t* meta = NULL; - fid = unifycr_fid_create_file(path); + fid = unifyfs_fid_create_file(path); if (fid < 0) { LOGERR("failed to create a new file %s", path); - /* FIXME: UNIFYCR_ERROR_NFILE or UNIFYCR_ERROR_IO ? */ - return (int) UNIFYCR_ERROR_IO; + /* FIXME: UNIFYFS_ERROR_NFILE or UNIFYFS_ERROR_IO ? */ + return (int) UNIFYFS_ERROR_IO; } - ret = unifycr_fid_store_alloc(fid); - if (ret != UNIFYCR_SUCCESS) { + ret = unifyfs_fid_store_alloc(fid); + if (ret != UNIFYFS_SUCCESS) { LOGERR("failed to allocate storage space for file %s (fid=%d)", path, fid); - return (int) UNIFYCR_ERROR_IO; + return (int) UNIFYFS_ERROR_IO; } - meta = unifycr_get_meta_from_fid(fid); + meta = unifyfs_get_meta_from_fid(fid); - meta->size = gfattr.file_attr.st_size; + meta->size = gfattr.size; gfattr.fid = fid; gfattr.gfid = gfid; - ins_file_meta(&unifycr_fattrs, &gfattr); + ins_file_meta(&unifyfs_fattrs, &gfattr); } else if (found_local && found_global) { /* file exists and is valid. */ if ((flags & O_CREAT) && (flags & O_EXCL)) { - return (int)UNIFYCR_ERROR_EXIST; + return (int)UNIFYFS_ERROR_EXIST; } - if ((flags & O_DIRECTORY) && !unifycr_fid_is_dir(fid)) { - return (int)UNIFYCR_ERROR_NOTDIR; + if ((flags & O_DIRECTORY) && !unifyfs_fid_is_dir(fid)) { + return (int)UNIFYFS_ERROR_NOTDIR; } - if (!(flags & O_DIRECTORY) && unifycr_fid_is_dir(fid)) { - return (int)UNIFYCR_ERROR_NOTDIR; + if (!(flags & O_DIRECTORY) && unifyfs_fid_is_dir(fid)) { + return (int)UNIFYFS_ERROR_NOTDIR; } if ((flags & O_TRUNC) && (flags & (O_RDWR | O_WRONLY))) { - unifycr_fid_truncate(fid, 0); + unifyfs_fid_truncate(fid, 0); } if (flags & O_APPEND) { - unifycr_filemeta_t* meta = unifycr_get_meta_from_fid(fid); + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); pos = meta->size; } } else { @@ -1380,37 +1372,35 @@ int unifycr_fid_open(const char* path, int flags, mode_t mode, int* outfid, if (!(flags & O_CREAT)) { LOGERR("%s does not exist (O_CREAT not given).", path); - return (int) UNIFYCR_ERROR_NOENT; + return (int) UNIFYFS_ERROR_NOENT; } LOGDBG("Creating a new entry for %s.", path); LOGDBG("shm_super_buf = %p; free_fid_stack = %p; " - "free_chunk_stack = %p; unifycr_filelist = %p; " + "free_chunk_stack = %p; unifyfs_filelist = %p; " "chunks = %p", shm_super_buf, free_fid_stack, - free_chunk_stack, unifycr_filelist, unifycr_chunks); + free_chunk_stack, unifyfs_filelist, unifyfs_chunks); /* allocate a file id slot for this new file */ - fid = unifycr_fid_create_file(path); + fid = unifyfs_fid_create_file(path); if (fid < 0) { LOGERR("Failed to create new file %s", path); - return (int) UNIFYCR_ERROR_NFILE; + return (int) UNIFYFS_ERROR_NFILE; } /* initialize the storage for the file */ - int store_rc = unifycr_fid_store_alloc(fid); - if (store_rc != UNIFYCR_SUCCESS) { + int store_rc = unifyfs_fid_store_alloc(fid); + if (store_rc != UNIFYFS_SUCCESS) { LOGERR("Failed to create storage for file %s", path); - return (int) UNIFYCR_ERROR_IO; + return (int) UNIFYFS_ERROR_IO; } - unifycr_init_file_attr(&sb, gfid); - /*create a file and send its attribute to key-value store*/ - ret = unifycr_set_global_file_meta(path, fid, gfid, &sb); + ret = unifyfs_set_global_file_meta(fid, gfid); if (ret) { LOGERR("Failed to populate the global meta entry for %s (fid:%d)", path, fid); - return (int) UNIFYCR_ERROR_IO; + return (int) UNIFYFS_ERROR_IO; } } @@ -1420,33 +1410,33 @@ int unifycr_fid_open(const char* path, int flags, mode_t mode, int* outfid, *outfid = fid; *outpos = pos; - LOGDBG("UNIFYCR_open generated fd %d for file %s", fid, path); + LOGDBG("UNIFYFS_open generated fd %d for file %s", fid, path); - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } -int unifycr_fid_close(int fid) +int unifyfs_fid_close(int fid) { /* TODO: clear any held locks */ /* nothing to do here, just a place holder */ - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /* delete a file id and return file its resources to free pools */ -int unifycr_fid_unlink(int fid) +int unifyfs_fid_unlink(int fid) { /* return data to free pools */ - int rc = unifycr_fid_truncate(fid, 0); - if (rc != UNIFYCR_SUCCESS) { + int rc = unifyfs_fid_truncate(fid, 0); + if (rc != UNIFYFS_SUCCESS) { /* failed to release storage for the file, * so bail out to keep its file id active */ return rc; } /* finalize the storage we're using for this file */ - rc = unifycr_fid_store_free(fid); - if (rc != UNIFYCR_SUCCESS) { + rc = unifyfs_fid_store_free(fid); + if (rc != UNIFYFS_SUCCESS) { /* released strorage for file, but failed to release * structures tracking storage, again bail out to keep * its file id active */ @@ -1458,17 +1448,17 @@ int unifycr_fid_unlink(int fid) * release the file id itself */ /* set this file id as not in use */ - unifycr_filelist[fid].in_use = 0; + unifyfs_filelist[fid].in_use = 0; /* add this id back to the free stack */ - rc = unifycr_fid_free(fid); - if (rc != UNIFYCR_SUCCESS) { + rc = unifyfs_fid_free(fid); + if (rc != UNIFYFS_SUCCESS) { /* storage for the file was released, but we hit * an error while freeing the file id */ return rc; } - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /* --------------------------------------- @@ -1486,15 +1476,15 @@ int unifycr_fid_unlink(int fid) * the local file id is used to index into other data * structures * - * - array of unifycr_filename structs, indexed by local + * - array of unifyfs_filename structs, indexed by local * file id, provides a field indicating whether file * slot is in use and if so, the current file name * - * - array of unifycr_filemeta structs, indexed by local + * - array of unifyfs_filemeta structs, indexed by local * file id, records list of storage blocks used to * store data for the file * - * - array of unifycr_chunkmeta structs, indexed by local + * - array of unifyfs_chunkmeta structs, indexed by local * file id and then by chunk id for recording metadata * of each chunk allocated to a file, including host * storage and id of that chunk within its storage @@ -1503,25 +1493,25 @@ int unifycr_fid_unlink(int fid) * * - stack to track free list of spillover chunks * - * - array of storage chunks of length unifycr_max_chunks, + * - array of storage chunks of length unifyfs_max_chunks, * if storing data in memory * * - count of number of active index entries * - array of index metadata to track physical offset - * of logical file data, of length unifycr_max_index_entries, + * of logical file data, of length unifyfs_max_index_entries, * entries added during write operations * * - count of number of active file metadata entries * - array of file metadata to track stat info for each - * file, of length unifycr_max_fattr_entries, filed + * file, of length unifyfs_max_fattr_entries, filed * in by client and read by server to record file meta * data */ /* compute memory size of superblock in bytes, * critical to keep this consistent with - * unifycr_init_pointers */ -static size_t unifycr_superblock_size(void) + * unifyfs_init_pointers */ +static size_t unifyfs_superblock_size(void) { size_t sb_size = 0; @@ -1530,58 +1520,68 @@ static size_t unifycr_superblock_size(void) sb_size += sizeof(uint32_t); /* free file id stack */ - sb_size += unifycr_stack_bytes(unifycr_max_files); + sb_size += unifyfs_stack_bytes(unifyfs_max_files); /* file name struct array */ - sb_size += unifycr_max_files * sizeof(unifycr_filename_t); + sb_size += unifyfs_max_files * sizeof(unifyfs_filename_t); /* file metadata struct array */ - sb_size += unifycr_max_files * sizeof(unifycr_filemeta_t); + sb_size += unifyfs_max_files * sizeof(unifyfs_filemeta_t); - /* memory chunk metadata struct array for each file, - * enables a file to use all space in memory */ - if (unifycr_use_memfs) { - sb_size += unifycr_max_files * unifycr_max_chunks * - sizeof(unifycr_chunkmeta_t); + if (unifyfs_use_memfs) { + /* memory chunk metadata struct array for each file, + * enables a file to use all space in memory */ + sb_size += unifyfs_max_files * unifyfs_max_chunks * + sizeof(unifyfs_chunkmeta_t); } - - /* spillover chunk metadata struct array for each file, - * enables a file to use all space in spillover file */ - if (unifycr_use_spillover) { - sb_size += unifycr_max_files * unifycr_spillover_max_chunks * - sizeof(unifycr_chunkmeta_t); + if (unifyfs_use_spillover) { + /* spillover chunk metadata struct array for each file, + * enables a file to use all space in spillover file */ + sb_size += unifyfs_max_files * unifyfs_spillover_max_chunks * + sizeof(unifyfs_chunkmeta_t); } - /* free memory chunk stack */ - if (unifycr_use_memfs) { - sb_size += unifycr_stack_bytes(unifycr_max_chunks); + /* free chunk stack */ + if (unifyfs_use_memfs) { + sb_size += unifyfs_stack_bytes(unifyfs_max_chunks); } - - /* free spillover chunk stack */ - if (unifycr_use_spillover) { - sb_size += unifycr_stack_bytes(unifycr_spillover_max_chunks); + if (unifyfs_use_spillover) { + sb_size += unifyfs_stack_bytes(unifyfs_spillover_max_chunks); } /* space for memory chunks */ - if (unifycr_use_memfs) { - sb_size += unifycr_page_size + - (unifycr_max_chunks * unifycr_chunk_size); + if (unifyfs_use_memfs) { + sb_size += unifyfs_page_size; + sb_size += unifyfs_max_chunks * unifyfs_chunk_size; } /* index region size */ - sb_size += unifycr_page_size; - sb_size += unifycr_max_index_entries * sizeof(unifycr_index_t); + sb_size += unifyfs_page_size; + sb_size += unifyfs_max_index_entries * sizeof(unifyfs_index_t); /* attribute region size */ - sb_size += unifycr_page_size; - sb_size += unifycr_max_fattr_entries * sizeof(unifycr_file_attr_t); + sb_size += unifyfs_page_size; + sb_size += unifyfs_max_fattr_entries * sizeof(unifyfs_file_attr_t); /* return number of bytes */ return sb_size; } +static inline +char* next_page_align(char* ptr) +{ + intptr_t orig = (intptr_t) ptr; + intptr_t aligned = orig; + intptr_t offset = orig % unifyfs_page_size; + if (offset) { + aligned += (unifyfs_page_size - offset); + } + LOGDBG("orig=0x%p, next-page-aligned=0x%p", ptr, (char*)aligned); + return (char*) aligned; +} + /* initialize our global pointers into the given superblock */ -static void* unifycr_init_pointers(void* superblock) +static void* unifyfs_init_pointers(void* superblock) { char* ptr = (char*)superblock; @@ -1591,78 +1591,62 @@ static void* unifycr_init_pointers(void* superblock) /* stack to manage free file ids */ free_fid_stack = ptr; - ptr += unifycr_stack_bytes(unifycr_max_files); + ptr += unifyfs_stack_bytes(unifyfs_max_files); /* record list of file names */ - unifycr_filelist = (unifycr_filename_t*)ptr; - ptr += unifycr_max_files * sizeof(unifycr_filename_t); + unifyfs_filelist = (unifyfs_filename_t*)ptr; + ptr += unifyfs_max_files * sizeof(unifyfs_filename_t); /* array of file meta data structures */ - unifycr_filemetas = (unifycr_filemeta_t*)ptr; - ptr += unifycr_max_files * sizeof(unifycr_filemeta_t); + unifyfs_filemetas = (unifyfs_filemeta_t*)ptr; + ptr += unifyfs_max_files * sizeof(unifyfs_filemeta_t); /* array of chunk meta data strucutres for each file */ - unifycr_chunkmetas = (unifycr_chunkmeta_t*)ptr; - if (unifycr_use_memfs) { - ptr += unifycr_max_files * unifycr_max_chunks * - sizeof(unifycr_chunkmeta_t); + unifyfs_chunkmetas = (unifyfs_chunkmeta_t*)ptr; + if (unifyfs_use_memfs) { + ptr += unifyfs_max_files * unifyfs_max_chunks * + sizeof(unifyfs_chunkmeta_t); } - - /* if we're using spillover, reserve chunk meta data - * for spill over chunks */ - if (unifycr_use_spillover) { - ptr += unifycr_max_files * unifycr_spillover_max_chunks * - sizeof(unifycr_chunkmeta_t); + if (unifyfs_use_spillover) { + ptr += unifyfs_max_files * unifyfs_spillover_max_chunks * + sizeof(unifyfs_chunkmeta_t); } /* stack to manage free memory data chunks */ - if (unifycr_use_memfs) { + if (unifyfs_use_memfs) { free_chunk_stack = ptr; - ptr += unifycr_stack_bytes(unifycr_max_chunks); + ptr += unifyfs_stack_bytes(unifyfs_max_chunks); } - - /* stack to manage free spill-over data chunks */ - if (unifycr_use_spillover) { + if (unifyfs_use_spillover) { free_spillchunk_stack = ptr; - ptr += unifycr_stack_bytes(unifycr_spillover_max_chunks); + ptr += unifyfs_stack_bytes(unifyfs_spillover_max_chunks); } /* Only set this up if we're using memfs */ - if (unifycr_use_memfs) { - /* round ptr up to start of next page */ - unsigned long long ull_ptr = (unsigned long long)ptr; - unsigned long long ull_page = (unsigned long long)unifycr_page_size; - unsigned long long num_pages = ull_ptr / ull_page; - if (ull_ptr > num_pages * ull_page) { - ptr = (char*)((num_pages + 1) * ull_page); - } - + if (unifyfs_use_memfs) { /* pointer to start of memory data chunks */ - unifycr_chunks = ptr; - ptr += unifycr_max_chunks * unifycr_chunk_size; + ptr = next_page_align(ptr); + unifyfs_chunks = ptr; + ptr += unifyfs_max_chunks * unifyfs_chunk_size; } else { - unifycr_chunks = NULL; + unifyfs_chunks = NULL; } /* record pointer to number of index entries */ - unifycr_indices.ptr_num_entries = (off_t*)ptr; - - /* skip a full page? */ - ptr += unifycr_page_size; + unifyfs_indices.ptr_num_entries = (size_t*)ptr; /* pointer to array of index entries */ - unifycr_indices.index_entry = (unifycr_index_t*)ptr; - ptr += unifycr_max_index_entries * sizeof(unifycr_index_t); + ptr += unifyfs_page_size; + unifyfs_indices.index_entry = (unifyfs_index_t*)ptr; + ptr += unifyfs_max_index_entries * sizeof(unifyfs_index_t); /* pointer to number of file metadata entries */ - unifycr_fattrs.ptr_num_entries = (off_t*)ptr; - - /* skip a full page? */ - ptr += unifycr_page_size; + unifyfs_fattrs.ptr_num_entries = (size_t*)ptr; /* pointer to array of file metadata entries */ - unifycr_fattrs.meta_entry = (unifycr_file_attr_t*)ptr; - ptr += unifycr_max_fattr_entries * sizeof(unifycr_file_attr_t); + ptr += unifyfs_page_size; + unifyfs_fattrs.meta_entry = (unifyfs_file_attr_t*)ptr; + ptr += unifyfs_max_fattr_entries * sizeof(unifyfs_file_attr_t); /* compute size of memory we're using and check that * it matches what we allocated */ @@ -1675,58 +1659,57 @@ static void* unifycr_init_pointers(void* superblock) } /* initialize data structures for first use */ -static int unifycr_init_structures() +static int unifyfs_init_structures() { /* compute total number of storage chunks available */ int numchunks = 0; - if (unifycr_use_memfs) { - numchunks += unifycr_max_chunks; + if (unifyfs_use_memfs) { + numchunks += unifyfs_max_chunks; } - if (unifycr_use_spillover) { - numchunks += unifycr_spillover_max_chunks; + if (unifyfs_use_spillover) { + numchunks += unifyfs_spillover_max_chunks; } int i; - for (i = 0; i < unifycr_max_files; i++) { + for (i = 0; i < unifyfs_max_files; i++) { /* indicate that file id is not in use by setting flag to 0 */ - unifycr_filelist[i].in_use = 0; + unifyfs_filelist[i].in_use = 0; /* set pointer to array of chunkmeta data structures */ - unifycr_filemeta_t* filemeta = &unifycr_filemetas[i]; + unifyfs_filemeta_t* filemeta = &unifyfs_filemetas[i]; /* compute offset to start of chunk meta list for this file */ - unifycr_chunkmeta_t* chunkmetas = &(unifycr_chunkmetas[numchunks * i]); - filemeta->chunk_meta = chunkmetas; + filemeta->chunkmeta_idx = numchunks * i; } /* initialize stack of free file ids */ - unifycr_stack_init(free_fid_stack, unifycr_max_files); + unifyfs_stack_init(free_fid_stack, unifyfs_max_files); /* initialize list of free memory chunks */ - if (unifycr_use_memfs) { - unifycr_stack_init(free_chunk_stack, unifycr_max_chunks); + if (unifyfs_use_memfs) { + unifyfs_stack_init(free_chunk_stack, unifyfs_max_chunks); } /* initialize list of free spillover chunks */ - if (unifycr_use_spillover) { - unifycr_stack_init(free_spillchunk_stack, unifycr_spillover_max_chunks); + if (unifyfs_use_spillover) { + unifyfs_stack_init(free_spillchunk_stack, unifyfs_spillover_max_chunks); } /* initialize count of key/value entries */ - *(unifycr_indices.ptr_num_entries) = 0; + *(unifyfs_indices.ptr_num_entries) = 0; /* initialize count of file stat structures */ - *(unifycr_fattrs.ptr_num_entries) = 0; + *(unifyfs_fattrs.ptr_num_entries) = 0; LOGDBG("Meta-stacks initialized!"); - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } -static int unifycr_get_spillblock(size_t size, const char* path) +static int unifyfs_get_spillblock(size_t size, const char* path) { //MAP_OR_FAIL(open); - mode_t perms = unifycr_getmode(0); + mode_t perms = unifyfs_getmode(0); int spillblock_fd = __real_open(path, O_RDWR | O_CREAT | O_EXCL, perms); if (spillblock_fd < 0) { if (errno == EEXIST) { @@ -1752,7 +1735,7 @@ static int unifycr_get_spillblock(size_t size, const char* path) /* create superblock of specified size and name, or attach to existing * block if available */ -static void* unifycr_superblock_shmget(size_t size, key_t key) +static void* unifyfs_superblock_shmget(size_t size, key_t key) { /* define name for superblock shared memory region */ snprintf(shm_super_name, sizeof(shm_super_name), "%d-super-%d", @@ -1760,21 +1743,33 @@ static void* unifycr_superblock_shmget(size_t size, key_t key) LOGDBG("Key for superblock = %x", key); /* open shared memory file */ - void* addr = unifycr_shm_alloc(shm_super_name, size); + void* addr = unifyfs_shm_alloc(shm_super_name, size); if (addr == NULL) { LOGERR("Failed to create superblock"); return NULL; } /* init our global variables to point to spots in superblock */ - unifycr_init_pointers(addr); - unifycr_init_structures(); + unifyfs_init_pointers(addr); + + /* initialize structures in superblock if it's newly allocated, + * we depend on shm_open setting all bytes to 0 to know that + * it is not initialized */ + int32_t initialized = *(int32_t*)addr; + if (initialized == 0) { + /* not yet initialized, so initialize values within superblock */ + unifyfs_init_structures(); + + /* superblock structure has been initialized, + * so set flag to indicate that fact */ + *(int32_t*)addr = 0xDEADBEEF; + } /* return starting memory address of super block */ return addr; } -static int unifycr_init(int rank) +static int unifyfs_init(int rank) { int rc; int i; @@ -1783,30 +1778,30 @@ static int unifycr_init(int rank) unsigned long long bits; char* cfgval; - if (!unifycr_initialized) { - /* unifycr debug level default is zero */ - unifycr_log_level = 0; + if (!unifyfs_initialized) { + /* unifyfs debug level default is zero */ + unifyfs_log_level = 0; cfgval = client_cfg.log_verbosity; if (cfgval != NULL) { rc = configurator_int_val(cfgval, &l); if (rc == 0) { - unifycr_log_level = (int)l; + unifyfs_log_level = (int)l; } } -#ifdef UNIFYCR_GOTCHA +#ifdef UNIFYFS_GOTCHA /* insert our I/O wrappers using gotcha */ enum gotcha_error_t result; - result = gotcha_wrap(wrap_unifycr_list, GOTCHA_NFUNCS, "unifycr"); + result = gotcha_wrap(wrap_unifyfs_list, GOTCHA_NFUNCS, "unifyfs"); if (result != GOTCHA_SUCCESS) { LOGERR("gotcha_wrap returned %d", (int) result); } /* check for an errors when registering functions with gotcha */ for (i = 0; i < GOTCHA_NFUNCS; i++) { - if (*(void**)(wrap_unifycr_list[i].function_address_pointer) == 0) { + if (*(void**)(wrap_unifyfs_list[i].function_address_pointer) == 0) { LOGERR("This function name failed to be wrapped: %s", - wrap_unifycr_list[i].name); + wrap_unifyfs_list[i].name); } } #endif @@ -1816,125 +1811,125 @@ static int unifycr_init(int rank) * print a message if this won't work */ if (sizeof(fpos_t) < sizeof(void*)) { LOGERR("fgetpos/fsetpos will not work correctly"); - unifycr_fpos_enabled = 0; + unifyfs_fpos_enabled = 0; } /* look up page size for buffer alignment */ - unifycr_page_size = getpagesize(); + unifyfs_page_size = getpagesize(); /* compute min and max off_t values */ bits = sizeof(off_t) * 8; - unifycr_max_offt = (off_t)((1ULL << (bits - 1ULL)) - 1ULL); - unifycr_min_offt = (off_t)(-(1ULL << (bits - 1ULL))); + unifyfs_max_offt = (off_t)((1ULL << (bits - 1ULL)) - 1ULL); + unifyfs_min_offt = (off_t)(-(1ULL << (bits - 1ULL))); /* compute min and max long values */ - unifycr_max_long = LONG_MAX; - unifycr_min_long = LONG_MIN; + unifyfs_max_long = LONG_MAX; + unifyfs_min_long = LONG_MIN; /* will we use spillover to store the files? */ - unifycr_use_spillover = 1; + unifyfs_use_spillover = 1; cfgval = client_cfg.spillover_enabled; if (cfgval != NULL) { rc = configurator_bool_val(cfgval, &b); if ((rc == 0) && !b) { - unifycr_use_spillover = 0; + unifyfs_use_spillover = 0; } } - LOGDBG("are we using spillover? %d", unifycr_use_spillover); + LOGDBG("are we using spillover? %d", unifyfs_use_spillover); /* determine maximum number of bytes of spillover for chunk storage */ - unifycr_spillover_size = UNIFYCR_SPILLOVER_SIZE; + unifyfs_spillover_size = UNIFYFS_SPILLOVER_SIZE; cfgval = client_cfg.spillover_size; if (cfgval != NULL) { rc = configurator_int_val(cfgval, &l); if (rc == 0) { - unifycr_spillover_size = (size_t)l; + unifyfs_spillover_size = (size_t)l; } } /* determine max number of files to store in file system */ - unifycr_max_files = UNIFYCR_MAX_FILES; + unifyfs_max_files = UNIFYFS_MAX_FILES; cfgval = client_cfg.client_max_files; if (cfgval != NULL) { rc = configurator_int_val(cfgval, &l); if (rc == 0) { - unifycr_max_files = (int)l; + unifyfs_max_files = (int)l; } } /* determine number of bits for chunk size */ - unifycr_chunk_bits = UNIFYCR_CHUNK_BITS; + unifyfs_chunk_bits = UNIFYFS_CHUNK_BITS; cfgval = client_cfg.shmem_chunk_bits; if (cfgval != NULL) { rc = configurator_int_val(cfgval, &l); if (rc == 0) { - unifycr_chunk_bits = (int)l; + unifyfs_chunk_bits = (int)l; } } /* determine maximum number of bytes of memory for chunk storage */ - unifycr_chunk_mem = UNIFYCR_CHUNK_MEM; + unifyfs_chunk_mem = UNIFYFS_CHUNK_MEM; cfgval = client_cfg.shmem_chunk_mem; if (cfgval != NULL) { rc = configurator_int_val(cfgval, &l); if (rc == 0) { - unifycr_chunk_mem = (size_t)l; + unifyfs_chunk_mem = (size_t)l; } } /* set chunk size, set chunk offset mask, and set total number * of chunks */ - unifycr_chunk_size = 1 << unifycr_chunk_bits; - unifycr_chunk_mask = unifycr_chunk_size - 1; - unifycr_max_chunks = unifycr_chunk_mem >> unifycr_chunk_bits; + unifyfs_chunk_size = 1 << unifyfs_chunk_bits; + unifyfs_chunk_mask = unifyfs_chunk_size - 1; + unifyfs_max_chunks = unifyfs_chunk_mem >> unifyfs_chunk_bits; /* set number of chunks in spillover device */ - unifycr_spillover_max_chunks = unifycr_spillover_size >> unifycr_chunk_bits; + unifyfs_spillover_max_chunks = unifyfs_spillover_size >> unifyfs_chunk_bits; /* define size of buffer used to cache key/value pairs for * data offsets before passing them to the server */ - unifycr_index_buf_size = UNIFYCR_INDEX_BUF_SIZE; + unifyfs_index_buf_size = UNIFYFS_INDEX_BUF_SIZE; cfgval = client_cfg.logfs_index_buf_size; if (cfgval != NULL) { rc = configurator_int_val(cfgval, &l); if (rc == 0) { - unifycr_index_buf_size = (size_t)l; + unifyfs_index_buf_size = (size_t)l; } } - unifycr_max_index_entries = - unifycr_index_buf_size / sizeof(unifycr_index_t); + unifyfs_max_index_entries = + unifyfs_index_buf_size / sizeof(unifyfs_index_t); /* define size of buffer used to cache stat structures * for files we create before passing this info * to the server */ - unifycr_fattr_buf_size = UNIFYCR_FATTR_BUF_SIZE; + unifyfs_fattr_buf_size = UNIFYFS_FATTR_BUF_SIZE; cfgval = client_cfg.logfs_attr_buf_size; if (cfgval != NULL) { rc = configurator_int_val(cfgval, &l); if (rc == 0) { - unifycr_fattr_buf_size = (size_t)l; + unifyfs_fattr_buf_size = (size_t)l; } } - unifycr_max_fattr_entries = - unifycr_fattr_buf_size / sizeof(unifycr_file_attr_t); + unifyfs_max_fattr_entries = + unifyfs_fattr_buf_size / sizeof(unifyfs_file_attr_t); /* if we're using NUMA, process some configuration settings */ #ifdef HAVE_LIBNUMA - char* env = getenv("UNIFYCR_NUMA_POLICY"); + char* env = getenv("UNIFYFS_NUMA_POLICY"); if (env) { - sprintf(unifycr_numa_policy, env); - LOGDBG("NUMA policy used: %s", unifycr_numa_policy); + sprintf(unifyfs_numa_policy, env); + LOGDBG("NUMA policy used: %s", unifyfs_numa_policy); } else { - sprintf(unifycr_numa_policy, "default"); + sprintf(unifyfs_numa_policy, "default"); } - env = getenv("UNIFYCR_USE_NUMA_BANK"); + env = getenv("UNIFYFS_USE_NUMA_BANK"); if (env) { int val = atoi(env); if (val >= 0) { - unifycr_numa_bank = val; + unifyfs_numa_bank = val; } else { - LOGERR("Incorrect NUMA bank specified in UNIFYCR_USE_NUMA_BANK." + LOGERR("Incorrect NUMA bank specified in UNIFYFS_USE_NUMA_BANK." " Proceeding with default allocation policy."); } } @@ -1947,81 +1942,81 @@ static int unifycr_init(int rank) if (getrlimit(RLIMIT_NOFILE, &r_limit) < 0) { LOGERR("getrlimit failed: errno=%d (%s)", errno, strerror(errno)); - return UNIFYCR_FAILURE; + return UNIFYFS_FAILURE; } - unifycr_fd_limit = r_limit.rlim_cur; - LOGDBG("FD limit for system = %ld", unifycr_fd_limit); + unifyfs_fd_limit = r_limit.rlim_cur; + LOGDBG("FD limit for system = %ld", unifyfs_fd_limit); /* initialize file descriptor structures */ - int num_fds = UNIFYCR_MAX_FILEDESCS; + int num_fds = UNIFYFS_MAX_FILEDESCS; for (i = 0; i < num_fds; i++) { - unifycr_fd_init(i); + unifyfs_fd_init(i); } /* initialize file stream structures */ - int num_streams = UNIFYCR_MAX_FILEDESCS; + int num_streams = UNIFYFS_MAX_FILEDESCS; for (i = 0; i < num_streams; i++) { - unifycr_stream_init(i); + unifyfs_stream_init(i); } /* initialize directory stream structures */ - int num_dirstreams = UNIFYCR_MAX_FILEDESCS; + int num_dirstreams = UNIFYFS_MAX_FILEDESCS; for (i = 0; i < num_dirstreams; i++) { - unifycr_dirstream_init(i); + unifyfs_dirstream_init(i); } /* initialize stack of free fd values */ - size_t free_fd_size = unifycr_stack_bytes(num_fds); - unifycr_fd_stack = malloc(free_fd_size); - unifycr_stack_init(unifycr_fd_stack, num_fds); + size_t free_fd_size = unifyfs_stack_bytes(num_fds); + unifyfs_fd_stack = malloc(free_fd_size); + unifyfs_stack_init(unifyfs_fd_stack, num_fds); /* initialize stack of free stream values */ - size_t free_stream_size = unifycr_stack_bytes(num_streams); - unifycr_stream_stack = malloc(free_stream_size); - unifycr_stack_init(unifycr_stream_stack, num_streams); + size_t free_stream_size = unifyfs_stack_bytes(num_streams); + unifyfs_stream_stack = malloc(free_stream_size); + unifyfs_stack_init(unifyfs_stream_stack, num_streams); /* initialize stack of free directory stream values */ - size_t free_dirstream_size = unifycr_stack_bytes(num_dirstreams); - unifycr_dirstream_stack = malloc(free_dirstream_size); - unifycr_stack_init(unifycr_dirstream_stack, num_dirstreams); + size_t free_dirstream_size = unifyfs_stack_bytes(num_dirstreams); + unifyfs_dirstream_stack = malloc(free_dirstream_size); + unifyfs_stack_init(unifyfs_dirstream_stack, num_dirstreams); /* determine the size of the superblock */ - shm_super_size = unifycr_superblock_size(); + shm_super_size = unifyfs_superblock_size(); /* get a superblock of shared memory and initialize our * global variables for this block */ - shm_super_buf = unifycr_superblock_shmget( - shm_super_size, unifycr_mount_shmget_key); + shm_super_buf = unifyfs_superblock_shmget( + shm_super_size, unifyfs_mount_shmget_key); if (shm_super_buf == NULL) { - LOGERR("unifycr_superblock_shmget() failed"); - return UNIFYCR_FAILURE; + LOGERR("unifyfs_superblock_shmget() failed"); + return UNIFYFS_FAILURE; } /* initialize spillover store */ - if (unifycr_use_spillover) { + if (unifyfs_use_spillover) { /* get directory in which to create spill over files */ cfgval = client_cfg.spillover_data_dir; if (cfgval != NULL) { strncpy(external_data_dir, cfgval, sizeof(external_data_dir)); } else { - LOGERR("UNIFYCR_SPILLOVER_DATA_DIR not set, must be an existing" + LOGERR("UNIFYFS_SPILLOVER_DATA_DIR not set, must be an existing" " writable path (e.g., /mnt/ssd):"); - return UNIFYCR_FAILURE; + return UNIFYFS_FAILURE; } /* define path to the spill over file for data chunks */ - char spillfile_prefix[UNIFYCR_MAX_FILENAME]; + char spillfile_prefix[UNIFYFS_MAX_FILENAME]; snprintf(spillfile_prefix, sizeof(spillfile_prefix), "%s/spill_%d_%d.log", external_data_dir, app_id, local_rank_idx); /* create the spill over file */ - unifycr_spilloverblock = - unifycr_get_spillblock(unifycr_spillover_size, + unifyfs_spilloverblock = + unifyfs_get_spillblock(unifyfs_spillover_size, spillfile_prefix); - if (unifycr_spilloverblock < 0) { - LOGERR("unifycr_get_spillblock() failed!"); - return UNIFYCR_FAILURE; + if (unifyfs_spilloverblock < 0) { + LOGERR("unifyfs_get_spillblock() failed!"); + return UNIFYFS_FAILURE; } /* get directory in which to create spill over files @@ -2030,9 +2025,9 @@ static int unifycr_init(int rank) if (cfgval != NULL) { strncpy(external_meta_dir, cfgval, sizeof(external_meta_dir)); } else { - LOGERR("UNIFYCR_SPILLOVER_META_DIR not set, must be an existing" + LOGERR("UNIFYFS_SPILLOVER_META_DIR not set, must be an existing" " writable path (e.g., /mnt/ssd):"); - return UNIFYCR_FAILURE; + return UNIFYFS_FAILURE; } /* define path to the spill over file for key/value pairs */ @@ -2041,77 +2036,62 @@ static int unifycr_init(int rank) external_meta_dir, app_id, local_rank_idx); /* create the spill over file for key value data */ - unifycr_spillmetablock = - unifycr_get_spillblock(unifycr_index_buf_size, + unifyfs_spillmetablock = + unifyfs_get_spillblock(unifyfs_index_buf_size, spillfile_prefix); - if (unifycr_spillmetablock < 0) { - LOGERR("unifycr_get_spillmetablock failed!"); - return UNIFYCR_FAILURE; + if (unifyfs_spillmetablock < 0) { + LOGERR("unifyfs_get_spillmetablock failed!"); + return UNIFYFS_FAILURE; } } /* remember that we've now initialized the library */ - unifycr_initialized = 1; + unifyfs_initialized = 1; } - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /* --------------------------------------- * APIs exposed to external libraries * --------------------------------------- */ -/** - * Transfer the client-side context information to the corresponding - * delegator on the server side. - */ -int unifycr_sync_to_del(unifycr_mount_in_t* in) +/* Fill mount rpc input struct with client-side context info */ +void fill_client_mount_info(unifyfs_mount_in_t* in) { - int num_procs_per_node = local_rank_cnt; - - size_t req_buf_sz = shm_req_size; - size_t recv_buf_sz = shm_recv_size; - size_t superblock_sz = shm_super_size; - - void* meta_start = (void*)unifycr_indices.ptr_num_entries; - size_t meta_offset = meta_start - shm_super_buf; - size_t meta_size = unifycr_max_index_entries - * sizeof(unifycr_index_t); - - void* fmeta_start = (void*)unifycr_fattrs.ptr_num_entries; - size_t fmeta_offset = fmeta_start - shm_super_buf; - size_t fmeta_size = unifycr_max_fattr_entries - * sizeof(unifycr_file_attr_t); + size_t meta_offset = (char*)unifyfs_indices.ptr_num_entries - + (char*)shm_super_buf; + size_t meta_size = unifyfs_max_index_entries + * sizeof(unifyfs_index_t); - void* data_start = (void*)unifycr_chunks; - size_t data_offset = data_start - shm_super_buf; - size_t data_size = (size_t)unifycr_max_chunks * unifycr_chunk_size; + size_t fmeta_offset = (char*)unifyfs_fattrs.ptr_num_entries - + (char*)shm_super_buf; + size_t fmeta_size = unifyfs_max_fattr_entries + * sizeof(unifyfs_file_attr_t); - char* external_spill_dir = malloc(UNIFYCR_MAX_FILENAME); - strcpy(external_spill_dir, external_data_dir); + size_t data_offset = (char*)unifyfs_chunks - (char*)shm_super_buf; + size_t data_size = (size_t)unifyfs_max_chunks * unifyfs_chunk_size; in->app_id = app_id; in->local_rank_idx = local_rank_idx; - in->dbg_rank = glb_rank; - in->num_procs_per_node = num_procs_per_node; - in->req_buf_sz = req_buf_sz; - in->recv_buf_sz = recv_buf_sz; - in->superblock_sz = superblock_sz; + in->dbg_rank = client_rank; + in->num_procs_per_node = local_rank_cnt; + in->req_buf_sz = shm_req_size; + in->recv_buf_sz = shm_recv_size; + in->superblock_sz = shm_super_size; in->meta_offset = meta_offset; in->meta_size = meta_size; in->fmeta_offset = fmeta_offset; in->fmeta_size = fmeta_size; in->data_offset = data_offset; in->data_size = data_size; - in->external_spill_dir = external_spill_dir; - - return UNIFYCR_SUCCESS; + in->external_spill_dir = strdup(external_data_dir); } /** * Initialize the shared recv memory buffer to receive data from the delegators */ -static int unifycr_init_recv_shm(int local_rank_idx, int app_id) +static int unifyfs_init_recv_shm(int local_rank_idx, int app_id) { /* get size of shared memory region from configuration */ char* cfgval = client_cfg.shmem_recv_size; @@ -2128,16 +2108,13 @@ static int unifycr_init_recv_shm(int local_rank_idx, int app_id) "%d-recv-%d", app_id, local_rank_idx); /* allocate memory for shared memory receive buffer */ - shm_recv_buf = unifycr_shm_alloc(shm_recv_name, shm_recv_size); + shm_recv_buf = unifyfs_shm_alloc(shm_recv_name, shm_recv_size); if (shm_recv_buf == NULL) { LOGERR("Failed to create buffer for read replies"); - return UNIFYCR_FAILURE; + return UNIFYFS_FAILURE; } - /* what is this for? */ - *((int*)shm_recv_buf) = app_id + 3; - - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } /** @@ -2150,7 +2127,7 @@ static int unifycr_init_recv_shm(int local_rank_idx, int app_id) * process is from * @return success/error code */ -static int unifycr_init_req_shm(int local_rank_idx, int app_id) +static int unifyfs_init_req_shm(int local_rank_idx, int app_id) { /* get size of shared memory region from configuration */ char* cfgval = client_cfg.shmem_req_size; @@ -2167,91 +2144,17 @@ static int unifycr_init_req_shm(int local_rank_idx, int app_id) "%d-req-%d", app_id, local_rank_idx); /* allocate memory for shared memory receive buffer */ - shm_req_buf = unifycr_shm_alloc(shm_req_name, shm_req_size); + shm_req_buf = unifyfs_shm_alloc(shm_req_name, shm_req_size); if (shm_req_buf == NULL) { LOGERR("Failed to create buffer for read requests"); - return UNIFYCR_FAILURE; + return UNIFYFS_FAILURE; } - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } -/* unifycr_init_rpc is replacing unifycr_init_socket */ -static int unifycr_client_rpc_init( - char* svr_addr_str, - unifycr_client_rpc_context_t** unifycr_rpc_context) -{ - *unifycr_rpc_context = malloc(sizeof(unifycr_client_rpc_context_t)); - assert(*unifycr_rpc_context); - - /* parse address string to determine the protocol */ - char proto[12] = {0}; - int i; - for (i = 0; - i < 11 && svr_addr_str[i] != '\0' && svr_addr_str[i] != ':'; - i++) { - proto[i] = svr_addr_str[i]; - } - - /* initialize margo */ - LOGDBG("svr_addr_str:%s", svr_addr_str); - (*unifycr_rpc_context)->mid = margo_init(proto, MARGO_CLIENT_MODE, - 1, 0); - assert((*unifycr_rpc_context)->mid); - margo_diag_start((*unifycr_rpc_context)->mid); - - /* register read rpc with mercury */ - (*unifycr_rpc_context)->unifycr_read_rpc_id = - MARGO_REGISTER((*unifycr_rpc_context)->mid, "unifycr_read_rpc", - unifycr_read_in_t, - unifycr_read_out_t, - NULL); - - (*unifycr_rpc_context)->unifycr_mread_rpc_id = - MARGO_REGISTER((*unifycr_rpc_context)->mid, "unifycr_mread_rpc", - unifycr_mread_in_t, - unifycr_mread_out_t, - NULL); - - (*unifycr_rpc_context)->unifycr_mount_rpc_id = - MARGO_REGISTER((*unifycr_rpc_context)->mid, "unifycr_mount_rpc", - unifycr_mount_in_t, - unifycr_mount_out_t, NULL); - - (*unifycr_rpc_context)->unifycr_unmount_rpc_id = - MARGO_REGISTER((*unifycr_rpc_context)->mid, "unifycr_unmount_rpc", - unifycr_unmount_in_t, - unifycr_unmount_out_t, NULL); - - (*unifycr_rpc_context)->unifycr_metaget_rpc_id = - MARGO_REGISTER((*unifycr_rpc_context)->mid, "unifycr_metaget_rpc", - unifycr_metaget_in_t, unifycr_metaget_out_t, - NULL); - - (*unifycr_rpc_context)->unifycr_metaset_rpc_id = - MARGO_REGISTER((*unifycr_rpc_context)->mid, "unifycr_metaset_rpc", - unifycr_metaset_in_t, unifycr_metaset_out_t, - NULL); - - (*unifycr_rpc_context)->unifycr_fsync_rpc_id = - MARGO_REGISTER((*unifycr_rpc_context)->mid, "unifycr_fsync_rpc", - unifycr_fsync_in_t, unifycr_fsync_out_t, - NULL); - - (*unifycr_rpc_context)->unifycr_filesize_rpc_id = - MARGO_REGISTER((*unifycr_rpc_context)->mid, "unifycr_filesize_rpc", - unifycr_filesize_in_t, - unifycr_filesize_out_t, - NULL); - - /* resolve server address */ - (*unifycr_rpc_context)->svr_addr = HG_ADDR_NULL; - int ret = margo_addr_lookup((*unifycr_rpc_context)->mid, svr_addr_str, - &((*unifycr_rpc_context)->svr_addr)); - - return UNIFYCR_SUCCESS; -} +#if defined(UNIFYFS_USE_DOMAIN_SOCKET) /** * initialize the client-side socket * used to communicate with the server-side @@ -2264,8 +2167,7 @@ static int unifycr_client_rpc_init( * delegators on the same node * @return success/error code */ - -static int unifycr_init_socket(int proc_id, int l_num_procs_per_node, +static int unifyfs_init_socket(int proc_id, int l_num_procs_per_node, int l_num_del_per_node) { int rc = -1; @@ -2274,10 +2176,8 @@ static int unifycr_init_socket(int proc_id, int l_num_procs_per_node, int result; int flag; struct sockaddr_un serv_addr; - char tmp_path[UNIFYCR_MAX_FILENAME] = {0}; -#ifdef HAVE_PMIX_H - char* pmix_path = NULL; -#endif + char tmp_path[UNIFYFS_MAX_FILENAME] = {0}; + char* pmi_path = NULL; client_sockfd = socket(AF_UNIX, SOCK_STREAM, 0); if (client_sockfd < 0) { @@ -2293,14 +2193,12 @@ static int unifycr_init_socket(int proc_id, int l_num_procs_per_node, snprintf(tmp_path, sizeof(tmp_path), "%s.%d.%d", SOCKET_PATH, getuid(), (proc_id / nprocs_per_del)); -#ifdef HAVE_PMIX_H - // lookup domain socket path in PMIx - if (unifycr_pmix_lookup(pmix_key_unifycrd_socket, 0, &pmix_path) == 0) { + // lookup domain socket path in key-val store + if (unifyfs_keyval_lookup_local(key_unifyfsd_socket, &pmi_path) == 0) { memset(tmp_path, 0, sizeof(tmp_path)); - snprintf(tmp_path, sizeof(tmp_path), "%s", pmix_path); - free(pmix_path); + snprintf(tmp_path, sizeof(tmp_path), "%s", pmi_path); + free(pmi_path); } -#endif memset(&serv_addr, 0, sizeof(serv_addr)); serv_addr.sun_family = AF_UNIX; @@ -2324,11 +2222,12 @@ static int unifycr_init_socket(int proc_id, int l_num_procs_per_node, return 0; } +#endif // UNIFYFS_USE_DOMAIN_SOCKET int compare_fattr(const void* a, const void* b) { - const unifycr_file_attr_t* ptr_a = a; - const unifycr_file_attr_t* ptr_b = b; + const unifyfs_file_attr_t* ptr_a = a; + const unifyfs_file_attr_t* ptr_b = b; if (ptr_a->fid > ptr_b->fid) { return 1; @@ -2374,234 +2273,166 @@ static int compare_name_rank_pair(const void* a, const void* b) } /** - * find the local index of a given rank among all ranks - * collocated on the same node - * @param local_rank_lst: a list of local ranks - * @param local_rank_cnt: number of local ranks - * @return index of rank in local_rank_lst - */ -static int find_rank_idx(int rank, int* local_rank_lst, int local_rank_cnt) -{ - int i; - - for (i = 0; i < local_rank_cnt; i++) { - if (local_rank_lst[i] == rank) { - return i; - } - } - - return -1; -} - -/** - * calculate the number of ranks per node, + * calculate the number of ranks per node + * + * sets global variables local_rank_cnt & local_rank_idx * * @param numTasks: number of tasks in the application * @return success/error code - * @return local_rank_lst: a list of local ranks - * @return local_rank_cnt: number of local ranks */ static int CountTasksPerNode(int rank, int numTasks) { - char hostname[HOST_NAME_MAX]; - char localhost[HOST_NAME_MAX]; - int resultsLen = 30; + char hostname[UNIFYFS_MAX_HOSTNAME]; + char localhost[UNIFYFS_MAX_HOSTNAME]; + int resultsLen = UNIFYFS_MAX_HOSTNAME; MPI_Status status; - int rc; + int i, j, rc; + int* local_rank_lst; + + if (numTasks <= 0) { + LOGERR("invalid number of tasks"); + return -1; + } rc = MPI_Get_processor_name(localhost, &resultsLen); if (rc != 0) { LOGERR("failed to get the processor's name"); } - if (numTasks > 0) { - if (rank == 0) { - int i; - /* a container of (rank, host) mappings*/ - name_rank_pair_t* host_set = - (name_rank_pair_t*)malloc(numTasks - * sizeof(name_rank_pair_t)); - /* - * MPI_receive all hostnames, and compare to local hostname - * TODO: handle the case when the length of hostname is larger - * than 30 - */ - for (i = 1; i < numTasks; i++) { - rc = MPI_Recv(hostname, HOST_NAME_MAX, - MPI_CHAR, MPI_ANY_SOURCE, - MPI_ANY_TAG, MPI_COMM_WORLD, - &status); - - if (rc != 0) { - LOGERR("cannot receive hostnames"); - return -1; - } - strcpy(host_set[i].hostname, hostname); - host_set[i].rank = status.MPI_SOURCE; - } - strcpy(host_set[0].hostname, localhost); - host_set[0].rank = 0; - - /*sort according to the hostname*/ - qsort(host_set, numTasks, sizeof(name_rank_pair_t), - compare_name_rank_pair); - - /* - * rank_cnt: records the number of processes on each node - * rank_set: the list of ranks for each node - */ - int** rank_set = (int**)malloc(numTasks * sizeof(int*)); - int* rank_cnt = (int*)malloc(numTasks * sizeof(int)); - int cursor = 0, set_counter = 0; - - for (i = 1; i < numTasks; i++) { - if (strcmp(host_set[i].hostname, - host_set[i - 1].hostname) == 0) { - /*do nothing*/ - } else { - // find a different rank, so switch to a new set - int j, k = 0; - - rank_set[set_counter] = - (int*)malloc((i - cursor) * sizeof(int)); - rank_cnt[set_counter] = i - cursor; - for (j = cursor; j <= i - 1; j++) { - - rank_set[set_counter][k] = host_set[j].rank; - k++; - } - - set_counter++; - cursor = i; - } - - } + if (rank == 0) { + /* a container of (rank, host) mappings*/ + name_rank_pair_t* host_set = + (name_rank_pair_t*)calloc(numTasks, + sizeof(name_rank_pair_t)); - /* fill rank_cnt and rank_set entry for the last node */ - int j = 0; + strcpy(host_set[0].hostname, localhost); + host_set[0].rank = 0; - rank_set[set_counter] = malloc((i - cursor) * sizeof(int)); - rank_cnt[set_counter] = numTasks - cursor; - for (i = cursor; i <= numTasks - 1; i++) { - rank_set[set_counter][j] = host_set[i].rank; - j++; - } - set_counter++; - - /* broadcast the rank_cnt and rank_set information to each rank */ - int root_set_no = -1; - - for (i = 0; i < set_counter; i++) { - for (j = 0; j < rank_cnt[i]; j++) { - if (rank_set[i][j] != 0) { - rc = MPI_Send(&rank_cnt[i], 1, MPI_INT, rank_set[i][j], - 0, MPI_COMM_WORLD); - if (rc != 0) { - LOGERR("cannot send local rank cnt"); - return -1; - } - - /*send the local rank set to the corresponding rank*/ - rc = MPI_Send(rank_set[i], rank_cnt[i], MPI_INT, - rank_set[i][j], 0, MPI_COMM_WORLD); - if (rc != 0) { - LOGERR("cannot send local rank list"); - return -1; - } - } else { - root_set_no = i; - } - } + /* + * MPI_Recv all hostnames, and compare to local hostname + */ + for (i = 1; i < numTasks; i++) { + rc = MPI_Recv(hostname, UNIFYFS_MAX_HOSTNAME, + MPI_CHAR, MPI_ANY_SOURCE, + MPI_ANY_TAG, MPI_COMM_WORLD, + &status); + if (rc != 0) { + LOGERR("cannot receive hostnames"); + return -1; } + strcpy(host_set[i].hostname, hostname); + host_set[i].rank = status.MPI_SOURCE; + } + /* sort by hostname */ + qsort(host_set, numTasks, sizeof(name_rank_pair_t), + compare_name_rank_pair); - /* root process set its own local rank set and rank_cnt*/ - if (root_set_no >= 0) { - local_rank_lst = malloc(rank_cnt[root_set_no] * sizeof(int)); - for (i = 0; i < rank_cnt[root_set_no]; i++) { - local_rank_lst[i] = rank_set[root_set_no][i]; + /* + * rank_cnt: records the number of processes on each node + * rank_set: the list of ranks for each node + */ + int** rank_set = (int**)calloc(numTasks, sizeof(int*)); + int* rank_cnt = (int*)calloc(numTasks, sizeof(int)); + int cursor = 0; + int set_counter = 0; + + for (i = 1; i < numTasks; i++) { + if (strcmp(host_set[i].hostname, + host_set[i - 1].hostname) != 0) { + // found a different host, so switch to a new set + rank_set[set_counter] = + (int*)calloc((i - cursor), sizeof(int)); + rank_cnt[set_counter] = i - cursor; + int hiter, riter = 0; + for (hiter = cursor; hiter < i; hiter++, riter++) { + rank_set[set_counter][riter] = host_set[hiter].rank; } - local_rank_cnt = rank_cnt[root_set_no]; - } - - for (i = 0; i < set_counter; i++) { - free(rank_set[i]); - } - - free(rank_cnt); - free(host_set); - free(rank_set); - } else { - /* - * non-root process performs MPI_send to send hostname to root node - */ - rc = MPI_Send(localhost, HOST_NAME_MAX, MPI_CHAR, - 0, 0, MPI_COMM_WORLD); - if (rc != 0) { - LOGERR("cannot send host name"); - return -1; - } - /*receive the local rank count */ - rc = MPI_Recv(&local_rank_cnt, 1, MPI_INT, - 0, 0, MPI_COMM_WORLD, &status); - if (rc != 0) { - LOGERR("cannot receive local rank cnt"); - return -1; + set_counter++; + cursor = i; } + } - /* receive the the local rank list */ - local_rank_lst = (int*)malloc(local_rank_cnt * sizeof(int)); - rc = MPI_Recv(local_rank_lst, local_rank_cnt, MPI_INT, - 0, 0, MPI_COMM_WORLD, &status); - if (rc != 0) { - free(local_rank_lst); - LOGERR("cannot receive local rank list"); - return -1; + /* fill rank_cnt and rank_set entry for the last node */ + rank_set[set_counter] = (int*)calloc((i - cursor), sizeof(int)); + rank_cnt[set_counter] = numTasks - cursor; + j = 0; + for (i = cursor; i < numTasks; i++, j++) { + rank_set[set_counter][j] = host_set[i].rank; + } + set_counter++; + + /* broadcast the rank_cnt and rank_set information to each rank */ + int root_set_no = -1; + for (i = 0; i < set_counter; i++) { + /* send each rank set to all of its ranks */ + for (j = 0; j < rank_cnt[i]; j++) { + if (rank_set[i][j] != 0) { + rc = MPI_Send(&rank_cnt[i], 1, MPI_INT, rank_set[i][j], + 0, MPI_COMM_WORLD); + if (rc != 0) { + LOGERR("cannot send local rank cnt"); + return -1; + } + rc = MPI_Send(rank_set[i], rank_cnt[i], MPI_INT, + rank_set[i][j], 0, MPI_COMM_WORLD); + if (rc != 0) { + LOGERR("cannot send local rank list"); + return -1; + } + } else { + root_set_no = i; + local_rank_cnt = rank_cnt[i]; + local_rank_lst = (int*)calloc(rank_cnt[i], sizeof(int)); + memcpy(local_rank_lst, rank_set[i], + (local_rank_cnt * sizeof(int))); + } } - } - qsort(local_rank_lst, local_rank_cnt, sizeof(int), - compare_int); - - // scatter ranks out + for (i = 0; i < set_counter; i++) { + free(rank_set[i]); + } + free(rank_cnt); + free(host_set); + free(rank_set); } else { - LOGERR("number of tasks is smaller than 0"); - return -1; - } - - return 0; -} - -/* returns server rpc address in newly allocated string - * to be freed by caller, returns NULL if not found */ -static char* addr_lookup_server_rpc() -{ - /* returns NULL if we can't find server address */ - char* str = NULL; - - /* TODO: support other lookup methods here like PMIX */ - - /* read server address string from well-known file name in ramdisk */ - FILE* fp = fopen("/dev/shm/svr_id", "r"); - if (fp != NULL) { - /* opened the file, now read the address string */ - char addr_string[50]; - int rc = fscanf(fp, "%s", addr_string); - if (rc == 1) { - /* read the server address, dup a copy of it */ - str = strdup(addr_string); + /* non-root process - MPI_Send hostname to root node */ + rc = MPI_Send(localhost, UNIFYFS_MAX_HOSTNAME, MPI_CHAR, + 0, 0, MPI_COMM_WORLD); + if (rc != 0) { + LOGERR("cannot send host name"); + return -1; + } + /* receive the local rank set count */ + rc = MPI_Recv(&local_rank_cnt, 1, MPI_INT, + 0, 0, MPI_COMM_WORLD, &status); + if (rc != 0) { + LOGERR("cannot receive local rank cnt"); + return -1; + } + /* receive the the local rank set */ + local_rank_lst = (int*)calloc(local_rank_cnt, sizeof(int)); + rc = MPI_Recv(local_rank_lst, local_rank_cnt, MPI_INT, + 0, 0, MPI_COMM_WORLD, &status); + if (rc != 0) { + free(local_rank_lst); + LOGERR("cannot receive local rank list"); + return -1; } - fclose(fp); } - /* print server address (debugging) */ - if (str != NULL) { - LOGDBG("rpc address: %s", str); + /* sort local ranks by rank */ + qsort(local_rank_lst, local_rank_cnt, sizeof(int), compare_int); + for (i = 0; i < local_rank_cnt; i++) { + if (local_rank_lst[i] == rank) { + local_rank_idx = i; + break; + } } - - return str; + free(local_rank_lst); + return 0; } /** @@ -2613,38 +2444,62 @@ static char* addr_lookup_server_rpc() * @param l_app_id: application ID * @return success/error code */ -int unifycr_mount(const char prefix[], int rank, size_t size, +int unifyfs_mount(const char prefix[], int rank, size_t size, int l_app_id) { int rc; + int kv_rank, kv_nranks; bool b; char* cfgval; - glb_rank = rank; - - /* print log messages to stderr */ - unifycr_log_open(NULL); + if (-1 != unifyfs_mounted) { + if (l_app_id != unifyfs_mounted) { + LOGERR("multiple mount support not yet implemented"); + return UNIFYFS_FAILURE; + } else { + LOGDBG("already mounted"); + return UNIFYFS_SUCCESS; + } + } /* record our rank for debugging messages, * record the value we should use for an app_id */ app_id = l_app_id; + client_rank = rank; + global_rank_cnt = (int)size; + + /* print log messages to stderr */ + unifyfs_log_open(NULL); /************************ * read configuration values ************************/ // initialize configuration - rc = unifycr_config_init(&client_cfg, 0, NULL); + rc = unifyfs_config_init(&client_cfg, 0, NULL); if (rc) { LOGERR("failed to initialize configuration."); - return UNIFYCR_FAILURE; + return UNIFYFS_FAILURE; } + client_cfg.ptype = UNIFYFS_CLIENT; // update configuration from runstate file - rc = unifycr_read_runstate(&client_cfg, NULL); + rc = unifyfs_read_runstate(&client_cfg, NULL); + if (rc) { + LOGERR("failed to update configuration from runstate."); + return UNIFYFS_FAILURE; + } + + // initialize k-v store access + kv_rank = client_rank; + kv_nranks = size; + rc = unifyfs_keyval_init(&client_cfg, &kv_rank, &kv_nranks); if (rc) { LOGERR("failed to update configuration from runstate."); - return UNIFYCR_FAILURE; + return UNIFYFS_FAILURE; + } + if ((client_rank != kv_rank) || (size != kv_nranks)) { + LOGDBG("mismatch on mount vs kvstore rank/size"); } /************************ @@ -2654,15 +2509,15 @@ int unifycr_mount(const char prefix[], int rank, size_t size, /* record a copy of the prefix string defining the mount point * we should intercept */ - unifycr_mount_prefix = strdup(prefix); - unifycr_mount_prefixlen = strlen(unifycr_mount_prefix); + unifyfs_mount_prefix = strdup(prefix); + unifyfs_mount_prefixlen = strlen(unifyfs_mount_prefix); /* - * unifycr_mount_shmget_key marks the start of + * unifyfs_mount_shmget_key marks the start of * the superblock shared memory of each rank * each process has three types of shared memory: * request memory, recv memory and superblock - * memory. We set unifycr_mount_shmget_key in + * memory. We set unifyfs_mount_shmget_key in * this way to avoid different ranks conflicting * on the same name in shm_open. */ @@ -2670,164 +2525,128 @@ int unifycr_mount(const char prefix[], int rank, size_t size, if (cfgval != NULL) { rc = configurator_bool_val(cfgval, &b); if ((rc == 0) && b) { - unifycr_use_single_shm = 1; + unifyfs_use_single_shm = 1; } } /* compute our local rank on the node, - * the following call initializes local_rank_{lst,cnt} */ + * the following call initializes local_rank_{cnt,ndx} */ rc = CountTasksPerNode(rank, size); if (rc < 0) { LOGERR("cannot get the local rank list."); return -1; } - local_rank_idx = find_rank_idx(rank, - local_rank_lst, local_rank_cnt); /* use our local rank on the node in shared memory and file * names to avoid conflicting with other procs on our node */ - unifycr_mount_shmget_key = local_rank_idx; + unifyfs_mount_shmget_key = local_rank_idx; /* initialize our library, creates superblock and spillover files */ - int ret = unifycr_init(rank); - if (ret != UNIFYCR_SUCCESS) { + int ret = unifyfs_init(rank); + if (ret != UNIFYFS_SUCCESS) { return ret; } - /************************ - * establish connection to server - ************************/ - /* open rpc connection to server */ - char* addr_string = addr_lookup_server_rpc(); - if (addr_string == NULL) { - LOGERR("Failed to find server address"); - return UNIFYCR_FAILURE; + ret = unifyfs_client_rpc_init(); + if (ret != UNIFYFS_SUCCESS) { + LOGERR("Failed to initialize client RPC"); + return ret; } - unifycr_client_rpc_init(addr_string, &unifycr_rpc_context); - free(addr_string); - /* call client rpc function here (which calls unifycr_sync_to_del) + /* call client mount rpc function here * to register our shared memory and files with server */ LOGDBG("calling mount"); - unifycr_client_mount_rpc_invoke(&unifycr_rpc_context); + invoke_client_mount_rpc(); +#if defined(UNIFYFS_USE_DOMAIN_SOCKET) /* open a socket to the server */ - rc = unifycr_init_socket(local_rank_idx, local_rank_cnt, + rc = unifyfs_init_socket(local_rank_idx, local_rank_cnt, local_del_cnt); if (rc < 0) { LOGERR("failed to initialize socket, rc == %d", rc); - return UNIFYCR_FAILURE; + return UNIFYFS_FAILURE; } +#endif /* create shared memory region for read requests */ - rc = unifycr_init_req_shm(local_rank_idx, app_id); + rc = unifyfs_init_req_shm(local_rank_idx, app_id); if (rc < 0) { LOGERR("failed to init shared request memory"); - return UNIFYCR_FAILURE; + return UNIFYFS_FAILURE; } /* create shared memory region for holding data for read replies */ - rc = unifycr_init_recv_shm(local_rank_idx, app_id); + rc = unifyfs_init_recv_shm(local_rank_idx, app_id); if (rc < 0) { LOGERR("failed to init shared receive memory"); - return UNIFYCR_FAILURE; + return UNIFYFS_FAILURE; } - /************************ - * create a local entry for our mount point directory - ************************/ - /* add mount point as a new directory in the file list */ - if (unifycr_get_fid_from_path(prefix) >= 0) { - /* we can't mount this location, because it already exists */ - LOGERR("can't mount this location, because it already exists"); - errno = EEXIST; - return -1; - } else { - /* claim an entry in our file list */ - int fid = unifycr_fid_create_directory(prefix); + if (unifyfs_get_fid_from_path(prefix) < 0) { + /* no entry exists for mount point, so create one */ + int fid = unifyfs_fid_create_directory(prefix); if (fid < 0) { /* if there was an error, return it */ - LOGERR("fid %d < 0", fid); - return fid; + LOGERR("failed to create directory entry for mount point: `%s'", + prefix); + return UNIFYFS_FAILURE; } } - return rc; -} - -/* free resources allocated in corresponding call - * to unifycr_client_rpc_init, frees structure - * allocated and sets pcontect to NULL */ -static int unifycr_client_rpc_finalize( - unifycr_client_rpc_context_t** pcontext) -{ - if (pcontext != NULL && *pcontext != NULL) { - /* define a temporary to refer to context */ - unifycr_client_rpc_context_t* ctx = *pcontext; - - /* free margo address to server */ - margo_addr_free(ctx->mid, ctx->svr_addr); - - /* shut down margo */ - margo_finalize(ctx->mid); + /* record client state as mounted for specific app_id */ + unifyfs_mounted = app_id; - /* free memory allocated for context structure, - * and set caller's pointer to NULL */ - free(ctx); - *pcontext = NULL; - } - - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } -/* free resources allocated during unifycr_init, +/* free resources allocated during unifyfs_init, * generally we do this in reverse order that * things were initailized in */ -static int unifycr_finalize(void) +static int unifyfs_finalize(void) { - int rc = UNIFYCR_SUCCESS; + int rc = UNIFYFS_SUCCESS; - if (!unifycr_initialized) { + if (!unifyfs_initialized) { /* not initialized yet, so we shouldn't call finalize */ - return UNIFYCR_FAILURE; + return UNIFYFS_FAILURE; } /* close spillover files */ - if (unifycr_spilloverblock != 0) { - close(unifycr_spilloverblock); - unifycr_spilloverblock = 0; + if (unifyfs_spilloverblock != 0) { + close(unifyfs_spilloverblock); + unifyfs_spilloverblock = 0; } - if (unifycr_spillmetablock != 0) { - close(unifycr_spillmetablock); - unifycr_spillmetablock = 0; + if (unifyfs_spillmetablock != 0) { + close(unifyfs_spillmetablock); + unifyfs_spillmetablock = 0; } /* detach from superblock */ - unifycr_shm_free(shm_super_name, shm_super_size, &shm_super_buf); + unifyfs_shm_free(shm_super_name, shm_super_size, &shm_super_buf); /* free directory stream stack */ - if (unifycr_dirstream_stack != NULL) { - free(unifycr_dirstream_stack); - unifycr_dirstream_stack = NULL; + if (unifyfs_dirstream_stack != NULL) { + free(unifyfs_dirstream_stack); + unifyfs_dirstream_stack = NULL; } /* free file stream stack */ - if (unifycr_stream_stack != NULL) { - free(unifycr_stream_stack); - unifycr_stream_stack = NULL; + if (unifyfs_stream_stack != NULL) { + free(unifyfs_stream_stack); + unifyfs_stream_stack = NULL; } /* free file descriptor stack */ - if (unifycr_fd_stack != NULL) { - free(unifycr_fd_stack); - unifycr_fd_stack = NULL; + if (unifyfs_fd_stack != NULL) { + free(unifyfs_fd_stack); + unifyfs_fd_stack = NULL; } /* no longer initialized, so update the flag */ - unifycr_initialized = 0; + unifyfs_initialized = 0; return rc; } @@ -2838,46 +2657,57 @@ static int unifycr_finalize(void) * one filesystem. * @return success/error code */ -int unifycr_unmount(void) +int unifyfs_unmount(void) { + int rc; + int ret = UNIFYFS_SUCCESS; + + if (-1 == unifyfs_mounted) { + return UNIFYFS_SUCCESS; + } + /************************ * tear down connection to server ************************/ /* detach from shared memory regions */ - unifycr_shm_free(shm_req_name, shm_req_size, &shm_req_buf); - unifycr_shm_free(shm_recv_name, shm_recv_size, &shm_recv_buf); + unifyfs_shm_free(shm_req_name, shm_req_size, &shm_req_buf); + unifyfs_shm_free(shm_recv_name, shm_recv_size, &shm_recv_buf); /* close socket to server */ if (client_sockfd >= 0) { errno = 0; - int close_rc = close(client_sockfd); - if (close_rc != 0) { - LOGERR("Failed to close socket to server rc=%d (%s)", + rc = close(client_sockfd); + if (rc != 0) { + LOGERR("Failed to close() socket to server errno=%d (%s)", errno, strerror(errno)); } } /* invoke unmount rpc to tell server we're disconnecting */ LOGDBG("calling unmount"); - int ret = unifycr_client_unmount_rpc_invoke(&unifycr_rpc_context); + rc = invoke_client_unmount_rpc(); + if (rc) { + LOGERR("client unmount rpc failed"); + ret = UNIFYFS_FAILURE; + } /* free resources allocated in client_rpc_init */ - unifycr_client_rpc_finalize(&unifycr_rpc_context); + unifyfs_client_rpc_finalize(); /************************ * free our mount point, and detach from structures * storing data ************************/ - /* free resources allocated in unifycr_init */ - unifycr_finalize(); + /* free resources allocated in unifyfs_init */ + unifyfs_finalize(); /* free memory tracking our mount prefix string */ - if (unifycr_mount_prefix != NULL) { - free(unifycr_mount_prefix); - unifycr_mount_prefix = NULL; - unifycr_mount_prefixlen = 0; + if (unifyfs_mount_prefix != NULL) { + free(unifyfs_mount_prefix); + unifyfs_mount_prefix = NULL; + unifyfs_mount_prefixlen = 0; } /************************ @@ -2885,13 +2715,255 @@ int unifycr_unmount(void) ************************/ /* clean up configuration */ - int tmp_rc = unifycr_config_fini(&client_cfg); - if (tmp_rc) { - ret = UNIFYCR_FAILURE; + rc = unifyfs_config_fini(&client_cfg); + if (rc) { + LOGERR("unifyfs_config_fini() failed"); + ret = UNIFYFS_FAILURE; } /* shut down our logging */ - unifycr_log_close(); + unifyfs_log_close(); + + unifyfs_mounted = -1; + + return ret; +} + +#define UNIFYFS_TX_BUFSIZE (64*(1<<10)) + +enum { + UNIFYFS_TX_STAGE_OUT = 0, + UNIFYFS_TX_STAGE_IN = 1, + UNIFYFS_TX_SERIAL = 0, + UNIFYFS_TX_PARALLEL = 1, +}; + +static +ssize_t do_transfer_data(int fd_src, int fd_dst, off_t offset, size_t count) +{ + ssize_t ret = 0; + off_t pos = 0; + ssize_t n_written = 0; + ssize_t n_left = 0; + ssize_t n_processed = 0; + size_t len = UNIFYFS_TX_BUFSIZE; + char buf[UNIFYFS_TX_BUFSIZE] = { 0, }; + + pos = lseek(fd_src, offset, SEEK_SET); + if (pos == (off_t) -1) { + LOGERR("lseek failed (%d: %s)\n", errno, strerror(errno)); + ret = -1; + goto out; + } + + pos = lseek(fd_dst, offset, SEEK_SET); + if (pos == (off_t) -1) { + LOGERR("lseek failed (%d: %s)\n", errno, strerror(errno)); + ret = -1; + goto out; + } + + while (count > n_processed) { + if (len > count) { + len = count; + } + + n_left = read(fd_src, buf, len); + + if (n_left == 0) { /* EOF */ + break; + } else if (n_left < 0) { /* error */ + ret = errno; + goto out; + } + + do { + n_written = write(fd_dst, buf, n_left); + + if (n_written < 0) { + ret = errno; + goto out; + } else if (n_written == 0 && errno && errno != EAGAIN) { + ret = errno; + goto out; + } + + n_left -= n_written; + n_processed += n_written; + } while (n_left); + } + +out: + return ret; +} + +static int do_transfer_file_serial(const char* src, const char* dst, + struct stat* sb_src, int dir) +{ + int ret = 0; + int fd_src = 0; + int fd_dst = 0; + char buf[UNIFYFS_TX_BUFSIZE] = { 0, }; + + /* + * for now, we do not use the @dir hint. + */ + + fd_src = open(src, O_RDONLY); + if (fd_src < 0) { + return errno; + } + + fd_dst = open(dst, O_CREAT | O_WRONLY | O_TRUNC, 0644); + if (fd_dst < 0) { + ret = errno; + goto out_close_src; + } + + ret = do_transfer_data(fd_src, fd_dst, 0, sb_src->st_size); + if (ret < 0) { + LOGERR("do_transfer_data failed!"); + } else { + fsync(fd_dst); + } + + close(fd_dst); +out_close_src: + close(fd_src); return ret; } + +static int do_transfer_file_parallel(const char* src, const char* dst, + struct stat* sb_src, int dir) +{ + int ret = 0; + int fd_src = 0; + int fd_dst = 0; + uint64_t total_chunks = 0; + uint64_t chunk_start = 0; + uint64_t remainder = 0; + uint64_t n_chunks = 0; + uint64_t offset = 0; + uint64_t len = 0; + uint64_t size = sb_src->st_size; + + fd_src = open(src, O_RDONLY); + if (fd_src < 0) { + return errno; + } + + fd_dst = open(dst, O_CREAT | O_WRONLY | O_TRUNC, 0644); + if (fd_dst < 0) { + ret = errno; + goto out_close_src; + } + + /* + * if the file is smaller than (rankcount*buffersize), just do with the + * serial mode. + * + * FIXME: is this assumtion fair even for the large rank count? + */ + if ((UNIFYFS_TX_BUFSIZE * global_rank_cnt) > size) { + if (client_rank == 0) { + ret = do_transfer_file_serial(src, dst, sb_src, dir); + if (ret) { + LOGERR("do_transfer_file_parallel failed"); + } + + return ret; + } + } + + total_chunks = size / UNIFYFS_TX_BUFSIZE; + if (size % UNIFYFS_TX_BUFSIZE) { + total_chunks++; + } + + n_chunks = total_chunks / global_rank_cnt; + remainder = total_chunks % global_rank_cnt; + + chunk_start = n_chunks * client_rank; + if (client_rank < remainder) { + chunk_start += client_rank; + n_chunks += 1; + } else { + chunk_start += remainder; + } + + offset = chunk_start * UNIFYFS_TX_BUFSIZE; + + if (client_rank == (global_rank_cnt - 1)) { + len = (n_chunks - 1) * UNIFYFS_TX_BUFSIZE; + len += size % UNIFYFS_TX_BUFSIZE; + } else { + len = n_chunks * UNIFYFS_TX_BUFSIZE; + } + + LOGDBG("parallel transfer (%d/%d): offset=%lu, length=%lu", + client_rank, global_rank_cnt, + (unsigned long) offset, (unsigned long) len); + + ret = do_transfer_data(fd_src, fd_dst, offset, len); + + close(fd_dst); +out_close_src: + close(fd_src); + + return ret; +} + +int unifyfs_transfer_file(const char* src, const char* dst, int parallel) +{ + int ret = 0; + int dir = 0; + struct stat sb_src = { 0, }; + struct stat sb_dst = { 0, }; + int unify_src = 0; + int unify_dst = 0; + char dst_path[PATH_MAX] = { 0, }; + char* pos = dst_path; + char* src_path = strdup(src); + + if (!src_path) { + return -ENOMEM; + } + + if (unifyfs_intercept_path(src)) { + dir = UNIFYFS_TX_STAGE_OUT; + unify_src = 1; + } + + ret = UNIFYFS_WRAP(stat)(src, &sb_src); + if (ret < 0) { + return -errno; + } + + pos += sprintf(pos, "%s", dst); + + if (unifyfs_intercept_path(dst)) { + dir = UNIFYFS_TX_STAGE_IN; + unify_dst = 1; + } + + ret = UNIFYFS_WRAP(stat)(dst, &sb_dst); + if (ret == 0 && !S_ISREG(sb_dst.st_mode)) { + if (S_ISDIR(sb_dst.st_mode)) { + sprintf(pos, "/%s", basename((char*) src_path)); + } else { + return -EEXIST; + } + } + + if (unify_src + unify_dst != 1) { + return -EINVAL; + } + + if (parallel) { + return do_transfer_file_parallel(src_path, dst_path, &sb_src, dir); + } else { + return do_transfer_file_serial(src_path, dst_path, &sb_src, dir); + } +} + diff --git a/client/src/unifycr.h b/client/src/unifyfs.h similarity index 58% rename from client/src/unifycr.h rename to client/src/unifyfs.h index 8b2e02b08..a2a11b8d1 100644 --- a/client/src/unifycr.h +++ b/client/src/unifyfs.h @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -40,24 +40,21 @@ * Please also read this file LICENSE.CRUISE */ -#ifndef UNIFYCR_H -#define UNIFYCR_H +#ifndef UNIFYFS_H +#define UNIFYFS_H +#include #include // size_t #include // off_t -#include - -#include "unifycr_const.h" +#include "unifyfs_const.h" -#ifndef HOST_NAME_MAX -# define HOST_NAME_MAX 256 +#ifdef __cplusplus +extern "C" { #endif -/* TODO: namespace C */ - /* linked list of chunk information given to an external library wanting - * to RDMA out a file from UNIFYCR */ + * to RDMA out a file from UNIFYFS */ typedef struct { off_t chunk_id; int location; @@ -66,20 +63,45 @@ typedef struct { struct chunk_list_t* next; } chunk_list_t; -/*data structures defined for unifycr********************/ +/*data structures defined for unifyfs********************/ typedef struct { - char hostname[HOST_NAME_MAX]; + char hostname[UNIFYFS_MAX_HOSTNAME]; int rank; } name_rank_pair_t; -#if 0 -int unifycr_mount(const char prefix[], int rank, size_t size, - int l_app_id, int subtype); -#endif -int unifycr_mount(const char prefix[], int rank, size_t size, +int unifyfs_mount(const char prefix[], int rank, size_t size, int l_app_id); -int unifycr_unmount(void); +int unifyfs_unmount(void); int compare_fattr(const void* a, const void* b); -#endif /* UNIFYCR_H */ +/** + * @brief transfer a single file between unifyfs and other file system. either + * @src or @dst should (not both) specify a unifyfs pathname, i.e., /unifyfs/.. + * + * @param src source file path + * @param dst destination file path + * @param parallel parallel transfer if set (parallel=1) + * + * @return 0 on success, negative errno otherwise. + */ +int unifyfs_transfer_file(const char* src, const char* dst, int parallel); + +static inline +int unifyfs_transfer_file_serial(const char* src, const char* dst) +{ + return unifyfs_transfer_file(src, dst, 0); +} + +static inline +int unifyfs_transfer_file_parallel(const char* src, const char* dst) +{ + return unifyfs_transfer_file(src, dst, 1); +} + + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif /* UNIFYFS_H */ diff --git a/client/src/unifyfsf.c b/client/src/unifyfsf.c new file mode 100644 index 000000000..ffc39d8f5 --- /dev/null +++ b/client/src/unifyfsf.c @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2017-2019, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +/* This file compiles a normalized interface for Fortran in which: + * - Fortran link names are in lower case + * - Fortran link names have a single trailing underscore + * - boolean true is expected to be 1 + */ + +#include +#include +#include + +#include "unifyfs.h" + +/* TODO: enable UNIFYFS_Fint to be configured to be different type */ +typedef int UNIFYFS_Fint; + +#ifdef USE_FORT_STDCALL +# define FORT_CALL __stdcall +#elif defined (USE_FORT_CDECL) +# define FORT_CALL __cdecl +#else +# define FORT_CALL +#endif + +#ifdef USE_FORT_MIXED_STR_LEN +# define FORT_MIXED_LEN_DECL , UNIFYFS_Fint +# define FORT_END_LEN_DECL +# define FORT_MIXED_LEN(a) , UNIFYFS_Fint a +# define FORT_END_LEN(a) +#else +# define FORT_MIXED_LEN_DECL +# define FORT_END_LEN_DECL , UNIFYFS_Fint +# define FORT_MIXED_LEN(a) +# define FORT_END_LEN(a) , UNIFYFS_Fint a +#endif + +#ifdef HAVE_FORTRAN_API +# ifdef FORTRAN_EXPORTS +# define FORTRAN_API __declspec(dllexport) +# else +# define FORTRAN_API __declspec(dllimport) +# endif +#else +# define FORTRAN_API +#endif + +/* convert a Fortran string to a C string, by removing any trailing + * spaces and terminating with a NULL */ +static int unifyfs_fstr2cstr(const char* fstr, int flen, char* cstr, int clen) +{ + int rc = 0; + + /* check that our pointers aren't NULL */ + if ((fstr == NULL) || (cstr == NULL)) { + return 1; + } + + /* determine length of Fortran string after removing trailing spaces */ + while ((flen > 0) && (fstr[flen-1] == ' ')) { + flen--; + } + + /* assume we can copy the whole string */ + int len = flen; + if (flen > (clen - 1)) { + /* Fortran string is longer than C buffer, trucnated copy */ + len = clen - 1; + rc = 1; + } + + /* copy the Fortran string to the C string */ + if (len > 0) { + strncpy(cstr, fstr, len); + } + + /* null-terminate the C string */ + if (len >= 0) { + cstr[len] = '\0'; + } + + return rc; +} + +/* convert a C string to a Fortran string, adding trailing spaces + * as necessary */ +static int unifyfs_cstr2fstr(const char* cstr, char* fstr, int flen) +{ + int rc = 0; + + /* check that our pointers aren't NULL */ + if ((cstr == NULL) || (fstr == NULL)) { + return 1; + } + + /* determine length of C string */ + int clen = strlen(cstr); + + /* copy the characters from the Fortran string to the C string */ + if (clen <= flen) { + /* C string will fit within our Fortran buffer, copy it over */ + if (clen > 0) { + strncpy(fstr, cstr, clen); + } + + /* fill in trailing spaces */ + while (clen < flen) { + fstr[clen] = ' '; + clen++; + } + } else { + /* C string is longer than Fortran buffer, truncated copy */ + strncpy(fstr, cstr, flen); + rc = 1; + } + + return rc; +} + +/*================================================ + * Mount, Unmount + *================================================*/ + +FORTRAN_API +void FORT_CALL unifyfs_mount_(char* prefix FORT_MIXED_LEN(prefix_len), + int* rank, int* size, int* app_id, + int* ierror FORT_END_LEN(prefix_len)) +{ + /* convert name from a Fortran string to C string */ + char prefix_tmp[1024]; + int rc = unifyfs_fstr2cstr(prefix, prefix_len, + prefix_tmp, sizeof(prefix_tmp)); + if (rc != 0) { + *ierror = 1; // !UNIFYFS_SUCCESS + return; + } + + int rank_tmp = *rank; + int size_tmp = *size; + int app_id_tmp = *app_id; + *ierror = unifyfs_mount(prefix_tmp, rank_tmp, size_tmp, app_id_tmp); + return; +} + +FORTRAN_API +void FORT_CALL unifyfs_unmount_(int* ierror) +{ + *ierror = unifyfs_unmount(); + return; +} diff --git a/client/src/unifyfsf.h b/client/src/unifyfsf.h new file mode 100644 index 000000000..645fb2ce7 --- /dev/null +++ b/client/src/unifyfsf.h @@ -0,0 +1,15 @@ +! Copyright (c) 2017, Lawrence Livermore National Security, LLC. +! Produced at the Lawrence Livermore National Laboratory. +! +! Copyright 2017-2019, UT-Battelle, LLC. +! +! LLNL-CODE-741539 +! All rights reserved. +! +! This is the license for UnifyFS. +! For details, see https://github.com/LLNL/UnifyFS. +! Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. +! +! return codes + INTEGER UNIFYFS_SUCCESS + PARAMETER (UNIFYFS_SUCCESS=0) diff --git a/client/unifycr-config.in b/client/unifyfs-config.in similarity index 68% rename from client/unifycr-config.in rename to client/unifyfs-config.in index c3395aa5c..28730f87c 100644 --- a/client/unifycr-config.in +++ b/client/unifyfs-config.in @@ -8,15 +8,15 @@ CP_WRAPPERS="@CP_WRAPPERS@" -UNIFYCR_LIB_PATH="@unifycr_lib_path@" -UNIFYCR_LD_FLAGS="@LDFLAGS@" +UNIFYFS_LIB_PATH="@unifyfs_lib_path@" +UNIFYFS_LD_FLAGS="@LDFLAGS@" -PRE_LD_FLAGS="-L$UNIFYCR_LIB_PATH $UNIFYCR_LD_FLAGS -lz $CP_WRAPPERS" -POST_LD_FLAGS="$UNIFYCR_LIB_PATH/libunifycr.a -lcrypto -lm -lrt -lpthread" +PRE_LD_FLAGS="-L$UNIFYFS_LIB_PATH $UNIFYFS_LD_FLAGS -lz $CP_WRAPPERS" +POST_LD_FLAGS="$UNIFYFS_LIB_PATH/libunifyfs.a -lcrypto -lm -lrt -lpthread" usage="\ -Usage: unifycr-config [--pre-ld-flags] [--post-ld-flags]" +Usage: unifyfs-config [--pre-ld-flags] [--post-ld-flags]" if test $# -eq 0; then echo "${usage}" 1>&2 diff --git a/client/unifycr.pc.in b/client/unifyfs.pc.in similarity index 51% rename from client/unifycr.pc.in rename to client/unifyfs.pc.in index 81c3ca878..04d6e2026 100644 --- a/client/unifycr.pc.in +++ b/client/unifyfs.pc.in @@ -3,10 +3,10 @@ exec_prefix=@exec_prefix@ libdir=@libdir@ includedir=@includedir@ -Name: unifycr -Description: client library for unifycr userspace checkpoint file system -Version: @LIBUNIFYCR_API_VERSION@ +Name: unifyfs +Description: client library for unifyfs userspace checkpoint file system +Version: @LIBUNIFYFS_API_VERSION@ Requires: -Libs: -L${libdir} -lunifycr +Libs: -L${libdir} -lunifyfs Cflags: -I${includedir} -D__FILE_OFFSET_BITS=64 diff --git a/common/src/Makefile.am b/common/src/Makefile.am index c68ee8f38..00dc6ae0a 100644 --- a/common/src/Makefile.am +++ b/common/src/Makefile.am @@ -1,6 +1,8 @@ -lib_LTLIBRARIES = libunifycr_common.la +lib_LTLIBRARIES = libunifyfs_common.la -libunifycr_commondir = $(includedir) +include_HEADERS = unifyfs_const.h err_enumerator.h + +libunifyfs_commondir = $(includedir) BASE_SRCS = \ ini.h \ @@ -11,32 +13,56 @@ BASE_SRCS = \ cm_enumerator.c \ rm_enumerator.h \ rm_enumerator.c \ + flatbuffers_common_builder.h \ + flatbuffers_common_reader.h \ + ucr_read_builder.h \ + ucr_read_reader.h \ tinyexpr.h \ tinyexpr.c \ - unifycr_const.h \ - unifycr_configurator.h \ - unifycr_configurator.c \ - unifycr_runstate.h \ - unifycr_runstate.c \ - unifycr_meta.h \ - unifycr_pmix.h \ - unifycr_log.h \ - unifycr_log.c \ - unifycr_shm.h \ - unifycr_shm.c + unifyfs_const.h \ + unifyfs_configurator.h \ + unifyfs_configurator.c \ + unifyfs_keyval.h \ + unifyfs_keyval.c \ + unifyfs_log.h \ + unifyfs_log.c \ + unifyfs_meta.h \ + unifyfs_rpc_util.h \ + unifyfs_rpc_util.c \ + unifyfs_client_rpcs.h \ + unifyfs_server_rpcs.h \ + unifyfs_runstate.h \ + unifyfs_runstate.c \ + unifyfs_shm.h \ + unifyfs_shm.c -OPT_SRCS = +OPT_FLAGS = OPT_LIBS = +OPT_SRCS = if USE_PMIX - OPT_SRCS += unifycr_pmix.c + OPT_FLAGS += -DUSE_PMIX OPT_LIBS += -lpmix endif -libunifycr_common_la_SOURCES = $(BASE_SRCS) $(OPT_SRCS) +if USE_PMI2 + OPT_FLAGS += -DUSE_PMI2 + OPT_LIBS += -lpmi2 +endif + +libunifyfs_common_la_SOURCES = $(BASE_SRCS) $(OPT_SRCS) + +libunifyfs_common_la_CPPFLAGS = \ + $(OPT_FLAGS) \ + $(MERCURY_CFLAGS) \ + $(ARGOBOTS_CFLAGS) \ + $(MARGO_CFLAGS) \ + $(FLATCC_CFLAGS) + +libunifyfs_common_la_LDFLAGS = \ + -version-info $(LIBUNIFYFS_LT_VERSION) -libunifycr_common_la_CPPFLAGS = -I$(top_builddir)/client -libunifycr_common_la_LDFLAGS = -version-info $(LIBUNIFYCR_LT_VERSION) -libunifycr_common_la_LIBADD = -lm $(OPT_LIBS) -lrt +libunifyfs_common_la_LIBADD = \ + $(OPT_LIBS) -lm -lrt AM_CFLAGS = -Wall -Wno-strict-aliasing diff --git a/common/src/cm_enumerator.c b/common/src/cm_enumerator.c index 13d97d750..23e46c82e 100644 --- a/common/src/cm_enumerator.c +++ b/common/src/cm_enumerator.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* Copyright (c) 2018 - Michael J. Brim @@ -27,19 +27,19 @@ /* c-strings for enum names */ #define ENUMITEM(name, desc) \ - const char *UNIFYCR_CM_ENUM_ ## name ## _NAME_STR = #name; -UNIFYCR_CM_ENUMERATOR + const char *UNIFYFS_CM_ENUM_ ## name ## _NAME_STR = #name; +UNIFYFS_CM_ENUMERATOR #undef ENUMITEM -const char *unifycr_cm_enum_str(unifycr_cm_e e) +const char *unifyfs_cm_enum_str(unifyfs_cm_e e) { switch (e) { - case UNIFYCR_CM_INVALID: - return "UNIFYCR_CM_INVALID"; + case UNIFYFS_CM_INVALID: + return "UNIFYFS_CM_INVALID"; #define ENUMITEM(name, desc) \ - case UNIFYCR_CM_ ## name: \ - return UNIFYCR_CM_ENUM_ ## name ## _NAME_STR; - UNIFYCR_CM_ENUMERATOR + case UNIFYFS_CM_ ## name: \ + return UNIFYFS_CM_ENUM_ ## name ## _NAME_STR; + UNIFYFS_CM_ENUMERATOR #undef ENUMITEM default : break; @@ -50,19 +50,19 @@ const char *unifycr_cm_enum_str(unifycr_cm_e e) /* c-strings for enum descriptions */ #define ENUMITEM(name, desc) \ - const char *UNIFYCR_CM_ENUM_ ## name ## _DESC_STR = #desc; -UNIFYCR_CM_ENUMERATOR + const char *UNIFYFS_CM_ENUM_ ## name ## _DESC_STR = #desc; +UNIFYFS_CM_ENUMERATOR #undef ENUMITEM -const char *unifycr_cm_enum_description(unifycr_cm_e e) +const char *unifyfs_cm_enum_description(unifyfs_cm_e e) { switch (e) { - case UNIFYCR_CM_INVALID: - return "invalid unifycr_cm_e value"; + case UNIFYFS_CM_INVALID: + return "invalid unifyfs_cm_e value"; #define ENUMITEM(name, desc) \ - case UNIFYCR_CM_ ## name: \ - return UNIFYCR_CM_ENUM_ ## name ## _DESC_STR; - UNIFYCR_CM_ENUMERATOR + case UNIFYFS_CM_ ## name: \ + return UNIFYFS_CM_ENUM_ ## name ## _DESC_STR; + UNIFYFS_CM_ENUMERATOR #undef ENUMITEM default : break; @@ -70,24 +70,24 @@ const char *unifycr_cm_enum_description(unifycr_cm_e e) return NULL; } -unifycr_cm_e unifycr_cm_enum_from_str(const char *s) +unifyfs_cm_e unifyfs_cm_enum_from_str(const char *s) { if (0) ; #define ENUMITEM(name, desc) \ else if (strcmp(s, #name) == 0) \ - return UNIFYCR_CM_ ## name; - UNIFYCR_CM_ENUMERATOR; + return UNIFYFS_CM_ ## name; + UNIFYFS_CM_ENUMERATOR; #undef ENUMITEM - return UNIFYCR_CM_INVALID; + return UNIFYFS_CM_INVALID; } /* validity check */ -int check_valid_unifycr_cm_enum(unifycr_cm_e e) +int check_valid_unifyfs_cm_enum(unifyfs_cm_e e) { - return ((e > UNIFYCR_CM_INVALID) && - (e < UNIFYCR_CM_ENUM_MAX) && - (unifycr_cm_enum_str(e) != NULL)); + return ((e > UNIFYFS_CM_INVALID) && + (e < UNIFYFS_CM_ENUM_MAX) && + (unifyfs_cm_enum_str(e) != NULL)); } diff --git a/common/src/cm_enumerator.h b/common/src/cm_enumerator.h index 1687612a8..50b611b73 100644 --- a/common/src/cm_enumerator.h +++ b/common/src/cm_enumerator.h @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* Copyright (c) 2018 - Michael J. Brim @@ -19,8 +19,8 @@ * MIT License - See LICENSE.tedium */ -#ifndef _UNIFYCR_CM_ENUMERATOR_H_ -#define _UNIFYCR_CM_ENUMERATOR_H_ +#ifndef _UNIFYFS_CM_ENUMERATOR_H_ +#define _UNIFYFS_CM_ENUMERATOR_H_ /** * @brief enumerator list expanded many times with varied ENUMITEM() definitions @@ -28,9 +28,9 @@ * @param item name * @param item short description */ -#define UNIFYCR_CM_ENUMERATOR \ +#define UNIFYFS_CM_ENUMERATOR \ ENUMITEM(NONE, "no consistency") \ - ENUMITEM(LAMINATED, "UnifyCR laminated consistency model") \ + ENUMITEM(LAMINATED, "UnifyFS laminated consistency model") \ ENUMITEM(POSIX, "POSIX I/O consistency model") #ifdef __cplusplus @@ -41,36 +41,36 @@ extern "C" { * @brief supported consistency models */ typedef enum { - UNIFYCR_CM_INVALID = 0, + UNIFYFS_CM_INVALID = 0, #define ENUMITEM(name, desc) \ - UNIFYCR_CM_ ## name, - UNIFYCR_CM_ENUMERATOR + UNIFYFS_CM_ ## name, + UNIFYFS_CM_ENUMERATOR #undef ENUMITEM - UNIFYCR_CM_ENUM_MAX -} unifycr_cm_e; + UNIFYFS_CM_ENUM_MAX +} unifyfs_cm_e; /** * @brief get C-string for given consistency model enum value */ -const char *unifycr_cm_enum_str(unifycr_cm_e e); +const char *unifyfs_cm_enum_str(unifyfs_cm_e e); /** * @brief get description for given consistency model enum value */ -const char *unifycr_cm_enum_description(unifycr_cm_e e); +const char *unifyfs_cm_enum_description(unifyfs_cm_e e); /** * @brief check validity of given consistency model enum value */ -int check_valid_unifycr_cm_enum(unifycr_cm_e e); +int check_valid_unifyfs_cm_enum(unifyfs_cm_e e); /** * @brief get enum value for given consistency model C-string */ -unifycr_cm_e unifycr_cm_enum_from_str(const char *s); +unifyfs_cm_e unifyfs_cm_enum_from_str(const char *s); #ifdef __cplusplus } /* extern C */ #endif -#endif /* UNIFYCR_CM_ENUMERATOR_H */ +#endif /* UNIFYFS_CM_ENUMERATOR_H */ diff --git a/common/src/err_enumerator.c b/common/src/err_enumerator.c index d9f6793d2..8f978f175 100644 --- a/common/src/err_enumerator.c +++ b/common/src/err_enumerator.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -42,21 +42,21 @@ /* c-strings for enum names */ #define ENUMITEM(name, desc) \ - const char *UNIFYCR_ERROR_ ## name ## _NAME_STR = #name; -UNIFYCR_ERROR_ENUMERATOR + const char *UNIFYFS_ERROR_ ## name ## _NAME_STR = #name; +UNIFYFS_ERROR_ENUMERATOR #undef ENUMITEM -const char *unifycr_error_enum_str(unifycr_error_e e) +const char *unifyfs_error_enum_str(unifyfs_error_e e) { switch (e) { - case UNIFYCR_FAILURE: - return "UNIFYCR_FAILURE"; - case UNIFYCR_SUCCESS: - return "UNIFYCR_SUCCESS"; + case UNIFYFS_FAILURE: + return "UNIFYFS_FAILURE"; + case UNIFYFS_SUCCESS: + return "UNIFYFS_SUCCESS"; #define ENUMITEM(name, desc) \ - case UNIFYCR_ERROR_ ## name: \ - return UNIFYCR_ERROR_ ## name ## _NAME_STR; - UNIFYCR_ERROR_ENUMERATOR + case UNIFYFS_ERROR_ ## name: \ + return UNIFYFS_ERROR_ ## name ## _NAME_STR; + UNIFYFS_ERROR_ENUMERATOR #undef ENUMITEM default : break; @@ -67,21 +67,21 @@ const char *unifycr_error_enum_str(unifycr_error_e e) /* c-strings for enum descriptions */ #define ENUMITEM(name, desc) \ - const char *UNIFYCR_ERROR_ ## name ## _DESC_STR = #desc; -UNIFYCR_ERROR_ENUMERATOR + const char *UNIFYFS_ERROR_ ## name ## _DESC_STR = #desc; +UNIFYFS_ERROR_ENUMERATOR #undef ENUMITEM -const char *unifycr_error_enum_description(unifycr_error_e e) +const char *unifyfs_error_enum_description(unifyfs_error_e e) { switch (e) { - case UNIFYCR_FAILURE: + case UNIFYFS_FAILURE: return "Failure"; - case UNIFYCR_SUCCESS: + case UNIFYFS_SUCCESS: return "Success"; #define ENUMITEM(name, desc) \ - case UNIFYCR_ERROR_ ## name: \ - return UNIFYCR_ERROR_ ## name ## _DESC_STR; - UNIFYCR_ERROR_ENUMERATOR + case UNIFYFS_ERROR_ ## name: \ + return UNIFYFS_ERROR_ ## name ## _DESC_STR; + UNIFYFS_ERROR_ENUMERATOR #undef ENUMITEM default : break; @@ -89,25 +89,25 @@ const char *unifycr_error_enum_description(unifycr_error_e e) return NULL; } -unifycr_error_e unifycr_error_enum_from_str(const char *s) +unifyfs_error_e unifyfs_error_enum_from_str(const char *s) { if (0) ; #define ENUMITEM(name, desc) \ else if (strcmp(s, #name) == 0) \ - return UNIFYCR_ERROR_ ## name; - UNIFYCR_ERROR_ENUMERATOR; + return UNIFYFS_ERROR_ ## name; + UNIFYFS_ERROR_ENUMERATOR; #undef ENUMITEM - return UNIFYCR_INVALID_ERROR; + return UNIFYFS_INVALID_ERROR; } /* validity check */ -int check_valid_unifycr_error_enum(unifycr_error_e e) +int check_valid_unifyfs_error_enum(unifyfs_error_e e) { - return ((e > UNIFYCR_INVALID_ERROR) && - (e < UNIFYCR_ERROR_MAX) && - (unifycr_error_enum_str(e) != NULL)); + return ((e > UNIFYFS_INVALID_ERROR) && + (e < UNIFYFS_ERROR_MAX) && + (unifyfs_error_enum_str(e) != NULL)); } diff --git a/common/src/err_enumerator.h b/common/src/err_enumerator.h index 6b55fba96..021262ebd 100644 --- a/common/src/err_enumerator.h +++ b/common/src/err_enumerator.h @@ -1,30 +1,15 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2019, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2019, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. - */ - -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * Copyright (c) 2017, Florida State University. Contributions from - * the Computer Architecture and Systems Research Laboratory (CASTL) - * at the Department of Computer Science. - * - * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror - * LLNL-CODE-728877. All rights reserved. - * - * This file is part of burstfs. - * For details, see https://github.com/llnl/burstfs - * Please read https://github.com/llnl/burstfs/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* Copyright (c) 2018 - Michael J. Brim @@ -34,8 +19,9 @@ * MIT License - See LICENSE.tedium */ -#ifndef _UNIFYCR_ERROR_ENUMERATOR_H_ -#define _UNIFYCR_ERROR_ENUMERATOR_H_ +#ifndef _UNIFYFS_ERROR_ENUMERATOR_H_ +#define _UNIFYFS_ERROR_ENUMERATOR_H_ +#include /** * @brief enumerator list expanded many times with varied ENUMITEM() definitions @@ -43,7 +29,7 @@ * @param item name * @param item short description */ -#define UNIFYCR_ERROR_ENUMERATOR \ +#define UNIFYFS_ERROR_ENUMERATOR \ ENUMITEM(ACCEPT, "Failed to accept RDMA connection.") \ ENUMITEM(ADDR, "Failed to parse IP address and port.") \ ENUMITEM(APPCONFIG, "Failed to initialize application config.") \ @@ -64,6 +50,7 @@ ENUMITEM(INVAL, "Invalid argument.") \ ENUMITEM(IO, "Generic I/O error.") \ ENUMITEM(ISDIR, "Invalid operation for directory.") \ + ENUMITEM(MARGO, "Mercury/Argobots operation error.") \ ENUMITEM(MDHIM, "MDHIM operation error.") \ ENUMITEM(MDINIT, "MDHIM initialization error.") \ ENUMITEM(NAMETOOLONG, "Filename is too long.") \ @@ -105,42 +92,49 @@ extern "C" { #endif +/* #define __ELASTERROR if our errno.h doesn't define it for us */ +#ifndef __ELASTERROR +#define __ELASTERROR 2000 +#endif + /** - * @brief supported consistency models + * @brief enum for error codes */ typedef enum { - UNIFYCR_INVALID_ERROR = -2, - UNIFYCR_FAILURE = -1, - UNIFYCR_SUCCESS = 0, + UNIFYFS_INVALID_ERROR = -2, + UNIFYFS_FAILURE = -1, + UNIFYFS_SUCCESS = 0, + /* Start our error numbers after the standard errno.h ones */ + UNIFRFS_START_OF_ERRORS = __ELASTERROR, #define ENUMITEM(name, desc) \ - UNIFYCR_ERROR_ ## name, - UNIFYCR_ERROR_ENUMERATOR + UNIFYFS_ERROR_ ## name, + UNIFYFS_ERROR_ENUMERATOR #undef ENUMITEM - UNIFYCR_ERROR_MAX -} unifycr_error_e; + UNIFYFS_ERROR_MAX +} unifyfs_error_e; /** - * @brief get C-string for given consistency model enum value + * @brief get C-string for given error enum value */ -const char *unifycr_error_enum_str(unifycr_error_e e); +const char *unifyfs_error_enum_str(unifyfs_error_e e); /** - * @brief get description for given consistency model enum value + * @brief get description for given error enum value */ -const char *unifycr_error_enum_description(unifycr_error_e e); +const char *unifyfs_error_enum_description(unifyfs_error_e e); /** - * @brief check validity of given consistency model enum value + * @brief check validity of given error enum value */ -int check_valid_unifycr_error_enum(unifycr_error_e e); +int check_valid_unifyfs_error_enum(unifyfs_error_e e); /** - * @brief get enum value for given consistency model C-string + * @brief get enum value for given error C-string */ -unifycr_error_e unifycr_error_enum_from_str(const char *s); +unifyfs_error_e unifyfs_error_enum_from_str(const char *s); #ifdef __cplusplus } /* extern C */ #endif -#endif /* UNIFYCR_ERROR_ENUMERATOR_H */ +#endif /* UNIFYFS_ERROR_ENUMERATOR_H */ diff --git a/client/src/flatbuffers_common_builder.h b/common/src/flatbuffers_common_builder.h similarity index 100% rename from client/src/flatbuffers_common_builder.h rename to common/src/flatbuffers_common_builder.h diff --git a/client/src/flatbuffers_common_reader.h b/common/src/flatbuffers_common_reader.h similarity index 100% rename from client/src/flatbuffers_common_reader.h rename to common/src/flatbuffers_common_reader.h diff --git a/common/src/rm_enumerator.c b/common/src/rm_enumerator.c index 2b7b88388..efd3a8922 100644 --- a/common/src/rm_enumerator.c +++ b/common/src/rm_enumerator.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* Copyright (c) 2018 - Michael J. Brim @@ -27,19 +27,19 @@ /* c-strings for enum names */ #define ENUMITEM(name, desc) \ - const char *UNIFYCR_RM_ENUM_ ## name ## _NAME_STR = #name; -UNIFYCR_RM_ENUMERATOR + const char *UNIFYFS_RM_ENUM_ ## name ## _NAME_STR = #name; +UNIFYFS_RM_ENUMERATOR #undef ENUMITEM -const char *unifycr_rm_enum_str(unifycr_rm_e e) +const char *unifyfs_rm_enum_str(unifyfs_rm_e e) { switch (e) { - case UNIFYCR_RM_INVALID: - return "UNIFYCR_RM_INVALID"; + case UNIFYFS_RM_INVALID: + return "UNIFYFS_RM_INVALID"; #define ENUMITEM(name, desc) \ - case UNIFYCR_RM_ ## name: \ - return UNIFYCR_RM_ENUM_ ## name ## _NAME_STR; - UNIFYCR_RM_ENUMERATOR + case UNIFYFS_RM_ ## name: \ + return UNIFYFS_RM_ENUM_ ## name ## _NAME_STR; + UNIFYFS_RM_ENUMERATOR #undef ENUMITEM default : break; @@ -50,19 +50,19 @@ const char *unifycr_rm_enum_str(unifycr_rm_e e) /* c-strings for enum descriptions */ #define ENUMITEM(name, desc) \ - const char *UNIFYCR_RM_ENUM_ ## name ## _DESC_STR = #desc; -UNIFYCR_RM_ENUMERATOR + const char *UNIFYFS_RM_ENUM_ ## name ## _DESC_STR = #desc; +UNIFYFS_RM_ENUMERATOR #undef ENUMITEM -const char *unifycr_rm_enum_description(unifycr_rm_e e) +const char *unifyfs_rm_enum_description(unifyfs_rm_e e) { switch (e) { - case UNIFYCR_RM_INVALID: - return "invalid unifycr_rm_e value"; + case UNIFYFS_RM_INVALID: + return "invalid unifyfs_rm_e value"; #define ENUMITEM(name, desc) \ - case UNIFYCR_RM_ ## name: \ - return UNIFYCR_RM_ENUM_ ## name ## _DESC_STR; - UNIFYCR_RM_ENUMERATOR + case UNIFYFS_RM_ ## name: \ + return UNIFYFS_RM_ENUM_ ## name ## _DESC_STR; + UNIFYFS_RM_ENUMERATOR #undef ENUMITEM default : break; @@ -70,24 +70,24 @@ const char *unifycr_rm_enum_description(unifycr_rm_e e) return NULL; } -unifycr_rm_e unifycr_rm_enum_from_str(const char *s) +unifyfs_rm_e unifyfs_rm_enum_from_str(const char *s) { if (0) ; #define ENUMITEM(name, desc) \ else if (strcmp(s, #name) == 0) \ - return UNIFYCR_RM_ ## name; - UNIFYCR_RM_ENUMERATOR; + return UNIFYFS_RM_ ## name; + UNIFYFS_RM_ENUMERATOR; #undef ENUMITEM - return UNIFYCR_RM_INVALID; + return UNIFYFS_RM_INVALID; } /* validity check */ -int check_valid_unifycr_rm_enum(unifycr_rm_e e) +int check_valid_unifyfs_rm_enum(unifyfs_rm_e e) { - return ((e > UNIFYCR_RM_INVALID) && - (e < UNIFYCR_RM_ENUM_MAX) && - (unifycr_rm_enum_str(e) != NULL)); + return ((e > UNIFYFS_RM_INVALID) && + (e < UNIFYFS_RM_ENUM_MAX) && + (unifyfs_rm_enum_str(e) != NULL)); } diff --git a/common/src/rm_enumerator.h b/common/src/rm_enumerator.h index 799177176..4019a5b55 100644 --- a/common/src/rm_enumerator.h +++ b/common/src/rm_enumerator.h @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* Copyright (c) 2018 - Michael J. Brim @@ -19,8 +19,8 @@ * MIT License - See LICENSE.tedium */ -#ifndef _UNIFYCR_RM_ENUMERATOR_H_ -#define _UNIFYCR_RM_ENUMERATOR_H_ +#ifndef _UNIFYFS_RM_ENUMERATOR_H_ +#define _UNIFYFS_RM_ENUMERATOR_H_ /** * @brief enumerator list expanded many times with varied ENUMITEM() definitions @@ -28,7 +28,7 @@ * @param item name * @param item short description */ -#define UNIFYCR_RM_ENUMERATOR \ +#define UNIFYFS_RM_ENUMERATOR \ ENUMITEM(PBS, "Portable Batch System / TORQUE") \ ENUMITEM(SLURM, "SchedMD SLURM") \ ENUMITEM(LSF, "IBM Spectrum LSF") \ @@ -42,36 +42,36 @@ extern "C" { * @brief supported resource managers */ typedef enum { - UNIFYCR_RM_INVALID = 0, + UNIFYFS_RM_INVALID = 0, #define ENUMITEM(name, desc) \ - UNIFYCR_RM_ ## name, - UNIFYCR_RM_ENUMERATOR + UNIFYFS_RM_ ## name, + UNIFYFS_RM_ENUMERATOR #undef ENUMITEM - UNIFYCR_RM_ENUM_MAX -} unifycr_rm_e; + UNIFYFS_RM_ENUM_MAX +} unifyfs_rm_e; /** * @brief get resource manager C-string for given enum value */ -const char *unifycr_rm_enum_str(unifycr_rm_e e); +const char *unifyfs_rm_enum_str(unifyfs_rm_e e); /** * @brief get resource manager description for given enum value */ -const char *unifycr_rm_enum_description(unifycr_rm_e e); +const char *unifyfs_rm_enum_description(unifyfs_rm_e e); /** * @brief check validity of given resource manager enum value */ -int check_valid_unifycr_rm_enum(unifycr_rm_e e); +int check_valid_unifyfs_rm_enum(unifyfs_rm_e e); /** * @brief get resource manager enum value for given C-string */ -unifycr_rm_e unifycr_rm_enum_from_str(const char *s); +unifyfs_rm_e unifyfs_rm_enum_from_str(const char *s); #ifdef __cplusplus } /* extern C */ #endif -#endif /* UNIFYCR_RM_ENUMERATOR_H */ +#endif /* UNIFYFS_RM_ENUMERATOR_H */ diff --git a/common/src/ucr_read_builder.h b/common/src/ucr_read_builder.h new file mode 100644 index 000000000..5b347277a --- /dev/null +++ b/common/src/ucr_read_builder.h @@ -0,0 +1,74 @@ +#ifndef UCR_READ_BUILDER_H +#define UCR_READ_BUILDER_H + +/* Generated by flatcc 0.5.3-pre FlatBuffers schema compiler for C by dvide.com */ + +#ifndef UCR_READ_READER_H +#include "ucr_read_reader.h" +#endif +#ifndef FLATBUFFERS_COMMON_BUILDER_H +#include "flatbuffers_common_builder.h" +#endif +#include "flatcc/flatcc_prologue.h" +#ifndef flatbuffers_identifier +#define flatbuffers_identifier 0 +#endif +#ifndef flatbuffers_extension +#define flatbuffers_extension ".bin" +#endif + +#define __unifyfs_Extent_formal_args , uint32_t v0, uint64_t v1, uint64_t v2 +#define __unifyfs_Extent_call_args , v0, v1, v2 +static inline unifyfs_Extent_t *unifyfs_Extent_assign(unifyfs_Extent_t *p, uint32_t v0, uint64_t v1, uint64_t v2) +{ p->fid = v0; p->offset = v1; p->length = v2; + return p; } +static inline unifyfs_Extent_t *unifyfs_Extent_copy(unifyfs_Extent_t *p, const unifyfs_Extent_t *p2) +{ p->fid = p2->fid; p->offset = p2->offset; p->length = p2->length; + return p; } +static inline unifyfs_Extent_t *unifyfs_Extent_assign_to_pe(unifyfs_Extent_t *p, uint32_t v0, uint64_t v1, uint64_t v2) +{ flatbuffers_uint32_assign_to_pe(&p->fid, v0); flatbuffers_uint64_assign_to_pe(&p->offset, v1); flatbuffers_uint64_assign_to_pe(&p->length, v2); + return p; } +static inline unifyfs_Extent_t *unifyfs_Extent_copy_to_pe(unifyfs_Extent_t *p, const unifyfs_Extent_t *p2) +{ flatbuffers_uint32_copy_to_pe(&p->fid, &p2->fid); flatbuffers_uint64_copy_to_pe(&p->offset, &p2->offset); flatbuffers_uint64_copy_to_pe(&p->length, &p2->length); + return p; } +static inline unifyfs_Extent_t *unifyfs_Extent_assign_from_pe(unifyfs_Extent_t *p, uint32_t v0, uint64_t v1, uint64_t v2) +{ flatbuffers_uint32_assign_from_pe(&p->fid, v0); flatbuffers_uint64_assign_from_pe(&p->offset, v1); flatbuffers_uint64_assign_from_pe(&p->length, v2); + return p; } +static inline unifyfs_Extent_t *unifyfs_Extent_copy_from_pe(unifyfs_Extent_t *p, const unifyfs_Extent_t *p2) +{ flatbuffers_uint32_copy_from_pe(&p->fid, &p2->fid); flatbuffers_uint64_copy_from_pe(&p->offset, &p2->offset); flatbuffers_uint64_copy_from_pe(&p->length, &p2->length); + return p; } +__flatbuffers_build_struct(flatbuffers_, unifyfs_Extent, 24, 8, unifyfs_Extent_identifier, unifyfs_Extent_type_identifier) + +static const flatbuffers_voffset_t __unifyfs_ReadRequest_required[] = { 0 }; +typedef flatbuffers_ref_t unifyfs_ReadRequest_ref_t; +static unifyfs_ReadRequest_ref_t unifyfs_ReadRequest_clone(flatbuffers_builder_t *B, unifyfs_ReadRequest_table_t t); +__flatbuffers_build_table(flatbuffers_, unifyfs_ReadRequest, 1) + +#define __unifyfs_ReadRequest_formal_args , unifyfs_Extent_vec_ref_t v0 +#define __unifyfs_ReadRequest_call_args , v0 +static inline unifyfs_ReadRequest_ref_t unifyfs_ReadRequest_create(flatbuffers_builder_t *B __unifyfs_ReadRequest_formal_args); +__flatbuffers_build_table_prolog(flatbuffers_, unifyfs_ReadRequest, unifyfs_ReadRequest_identifier, unifyfs_ReadRequest_type_identifier) + +__flatbuffers_build_vector_field(0, flatbuffers_, unifyfs_ReadRequest_extents, unifyfs_Extent, unifyfs_Extent_t, unifyfs_ReadRequest) + +static inline unifyfs_ReadRequest_ref_t unifyfs_ReadRequest_create(flatbuffers_builder_t *B __unifyfs_ReadRequest_formal_args) +{ + if (unifyfs_ReadRequest_start(B) + || unifyfs_ReadRequest_extents_add(B, v0)) { + return 0; + } + return unifyfs_ReadRequest_end(B); +} + +static unifyfs_ReadRequest_ref_t unifyfs_ReadRequest_clone(flatbuffers_builder_t *B, unifyfs_ReadRequest_table_t t) +{ + __flatbuffers_memoize_begin(B, t); + if (unifyfs_ReadRequest_start(B) + || unifyfs_ReadRequest_extents_pick(B, t)) { + return 0; + } + __flatbuffers_memoize_end(B, t, unifyfs_ReadRequest_end(B)); +} + +#include "flatcc/flatcc_epilogue.h" +#endif /* UCR_READ_BUILDER_H */ diff --git a/common/src/ucr_read_reader.h b/common/src/ucr_read_reader.h new file mode 100644 index 000000000..eb26b9d5f --- /dev/null +++ b/common/src/ucr_read_reader.h @@ -0,0 +1,74 @@ +#ifndef UCR_READ_READER_H +#define UCR_READ_READER_H + +/* Generated by flatcc 0.5.3-pre FlatBuffers schema compiler for C by dvide.com */ + +#ifndef FLATBUFFERS_COMMON_READER_H +#include "flatbuffers_common_reader.h" +#endif +#include "flatcc/flatcc_flatbuffers.h" +#ifndef __alignas_is_defined +#include +#endif +#include "flatcc/flatcc_prologue.h" +#ifndef flatbuffers_identifier +#define flatbuffers_identifier 0 +#endif +#ifndef flatbuffers_extension +#define flatbuffers_extension ".bin" +#endif + +typedef struct unifyfs_Extent unifyfs_Extent_t; +typedef const unifyfs_Extent_t *unifyfs_Extent_struct_t; +typedef unifyfs_Extent_t *unifyfs_Extent_mutable_struct_t; +typedef const unifyfs_Extent_t *unifyfs_Extent_vec_t; +typedef unifyfs_Extent_t *unifyfs_Extent_mutable_vec_t; + +typedef const struct unifyfs_ReadRequest_table *unifyfs_ReadRequest_table_t; +typedef const flatbuffers_uoffset_t *unifyfs_ReadRequest_vec_t; +typedef flatbuffers_uoffset_t *unifyfs_ReadRequest_mutable_vec_t; +#ifndef unifyfs_Extent_identifier +#define unifyfs_Extent_identifier flatbuffers_identifier +#endif +#define unifyfs_Extent_type_hash ((flatbuffers_thash_t)0xfe153735) +#define unifyfs_Extent_type_identifier "\x35\x37\x15\xfe" +#ifndef unifyfs_ReadRequest_identifier +#define unifyfs_ReadRequest_identifier flatbuffers_identifier +#endif +#define unifyfs_ReadRequest_type_hash ((flatbuffers_thash_t)0x70b2f5ee) +#define unifyfs_ReadRequest_type_identifier "\xee\xf5\xb2\x70" + + +struct unifyfs_Extent { + alignas(8) uint32_t fid; + alignas(8) uint64_t offset; + alignas(8) uint64_t length; +}; +static_assert(sizeof(unifyfs_Extent_t) == 24, "struct size mismatch"); + +static inline const unifyfs_Extent_t *unifyfs_Extent__const_ptr_add(const unifyfs_Extent_t *p, size_t i) { return p + i; } +static inline unifyfs_Extent_t *unifyfs_Extent__ptr_add(unifyfs_Extent_t *p, size_t i) { return p + i; } +static inline unifyfs_Extent_struct_t unifyfs_Extent_vec_at(unifyfs_Extent_vec_t vec, size_t i) +__flatbuffers_struct_vec_at(vec, i) +static inline size_t unifyfs_Extent__size() { return 24; } +static inline size_t unifyfs_Extent_vec_len(unifyfs_Extent_vec_t vec) +__flatbuffers_vec_len(vec) +__flatbuffers_struct_as_root(unifyfs_Extent) + +__flatbuffers_define_struct_scalar_field(unifyfs_Extent, fid, flatbuffers_uint32, uint32_t) +__flatbuffers_define_struct_scalar_field(unifyfs_Extent, offset, flatbuffers_uint64, uint64_t) +__flatbuffers_define_struct_scalar_field(unifyfs_Extent, length, flatbuffers_uint64, uint64_t) + + +struct unifyfs_ReadRequest_table { uint8_t unused__; }; + +static inline size_t unifyfs_ReadRequest_vec_len(unifyfs_ReadRequest_vec_t vec) +__flatbuffers_vec_len(vec) +static inline unifyfs_ReadRequest_table_t unifyfs_ReadRequest_vec_at(unifyfs_ReadRequest_vec_t vec, size_t i) +__flatbuffers_offset_vec_at(unifyfs_ReadRequest_table_t, vec, i, 0) +__flatbuffers_table_as_root(unifyfs_ReadRequest) + +__flatbuffers_define_vector_field(0, unifyfs_ReadRequest, extents, unifyfs_Extent_vec_t, 0) + +#include "flatcc/flatcc_epilogue.h" +#endif /* UCR_READ_READER_H */ diff --git a/common/src/unifycr_const.h b/common/src/unifycr_const.h deleted file mode 100644 index 136e3df91..000000000 --- a/common/src/unifycr_const.h +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * - * Copyright 2017, UT-Battelle, LLC. - * - * LLNL-CODE-741539 - * All rights reserved. - * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. - */ - -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * Copyright (c) 2017, Florida State University. Contributions from - * the Computer Architecture and Systems Research Laboratory (CASTL) - * at the Department of Computer Science. - * - * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror - * LLNL-CODE-728877. All rights reserved. - * - * This file is part of burstfs. - * For details, see https://github.com/llnl/burstfs - * Please read https://github.com/llnl/burstfs/LICENSE for full license text. - */ - -#ifndef UNIFYCR_CONST_H -#define UNIFYCR_CONST_H - -#include -#include // provides HOST_NAME_MAX - -/* ********************** ERROR CODES ************************ */ -#include "err_enumerator.h" -#define ULFS_SUCCESS ((int)UNIFYCR_SUCCESS) - -/* ********************** STRING CONSTANTS ************************ */ -#define DEFAULT_INTERFACE "ib0" -#define SOCKET_PATH "/tmp/unifycr_server_sock" - -/* ********************** INT CONSTANTS ************************ */ - -// Byte counts -#define KIB 1024 -#define MIB 1048576 -#define GIB 1073741824 - -// Generic -#define GEN_STR_LEN KIB -#define UNIFYCR_MAX_FILENAME KIB - -// Metadata -#define MAX_META_PER_SEND (512*KIB) -#define MAX_FILE_CNT_PER_NODE KIB - -// Request Manager -#define RECV_BUF_CNT 1 -#define RECV_BUF_LEN (MIB + 128*KIB) -#define REQ_BUF_LEN (128*MIB + 4*KIB + 128*KIB) -#define SHM_WAIT_INTERVAL 10 - -// Service Manager -#define MIN_SLEEP_INTERVAL 10 -#define SLEEP_INTERVAL 100 /* unit: us */ -#define SLEEP_SLICE_PER_UNIT 10 -#define READ_BLOCK_SIZE MIB -#define SEND_BLOCK_SIZE (MIB + 128*KIB) -#define READ_BUF_SZ GIB -#define LARGE_BURSTY_DATA (500 * MIB) -#define MAX_BURSTY_INTERVAL 10000 /* unit: us */ - -// Request and Service Managers, Command Handler -#define MAX_NUM_CLIENTS 64 /* app processes per server */ -#define CLI_DATA_TAG 5001 -#define SER_DATA_TAG 6001 - -// Client and Command Handler -#define CMD_BUF_SIZE (2 * KIB) -#define MMAP_OPEN_FLAG (O_RDWR|O_CREAT) -#define MMAP_OPEN_MODE 00777 - -// Client -#define UNIFYCR_MAX_FILES 128 -#define UNIFYCR_MAX_FILEDESCS UNIFYCR_MAX_FILES -#define UNIFYCR_STREAM_BUFSIZE MIB -#define UNIFYCR_CHUNK_BITS 24 -#define UNIFYCR_CHUNK_MEM (256 * MIB) -#define UNIFYCR_SPILLOVER_SIZE (KIB * MIB) -#define UNIFYCR_SUPERBLOCK_KEY 4321 -#define UNIFYCR_SHMEM_REQ_SIZE (MIB*128 + 128*KIB) -#define UNIFYCR_SHMEM_RECV_SIZE (MIB + 128*KIB) -#define UNIFYCR_INDEX_BUF_SIZE (20 * MIB) -#define UNIFYCR_FATTR_BUF_SIZE MIB -#define UNIFYCR_MAX_SPLIT_CNT MIB -#define UNIFYCR_MAX_READ_CNT MIB - -// Following are not currently used: -// #define ACK_MSG_SZ 8 -// #define C_CLI_SEND_BUF_SIZE 1048576 -// #define C_CLI_RECV_BUF_SIZE 1048576 -// #define DATA_BUF_LEN (1048576 + 4096 + 131072) -// #define IP_STR_LEN 20 -// #define MAX_CQ_SIZE 2*NUM_CLI -// #define MAX_NUM_CONNS 10 -// /* number of messages one server sends to each of another server */ -// #define NUM_MSG 32768 -// #define NUM_OF_READ_TASKS 65536 -// #define PORT_STR_LEN 10 -// #define SH_BUF_SIZE 1048576 -// /* buffer size on the service manager for read clustering and pipelining */ -// #define SERVICE_MEM_POOL_SIZE 2147483648 - -/* ****************** Metadata/MDHIM Default Values ******************** */ -#define META_DEFAULT_DB_NAME unifycr_db -#define META_DEFAULT_SERVER_RATIO 1 -#define META_DEFAULT_RANGE_SZ MIB - -#endif - - diff --git a/common/src/unifycr_log.c b/common/src/unifycr_log.c deleted file mode 100644 index ffb6f658b..000000000 --- a/common/src/unifycr_log.c +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * - * Copyright 2017, UT-Battelle, LLC. - * - * LLNL-CODE-741539 - * All rights reserved. - * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. - */ - -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * Copyright (c) 2017, Florida State University. Contributions from - * the Computer Architecture and Systems Research Laboratory (CASTL) - * at the Department of Computer Science. - * - * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror - * LLNL-CODE-728877. All rights reserved. - * - * This file is part of burstfs. - * For details, see https://github.com/llnl/burstfs - * Please read https://github.com/llnl/burstfs/LICENSE for full license text. - */ - -#include -#include "unifycr_log.h" -#include "unifycr_const.h" - -/* one of the loglevel values */ -unifycr_log_level_t unifycr_log_level = 5; - -/* pointer to log file stream */ -FILE* unifycr_log_stream; - -int glb_rank; - -/* used within LOG macro to build a timestamp */ -time_t unifycr_log_time; -struct tm* unifycr_log_ltime; -char unifycr_log_timestamp[256]; - -/* open specified file as log file stream, - * returns UNIFYCR_SUCCESS on success */ -int unifycr_log_open(const char* file) -{ - unifycr_log_stream = fopen(file, "a"); - if (unifycr_log_stream == NULL) { - /* failed to open file name, fall back to stderr */ - unifycr_log_stream = stderr; - return (int)UNIFYCR_ERROR_DBG; - } else { - return UNIFYCR_SUCCESS; - } -} - -/* close our log file stream, - * returns UNIFYCR_SUCCESS on success */ -int unifycr_log_close(void) -{ - if (unifycr_log_stream == NULL) { - /* nothing to close */ - return (int)UNIFYCR_ERROR_DBG; - } else { - /* if stream is open, and its not stderr, close it */ - if (unifycr_log_stream != stderr && - fclose(unifycr_log_stream) == 0) { - return UNIFYCR_SUCCESS; - } - return (int)UNIFYCR_ERROR_DBG; - } -} diff --git a/common/src/unifycr_meta.h b/common/src/unifycr_meta.h deleted file mode 100644 index 5d13f6f34..000000000 --- a/common/src/unifycr_meta.h +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2018, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * - * Copyright 2018, UT-Battelle, LLC. - * - * LLNL-CODE-741539 - * All rights reserved. - * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. - */ - -#ifndef UNIFYCR_META_H -#define UNIFYCR_META_H - -#include - -#include "unifycr_const.h" - -/** - * Server commands - */ -typedef enum { - COMM_MOUNT, - COMM_META_FSYNC, - COMM_META_GET, - COMM_META_SET, - COMM_READ, - COMM_UNMOUNT, - COMM_DIGEST, - COMM_SYNC_DEL, -} cmd_lst_t; - -typedef struct { - int fid; - int gfid; - char filename[UNIFYCR_MAX_FILENAME]; - struct stat file_attr; -} unifycr_file_attr_t; - -typedef struct { - off_t file_pos; - off_t mem_pos; - size_t length; - int fid; -} unifycr_index_t; - -/* defines header for read reply as written by request manager - * back to application client via shared memory, the data - * payload of length bytes immediately follows the header */ -typedef struct { - size_t offset; /* offset within file */ - size_t length; /* number of bytes */ - int src_fid; /* global file id */ - int errcode; /* indicates whether read encountered error */ -} shm_meta_t; - -#endif /* UNIFYCR_META_H */ diff --git a/common/src/unifycr_pmix.c b/common/src/unifycr_pmix.c deleted file mode 100644 index ba8218b40..000000000 --- a/common/src/unifycr_pmix.c +++ /dev/null @@ -1,282 +0,0 @@ -/* - * Copyright (c) 2018, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * - * Copyright 2018, UT-Battelle, LLC. - * - * LLNL-CODE-741539 - * All rights reserved. - * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. - */ - -#include "unifycr_const.h" -#include "unifycr_pmix.h" - -#include -#include -#include -#include - -static int initialized; // = 0 -static int published; // = 0 -static size_t univ_nprocs; // = 0 -static pmix_proc_t myproc; - -// PMIx host -char myhost[80]; - -// PMIx keys we use -const char* pmix_key_runstate = "unifycr.runstate"; -const char* pmix_key_unifycrd_socket = "unifycrd.socket"; - - -// initialize PMIx -int unifycr_pmix_init(int* orank, - size_t* ouniv) -{ - int rc; - pmix_value_t value; - pmix_value_t* valp = &value; - pmix_proc_t proc; - - if (!initialized) { - gethostname(myhost, sizeof(myhost)); - - /* init PMIx */ - PMIX_PROC_CONSTRUCT(&myproc); - rc = PMIx_Init(&myproc, NULL, 0); - if (rc != PMIX_SUCCESS) { - fprintf(stderr, "ERROR [%s]: %s() - PMIx_Init failed: %s\n", - myhost, __func__, PMIx_Error_string(rc)); - return (int)UNIFYCR_FAILURE; - } - - /* get PMIx universe size */ - PMIX_PROC_CONSTRUCT(&proc); - (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN); - proc.rank = PMIX_RANK_WILDCARD; - rc = PMIx_Get(&proc, PMIX_UNIV_SIZE, NULL, 0, &valp); - if (rc != PMIX_SUCCESS) { - fprintf(stderr, "ERROR [%s]: %s() - PMIx rank %d: " - "PMIx_Get(UNIV_SIZE) failed: %s\n", - myhost, __func__, myproc.rank, PMIx_Error_string(rc)); - return (int)UNIFYCR_FAILURE; - } - univ_nprocs = (size_t) valp->data.uint32; - PMIX_VALUE_RELEASE(valp); - - initialized = 1; - } - - if (orank != NULL) { - *orank = myproc.rank; - } - if (ouniv != NULL) { - *ouniv = univ_nprocs; - } - - return (int)UNIFYCR_SUCCESS; -} - -// finalize PMIx -int unifycr_pmix_fini(void) -{ - int rc; - size_t ninfo; - pmix_info_t* info; - pmix_data_range_t range; - - rc = (int) UNIFYCR_SUCCESS; - - if (initialized) { - - if (published) { - /* unpublish everything I published */ - range = PMIX_RANGE_GLOBAL; - ninfo = 1; - PMIX_INFO_CREATE(info, ninfo); - PMIX_INFO_LOAD(&info[0], PMIX_RANGE, &range, PMIX_DATA_RANGE); - rc = PMIx_Unpublish(NULL, info, ninfo); - if (rc != PMIX_SUCCESS) { - fprintf(stderr, "ERROR [%s]: %s() - PMIx rank %d: " - "PMIx_Unpublish failed: %s\n", - myhost, __func__, myproc.rank, PMIx_Error_string(rc)); - } - published = 0; - } - - /* fini PMIx */ - rc = PMIx_Finalize(NULL, 0); - if (rc != PMIX_SUCCESS) { - fprintf(stderr, "ERROR [%s]: %s() PMIx rank %d: " - "PMIx_Finalize() failed: %s\n", - myhost, __func__, myproc.rank, PMIx_Error_string(rc)); - rc = (int) UNIFYCR_FAILURE; - } - PMIX_PROC_DESTRUCT(&myproc); - univ_nprocs = 0; - initialized = 0; - } - return rc; -} - -// publish a key-value pair -int unifycr_pmix_publish(const char* key, - const char* val) -{ - int rc; - size_t len, hlen, ninfo; - pmix_info_t* info; - pmix_data_range_t range; - char pmix_key[PMIX_MAX_KEYLEN]; - - if (!initialized) { - rc = unifycr_pmix_init(NULL, NULL); - if (rc != (int)UNIFYCR_SUCCESS) { - return rc; - } - } - - if ((key == NULL) || (val == NULL)) { - fprintf(stderr, "ERROR [%s]: %s() - NULL key or value\n", - myhost, __func__); - return (int)UNIFYCR_ERROR_INVAL; - } - - len = strlen(key); - hlen = strlen(myhost); - if ((len + hlen) >= sizeof(pmix_key)) { - fprintf(stderr, "ERROR [%s]: %s() - " - "length of key (%zd) exceeds max %zd\n", - myhost, __func__, len, sizeof(pmix_key)); - return (int)UNIFYCR_ERROR_INVAL; - } else { - snprintf(pmix_key, sizeof(pmix_key), "%s.%s", key, myhost); - } - - /* set key-val and modify publish behavior */ - range = PMIX_RANGE_GLOBAL; - ninfo = 2; - PMIX_INFO_CREATE(info, ninfo); - PMIX_INFO_LOAD(&info[0], pmix_key, val, PMIX_STRING); - PMIX_INFO_LOAD(&info[1], PMIX_RANGE, &range, PMIX_DATA_RANGE); - - /* try to publish */ - rc = PMIx_Publish(info, ninfo); - if (rc != PMIX_SUCCESS) { - fprintf(stderr, "ERROR [%s]: %s() - PMIx rank %d: " - "PMIx_Publish failed: %s\n", - myhost, __func__, myproc.rank, PMIx_Error_string(rc)); - rc = (int)UNIFYCR_FAILURE; - } else { - published = 1; - rc = (int)UNIFYCR_SUCCESS; - } - /* cleanup */ - PMIX_INFO_FREE(info, ninfo); - - return rc; -} - -// lookup a key-value pair -int unifycr_pmix_lookup_common(const char* pmix_key, - int keywait, - char** oval) -{ - int rc, wait; - size_t ndir; - pmix_data_range_t range; - pmix_info_t* directives; - pmix_pdata_t* pdata; - - if ((pmix_key == NULL) || (oval == NULL)) { - fprintf(stderr, "ERROR [%s]: %s() - NULL key or value\n", - myhost, __func__); - return (int)UNIFYCR_ERROR_INVAL; - } - - /* set key to lookup */ - PMIX_PDATA_CREATE(pdata, 1); - PMIX_PDATA_LOAD(&pdata[0], &myproc, pmix_key, NULL, PMIX_STRING); - - /* modify lookup behavior */ - wait = 0; - ndir = 1; - if (keywait) { - ndir++; - wait = 1; - } - range = PMIX_RANGE_GLOBAL; - PMIX_INFO_CREATE(directives, ndir); - PMIX_INFO_LOAD(&directives[0], PMIX_RANGE, &range, PMIX_DATA_RANGE); - if (keywait) { - PMIX_INFO_LOAD(&directives[1], PMIX_WAIT, &wait, PMIX_INT); - } - - /* try lookup */ - rc = PMIx_Lookup(pdata, 1, directives, ndir); - if (rc != PMIX_SUCCESS) { - fprintf(stderr, "ERROR [%s]: %s() - PMIx rank %d: " - "PMIx_Lookup(%s) failed: %s\n", - myhost, __func__, myproc.rank, pmix_key, PMIx_Error_string(rc)); - *oval = NULL; - rc = (int)UNIFYCR_FAILURE; - } else { - if (pdata[0].value.data.string != NULL) { - *oval = strdup(pdata[0].value.data.string); - rc = (int)UNIFYCR_SUCCESS; - } else { - fprintf(stderr, "ERROR [%s]: %s() - PMIx rank %d: " - "PMIx_Lookup(%s) returned NULL string\n", - myhost, __func__, myproc.rank, pmix_key); - *oval = NULL; - rc = (int)UNIFYCR_FAILURE; - } - } - /* cleanup */ - PMIX_PDATA_FREE(pdata, 1); - PMIX_INFO_FREE(directives, ndir); - - return rc; -} - -int unifycr_pmix_lookup(const char* key, - int keywait, - char** oval) -{ - int rc; - char full_key[PMIX_MAX_KEYLEN]; - - if (!initialized) { - rc = unifycr_pmix_init(NULL, NULL); - if (rc != (int)UNIFYCR_SUCCESS) { - return rc; - } - } - - snprintf(full_key, sizeof(full_key), "%s.%s", key, myhost); - return unifycr_pmix_lookup_common(full_key, keywait, oval); -} - -int unifycr_pmix_lookup_remote(const char* host, - const char* key, - int keywait, - char** oval) -{ - int rc; - char full_key[PMIX_MAX_KEYLEN]; - - if (!initialized) { - rc = unifycr_pmix_init(NULL, NULL); - if (rc != (int)UNIFYCR_SUCCESS) { - return rc; - } - } - - snprintf(full_key, sizeof(full_key), "%s.%s", key, host); - return unifycr_pmix_lookup_common(full_key, keywait, oval); -} - diff --git a/common/src/unifycr_pmix.h b/common/src/unifycr_pmix.h deleted file mode 100644 index a64ad837c..000000000 --- a/common/src/unifycr_pmix.h +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2018, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * - * Copyright 2018, UT-Battelle, LLC. - * - * LLNL-CODE-741539 - * All rights reserved. - * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. - */ - -// NOTE: this file compiles to empty if PMIx not detected at configure time -#include -#ifdef HAVE_PMIX_H - -#ifndef UNIFYCR_PMIX_H -#define UNIFYCR_PMIX_H - -#include -#include - -// PMIx keys we use -const char* pmix_key_runstate; // path to runstate file -const char* pmix_key_unifycrd_socket; // server domain socket path - -// initialize PMIx -int unifycr_pmix_init(int* orank, - size_t* ouniv); - -// finalize PMIx -int unifycr_pmix_fini(void); - -// publish a key-value pair -int unifycr_pmix_publish(const char* key, - const char* val); - -// lookup a key-value pair -int unifycr_pmix_lookup(const char* key, - int wait, - char** oval); - -int unifycr_pmix_lookup_remote(const char* host, - const char* key, - int wait, - char** oval); - -#endif // UNIFYCR_PMIX_H - -#endif // HAVE_PMIX_H - diff --git a/common/src/unifycr_runstate.c b/common/src/unifycr_runstate.c deleted file mode 100644 index fe23a4a89..000000000 --- a/common/src/unifycr_runstate.c +++ /dev/null @@ -1,124 +0,0 @@ -#include -#include -#include -#include -#include - -#include "unifycr_runstate.h" -#include "unifycr_pmix.h" - -const char* runstate_file = "unifycr-runstate.conf"; - -int unifycr_read_runstate(unifycr_cfg_t* cfg, - const char* runstate_path) -{ - int rc = (int)UNIFYCR_SUCCESS; - int have_path = 0; - int uid = (int)getuid(); - char runstate_fname[UNIFYCR_MAX_FILENAME] = {0}; -#ifdef HAVE_PMIX_H - char* pmix_path = NULL; -#endif - - if (cfg == NULL) { - fprintf(stderr, "%s() - invalid args\n", __func__); - return (int)UNIFYCR_ERROR_INVAL; - } - - if (runstate_path == NULL) { - -#ifdef HAVE_PMIX_H - // lookup runstate file path in PMIx - if (unifycr_pmix_lookup(pmix_key_runstate, 0, &pmix_path) == 0) { - have_path = 1; - snprintf(runstate_fname, sizeof(runstate_fname), - "%s", pmix_path); - free(pmix_path); - } -#endif - - if (!have_path) { - if (cfg->runstate_dir == NULL) { - fprintf(stderr, - "%s() - bad runstate dir config setting\n", __func__); - return (int)UNIFYCR_ERROR_APPCONFIG; - } - snprintf(runstate_fname, sizeof(runstate_fname), - "%s/%s.%d", cfg->runstate_dir, runstate_file, uid); - } - } else { - snprintf(runstate_fname, sizeof(runstate_fname), - "%s", runstate_path); - } - - if (unifycr_config_process_ini_file(cfg, runstate_fname) != 0) { - fprintf(stderr, - "%s() - failed to process runstate file %s\n", - __func__, runstate_fname); - rc = (int)UNIFYCR_ERROR_APPCONFIG; - } - - return rc; -} - -int unifycr_write_runstate(unifycr_cfg_t* cfg) -{ - int rc = (int)UNIFYCR_SUCCESS; - int uid = (int)getuid(); - FILE* runstate_fp = NULL; - char runstate_fname[UNIFYCR_MAX_FILENAME] = {0}; - - if (cfg == NULL) { - fprintf(stderr, "%s() - invalid config arg\n", __func__); - return (int)UNIFYCR_ERROR_INVAL; - } - - snprintf(runstate_fname, sizeof(runstate_fname), - "%s/%s.%d", cfg->runstate_dir, runstate_file, uid); - - runstate_fp = fopen(runstate_fname, "w"); - if (runstate_fp == NULL) { - fprintf(stderr, - "%s() - failed to create file %s - %s\n", - __func__, runstate_fname, strerror(errno)); - rc = (int)UNIFYCR_ERROR_FILE; - } else { - unifycr_config_print_ini(cfg, runstate_fp); - fclose(runstate_fp); - -#ifdef HAVE_PMIX_H - // publish runstate file path to PMIx - if (unifycr_pmix_publish(pmix_key_runstate, runstate_fname) != 0) { - fprintf(stderr, "%s() - failed to publish %s k-v pair\n", - __func__, pmix_key_runstate); - rc = (int)UNIFYCR_ERROR_PMIX; - } -#endif - } - - return rc; -} - -int unifycr_clean_runstate(unifycr_cfg_t* cfg) -{ - int rc = (int)UNIFYCR_SUCCESS; - char runstate_fname[UNIFYCR_MAX_FILENAME] = {0}; - - if (cfg == NULL) { - fprintf(stderr, "%s() - invalid config arg\n", __func__); - return (int)UNIFYCR_ERROR_INVAL; - } - - snprintf(runstate_fname, sizeof(runstate_fname), - "%s/%s", cfg->runstate_dir, runstate_file); - - rc = unlink(runstate_fname); - if (rc != 0) { - fprintf(stderr, - "%s() - failed to remove file %s - %s\n", - __func__, runstate_fname, strerror(errno)); - rc = (int)UNIFYCR_ERROR_FILE; - } - - return rc; -} diff --git a/common/src/unifycr_runstate.h b/common/src/unifycr_runstate.h deleted file mode 100644 index faa41c622..000000000 --- a/common/src/unifycr_runstate.h +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef _UNIFYCR_RUNSTATE_H_ -#define _UNIFYCR_RUNSTATE_H_ - -#include "unifycr_configurator.h" - -int unifycr_read_runstate(unifycr_cfg_t* cfg, - const char* runstate_path); - -int unifycr_write_runstate(unifycr_cfg_t* cfg); - -int unifycr_clean_runstate(unifycr_cfg_t* cfg); - -#endif // UNIFYCR_RUNSTATE_H diff --git a/common/src/unifyfs_client_rpcs.h b/common/src/unifyfs_client_rpcs.h new file mode 100644 index 000000000..546810937 --- /dev/null +++ b/common/src/unifyfs_client_rpcs.h @@ -0,0 +1,154 @@ +#ifndef __UNIFYFS_CLIENT_RPCS_H +#define __UNIFYFS_CLIENT_RPCS_H + +/* + * Declarations for client-server margo RPCs (shared-memory) + */ + +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* unifyfs_mount_rpc (client => server) + * + * connect application client to the server, and + * initialize shared memory state */ +MERCURY_GEN_PROC(unifyfs_mount_in_t, + ((int32_t)(app_id)) + ((int32_t)(local_rank_idx)) + ((int32_t)(dbg_rank)) + ((int32_t)(num_procs_per_node)) + ((hg_const_string_t)(client_addr_str)) + ((hg_size_t)(req_buf_sz)) + ((hg_size_t)(recv_buf_sz)) + ((hg_size_t)(superblock_sz)) + ((hg_size_t)(meta_offset)) + ((hg_size_t)(meta_size)) + ((hg_size_t)(fmeta_offset)) + ((hg_size_t)(fmeta_size)) + ((hg_size_t)(data_offset)) + ((hg_size_t)(data_size)) + ((hg_const_string_t)(external_spill_dir))) +MERCURY_GEN_PROC(unifyfs_mount_out_t, + ((hg_size_t)(max_recs_per_slice)) + ((int32_t)(ret))) +DECLARE_MARGO_RPC_HANDLER(unifyfs_mount_rpc) + +/* unifyfs_unmount_rpc (client => server) + * + * disconnect client from server */ +MERCURY_GEN_PROC(unifyfs_unmount_in_t, + ((int32_t)(app_id)) + ((int32_t)(local_rank_idx))) +MERCURY_GEN_PROC(unifyfs_unmount_out_t, ((int32_t)(ret))) +DECLARE_MARGO_RPC_HANDLER(unifyfs_unmount_rpc) + +/* need to transfer timespec structs */ +typedef struct timespec sys_timespec_t; +MERCURY_GEN_STRUCT_PROC(sys_timespec_t, + ((uint64_t)(tv_sec)) + ((uint64_t)(tv_nsec))) + +/* unifyfs_metaset_rpc (client => server) + * + * given a global file id and a file name, + * record key/value entry for this file */ +MERCURY_GEN_PROC(unifyfs_metaset_in_t, + ((hg_const_string_t)(filename)) + ((int32_t)(fid)) + ((int32_t)(gfid)) + ((uint32_t)(mode)) + ((uint32_t)(uid)) + ((uint32_t)(gid)) + ((uint64_t)(size)) + ((sys_timespec_t)(atime)) + ((sys_timespec_t)(mtime)) + ((sys_timespec_t)(ctime)) + ((uint32_t)(is_laminated))) +MERCURY_GEN_PROC(unifyfs_metaset_out_t, ((int32_t)(ret))) +DECLARE_MARGO_RPC_HANDLER(unifyfs_metaset_rpc) + +/* unifyfs_metaget_rpc (client => server) + * + * returns file metadata including size and name + * given a global file id */ +MERCURY_GEN_PROC(unifyfs_metaget_in_t, + ((int32_t)(gfid))) +MERCURY_GEN_PROC(unifyfs_metaget_out_t, + ((int32_t)(ret)) + ((hg_const_string_t)(filename)) + ((int32_t)(fid)) + ((int32_t)(gfid)) + ((uint32_t)(mode)) + ((uint32_t)(uid)) + ((uint32_t)(gid)) + ((uint64_t)(size)) + ((sys_timespec_t)(atime)) + ((sys_timespec_t)(mtime)) + ((sys_timespec_t)(ctime)) + ((uint32_t)(is_laminated))) +DECLARE_MARGO_RPC_HANDLER(unifyfs_metaget_rpc) + +/* unifyfs_fsync_rpc (client => server) + * + * given app_id, client_id, and a global file id as input, + * read extent location metadata from client shared memory + * and insert corresponding key/value pairs into global index */ +MERCURY_GEN_PROC(unifyfs_fsync_in_t, + ((int32_t)(app_id)) + ((int32_t)(local_rank_idx)) + ((int32_t)(gfid))) +MERCURY_GEN_PROC(unifyfs_fsync_out_t, ((int32_t)(ret))) +DECLARE_MARGO_RPC_HANDLER(unifyfs_fsync_rpc) + +/* unifyfs_filesize_rpc (client => server) + * + * given an app_id, client_id, global file id, + * return filesize for given file */ +MERCURY_GEN_PROC(unifyfs_filesize_in_t, + ((int32_t)(app_id)) + ((int32_t)(local_rank_idx)) + ((int32_t)(gfid))) +MERCURY_GEN_PROC(unifyfs_filesize_out_t, + ((int32_t)(ret)) + ((hg_size_t)(filesize))) +DECLARE_MARGO_RPC_HANDLER(unifyfs_filesize_rpc) + +/* unifyfs_read_rpc (client => server) + * + * given an app_id, client_id, global file id, an offset, and a length, + * initiate read request for data */ +MERCURY_GEN_PROC(unifyfs_read_in_t, + ((int32_t)(app_id)) + ((int32_t)(local_rank_idx)) + ((int32_t)(gfid)) + ((hg_size_t)(offset)) + ((hg_size_t)(length))) +MERCURY_GEN_PROC(unifyfs_read_out_t, ((int32_t)(ret))) +DECLARE_MARGO_RPC_HANDLER(unifyfs_read_rpc) + +/* unifyfs_mread_rpc (client => server) + * + * given an app_id, client_id, global file id, and a count + * of read requests, followed by list of offset/length tuples + * initiate read requests for data */ +MERCURY_GEN_PROC(unifyfs_mread_in_t, + ((int32_t)(app_id)) + ((int32_t)(local_rank_idx)) + ((int32_t)(read_count)) + ((hg_size_t)(bulk_size)) + ((hg_bulk_t)(bulk_handle))) +MERCURY_GEN_PROC(unifyfs_mread_out_t, ((int32_t)(ret))) +DECLARE_MARGO_RPC_HANDLER(unifyfs_mread_rpc) + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // UNIFYFS_CLIENT_RPCS_H diff --git a/common/src/unifycr_configurator.c b/common/src/unifyfs_configurator.c similarity index 81% rename from common/src/unifycr_configurator.c rename to common/src/unifyfs_configurator.c index a64781097..56e1bba34 100644 --- a/common/src/unifycr_configurator.c +++ b/common/src/unifyfs_configurator.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* Copyright (c) 2018 - Michael J. Brim @@ -41,16 +41,16 @@ #include "ini.h" #include "tinyexpr.h" -#include "unifycr_configurator.h" +#include "unifyfs_configurator.h" -#define UNIFYCR_CFG_MAX_MSG 1024 +#define UNIFYFS_CFG_MAX_MSG 1024 #define stringify_indirect(x) #x #define stringify(x) stringify_indirect(x) // initialize configuration using all available methods -int unifycr_config_init(unifycr_cfg_t *cfg, +int unifyfs_config_init(unifyfs_cfg_t *cfg, int argc, char **argv) { @@ -60,44 +60,44 @@ int unifycr_config_init(unifycr_cfg_t *cfg, if (cfg == NULL) return -1; - memset((void *)cfg, 0, sizeof(unifycr_cfg_t)); + memset((void *)cfg, 0, sizeof(unifyfs_cfg_t)); // set default configuration - rc = unifycr_config_set_defaults(cfg); + rc = unifyfs_config_set_defaults(cfg); if (rc) return rc; // process system config file (if available) - syscfg = cfg->unifycr_configfile; + syscfg = cfg->unifyfs_configfile; rc = configurator_file_check(NULL, NULL, syscfg, NULL); if (rc == 0) { - rc = unifycr_config_process_ini_file(cfg, syscfg); + rc = unifyfs_config_process_ini_file(cfg, syscfg); if (rc) return rc; } if (syscfg != NULL) free(syscfg); - cfg->unifycr_configfile = NULL; + cfg->unifyfs_configfile = NULL; // process environment (overrides defaults and system config) - rc = unifycr_config_process_environ(cfg); + rc = unifyfs_config_process_environ(cfg); if (rc) return rc; // process command-line args (overrides all previous) - rc = unifycr_config_process_cli_args(cfg, argc, argv); + rc = unifyfs_config_process_cli_args(cfg, argc, argv); if (rc) return rc; // read config file passed on command-line (does not override cli args) - if (cfg->unifycr_configfile != NULL) { - rc = unifycr_config_process_ini_file(cfg, cfg->unifycr_configfile); + if (cfg->unifyfs_configfile != NULL) { + rc = unifyfs_config_process_ini_file(cfg, cfg->unifyfs_configfile); if (rc) return rc; } // validate settings - rc = unifycr_config_validate(cfg); + rc = unifyfs_config_validate(cfg); if (rc) return rc; @@ -105,24 +105,24 @@ int unifycr_config_init(unifycr_cfg_t *cfg, } // cleanup allocated state -int unifycr_config_fini(unifycr_cfg_t *cfg) +int unifyfs_config_fini(unifyfs_cfg_t *cfg) { if (cfg == NULL) return -1; -#define UNIFYCR_CFG(sec, key, typ, dv, desc, vfn) \ +#define UNIFYFS_CFG(sec, key, typ, dv, desc, vfn) \ if (cfg->sec##_##key != NULL) { \ free(cfg->sec##_##key); \ cfg->sec##_##key = NULL; \ } -#define UNIFYCR_CFG_CLI(sec, key, typ, dv, desc, vfn, opt, use) \ +#define UNIFYFS_CFG_CLI(sec, key, typ, dv, desc, vfn, opt, use) \ if (cfg->sec##_##key != NULL) { \ free(cfg->sec##_##key); \ cfg->sec##_##key = NULL; \ } -#define UNIFYCR_CFG_MULTI(sec, key, typ, desc, vfn, me) \ +#define UNIFYFS_CFG_MULTI(sec, key, typ, desc, vfn, me) \ for (u = 0; u < me; u++) { \ if (cfg->sec##_##key[u] != NULL) { \ free(cfg->sec##_##key[u]); \ @@ -131,7 +131,7 @@ int unifycr_config_fini(unifycr_cfg_t *cfg) } \ cfg->n_##sec##_##key = 0; -#define UNIFYCR_CFG_MULTI_CLI(sec, key, typ, desc, vfn, me, opt, use) \ +#define UNIFYFS_CFG_MULTI_CLI(sec, key, typ, desc, vfn, me, opt, use) \ for (u = 0; u < me; u++) { \ if (cfg->sec##_##key[u] != NULL) { \ free(cfg->sec##_##key[u]); \ @@ -140,67 +140,67 @@ int unifycr_config_fini(unifycr_cfg_t *cfg) } \ cfg->n_##sec##_##key = 0; - UNIFYCR_CONFIGS; -#undef UNIFYCR_CFG -#undef UNIFYCR_CFG_CLI -#undef UNIFYCR_CFG_MULTI -#undef UNIFYCR_CFG_MULTI_CLI + UNIFYFS_CONFIGS; +#undef UNIFYFS_CFG +#undef UNIFYFS_CFG_CLI +#undef UNIFYFS_CFG_MULTI +#undef UNIFYFS_CFG_MULTI_CLI return 0; } // print configuration to specified file (or stderr) -void unifycr_config_print(unifycr_cfg_t *cfg, +void unifyfs_config_print(unifyfs_cfg_t *cfg, FILE *fp) { - char msg[UNIFYCR_CFG_MAX_MSG]; + char msg[UNIFYFS_CFG_MAX_MSG]; if (fp == NULL) fp = stderr; -#define UNIFYCR_CFG(sec, key, typ, dv, desc, vfn) \ +#define UNIFYFS_CFG(sec, key, typ, dv, desc, vfn) \ if (cfg->sec##_##key != NULL) { \ - snprintf(msg, sizeof(msg), "UNIFYCR CONFIG: %s.%s = %s", \ + snprintf(msg, sizeof(msg), "UNIFYFS CONFIG: %s.%s = %s", \ #sec, #key, cfg->sec##_##key); \ fprintf(fp, "%s\n", msg); \ } -#define UNIFYCR_CFG_CLI(sec, key, typ, dv, desc, vfn, opt, use) \ +#define UNIFYFS_CFG_CLI(sec, key, typ, dv, desc, vfn, opt, use) \ if (cfg->sec##_##key != NULL) { \ - snprintf(msg, sizeof(msg), "UNIFYCR CONFIG: %s.%s = %s", \ + snprintf(msg, sizeof(msg), "UNIFYFS CONFIG: %s.%s = %s", \ #sec, #key, cfg->sec##_##key); \ fprintf(fp, "%s\n", msg); \ } -#define UNIFYCR_CFG_MULTI(sec, key, typ, desc, vfn, me) \ +#define UNIFYFS_CFG_MULTI(sec, key, typ, desc, vfn, me) \ for (u = 0; u < me; u++) { \ if (cfg->sec##_##key[u] != NULL) { \ - snprintf(msg, sizeof(msg), "UNIFYCR CONFIG: %s.%s[%u] = %s", \ + snprintf(msg, sizeof(msg), "UNIFYFS CONFIG: %s.%s[%u] = %s", \ #sec, #key, u+1, cfg->sec##_##key[u]); \ fprintf(fp, "%s\n", msg); \ } \ } -#define UNIFYCR_CFG_MULTI_CLI(sec, key, typ, desc, vfn, me, opt, use) \ +#define UNIFYFS_CFG_MULTI_CLI(sec, key, typ, desc, vfn, me, opt, use) \ for (u = 0; u < me; u++) { \ if (cfg->sec##_##key[u] != NULL) { \ - snprintf(msg, sizeof(msg), "UNIFYCR CONFIG: %s.%s[%u] = %s", \ + snprintf(msg, sizeof(msg), "UNIFYFS CONFIG: %s.%s[%u] = %s", \ #sec, #key, u+1, cfg->sec##_##key[u]); \ fprintf(fp, "%s\n", msg); \ } \ } - UNIFYCR_CONFIGS; -#undef UNIFYCR_CFG -#undef UNIFYCR_CFG_CLI -#undef UNIFYCR_CFG_MULTI -#undef UNIFYCR_CFG_MULTI_CLI + UNIFYFS_CONFIGS; +#undef UNIFYFS_CFG +#undef UNIFYFS_CFG_CLI +#undef UNIFYFS_CFG_MULTI +#undef UNIFYFS_CFG_MULTI_CLI fflush(fp); } // print configuration in .ini format to specified file (or stderr) -void unifycr_config_print_ini(unifycr_cfg_t *cfg, +void unifyfs_config_print_ini(unifyfs_cfg_t *cfg, FILE *inifp) { const char *curr_sec = NULL; @@ -209,7 +209,7 @@ void unifycr_config_print_ini(unifycr_cfg_t *cfg, if (inifp == NULL) inifp = stderr; -#define UNIFYCR_CFG(sec, key, typ, dv, desc, vfn) \ +#define UNIFYFS_CFG(sec, key, typ, dv, desc, vfn) \ if (cfg->sec##_##key != NULL) { \ curr_sec = #sec; \ if ((last_sec == NULL) || (strcmp(curr_sec, last_sec) != 0)) \ @@ -218,7 +218,7 @@ void unifycr_config_print_ini(unifycr_cfg_t *cfg, last_sec = curr_sec; \ } -#define UNIFYCR_CFG_CLI(sec, key, typ, dv, desc, vfn, opt, use) \ +#define UNIFYFS_CFG_CLI(sec, key, typ, dv, desc, vfn, opt, use) \ if (cfg->sec##_##key != NULL) { \ curr_sec = #sec; \ if ((last_sec == NULL) || (strcmp(curr_sec, last_sec) != 0)) \ @@ -227,7 +227,7 @@ void unifycr_config_print_ini(unifycr_cfg_t *cfg, last_sec = curr_sec; \ } -#define UNIFYCR_CFG_MULTI(sec, key, typ, desc, vfn, me) \ +#define UNIFYFS_CFG_MULTI(sec, key, typ, desc, vfn, me) \ for (u = 0; u < me; u++) { \ if (cfg->sec##_##key[u] != NULL) { \ curr_sec = #sec; \ @@ -239,7 +239,7 @@ void unifycr_config_print_ini(unifycr_cfg_t *cfg, } \ } -#define UNIFYCR_CFG_MULTI_CLI(sec, key, typ, desc, vfn, me, opt, use) \ +#define UNIFYFS_CFG_MULTI_CLI(sec, key, typ, desc, vfn, me, opt, use) \ for (u = 0; u < me; u++) { \ if (cfg->sec##_##key[u] != NULL) { \ curr_sec = #sec; \ @@ -251,105 +251,105 @@ void unifycr_config_print_ini(unifycr_cfg_t *cfg, } \ } - UNIFYCR_CONFIGS; -#undef UNIFYCR_CFG -#undef UNIFYCR_CFG_CLI -#undef UNIFYCR_CFG_MULTI -#undef UNIFYCR_CFG_MULTI_CLI + UNIFYFS_CONFIGS; +#undef UNIFYFS_CFG +#undef UNIFYFS_CFG_CLI +#undef UNIFYFS_CFG_MULTI +#undef UNIFYFS_CFG_MULTI_CLI fflush(inifp); } -// set default values given in UNIFYCR_CONFIGS -int unifycr_config_set_defaults(unifycr_cfg_t *cfg) +// set default values given in UNIFYFS_CONFIGS +int unifyfs_config_set_defaults(unifyfs_cfg_t *cfg) { char *val; if (cfg == NULL) return -1; -#define UNIFYCR_CFG(sec, key, typ, dv, desc, vfn) \ +#define UNIFYFS_CFG(sec, key, typ, dv, desc, vfn) \ val = stringify(dv); \ if (0 != strcmp(val, "NULLSTRING")) \ cfg->sec##_##key = strdup(val); -#define UNIFYCR_CFG_CLI(sec, key, typ, dv, desc, vfn, opt, use) \ +#define UNIFYFS_CFG_CLI(sec, key, typ, dv, desc, vfn, opt, use) \ val = stringify(dv); \ if (0 != strcmp(val, "NULLSTRING")) \ cfg->sec##_##key = strdup(val); -#define UNIFYCR_CFG_MULTI(sec, key, typ, desc, vfn, me) \ +#define UNIFYFS_CFG_MULTI(sec, key, typ, desc, vfn, me) \ cfg->n_##sec##_##key = 0; \ memset((void *)cfg->sec##_##key, 0, sizeof(cfg->sec##_##key)); -#define UNIFYCR_CFG_MULTI_CLI(sec, key, typ, desc, vfn, me, opt, use) \ +#define UNIFYFS_CFG_MULTI_CLI(sec, key, typ, desc, vfn, me, opt, use) \ cfg->n_##sec##_##key = 0; \ memset((void *)cfg->sec##_##key, 0, sizeof(cfg->sec##_##key)); - UNIFYCR_CONFIGS; -#undef UNIFYCR_CFG -#undef UNIFYCR_CFG_CLI -#undef UNIFYCR_CFG_MULTI -#undef UNIFYCR_CFG_MULTI_CLI + UNIFYFS_CONFIGS; +#undef UNIFYFS_CFG +#undef UNIFYFS_CFG_CLI +#undef UNIFYFS_CFG_MULTI +#undef UNIFYFS_CFG_MULTI_CLI return 0; } // utility routine to print CLI usage (and optional usage error message) -void unifycr_config_cli_usage(char *arg0) +void unifyfs_config_cli_usage(char *arg0) { fprintf(stderr, "USAGE: %s [options]\n", arg0); -#define UNIFYCR_CFG(sec, key, typ, dv, desc, vfn) +#define UNIFYFS_CFG(sec, key, typ, dv, desc, vfn) -#define UNIFYCR_CFG_CLI(sec, key, typ, dv, desc, vfn, opt, use) \ +#define UNIFYFS_CFG_CLI(sec, key, typ, dv, desc, vfn, opt, use) \ fprintf(stderr, " -%c,--%s-%s <%s>\t%s (default value: %s)\n", \ opt, #sec, #key, #typ, use, stringify(dv)); -#define UNIFYCR_CFG_MULTI(sec, key, typ, desc, vfn, me) +#define UNIFYFS_CFG_MULTI(sec, key, typ, desc, vfn, me) -#define UNIFYCR_CFG_MULTI_CLI(sec, key, typ, desc, vfn, me, opt, use) \ +#define UNIFYFS_CFG_MULTI_CLI(sec, key, typ, desc, vfn, me, opt, use) \ fprintf(stderr, " -%c,--%s-%s <%s>\t%s (multiple values supported - max %u entries)\n", \ opt, #sec, #key, #typ, use, me); - UNIFYCR_CONFIGS; -#undef UNIFYCR_CFG -#undef UNIFYCR_CFG_CLI -#undef UNIFYCR_CFG_MULTI -#undef UNIFYCR_CFG_MULTI_CLI + UNIFYFS_CONFIGS; +#undef UNIFYFS_CFG +#undef UNIFYFS_CFG_CLI +#undef UNIFYFS_CFG_MULTI +#undef UNIFYFS_CFG_MULTI_CLI fflush(stderr); } // print usage error message -void unifycr_config_cli_usage_error(char *arg0, +void unifyfs_config_cli_usage_error(char *arg0, char *err_msg) { if (err_msg != NULL) fprintf(stderr, "USAGE ERROR: %s : %s\n\n", arg0, err_msg); - unifycr_config_cli_usage(arg0); + unifyfs_config_cli_usage(arg0); } static struct option cli_options[] = { -#define UNIFYCR_CFG(sec, key, typ, dv, desc, vfn) -#define UNIFYCR_CFG_CLI(sec, key, typ, dv, desc, vfn, opt, use) \ +#define UNIFYFS_CFG(sec, key, typ, dv, desc, vfn) +#define UNIFYFS_CFG_CLI(sec, key, typ, dv, desc, vfn, opt, use) \ { #sec "-" #key, required_argument, NULL, opt }, -#define UNIFYCR_CFG_MULTI(sec, key, typ, desc, vfn, me) -#define UNIFYCR_CFG_MULTI_CLI(sec, key, typ, desc, vfn, me, opt, use) \ +#define UNIFYFS_CFG_MULTI(sec, key, typ, desc, vfn, me) +#define UNIFYFS_CFG_MULTI_CLI(sec, key, typ, desc, vfn, me, opt, use) \ { #sec "-" #key, required_argument, NULL, opt }, - UNIFYCR_CONFIGS -#undef UNIFYCR_CFG -#undef UNIFYCR_CFG_CLI -#undef UNIFYCR_CFG_MULTI -#undef UNIFYCR_CFG_MULTI_CLI + UNIFYFS_CONFIGS +#undef UNIFYFS_CFG +#undef UNIFYFS_CFG_CLI +#undef UNIFYFS_CFG_MULTI +#undef UNIFYFS_CFG_MULTI_CLI { NULL, 0, NULL, 0 } }; // update config struct based on command line args -int unifycr_config_process_cli_args(unifycr_cfg_t *cfg, +int unifyfs_config_process_cli_args(unifyfs_cfg_t *cfg, int argc, char **argv) { @@ -357,7 +357,7 @@ int unifycr_config_process_cli_args(unifycr_cfg_t *cfg, int usage_err = 0; int ondx = 0; int sndx = 0; - char errmsg[UNIFYCR_CFG_MAX_MSG]; + char errmsg[UNIFYFS_CFG_MAX_MSG]; char short_opts[256]; extern char *optarg; extern int optind, optopt; @@ -369,9 +369,9 @@ int unifycr_config_process_cli_args(unifycr_cfg_t *cfg, memset((void *)short_opts, 0, sizeof(short_opts)); short_opts[sndx++] = ':'; // report missing args -#define UNIFYCR_CFG(sec, key, typ, dv, desc, vfn) +#define UNIFYFS_CFG(sec, key, typ, dv, desc, vfn) -#define UNIFYCR_CFG_CLI(sec, key, typ, dv, desc, vfn, opt, use) \ +#define UNIFYFS_CFG_CLI(sec, key, typ, dv, desc, vfn, opt, use) \ short_opts[sndx++] = opt; \ if (strcmp(#typ, "BOOL") == 0) { \ short_opts[sndx++] = ':'; \ @@ -382,9 +382,9 @@ int unifycr_config_process_cli_args(unifycr_cfg_t *cfg, short_opts[sndx++] = ':'; \ cli_options[ondx++].has_arg = required_argument; \ } -#define UNIFYCR_CFG_MULTI(sec, key, typ, desc, vfn, me) +#define UNIFYFS_CFG_MULTI(sec, key, typ, desc, vfn, me) -#define UNIFYCR_CFG_MULTI_CLI(sec, key, typ, desc, vfn, me, opt, use) \ +#define UNIFYFS_CFG_MULTI_CLI(sec, key, typ, desc, vfn, me, opt, use) \ short_opts[sndx++] = opt; \ if (strcmp(#typ, "BOOL") == 0) { \ short_opts[sndx++] = ':'; \ @@ -396,21 +396,21 @@ int unifycr_config_process_cli_args(unifycr_cfg_t *cfg, cli_options[ondx++].has_arg = required_argument; \ } - UNIFYCR_CONFIGS; -#undef UNIFYCR_CFG -#undef UNIFYCR_CFG_CLI -#undef UNIFYCR_CFG_MULTI -#undef UNIFYCR_CFG_MULTI_CLI + UNIFYFS_CONFIGS; +#undef UNIFYFS_CFG +#undef UNIFYFS_CFG_CLI +#undef UNIFYFS_CFG_MULTI +#undef UNIFYFS_CFG_MULTI_CLI - //fprintf(stderr, "UNIFYCR CONFIG DEBUG: short-opts '%s'\n", short_opts); + //fprintf(stderr, "UNIFYFS CONFIG DEBUG: short-opts '%s'\n", short_opts); // process argv while ((c = getopt_long(argc, argv, short_opts, cli_options, NULL)) != -1) { switch (c) { -#define UNIFYCR_CFG(sec, key, typ, dv, desc, vfn) +#define UNIFYFS_CFG(sec, key, typ, dv, desc, vfn) -#define UNIFYCR_CFG_CLI(sec, key, typ, dv, desc, vfn, opt, use) \ +#define UNIFYFS_CFG_CLI(sec, key, typ, dv, desc, vfn, opt, use) \ case opt: { \ if (optarg) { \ if (cfg->sec##_##key != NULL) \ @@ -425,9 +425,9 @@ int unifycr_config_process_cli_args(unifycr_cfg_t *cfg, break; \ } -#define UNIFYCR_CFG_MULTI(sec, key, typ, desc, vfn, me) +#define UNIFYFS_CFG_MULTI(sec, key, typ, desc, vfn, me) -#define UNIFYCR_CFG_MULTI_CLI(sec, key, typ, desc, vfn, me, opt, use) \ +#define UNIFYFS_CFG_MULTI_CLI(sec, key, typ, desc, vfn, me, opt, use) \ case opt: { \ if (cfg->sec##_##key[cfg->n_##sec##_##key] != NULL) \ free(cfg->sec##_##key[cfg->n_##sec##_##key]; \ @@ -435,11 +435,11 @@ int unifycr_config_process_cli_args(unifycr_cfg_t *cfg, break; \ } - UNIFYCR_CONFIGS -#undef UNIFYCR_CFG -#undef UNIFYCR_CFG_CLI -#undef UNIFYCR_CFG_MULTI -#undef UNIFYCR_CFG_MULTI_CLI + UNIFYFS_CONFIGS +#undef UNIFYFS_CFG +#undef UNIFYFS_CFG_CLI +#undef UNIFYFS_CFG_MULTI +#undef UNIFYFS_CFG_MULTI_CLI case ':': usage_err = 1; @@ -453,7 +453,7 @@ int unifycr_config_process_cli_args(unifycr_cfg_t *cfg, break; default: // should not reach here - fprintf(stderr, "UNIFYCR CONFIG DEBUG: unhandled option '%s'\n", optarg); + fprintf(stderr, "UNIFYFS CONFIG DEBUG: unhandled option '%s'\n", optarg); break; } if (usage_err) @@ -464,7 +464,7 @@ int unifycr_config_process_cli_args(unifycr_cfg_t *cfg, rc = 0; else { rc = -1; - unifycr_config_cli_usage_error(argv[0], errmsg); + unifyfs_config_cli_usage_error(argv[0], errmsg); } return rc; @@ -483,9 +483,9 @@ char *getenv_helper(const char *section, memset((void *)envname, 0, sizeof(envname)); - ndx += sprintf(envname, "UNIFYCR_"); + ndx += sprintf(envname, "UNIFYFS_"); - if (strcmp(section, "unifycr") != 0) { + if (strcmp(section, "unifyfs") != 0) { len = strlen(section); for (u = 0; u < len; u++) envname[ndx + u] = toupper(section[u]); @@ -501,13 +501,13 @@ char *getenv_helper(const char *section, if (mentry) ndx += sprintf(envname + ndx, "_%u", mentry); - //fprintf(stderr, "UNIFYCR CONFIG DEBUG: checking env var %s\n", envname); + //fprintf(stderr, "UNIFYFS CONFIG DEBUG: checking env var %s\n", envname); return getenv(envname); } // update config struct based on environment variables -int unifycr_config_process_environ(unifycr_cfg_t *cfg) +int unifyfs_config_process_environ(unifyfs_cfg_t *cfg) { char *envval; @@ -515,7 +515,7 @@ int unifycr_config_process_environ(unifycr_cfg_t *cfg) return -1; -#define UNIFYCR_CFG(sec, key, typ, dv, desc, vfn) \ +#define UNIFYFS_CFG(sec, key, typ, dv, desc, vfn) \ envval = getenv_helper(#sec, #key, 0); \ if (envval != NULL) { \ if (cfg->sec##_##key != NULL) \ @@ -523,7 +523,7 @@ int unifycr_config_process_environ(unifycr_cfg_t *cfg) cfg->sec##_##key = strdup(envval); \ } -#define UNIFYCR_CFG_CLI(sec, key, typ, dv, desc, vfn, opt, use) \ +#define UNIFYFS_CFG_CLI(sec, key, typ, dv, desc, vfn, opt, use) \ envval = getenv_helper(#sec, #key, 0); \ if (envval != NULL) { \ if (cfg->sec##_##key != NULL) \ @@ -531,7 +531,7 @@ int unifycr_config_process_environ(unifycr_cfg_t *cfg) cfg->sec##_##key = strdup(envval); \ } -#define UNIFYCR_CFG_MULTI(sec, key, typ, desc, vfn, me) \ +#define UNIFYFS_CFG_MULTI(sec, key, typ, desc, vfn, me) \ for (u = 0; u < me; u++) { \ envval = getenv_helper(#sec, #key, u+1); \ if (envval != NULL) { \ @@ -542,7 +542,7 @@ int unifycr_config_process_environ(unifycr_cfg_t *cfg) } \ } -#define UNIFYCR_CFG_MULTI_CLI(sec, key, typ, desc, vfn, me, opt, use) \ +#define UNIFYFS_CFG_MULTI_CLI(sec, key, typ, desc, vfn, me, opt, use) \ for (u = 0; u < me; u++) { \ envval = getenv_helper(#sec, #key, u+1); \ if (envval != NULL) { \ @@ -553,11 +553,11 @@ int unifycr_config_process_environ(unifycr_cfg_t *cfg) } \ } - UNIFYCR_CONFIGS; -#undef UNIFYCR_CFG -#undef UNIFYCR_CFG_CLI -#undef UNIFYCR_CFG_MULTI -#undef UNIFYCR_CFG_MULTI_CLI + UNIFYFS_CONFIGS; +#undef UNIFYFS_CFG +#undef UNIFYFS_CFG_CLI +#undef UNIFYFS_CFG_MULTI +#undef UNIFYFS_CFG_MULTI_CLI return 0; } @@ -570,14 +570,14 @@ int inih_config_handler(void *user, { char *curval; char *defval; - unifycr_cfg_t *cfg = (unifycr_cfg_t *) user; + unifyfs_cfg_t *cfg = (unifyfs_cfg_t *) user; assert(cfg != NULL); // if not already set by CLI args, set cfg cfgs if (0) ; -#define UNIFYCR_CFG(sec, key, typ, dv, desc, vfn) \ +#define UNIFYFS_CFG(sec, key, typ, dv, desc, vfn) \ else if ((strcmp(section, #sec) == 0) && (strcmp(kee, #key) == 0)) { \ curval = cfg->sec##_##key; \ defval = stringify(dv); \ @@ -589,7 +589,7 @@ int inih_config_handler(void *user, } \ } -#define UNIFYCR_CFG_CLI(sec, key, typ, dv, desc, vfn, opt, use) \ +#define UNIFYFS_CFG_CLI(sec, key, typ, dv, desc, vfn, opt, use) \ else if ((strcmp(section, #sec) == 0) && (strcmp(kee, #key) == 0)) { \ curval = cfg->sec##_##key; \ defval = stringify(dv); \ @@ -601,31 +601,31 @@ int inih_config_handler(void *user, } \ } -#define UNIFYCR_CFG_MULTI(sec, key, typ, desc, vfn, me) \ +#define UNIFYFS_CFG_MULTI(sec, key, typ, desc, vfn, me) \ else if ((strcmp(section, #sec) == 0) && (strcmp(kee, #key) == 0)) { \ cfg->sec##_##key[cfg->n_##sec##_##key++] = strdup(val); \ } -#define UNIFYCR_CFG_MULTI_CLI(sec, key, typ, desc, vfn, me, opt, use) \ +#define UNIFYFS_CFG_MULTI_CLI(sec, key, typ, desc, vfn, me, opt, use) \ else if ((strcmp(section, #sec) == 0) && (strcmp(kee, #key) == 0)) { \ cfg->sec##_##key[cfg->n_##sec##_##key++] = strdup(val); \ } - UNIFYCR_CONFIGS; -#undef UNIFYCR_CFG -#undef UNIFYCR_CFG_CLI -#undef UNIFYCR_CFG_MULTI -#undef UNIFYCR_CFG_MULTI_CLI + UNIFYFS_CONFIGS; +#undef UNIFYFS_CFG +#undef UNIFYFS_CFG_CLI +#undef UNIFYFS_CFG_MULTI +#undef UNIFYFS_CFG_MULTI_CLI return 1; } // update config struct based on config file, using inih -int unifycr_config_process_ini_file(unifycr_cfg_t *cfg, +int unifyfs_config_process_ini_file(unifyfs_cfg_t *cfg, const char *file) { int rc, inih_rc; - char errmsg[UNIFYCR_CFG_MAX_MSG]; + char errmsg[UNIFYFS_CFG_MAX_MSG]; if (cfg == NULL) return EINVAL; @@ -642,14 +642,14 @@ int unifycr_config_process_ini_file(unifycr_cfg_t *cfg, snprintf(errmsg, sizeof(errmsg), "failed to open config file %s", file); - fprintf(stderr, "UNIFYCR CONFIG ERROR: %s\n", errmsg); + fprintf(stderr, "UNIFYFS CONFIG ERROR: %s\n", errmsg); rc = ENOENT; break; case -2: snprintf(errmsg, sizeof(errmsg), "failed to parse config file %s", file); - fprintf(stderr, "UNIFYCR CONFIG ERROR: %s\n", errmsg); + fprintf(stderr, "UNIFYFS CONFIG ERROR: %s\n", errmsg); rc = ENOMEM; break; default: @@ -663,7 +663,7 @@ int unifycr_config_process_ini_file(unifycr_cfg_t *cfg, "failed to parse config file %s", file); rc = EINVAL; - fprintf(stderr, "UNIFYCR CONFIG ERROR: %s\n", errmsg); + fprintf(stderr, "UNIFYFS CONFIG ERROR: %s\n", errmsg); break; } @@ -695,7 +695,7 @@ int validate_value(const char *section, // validate configuration -int unifycr_config_validate(unifycr_cfg_t *cfg) +int unifyfs_config_validate(unifyfs_cfg_t *cfg) { int rc = 0; int vrc; @@ -704,11 +704,11 @@ int unifycr_config_validate(unifycr_cfg_t *cfg) if (cfg == NULL) return EINVAL; -#define UNIFYCR_CFG(sec, key, typ, dv, desc, vfn) \ +#define UNIFYFS_CFG(sec, key, typ, dv, desc, vfn) \ vrc = validate_value(#sec, #key, cfg->sec##_##key, #typ, vfn, &new_val); \ if (vrc) { \ rc = vrc; \ - fprintf(stderr, "UNIFYCR CONFIG ERROR: value '%s' for %s.%s is INVALID %s\n", \ + fprintf(stderr, "UNIFYFS CONFIG ERROR: value '%s' for %s.%s is INVALID %s\n", \ cfg->sec##_##key, #sec, #key, #typ); \ } \ else if (new_val != NULL) { \ @@ -718,11 +718,11 @@ int unifycr_config_validate(unifycr_cfg_t *cfg) new_val = NULL; \ } -#define UNIFYCR_CFG_CLI(sec, key, typ, dv, desc, vfn, opt, use) \ +#define UNIFYFS_CFG_CLI(sec, key, typ, dv, desc, vfn, opt, use) \ vrc = validate_value(#sec, #key, cfg->sec##_##key, #typ, vfn, &new_val); \ if (vrc) { \ rc = vrc; \ - fprintf(stderr, "UNIFYCR CONFIG ERROR: value '%s' for %s.%s is INVALID %s\n", \ + fprintf(stderr, "UNIFYFS CONFIG ERROR: value '%s' for %s.%s is INVALID %s\n", \ cfg->sec##_##key, #sec, #key, #typ); \ } \ else if (new_val != NULL) { \ @@ -732,12 +732,12 @@ int unifycr_config_validate(unifycr_cfg_t *cfg) new_val = NULL; \ } -#define UNIFYCR_CFG_MULTI(sec, key, typ, desc, vfn, me) \ +#define UNIFYFS_CFG_MULTI(sec, key, typ, desc, vfn, me) \ for (u = 0; u < me; u++) { \ vrc = validate_value(#sec, #key, cfg->sec##_##key[u], #typ, vfn, &new_val); \ if (vrc) { \ rc = vrc; \ - fprintf(stderr, "UNIFYCR CONFIG ERROR: value[%u] '%s' for %s.%s is INVALID %s\n", \ + fprintf(stderr, "UNIFYFS CONFIG ERROR: value[%u] '%s' for %s.%s is INVALID %s\n", \ u+1, cfg->sec##_##key[u], #sec, #key, #typ); \ } \ else if (new_val != NULL) { \ @@ -748,12 +748,12 @@ int unifycr_config_validate(unifycr_cfg_t *cfg) } \ } -#define UNIFYCR_CFG_MULTI_CLI(sec, key, typ, desc, vfn, me, opt, use) \ +#define UNIFYFS_CFG_MULTI_CLI(sec, key, typ, desc, vfn, me, opt, use) \ for (u = 0; u < me; u++) { \ vrc = validate_value(#sec, #key, cfg->sec##_##key[u], #typ, vfn, &new_val); \ if (vrc) { \ rc = vrc; \ - fprintf(stderr, "UNIFYCR CONFIG ERROR: value[%u] '%s' for %s.%s is INVALID %s\n", \ + fprintf(stderr, "UNIFYFS CONFIG ERROR: value[%u] '%s' for %s.%s is INVALID %s\n", \ u+1, cfg->sec##_##key[u], #sec, #key, #typ); \ } \ else if (new_val != NULL) { \ @@ -764,11 +764,11 @@ int unifycr_config_validate(unifycr_cfg_t *cfg) } \ } - UNIFYCR_CONFIGS; -#undef UNIFYCR_CFG -#undef UNIFYCR_CFG_CLI -#undef UNIFYCR_CFG_MULTI -#undef UNIFYCR_CFG_MULTI_CLI + UNIFYFS_CONFIGS; +#undef UNIFYFS_CFG +#undef UNIFYFS_CFG_CLI +#undef UNIFYFS_CFG_MULTI +#undef UNIFYFS_CFG_MULTI_CLI return rc; } diff --git a/common/src/unifycr_configurator.h b/common/src/unifyfs_configurator.h similarity index 58% rename from common/src/unifycr_configurator.h rename to common/src/unifyfs_configurator.h index c6549fc8d..e2cb36f08 100644 --- a/common/src/unifycr_configurator.h +++ b/common/src/unifyfs_configurator.h @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* Copyright (c) 2018 - Michael J. Brim @@ -19,8 +19,8 @@ * MIT License - See LICENSE.tedium */ -#ifndef _UNIFYCR_CONFIGURATOR_H_ -#define _UNIFYCR_CONFIGURATOR_H_ +#ifndef _UNIFYFS_CONFIGURATOR_H_ +#define _UNIFYFS_CONFIGURATOR_H_ /* Configurator unifies config files, environment variables, and command-line * arguments into a set of simple preprocessor definitions that capture the @@ -41,7 +41,7 @@ # include #endif -#include "unifycr_const.h" +#include "unifyfs_const.h" #ifndef TMPDIR #define TMPDIR /tmp @@ -61,100 +61,110 @@ // NOTE: NULLSTRING is a sentinel token meaning "no default string value" -/* UNIFYCR_CONFIGS is the list of configuration settings, and should contain +/* UNIFYFS_CONFIGS is the list of configuration settings, and should contain one macro definition per setting */ -#define UNIFYCR_CONFIGS \ - UNIFYCR_CFG_CLI(unifycr, cleanup, BOOL, off, "cleanup storage on server exit", NULL, 'C', "on|off") \ - UNIFYCR_CFG_CLI(unifycr, configfile, STRING, SYSCONFDIR/unifycr/unifycr.conf, "path to configuration file", configurator_file_check, 'f', "specify full path to config file") \ - UNIFYCR_CFG_CLI(unifycr, consistency, STRING, LAMINATED, "consistency model", NULL, 'c', "specify consistency model (NONE | LAMINATED | POSIX)") \ - UNIFYCR_CFG_CLI(unifycr, daemonize, BOOL, on, "enable server daemonization", NULL, 'D', "on|off") \ - UNIFYCR_CFG_CLI(unifycr, debug, BOOL, off, "enable debug output", NULL, 'd', "on|off") \ - UNIFYCR_CFG_CLI(unifycr, mountpoint, STRING, /unifycr, "mountpoint directory", NULL, 'm', "specify full path to desired mountpoint") \ - UNIFYCR_CFG(client, max_files, INT, UNIFYCR_MAX_FILES, "client max file count", NULL) \ - UNIFYCR_CFG_CLI(log, verbosity, INT, 0, "log verbosity level", NULL, 'v', "specify logging verbosity level") \ - UNIFYCR_CFG_CLI(log, file, STRING, unifycrd.log, "log file name", NULL, 'l', "specify log file name") \ - UNIFYCR_CFG_CLI(log, dir, STRING, LOGDIR, "log file directory", configurator_directory_check, 'L', "specify full path to directory to contain log file") \ - UNIFYCR_CFG(logfs, index_buf_size, INT, UNIFYCR_INDEX_BUF_SIZE, "log file system index buffer size", NULL) \ - UNIFYCR_CFG(logfs, attr_buf_size, INT, UNIFYCR_FATTR_BUF_SIZE, "log file system file attributes buffer size", NULL) \ - UNIFYCR_CFG(meta, db_name, STRING, META_DEFAULT_DB_NAME, "metadata database name", NULL) \ - UNIFYCR_CFG(meta, db_path, STRING, /tmp, "metadata database path", NULL) \ - UNIFYCR_CFG(meta, server_ratio, INT, META_DEFAULT_SERVER_RATIO, "metadata server ratio", NULL) \ - UNIFYCR_CFG(meta, range_size, INT, META_DEFAULT_RANGE_SZ, "metadata range size", NULL) \ - UNIFYCR_CFG_CLI(runstate, dir, STRING, RUNDIR, "runstate file directory", configurator_directory_check, 'R', "specify full path to directory to contain server runstate file") \ - UNIFYCR_CFG(shmem, chunk_bits, INT, UNIFYCR_CHUNK_BITS, "shared memory data chunk size in bits (i.e., size=2^bits)", NULL) \ - UNIFYCR_CFG(shmem, chunk_mem, INT, UNIFYCR_CHUNK_MEM, "shared memory segment size for data chunks", NULL) \ - UNIFYCR_CFG(shmem, recv_size, INT, UNIFYCR_SHMEM_RECV_SIZE, "shared memory segment size in bytes for receiving data from delegators", NULL) \ - UNIFYCR_CFG(shmem, req_size, INT, UNIFYCR_SHMEM_REQ_SIZE, "shared memory segment size in bytes for sending requests to delegators", NULL) \ - UNIFYCR_CFG(shmem, single, BOOL, off, "use single shared memory region for all clients", NULL) \ - UNIFYCR_CFG(spillover, enabled, BOOL, on, "use local device for data chunk spillover", NULL) \ - UNIFYCR_CFG(spillover, data_dir, STRING, NULLSTRING, "spillover data directory", configurator_directory_check) \ - UNIFYCR_CFG(spillover, meta_dir, STRING, NULLSTRING, "spillover metadata directory", configurator_directory_check) \ - UNIFYCR_CFG(spillover, size, INT, UNIFYCR_SPILLOVER_SIZE, "spillover max data size in bytes", NULL) \ +#define UNIFYFS_CONFIGS \ + UNIFYFS_CFG_CLI(unifyfs, cleanup, BOOL, off, "cleanup storage on server exit", NULL, 'C', "on|off") \ + UNIFYFS_CFG_CLI(unifyfs, configfile, STRING, SYSCONFDIR/unifyfs/unifyfs.conf, "path to configuration file", configurator_file_check, 'f', "specify full path to config file") \ + UNIFYFS_CFG_CLI(unifyfs, consistency, STRING, LAMINATED, "consistency model", NULL, 'c', "specify consistency model (NONE | LAMINATED | POSIX)") \ + UNIFYFS_CFG_CLI(unifyfs, daemonize, BOOL, on, "enable server daemonization", NULL, 'D', "on|off") \ + UNIFYFS_CFG_CLI(unifyfs, mountpoint, STRING, /unifyfs, "mountpoint directory", NULL, 'm', "specify full path to desired mountpoint") \ + UNIFYFS_CFG(client, max_files, INT, UNIFYFS_MAX_FILES, "client max file count", NULL) \ + UNIFYFS_CFG_CLI(log, verbosity, INT, 0, "log verbosity level", NULL, 'v', "specify logging verbosity level") \ + UNIFYFS_CFG_CLI(log, file, STRING, unifyfsd.log, "log file name", NULL, 'l', "specify log file name") \ + UNIFYFS_CFG_CLI(log, dir, STRING, LOGDIR, "log file directory", configurator_directory_check, 'L', "specify full path to directory to contain log file") \ + UNIFYFS_CFG(logfs, index_buf_size, INT, UNIFYFS_INDEX_BUF_SIZE, "log file system index buffer size", NULL) \ + UNIFYFS_CFG(logfs, attr_buf_size, INT, UNIFYFS_FATTR_BUF_SIZE, "log file system file attributes buffer size", NULL) \ + UNIFYFS_CFG(margo, tcp, BOOL, on, "use TCP for server-server margo RPCs", NULL) \ + UNIFYFS_CFG(meta, db_name, STRING, META_DEFAULT_DB_NAME, "metadata database name", NULL) \ + UNIFYFS_CFG(meta, db_path, STRING, /tmp, "metadata database path", NULL) \ + UNIFYFS_CFG(meta, server_ratio, INT, META_DEFAULT_SERVER_RATIO, "metadata server ratio", NULL) \ + UNIFYFS_CFG(meta, range_size, INT, META_DEFAULT_RANGE_SZ, "metadata range size", NULL) \ + UNIFYFS_CFG_CLI(runstate, dir, STRING, RUNDIR, "runstate file directory", configurator_directory_check, 'R', "specify full path to directory to contain server runstate file") \ + UNIFYFS_CFG_CLI(server, hostfile, STRING, NULLSTRING, "server hostfile name", NULL, 'H', "specify full path to server hostfile") \ + UNIFYFS_CFG_CLI(sharedfs, dir, STRING, NULLSTRING, "shared file system directory", configurator_directory_check, 'S', "specify full path to directory to contain server shared files") \ + UNIFYFS_CFG(shmem, chunk_bits, INT, UNIFYFS_CHUNK_BITS, "shared memory data chunk size in bits (i.e., size=2^bits)", NULL) \ + UNIFYFS_CFG(shmem, chunk_mem, INT, UNIFYFS_CHUNK_MEM, "shared memory segment size for data chunks", NULL) \ + UNIFYFS_CFG(shmem, recv_size, INT, UNIFYFS_SHMEM_RECV_SIZE, "shared memory segment size in bytes for receiving data from delegators", NULL) \ + UNIFYFS_CFG(shmem, req_size, INT, UNIFYFS_SHMEM_REQ_SIZE, "shared memory segment size in bytes for sending requests to delegators", NULL) \ + UNIFYFS_CFG(shmem, single, BOOL, off, "use single shared memory region for all clients", NULL) \ + UNIFYFS_CFG(spillover, enabled, BOOL, on, "use local device for data chunk spillover", NULL) \ + UNIFYFS_CFG(spillover, data_dir, STRING, NULLSTRING, "spillover data directory", configurator_directory_check) \ + UNIFYFS_CFG(spillover, meta_dir, STRING, NULLSTRING, "spillover metadata directory", configurator_directory_check) \ + UNIFYFS_CFG(spillover, size, INT, UNIFYFS_SPILLOVER_SIZE, "spillover max data size in bytes", NULL) \ #ifdef __cplusplus extern "C" { #endif -/* unifycr_cfg_t struct */ +typedef enum { + INVALID_PROCESS_TYPE = 0, + UNIFYFS_CLIENT = 1, + UNIFYFS_SERVER = 2 +} unifyfs_proc_type_e; + +/* unifyfs_cfg_t struct */ typedef struct { -#define UNIFYCR_CFG(sec, key, typ, dv, desc, vfn) \ + unifyfs_proc_type_e ptype; + +#define UNIFYFS_CFG(sec, key, typ, dv, desc, vfn) \ char *sec##_##key; -#define UNIFYCR_CFG_CLI(sec, key, typ, dv, desc, vfn, opt, use) \ +#define UNIFYFS_CFG_CLI(sec, key, typ, dv, desc, vfn, opt, use) \ char *sec##_##key; -#define UNIFYCR_CFG_MULTI(sec, key, typ, dv, desc, vfn, me) \ +#define UNIFYFS_CFG_MULTI(sec, key, typ, dv, desc, vfn, me) \ char *sec##_##key[me]; \ unsigned n_##sec##_##key; -#define UNIFYCR_CFG_MULTI_CLI(sec, key, typ, dv, desc, vfn, me, opt, use) \ +#define UNIFYFS_CFG_MULTI_CLI(sec, key, typ, dv, desc, vfn, me, opt, use) \ char *sec##_##key[me]; \ unsigned n_##sec##_##key; - UNIFYCR_CONFIGS + UNIFYFS_CONFIGS -#undef UNIFYCR_CFG -#undef UNIFYCR_CFG_CLI -#undef UNIFYCR_CFG_MULTI -#undef UNIFYCR_CFG_MULTI_CLI -} unifycr_cfg_t; +#undef UNIFYFS_CFG +#undef UNIFYFS_CFG_CLI +#undef UNIFYFS_CFG_MULTI +#undef UNIFYFS_CFG_MULTI_CLI +} unifyfs_cfg_t; /* initialization and cleanup */ -int unifycr_config_init(unifycr_cfg_t *cfg, +int unifyfs_config_init(unifyfs_cfg_t *cfg, int argc, char **argv); -int unifycr_config_fini(unifycr_cfg_t *cfg); +int unifyfs_config_fini(unifyfs_cfg_t *cfg); /* print configuration to specified file (or stderr if fp==NULL) */ -void unifycr_config_print(unifycr_cfg_t *cfg, +void unifyfs_config_print(unifyfs_cfg_t *cfg, FILE *fp); /* print configuration in .INI format to specified file (or stderr) */ -void unifycr_config_print_ini(unifycr_cfg_t *cfg, +void unifyfs_config_print_ini(unifyfs_cfg_t *cfg, FILE *inifp); /* used internally, but may be useful externally */ -int unifycr_config_set_defaults(unifycr_cfg_t *cfg); +int unifyfs_config_set_defaults(unifyfs_cfg_t *cfg); -void unifycr_config_cli_usage(char *arg0); -void unifycr_config_cli_usage_error(char *arg0, +void unifyfs_config_cli_usage(char *arg0); +void unifyfs_config_cli_usage_error(char *arg0, char *err_msg); -int unifycr_config_process_cli_args(unifycr_cfg_t *cfg, +int unifyfs_config_process_cli_args(unifyfs_cfg_t *cfg, int argc, char **argv); -int unifycr_config_process_environ(unifycr_cfg_t *cfg); +int unifyfs_config_process_environ(unifyfs_cfg_t *cfg); -int unifycr_config_process_ini_file(unifycr_cfg_t *cfg, +int unifyfs_config_process_ini_file(unifyfs_cfg_t *cfg, const char *file); -int unifycr_config_validate(unifycr_cfg_t *cfg); +int unifyfs_config_validate(unifyfs_cfg_t *cfg); /* validate function prototype - Returns: 0 for valid input, non-zero otherwise. @@ -201,4 +211,4 @@ int configurator_directory_check(const char *section, } /* extern C */ #endif -#endif /* UNIFYCR_CONFIGURATOR_H */ +#endif /* UNIFYFS_CONFIGURATOR_H */ diff --git a/common/src/unifyfs_const.h b/common/src/unifyfs_const.h new file mode 100644 index 000000000..1474b35c6 --- /dev/null +++ b/common/src/unifyfs_const.h @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2017, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +/* + * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * Copyright (c) 2017, Florida State University. Contributions from + * the Computer Architecture and Systems Research Laboratory (CASTL) + * at the Department of Computer Science. + * + * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror + * LLNL-CODE-728877. All rights reserved. + * + * This file is part of burstfs. + * For details, see https://github.com/llnl/burstfs + * Please read https://github.com/llnl/burstfs/LICENSE for full license text. + */ + +#ifndef UNIFYFS_CONST_H +#define UNIFYFS_CONST_H + +/* ********************** ERROR CODES ************************ */ +#include "err_enumerator.h" +#define ULFS_SUCCESS ((int)UNIFYFS_SUCCESS) + +/* ********************** STRING CONSTANTS ************************ */ +#define DEFAULT_INTERFACE "ib0" +#define SOCKET_PATH "/tmp/unifyfs_server_sock" + +/* ********************** INT CONSTANTS ************************ */ + +// Byte counts +#define KIB 1024 +#define MIB 1048576 +#define GIB 1073741824 + +// Generic +#define GEN_STR_LEN KIB +#define UNIFYFS_MAX_FILENAME KIB +#define UNIFYFS_MAX_HOSTNAME 64 + +// Metadata +#define MAX_FILE_CNT_PER_NODE KIB + +// Request Manager +#define RECV_BUF_CNT 4 /* number of remote read buffers */ +#define SENDRECV_BUF_LEN (8 * MIB) /* remote read buffer size */ +#define MAX_META_PER_SEND (4 * KIB) /* max read request count per server */ +#define REQ_BUF_LEN (MAX_META_PER_SEND * 128) /* read requests (send_msg_t) */ +#define SHM_WAIT_INTERVAL 1000 /* unit: ns */ +#define RM_MAX_ACTIVE_REQUESTS 64 /* number of concurrent read requests */ + +// Service Manager +#define LARGE_BURSTY_DATA (512 * MIB) +#define MAX_BURSTY_INTERVAL 10000 /* unit: us */ +#define MIN_SLEEP_INTERVAL 10 /* unit: us */ +#define SLEEP_INTERVAL 500 /* unit: us */ +#define SLEEP_SLICE_PER_UNIT 50 /* unit: us */ +#define READ_BLOCK_SIZE MIB +#define READ_BUF_SZ GIB + +// Request and Service Managers, Command Handler +#define MAX_NUM_CLIENTS 64 /* app processes per server */ + +// Client and Command Handler +#define CMD_BUF_SIZE (2 * KIB) + +// Client +#define UNIFYFS_MAX_FILES 128 +#define UNIFYFS_MAX_FILEDESCS UNIFYFS_MAX_FILES +#define UNIFYFS_STREAM_BUFSIZE MIB +#define UNIFYFS_CHUNK_BITS 24 +#define UNIFYFS_CHUNK_MEM (256 * MIB) +#define UNIFYFS_SPILLOVER_SIZE (KIB * MIB) +#define UNIFYFS_SUPERBLOCK_KEY 4321 +#define UNIFYFS_SHMEM_REQ_SIZE (8 * MIB) +#define UNIFYFS_SHMEM_RECV_SIZE (32 * MIB) +#define UNIFYFS_INDEX_BUF_SIZE (20 * MIB) +#define UNIFYFS_FATTR_BUF_SIZE MIB +#define UNIFYFS_MAX_READ_CNT KIB + +/* max read size = UNIFYFS_MAX_SPLIT_CNT * META_DEFAULT_RANGE_SZ */ +#define UNIFYFS_MAX_SPLIT_CNT (4 * KIB) + +// Metadata/MDHIM Default Values +#define META_DEFAULT_DB_NAME unifyfs_db +#define META_DEFAULT_SERVER_RATIO 1 +#define META_DEFAULT_RANGE_SZ MIB + +#endif // UNIFYFS_CONST_H + diff --git a/common/src/unifyfs_keyval.c b/common/src/unifyfs_keyval.c new file mode 100644 index 000000000..e5f528f88 --- /dev/null +++ b/common/src/unifyfs_keyval.c @@ -0,0 +1,998 @@ +/* + * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2019, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#include "unifyfs_const.h" +#include "unifyfs_keyval.h" +#include "unifyfs_log.h" + +//#include "config.h" + +#if defined(USE_PMIX) +# include +#elif defined(USE_PMI2) +# include +#endif + +#include +#include +#include +#include +#include +#include +#include + +// UnifyFS keys +const char* key_runstate = "unifyfs.runstate"; +const char* key_unifyfsd_socket = "unifyfsd.socket"; +const char* key_unifyfsd_margo_shm = "unifyfsd.margo-shm"; +const char* key_unifyfsd_margo_svr = "unifyfsd.margo-svr"; +const char* key_unifyfsd_mpi_rank = "unifyfsd.mpi-rank"; + +// key-value store state +static int kv_initialized; // = 0 +static int kv_published; // = 0 +static int kv_myrank = -1; +static int kv_nranks = -1; +static char kv_myhost[80]; + +// max supported key and value length +static size_t kv_max_keylen; // = 0 +static size_t kv_max_vallen; // = 0 + +#ifndef UNIFYFS_MAX_KV_KEYLEN +# define UNIFYFS_MAX_KV_KEYLEN 256 +#endif + +#ifndef UNIFYFS_MAX_KV_VALLEN +# define UNIFYFS_MAX_KV_VALLEN 4096 +#endif + + +//--------------------- PMI2 K-V Store --------------------- +#if defined(USE_PMI2) +#include + +static int pmi2_initialized; // = 0 +static int pmi2_need_finalize; // = 0 +static int pmi2_has_nameserv; // = 0 + +static char pmi2_errstr[64]; + +static void unifyfs_pmi2_errstr(int rc) +{ + switch (rc) { + case PMI2_SUCCESS: + snprintf(pmi2_errstr, sizeof(pmi2_errstr), "PMI2 Success"); + break; + case PMI2_FAIL: + snprintf(pmi2_errstr, sizeof(pmi2_errstr), "PMI2 Failure"); + break; + case PMI2_ERR_NOMEM: + snprintf(pmi2_errstr, sizeof(pmi2_errstr), "PMI2 Out of Memory"); + break; + case PMI2_ERR_INIT: + snprintf(pmi2_errstr, sizeof(pmi2_errstr), "PMI2 Init Error"); + break; + case PMI2_ERR_INVALID_ARG: + snprintf(pmi2_errstr, sizeof(pmi2_errstr), "PMI2 Invalid Arg"); + break; + case PMI2_ERR_INVALID_KEY: + snprintf(pmi2_errstr, sizeof(pmi2_errstr), "PMI2 Invalid Key"); + break; + case PMI2_ERR_INVALID_KEY_LENGTH: + snprintf(pmi2_errstr, sizeof(pmi2_errstr), "PMI2 Invalid Key Length"); + break; + case PMI2_ERR_INVALID_VAL: + snprintf(pmi2_errstr, sizeof(pmi2_errstr), "PMI2 Invalid Val"); + break; + case PMI2_ERR_INVALID_VAL_LENGTH: + snprintf(pmi2_errstr, sizeof(pmi2_errstr), "PMI2 Invalid Val Length"); + break; + case PMI2_ERR_INVALID_LENGTH: + snprintf(pmi2_errstr, sizeof(pmi2_errstr), "PMI2 Invalid Length"); + break; + case PMI2_ERR_INVALID_NUM_ARGS: + snprintf(pmi2_errstr, sizeof(pmi2_errstr), "PMI2 Invalid Num Args"); + break; + case PMI2_ERR_INVALID_ARGS: + snprintf(pmi2_errstr, sizeof(pmi2_errstr), "PMI2 Invalid Args"); + break; + case PMI2_ERR_INVALID_NUM_PARSED: + snprintf(pmi2_errstr, sizeof(pmi2_errstr), "PMI2 Invalid Num Parsed"); + break; + case PMI2_ERR_INVALID_KEYVALP: + snprintf(pmi2_errstr, sizeof(pmi2_errstr), "PMI2 Invalid KeyVal Ptr"); + break; + case PMI2_ERR_INVALID_SIZE: + snprintf(pmi2_errstr, sizeof(pmi2_errstr), "PMI2 Invalid Size"); + break; + case PMI2_ERR_OTHER: + snprintf(pmi2_errstr, sizeof(pmi2_errstr), "PMI2 Other Error"); + break; + default: + snprintf(pmi2_errstr, sizeof(pmi2_errstr), "Unknown PMI2 Error Code"); + break; + } +} + +// initialize PMI2 +static int unifyfs_pmi2_init(void) +{ + int nprocs, rank, rc, val, len, found; + int pmi_world_rank = -1; + int pmi_world_nprocs = -1; + + kv_max_keylen = PMI2_MAX_KEYLEN; + kv_max_vallen = PMI2_MAX_VALLEN; + + rc = PMI2_Initialized(); + if (!rc) { + int spawned, appid; + rc = PMI2_Init(&spawned, &nprocs, &rank, &appid); + if (rc != PMI2_SUCCESS) { + unifyfs_pmi2_errstr(rc); + LOGERR("PMI2_Init() failed: %s", pmi2_errstr); + return (int)UNIFYFS_FAILURE; + } + pmi_world_rank = rank; + pmi_world_nprocs = nprocs; + pmi2_need_finalize = 1; + } + + if (-1 == pmi_world_rank) { + rc = PMI2_Job_GetRank(&rank); + if (rc != PMI2_SUCCESS) { + unifyfs_pmi2_errstr(rc); + LOGERR("PMI2_Job_GetRank() failed: %s", pmi2_errstr); + return (int)UNIFYFS_FAILURE; + } else { + pmi_world_rank = rank; + } + } + + if (-1 == pmi_world_nprocs) { + found = 0; + nprocs = 0; + rc = PMI2_Info_GetJobAttrIntArray("universeSize", &nprocs, 1, + &len, &found); + if (found) { + pmi_world_nprocs = nprocs; + } + } + + found = 0; + val = 0; + rc = PMI2_Info_GetJobAttrIntArray("hasNameServ", &val, 1, + &len, &found); + if (found) { + pmi2_has_nameserv = val; + } + + char pmi_jobid[64]; + memset(pmi_jobid, 0, sizeof(pmi_jobid)); + rc = PMI2_Job_GetId(pmi_jobid, sizeof(pmi_jobid)); + if (rc != PMI2_SUCCESS) { + unifyfs_pmi2_errstr(rc); + LOGERR("PMI2_Job_GetId() failed: %s", pmi2_errstr); + return (int)UNIFYFS_FAILURE; + } + + kv_myrank = pmi_world_rank; + kv_nranks = pmi_world_nprocs; + + LOGDBG("PMI2 Job Id: %s, Rank: %d of %d, hasNameServer=%d", pmi_jobid, + kv_myrank, kv_nranks, pmi2_has_nameserv); + + pmi2_initialized = 1; + + return (int)UNIFYFS_SUCCESS; +} + +// finalize PMI2 +static int unifyfs_pmi2_fini(void) +{ + int rc; + + if (pmi2_need_finalize) { + rc = PMI2_Finalize(); + if (rc != PMI2_SUCCESS) { + unifyfs_pmi2_errstr(rc); + LOGERR("PMI2_Finalize() failed: %s", pmi2_errstr); + return (int)UNIFYFS_FAILURE; + } + pmi2_need_finalize = 0; + pmi2_initialized = 0; + } + + return (int)UNIFYFS_SUCCESS; +} + +// lookup a key-value pair +static int unifyfs_pmi2_lookup(const char* key, + char** oval) +{ + int rc, vallen; + char pmi2_key[PMI2_MAX_KEYLEN] = {0}; + char pmi2_val[PMI2_MAX_VALLEN] = {0}; + + if (!pmi2_initialized) { + return (int)UNIFYFS_FAILURE; + } + + strncpy(pmi2_key, key, sizeof(pmi2_key)); + vallen = 0; + rc = PMI2_KVS_Get(NULL, PMI2_ID_NULL, pmi2_key, pmi2_val, + sizeof(pmi2_val), &vallen); + if (rc != PMI2_SUCCESS) { + unifyfs_pmi2_errstr(rc); + LOGERR("PMI2_KVS_Get(%s) failed: %s", key, pmi2_errstr); + return (int)UNIFYFS_FAILURE; + } + *oval = strdup(pmi2_val); + return (int)UNIFYFS_SUCCESS; +} + +// publish a key-value pair +static int unifyfs_pmi2_publish(const char* key, + const char* val) +{ + int rc; + char pmi2_key[PMI2_MAX_KEYLEN] = {0}; + char pmi2_val[PMI2_MAX_VALLEN] = {0}; + + if (!pmi2_initialized) { + return (int)UNIFYFS_FAILURE; + } + + strncpy(pmi2_key, key, sizeof(pmi2_key)); + strncpy(pmi2_val, val, sizeof(pmi2_val)); + rc = PMI2_KVS_Put(pmi2_key, pmi2_val); + if (rc != PMI2_SUCCESS) { + unifyfs_pmi2_errstr(rc); + LOGERR("PMI2_KVS_Put(%s) failed: %s", key, pmi2_errstr); + return (int)UNIFYFS_FAILURE; + } + rc = PMI2_KVS_Fence(); + if (rc != PMI2_SUCCESS) { + unifyfs_pmi2_errstr(rc); + LOGERR("PMI2_KVS_Fence() failed: %s", pmi2_errstr); + return (int)UNIFYFS_FAILURE; + } + return (int)UNIFYFS_SUCCESS; +} + +#endif // USE_PMI2 + + +//--------------------- PMIx K-V Store --------------------- +#if defined(USE_PMIX) +#include + +static int pmix_initialized; // = 0 +static pmix_proc_t pmix_myproc; + +#ifndef PMIX_MAX_VALLEN +# define PMIX_MAX_VALLEN UNIFYFS_MAX_KV_VALLEN +#endif + +// initialize PMIx +static int unifyfs_pmix_init(void) +{ + int rc; + size_t pmix_univ_nprocs; + pmix_value_t value; + pmix_value_t* valp = &value; + pmix_proc_t proc; + + /* init PMIx */ + PMIX_PROC_CONSTRUCT(&pmix_myproc); + rc = PMIx_Init(&pmix_myproc, NULL, 0); + if (rc != PMIX_SUCCESS) { + LOGERR("PMIx_Init() failed: %s", PMIx_Error_string(rc)); + return (int)UNIFYFS_FAILURE; + } + + kv_max_keylen = PMIX_MAX_KEYLEN; + kv_max_vallen = PMIX_MAX_VALLEN; + + /* get PMIx universe size */ + PMIX_PROC_CONSTRUCT(&proc); + (void)strncpy(proc.nspace, pmix_myproc.nspace, PMIX_MAX_NSLEN); + proc.rank = PMIX_RANK_WILDCARD; + rc = PMIx_Get(&proc, PMIX_UNIV_SIZE, NULL, 0, &valp); + if (rc != PMIX_SUCCESS) { + LOGERR("PMIx rank %d: PMIx_Get(UNIV_SIZE) failed: %s", + pmix_myproc.rank, PMIx_Error_string(rc)); + return (int)UNIFYFS_FAILURE; + } + pmix_univ_nprocs = (size_t) valp->data.uint32; + + kv_myrank = pmix_myproc.rank; + kv_nranks = (int)pmix_univ_nprocs; + + LOGDBG("PMIX Job Id: %s, Rank: %d of %d", pmix_myproc.nspace, + kv_myrank, kv_nranks); + + PMIX_VALUE_RELEASE(valp); + PMIX_PROC_DESTRUCT(&proc); + + pmix_initialized = 1; + + return (int)UNIFYFS_SUCCESS; +} + +// finalize PMIx +static int unifyfs_pmix_fini(void) +{ + int rc; + size_t ninfo; + pmix_info_t* info; + pmix_data_range_t range; + + if (kv_published) { + /* unpublish everything I published */ + range = PMIX_RANGE_GLOBAL; + ninfo = 1; + PMIX_INFO_CREATE(info, ninfo); + PMIX_INFO_LOAD(&info[0], PMIX_RANGE, &range, PMIX_DATA_RANGE); + rc = PMIx_Unpublish(NULL, info, ninfo); + if (rc != PMIX_SUCCESS) { + LOGERR("PMIx rank %d: PMIx_Unpublish() failed: %s", + pmix_myproc.rank, PMIx_Error_string(rc)); + } + } + + /* fini PMIx */ + rc = PMIx_Finalize(NULL, 0); + if (rc != PMIX_SUCCESS) { + LOGERR("PMIx rank %d: PMIx_Finalize() failed: %s", + pmix_myproc.rank, PMIx_Error_string(rc)); + rc = (int) UNIFYFS_FAILURE; + } else { + PMIX_PROC_DESTRUCT(&pmix_myproc); + pmix_initialized = 0; + rc = (int) UNIFYFS_SUCCESS; + } + return rc; +} + +static int unifyfs_pmix_lookup(const char* key, + char** oval) +{ + int rc, wait; + size_t ndir; + pmix_data_range_t range; + pmix_info_t* directives; + pmix_pdata_t* pdata; + char pmix_key[PMIX_MAX_KEYLEN+1]; + + if (!pmix_initialized) { + return (int)UNIFYFS_FAILURE; + } + + /* set key to lookup */ + strncpy(pmix_key, key, sizeof(pmix_key)); + PMIX_PDATA_CREATE(pdata, 1); + PMIX_PDATA_LOAD(&pdata[0], &pmix_myproc, pmix_key, NULL, PMIX_STRING); + + /* modify lookup behavior */ + wait = 1; + ndir = 2; + range = PMIX_RANGE_GLOBAL; + PMIX_INFO_CREATE(directives, ndir); + PMIX_INFO_LOAD(&directives[0], PMIX_RANGE, &range, PMIX_DATA_RANGE); + PMIX_INFO_LOAD(&directives[1], PMIX_WAIT, &wait, PMIX_INT); + + /* try lookup */ + rc = PMIx_Lookup(pdata, 1, directives, ndir); + if (rc != PMIX_SUCCESS) { + LOGERR("PMIx rank %d: PMIx_Lookup(%s) failed: %s", + pmix_myproc.rank, pmix_key, PMIx_Error_string(rc)); + *oval = NULL; + rc = (int)UNIFYFS_FAILURE; + } else { + if (pdata[0].value.data.string != NULL) { + *oval = strdup(pdata[0].value.data.string); + rc = (int)UNIFYFS_SUCCESS; + } else { + LOGERR("PMIx rank %d: PMIx_Lookup(%s) returned NULL string", + pmix_myproc.rank, pmix_key); + *oval = NULL; + rc = (int)UNIFYFS_FAILURE; + } + } + /* cleanup */ + PMIX_PDATA_FREE(pdata, 1); + PMIX_INFO_FREE(directives, ndir); + return rc; +} + +// publish a key-value pair +static int unifyfs_pmix_publish(const char* key, + const char* val) +{ + int rc; + size_t ninfo; + pmix_info_t* info; + pmix_data_range_t range; + char pmix_key[PMIX_MAX_KEYLEN+1]; + + if (!pmix_initialized) { + return (int)UNIFYFS_FAILURE; + } + + /* set key-val and modify publish behavior */ + strncpy(pmix_key, key, sizeof(pmix_key)); + range = PMIX_RANGE_GLOBAL; + ninfo = 2; + PMIX_INFO_CREATE(info, ninfo); + PMIX_INFO_LOAD(&info[0], pmix_key, val, PMIX_STRING); + PMIX_INFO_LOAD(&info[1], PMIX_RANGE, &range, PMIX_DATA_RANGE); + + /* try to publish */ + rc = PMIx_Publish(info, ninfo); + if (rc != PMIX_SUCCESS) { + LOGERR("PMIx rank %d: PMIx_Publish failed: %s", + pmix_myproc.rank, PMIx_Error_string(rc)); + rc = (int)UNIFYFS_FAILURE; + } else { + rc = (int)UNIFYFS_SUCCESS; + } + /* cleanup */ + PMIX_INFO_FREE(info, ninfo); + return rc; +} + +#endif // USE_PMIX + + +//--------------------- File system K-V Store --------------------- + +static char localfs_kvdir[UNIFYFS_MAX_FILENAME]; +static char sharedfs_kvdir[UNIFYFS_MAX_FILENAME]; +static char sharedfs_rank_kvdir[UNIFYFS_MAX_FILENAME]; +static int have_sharedfs_kvstore; // = 0 + +static int unifyfs_fskv_init(unifyfs_cfg_t* cfg) +{ + int rc, err; + struct stat s; + + + if (NULL == cfg) { + LOGERR("NULL config"); + return (int)UNIFYFS_ERROR_INVAL; + } + + memset(localfs_kvdir, 0, sizeof(localfs_kvdir)); + memset(sharedfs_kvdir, 0, sizeof(sharedfs_kvdir)); + memset(sharedfs_rank_kvdir, 0, sizeof(sharedfs_rank_kvdir)); + + // find or create local kvstore directory + if (NULL == cfg->runstate_dir) { + LOGERR("local file system k-v store requires cfg.runstate_dir"); + return (int)UNIFYFS_FAILURE; + } + snprintf(localfs_kvdir, sizeof(localfs_kvdir), "%s/kvstore", + cfg->runstate_dir); + memset(&s, 0, sizeof(struct stat)); + rc = stat(localfs_kvdir, &s); + if (rc != 0) { + if (UNIFYFS_SERVER == cfg->ptype) { + // try to create it + rc = mkdir(localfs_kvdir, 0770); + err = errno; + if ((rc != 0) && (err != EEXIST)) { + LOGERR("failed to create local kvstore directory %s - %s", + localfs_kvdir, strerror(err)); + return (int)UNIFYFS_FAILURE; + } + } else { + LOGERR("missing local kvstore directory %s", localfs_kvdir); + return (int)UNIFYFS_FAILURE; + } + } + + if ((UNIFYFS_SERVER == cfg->ptype) && (NULL != cfg->sharedfs_dir)) { + // find or create shared kvstore directory + snprintf(sharedfs_kvdir, sizeof(sharedfs_kvdir), "%s/kvstore", + cfg->sharedfs_dir); + memset(&s, 0, sizeof(struct stat)); + rc = stat(sharedfs_kvdir, &s); + if (rc != 0) { + // try to create it + rc = mkdir(sharedfs_kvdir, 0770); + err = errno; + if ((rc != 0) && (err != EEXIST)) { + LOGERR("failed to create kvstore directory %s - %s", + sharedfs_kvdir, strerror(err)); + return (int)UNIFYFS_FAILURE; + } + } + + // find or create rank-specific subdir + snprintf(sharedfs_rank_kvdir, sizeof(sharedfs_rank_kvdir), "%s/%d", + sharedfs_kvdir, kv_myrank); + memset(&s, 0, sizeof(struct stat)); + rc = stat(sharedfs_rank_kvdir, &s); + if (rc != 0) { + // try to create it + rc = mkdir(sharedfs_rank_kvdir, 0770); + err = errno; + if ((rc != 0) && (err != EEXIST)) { + LOGERR("failed to create rank kvstore directory %s - %s", + sharedfs_rank_kvdir, strerror(err)); + return (int)UNIFYFS_FAILURE; + } + } + have_sharedfs_kvstore = 1; + } + + kv_max_keylen = UNIFYFS_MAX_KV_KEYLEN; + kv_max_vallen = UNIFYFS_MAX_KV_VALLEN; + return (int)UNIFYFS_SUCCESS; +} + +static int unifyfs_fskv_fini(void) +{ + int rc; + struct stat s; + struct dirent* de; + + kv_max_keylen = 0; + kv_max_vallen = 0; + + // cleanup local k-v hierarchy + memset(&s, 0, sizeof(struct stat)); + rc = stat(localfs_kvdir, &s); + if (rc == 0) { + // remove local k-v files + char kvfile[UNIFYFS_MAX_FILENAME]; + DIR* lkv = opendir(localfs_kvdir); + if (NULL == lkv) { + LOGERR("failed to opendir(%s)", localfs_kvdir); + return (int)UNIFYFS_FAILURE; + } + while (NULL != (de = readdir(lkv))) { + if ((0 == strcmp(".", de->d_name)) || + (0 == strcmp("..", de->d_name))) { + continue; + } + memset(kvfile, 0, sizeof(kvfile)); + snprintf(kvfile, sizeof(kvfile), "%s/%s", + localfs_kvdir, de->d_name); + rc = remove(kvfile); + if (rc != 0) { + LOGERR("failed to remove local kvstore entry %s", + de->d_name); + } + } + closedir(lkv); + + // remove local k-v root + rc = rmdir(localfs_kvdir); + if (rc != 0) { + LOGERR("failed to remove local kvstore dir %s", + sharedfs_rank_kvdir); + return (int)UNIFYFS_FAILURE; + } + } + + if (have_sharedfs_kvstore) { + // cleanup rank-specific k-v hierarchy + memset(&s, 0, sizeof(struct stat)); + rc = stat(sharedfs_rank_kvdir, &s); + if (rc == 0) { + // remove rank-specific k-v files + char rank_kvfile[UNIFYFS_MAX_FILENAME]; + DIR* rkv = opendir(sharedfs_rank_kvdir); + if (NULL == rkv) { + LOGERR("failed to opendir(%s)", sharedfs_rank_kvdir); + return (int)UNIFYFS_FAILURE; + } + while (NULL != (de = readdir(rkv))) { + if ((0 == strcmp(".", de->d_name)) || + (0 == strcmp("..", de->d_name))) { + continue; + } + memset(rank_kvfile, 0, sizeof(rank_kvfile)); + snprintf(rank_kvfile, sizeof(rank_kvfile), "%s/%s", + sharedfs_rank_kvdir, de->d_name); + rc = remove(rank_kvfile); + if (rc != 0) { + LOGERR("failed to remove rank-specific kvstore entry %s", + de->d_name); + } + } + closedir(rkv); + + // remove rank-specific subdir + rc = rmdir(sharedfs_rank_kvdir); + if (rc != 0) { + LOGERR("failed to remove rank-specific kvstore dir %s", + sharedfs_rank_kvdir); + return (int)UNIFYFS_FAILURE; + } + } + + // cleanup sharedfs k-v hierarchy root + if (0 == kv_myrank) { + memset(&s, 0, sizeof(struct stat)); + rc = stat(sharedfs_kvdir, &s); + if (rc == 0) { + int entry_count = 0; + DIR* skv = opendir(sharedfs_kvdir); + if (NULL == skv) { + LOGERR("failed to opendir(%s)", sharedfs_kvdir); + return (int)UNIFYFS_FAILURE; + } + while (NULL != (de = readdir(skv))) { + if ((0 == strcmp(".", de->d_name)) || + (0 == strcmp("..", de->d_name))) { + continue; + } + entry_count++; + } + closedir(skv); + if (0 == entry_count) { + // all subdirs gone, remove root + rc = rmdir(sharedfs_kvdir); + if (rc != 0) { + LOGERR("failed to remove sharedfs kvstore dir %s", + sharedfs_kvdir); + return (int)UNIFYFS_FAILURE; + } + } + } + } + } + return (int)UNIFYFS_SUCCESS; +} + +// lookup a key-value pair +static int unifyfs_fskv_lookup_local(const char* key, + char** oval) +{ + FILE* kvf; + char kvfile[UNIFYFS_MAX_FILENAME]; + char kvalue[kv_max_vallen]; + + snprintf(kvfile, sizeof(kvfile), "%s/%s", + localfs_kvdir, key); + kvf = fopen(kvfile, "r"); + if (NULL == kvf) { + LOGERR("failed to open kvstore entry %s", kvfile); + return (int)UNIFYFS_FAILURE; + } + memset(kvalue, 0, sizeof(kvalue)); + fscanf(kvf, "%s\n", kvalue); + fclose(kvf); + + *oval = strdup(kvalue); + return (int)UNIFYFS_SUCCESS; +} + +static int unifyfs_fskv_lookup_remote(int rank, + const char* key, + char** oval) +{ + FILE* kvf; + char rank_kvfile[UNIFYFS_MAX_FILENAME]; + char kvalue[kv_max_vallen]; + + if (!have_sharedfs_kvstore) { + return (int)UNIFYFS_FAILURE; + } + + snprintf(rank_kvfile, sizeof(rank_kvfile), "%s/%d/%s", + sharedfs_kvdir, rank, key); + kvf = fopen(rank_kvfile, "r"); + if (NULL == kvf) { + LOGERR("failed to open kvstore entry %s", rank_kvfile); + return (int)UNIFYFS_FAILURE; + } + memset(kvalue, 0, sizeof(kvalue)); + fscanf(kvf, "%s\n", kvalue); + fclose(kvf); + + *oval = strdup(kvalue); + return (int)UNIFYFS_SUCCESS; +} + +// publish a key-value pair +static int unifyfs_fskv_publish_local(const char* key, + const char* val) +{ + FILE* kvf; + char kvfile[UNIFYFS_MAX_FILENAME]; + + snprintf(kvfile, sizeof(kvfile), "%s/%s", + localfs_kvdir, key); + kvf = fopen(kvfile, "w"); + if (NULL == kvf) { + LOGERR("failed to create kvstore entry %s", kvfile); + return (int)UNIFYFS_FAILURE; + } + fprintf(kvf, "%s\n", val); + fclose(kvf); + + return (int)UNIFYFS_SUCCESS; +} + +static int unifyfs_fskv_publish_remote(const char* key, + const char* val) +{ + FILE* kvf; + char rank_kvfile[UNIFYFS_MAX_FILENAME]; + + if (!have_sharedfs_kvstore) { + return (int)UNIFYFS_FAILURE; + } + + snprintf(rank_kvfile, sizeof(rank_kvfile), "%s/%s", + sharedfs_rank_kvdir, key); + kvf = fopen(rank_kvfile, "w"); + if (NULL == kvf) { + LOGERR("failed to create kvstore entry %s", rank_kvfile); + return (int)UNIFYFS_FAILURE; + } + fprintf(kvf, "%s\n", val); + fclose(kvf); + + return (int)UNIFYFS_SUCCESS; +} + +//--------------------- K-V Store API --------------------- + +// Initialize key-value store +int unifyfs_keyval_init(unifyfs_cfg_t* cfg, + int* rank, + int* nranks) +{ + int rc; + + if (!kv_initialized) { + gethostname(kv_myhost, sizeof(kv_myhost)); + +#if defined(USE_PMIX) + rc = unifyfs_pmix_init(); + if (rc != (int)UNIFYFS_SUCCESS) { + return rc; + } +#elif defined(USE_PMI2) + rc = unifyfs_pmi2_init(); + if (rc != (int)UNIFYFS_SUCCESS) { + return rc; + } +#else + // use passed-in rank, nranks + if (NULL != rank) { + kv_myrank = *rank; + } + if (NULL != nranks) { + kv_nranks = *nranks; + } +#endif + // NOTE: do this after getting rank/n_ranks info + rc = unifyfs_fskv_init(cfg); + if (rc != (int)UNIFYFS_SUCCESS) { + return rc; + } + + kv_initialized = 1; + } + + if (NULL != rank) { + *rank = kv_myrank; + } + if (NULL != nranks) { + *nranks = kv_nranks; + } + return (int)UNIFYFS_SUCCESS; +} + +// Finalize key-value store +int unifyfs_keyval_fini(void) +{ + int rc; + if (kv_initialized) { + rc = unifyfs_fskv_fini(); + if (rc != (int)UNIFYFS_SUCCESS) { + return rc; + } + +#if defined(USE_PMIX) + rc = unifyfs_pmix_fini(); + if (rc != (int)UNIFYFS_SUCCESS) { + return rc; + } +#elif defined(USE_PMI2) + rc = unifyfs_pmi2_fini(); + if (rc != (int)UNIFYFS_SUCCESS) { + return rc; + } +#endif + + kv_initialized = 0; + kv_published = 0; + kv_max_keylen = 0; + kv_max_vallen = 0; + } + return (int)UNIFYFS_SUCCESS; +} + +// Lookup a local key-value pair +int unifyfs_keyval_lookup_local(const char* key, + char** oval) +{ + int rc = UNIFYFS_FAILURE; + + if ((NULL == key) || (NULL == oval)) { + LOGERR("NULL parameter"); + return (int)UNIFYFS_ERROR_INVAL; + } + + if (!kv_initialized) { + rc = unifyfs_keyval_init(NULL, NULL, NULL); + if (rc != (int)UNIFYFS_SUCCESS) { + return rc; + } + } + + size_t len = strlen(key); + if (len >= (kv_max_keylen - 1)) { + LOGERR("length of key (%zd) exceeds max %zd", + len, kv_max_keylen); + return (int)UNIFYFS_ERROR_INVAL; + } + + // do the lookup + rc = unifyfs_fskv_lookup_local(key, oval); + if (rc != (int)UNIFYFS_SUCCESS) { + LOGERR("local keyval lookup for '%s' failed", key); + } + return rc; +} + +// Lookup a remote key-value pair +int unifyfs_keyval_lookup_remote(int rank, + const char* key, + char** oval) +{ + int rc = UNIFYFS_FAILURE; + + if ((NULL == key) || (NULL == oval)) { + LOGERR("NULL parameter"); + return (int)UNIFYFS_ERROR_INVAL; + } + + if (!kv_initialized) { + rc = unifyfs_keyval_init(NULL, NULL, NULL); + if (rc != (int)UNIFYFS_SUCCESS) { + return rc; + } + } + + // NOTE: assumes rank value fits in 10 characters + size_t len = 10 + strlen(key) + 1; + if (len >= (kv_max_keylen - 1)) { + LOGERR("length of key (%zd) exceeds max %zd", + len, kv_max_keylen); + return (int)UNIFYFS_ERROR_INVAL; + } + + // generate full key, which includes remote host + char rank_key[kv_max_keylen]; + memset(rank_key, 0, sizeof(rank_key)); + snprintf(rank_key, sizeof(rank_key), "%d.%s", rank, key); + + // do the lookup +#if defined(USE_PMIX) + rc = unifyfs_pmix_lookup(rank_key, oval); +#elif defined(USE_PMI2) + rc = unifyfs_pmi2_lookup(rank_key, oval); +#endif + if (rc != (int)UNIFYFS_SUCCESS) { + rc = unifyfs_fskv_lookup_remote(rank, key, oval); + } + if (rc != (int)UNIFYFS_SUCCESS) { + LOGERR("remote keyval lookup for '%s' failed", key); + } + return rc; +} + +// Publish a locally-visible key-value pair +int unifyfs_keyval_publish_local(const char* key, + const char* val) +{ + int rc = UNIFYFS_FAILURE; + + if (!kv_initialized) { + return (int)UNIFYFS_FAILURE; + } + + if ((key == NULL) || (val == NULL)) { + LOGERR("NULL key or value"); + return (int)UNIFYFS_ERROR_INVAL; + } + + size_t len = strlen(key); + if (len >= (kv_max_keylen - 1)) { + LOGERR("length of key (%zd) exceeds max %zd", + len, kv_max_keylen); + return (int)UNIFYFS_ERROR_INVAL; + } + + len = strlen(val); + if (len >= kv_max_vallen) { + LOGERR("length of val (%zd) exceeds max %zd", + len, kv_max_vallen); + return (int)UNIFYFS_ERROR_INVAL; + } + + // publish it + rc = unifyfs_fskv_publish_local(key, val); + if (rc != (int)UNIFYFS_SUCCESS) { + LOGERR("local keyval publish for '%s' failed", key); + } else { + kv_published = 1; + } + return rc; +} + +// Publish a remotely-visible key-value pair +int unifyfs_keyval_publish_remote(const char* key, + const char* val) +{ + int rc = UNIFYFS_FAILURE; + + if (!kv_initialized) { + return (int)UNIFYFS_FAILURE; + } + + if ((key == NULL) || (val == NULL)) { + LOGERR("NULL key or value"); + return (int)UNIFYFS_ERROR_INVAL; + } + + // NOTE: assumes rank value fits in 10 characters + size_t len = 10 + strlen(key) + 1; + if (len >= (kv_max_keylen - 1)) { + LOGERR("length of key (%zd) exceeds max %zd", + len, kv_max_keylen); + return (int)UNIFYFS_ERROR_INVAL; + } + + // generate full key, which includes remote host + char rank_key[kv_max_keylen]; + memset(rank_key, 0, sizeof(rank_key)); + snprintf(rank_key, sizeof(rank_key), "%d.%s", kv_myrank, key); + + // publish it + #if defined(USE_PMIX) + rc = unifyfs_pmix_publish(rank_key, val); +#elif defined(USE_PMI2) + rc = unifyfs_pmi2_publish(rank_key, val); +#endif + if (rc != (int)UNIFYFS_SUCCESS) { + rc = unifyfs_fskv_publish_remote(key, val); + } + + if (rc != (int)UNIFYFS_SUCCESS) { + LOGERR("remote keyval publish for '%s' failed", key); + } else { + kv_published = 1; + } + return rc; +} diff --git a/common/src/unifyfs_keyval.h b/common/src/unifyfs_keyval.h new file mode 100644 index 000000000..c98cfedae --- /dev/null +++ b/common/src/unifyfs_keyval.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2019, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#ifndef UNIFYFS_KEYVAL_H +#define UNIFYFS_KEYVAL_H + +#include "unifyfs_configurator.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// keys we use +const char* key_runstate; // path to runstate file +const char* key_unifyfsd_socket; // server domain socket path +const char* key_unifyfsd_margo_shm; // client-server margo address +const char* key_unifyfsd_margo_svr; // server-server margo address +const char* key_unifyfsd_mpi_rank; // server-server MPI rank + +// initialize key-value store +int unifyfs_keyval_init(unifyfs_cfg_t* cfg, + int* rank, + int* nranks); + +// finalize key-value store +int unifyfs_keyval_fini(void); + +// publish a key-value pair with local visibility +int unifyfs_keyval_publish_local(const char* key, + const char* val); + +// publish a key-value pair with remote visibility +int unifyfs_keyval_publish_remote(const char* key, + const char* val); + +// lookup a local key-value pair +int unifyfs_keyval_lookup_local(const char* key, + char** oval); + +// lookup a remote key-value pair +int unifyfs_keyval_lookup_remote(int rank, + const char* key, + char** oval); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // UNIFYFS_KEYVAL_H diff --git a/common/src/unifyfs_log.c b/common/src/unifyfs_log.c new file mode 100644 index 000000000..294409e36 --- /dev/null +++ b/common/src/unifyfs_log.c @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2017, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +/* + * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * Copyright (c) 2017, Florida State University. Contributions from + * the Computer Architecture and Systems Research Laboratory (CASTL) + * at the Department of Computer Science. + * + * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror + * LLNL-CODE-728877. All rights reserved. + * + * This file is part of burstfs. + * For details, see https://github.com/llnl/burstfs + * Please read https://github.com/llnl/burstfs/LICENSE for full license text. + */ + +#include +#include +#include "unifyfs_log.h" +#include "unifyfs_const.h" + +/* one of the loglevel values */ +unifyfs_log_level_t unifyfs_log_level = 5; + +/* pointer to log file stream */ +FILE* unifyfs_log_stream; // = NULL + +/* used within LOG macro to build a timestamp */ +time_t unifyfs_log_time; +struct tm* unifyfs_log_ltime; +char unifyfs_log_timestamp[256]; + +/* used to reduce source file name length */ +size_t unifyfs_log_source_base_len; // = 0 +static const char* this_file = __FILE__; + +/* open specified file as log file stream, + * returns UNIFYFS_SUCCESS on success */ +int unifyfs_log_open(const char* file) +{ + if (0 == unifyfs_log_source_base_len) { + // NOTE: if you change the source location of this file, update string + char* srcdir = strstr(this_file, "common/src/unifyfs_log.c"); + if (NULL != srcdir) { + unifyfs_log_source_base_len = srcdir - this_file; + } + } + + FILE* logf = fopen(file, "a"); + if (logf == NULL) { + /* failed to open file name, fall back to stderr */ + unifyfs_log_stream = stderr; + return (int)UNIFYFS_ERROR_DBG; + } else { + unifyfs_log_stream = logf; + return UNIFYFS_SUCCESS; + } +} + +/* close our log file stream, + * returns UNIFYFS_SUCCESS on success */ +int unifyfs_log_close(void) +{ + if (unifyfs_log_stream == NULL) { + /* nothing to close */ + return (int)UNIFYFS_ERROR_DBG; + } else { + /* if stream is open, and its not stderr, close it */ + if (unifyfs_log_stream != stderr && + fclose(unifyfs_log_stream) == 0) { + return UNIFYFS_SUCCESS; + } + return (int)UNIFYFS_ERROR_DBG; + } +} diff --git a/common/src/unifycr_log.h b/common/src/unifyfs_log.h similarity index 50% rename from common/src/unifycr_log.h rename to common/src/unifyfs_log.h index 10e5816fc..391d0f1f7 100644 --- a/common/src/unifycr_log.h +++ b/common/src/unifyfs_log.h @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -27,8 +27,8 @@ * Please read https://github.com/llnl/burstfs/LICENSE for full license text. */ -#ifndef __LOG_H__ -#define __LOG_H__ +#ifndef __UNIFYFS_LOG_H__ +#define __UNIFYFS_LOG_H__ #include #include @@ -36,22 +36,24 @@ #include #include +#ifdef __cplusplus +extern "C" { +#endif + typedef enum { LOG_FATAL = 1, LOG_ERR = 2, LOG_WARN = 3, LOG_INFO = 4, LOG_DBG = 5 -} unifycr_log_level_t; - -extern unifycr_log_level_t unifycr_log_level; +} unifyfs_log_level_t; -extern FILE* unifycr_log_stream; -extern int glb_rank; - -extern time_t unifycr_log_time; -extern struct tm* unifycr_log_ltime; -extern char unifycr_log_timestamp[256]; +extern unifyfs_log_level_t unifyfs_log_level; +extern FILE* unifyfs_log_stream; +extern time_t unifyfs_log_time; +extern struct tm* unifyfs_log_ltime; +extern char unifyfs_log_timestamp[256]; +extern size_t unifyfs_log_source_base_len; #if defined(__NR_gettid) #define gettid() syscall(__NR_gettid) @@ -62,17 +64,21 @@ extern char unifycr_log_timestamp[256]; #endif #define LOG(level, ...) \ - if (level <= unifycr_log_level) { \ - unifycr_log_time = time(NULL); \ - unifycr_log_ltime = localtime(&unifycr_log_time); \ - strftime(unifycr_log_timestamp, sizeof(unifycr_log_timestamp), \ - "%Y-%m-%dT%H:%M:%S", unifycr_log_ltime); \ - fprintf(unifycr_log_stream, "%s rank=%d tid=%ld @ %s:%d in %s: ", \ - unifycr_log_timestamp, glb_rank, (long)gettid(), \ - __FILE__, __LINE__, __func__); \ - fprintf(unifycr_log_stream, __VA_ARGS__); \ - fprintf(unifycr_log_stream, "\n"); \ - fflush(unifycr_log_stream); \ + if (level <= unifyfs_log_level) { \ + const char* srcfile = __FILE__ + unifyfs_log_source_base_len; \ + unifyfs_log_time = time(NULL); \ + unifyfs_log_ltime = localtime(&unifyfs_log_time); \ + strftime(unifyfs_log_timestamp, sizeof(unifyfs_log_timestamp), \ + "%Y-%m-%dT%H:%M:%S", unifyfs_log_ltime); \ + if (NULL == unifyfs_log_stream) { \ + unifyfs_log_stream = stderr; \ + } \ + fprintf(unifyfs_log_stream, "%s tid=%ld @ %s() [%s:%d] ", \ + unifyfs_log_timestamp, (long)gettid(), \ + __func__, srcfile, __LINE__); \ + fprintf(unifyfs_log_stream, __VA_ARGS__); \ + fprintf(unifyfs_log_stream, "\n"); \ + fflush(unifyfs_log_stream); \ } #define LOGERR(...) LOG(LOG_ERR, __VA_ARGS__) @@ -80,11 +86,15 @@ extern char unifycr_log_timestamp[256]; #define LOGDBG(...) LOG(LOG_DBG, __VA_ARGS__) /* open specified file as debug file stream, - * * returns UNIFYCR_SUCCESS on success */ -int unifycr_log_open(const char* file); + * returns UNIFYFS_SUCCESS on success */ +int unifyfs_log_open(const char* file); /* close our debug file stream, - * * returns UNIFYCR_SUCCESS on success */ -int unifycr_log_close(void); + * returns UNIFYFS_SUCCESS on success */ +int unifyfs_log_close(void); + +#ifdef __cplusplus +} // extern "C" +#endif -#endif /* LOG_H */ +#endif /* UNIFYFS_LOG_H */ diff --git a/common/src/unifyfs_meta.h b/common/src/unifyfs_meta.h new file mode 100644 index 000000000..ea5a159b6 --- /dev/null +++ b/common/src/unifyfs_meta.h @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2018, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2018, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#ifndef UNIFYFS_META_H +#define UNIFYFS_META_H + +#include +#include +#include +#include +#include + +#include "unifyfs_const.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Server commands + */ +typedef enum { + COMM_MOUNT, + COMM_META_FSYNC, + COMM_META_GET, + COMM_META_SET, + COMM_READ, + COMM_UNMOUNT, + COMM_DIGEST, + COMM_SYNC_DEL, +} cmd_lst_t; + +typedef struct { + int fid; + int gfid; + char filename[UNIFYFS_MAX_FILENAME]; + + /* essential stat fields */ + uint32_t mode; /* st_mode bits */ + uint32_t uid; + uint32_t gid; + uint64_t size; + struct timespec atime; + struct timespec mtime; + struct timespec ctime; + + /* Set when the file is laminated */ + uint32_t is_laminated; +} unifyfs_file_attr_t; + +enum { + UNIFYFS_STAT_DEFAULT_DEV = 0, + UNIFYFS_STAT_DEFAULT_BLKSIZE = 4096, + UNIFYFS_STAT_DEFAULT_FILE_MODE = S_IFREG | 0644, + UNIFYFS_STAT_DEFAULT_DIR_MODE = S_IFDIR | 0755, +}; + +static inline +void unifyfs_file_attr_to_stat(unifyfs_file_attr_t* fattr, struct stat* sb) +{ + if (fattr && sb) { + sb->st_dev = UNIFYFS_STAT_DEFAULT_DEV; + sb->st_ino = fattr->gfid; + sb->st_mode = fattr->mode; + sb->st_uid = fattr->uid; + sb->st_gid = fattr->gid; + sb->st_rdev = UNIFYFS_STAT_DEFAULT_DEV; + sb->st_size = fattr->size; + sb->st_blksize = UNIFYFS_STAT_DEFAULT_BLKSIZE; + sb->st_blocks = fattr->size / UNIFYFS_STAT_DEFAULT_BLKSIZE; + if (fattr->size % UNIFYFS_STAT_DEFAULT_BLKSIZE > 0) { + sb->st_blocks += 1; + } + + /* + * Re-purpose st_nlink to tell us if the file is laminated or not. + * That way, if we do eventually make /unifyfs mountable, we can easily + * see with 'ls -l' or stat if the file is laminated or not. + */ + sb->st_nlink = fattr->is_laminated ? 1 : 0; + + sb->st_atime = fattr->atime.tv_sec; + sb->st_mtime = fattr->mtime.tv_sec; + sb->st_ctime = fattr->ctime.tv_sec; + } +} + +typedef struct { + off_t file_pos; + off_t mem_pos; + size_t length; + int fid; +} unifyfs_index_t; + +/* Header for read request reply in client shared memory region. + * The associated data payload immediately follows the header in + * the shmem region. + * offset - offset within file + * length - data size + * gfid - global file id + * errcode - read error code (zero on success) */ +typedef struct { + size_t offset; + size_t length; + int gfid; + int errcode; +} shm_meta_t; + +/* State values for client shared memory region */ +typedef enum { + SHMEM_REGION_EMPTY = 0, // set by client to indicate drain complete + SHMEM_REGION_DATA_READY = 1, // set by server to initiate client drain + SHMEM_REGION_DATA_COMPLETE = 2 // set by server when done writing data +} shm_region_state_e; + +/* Header for client shared memory region. + * sync - for synchronizing updates/access by server threads + * meta_cnt - number of shm_meta_t (i.e., read replies) currently in shmem + * bytes - total bytes of shmem region in use (shm_meta_t + payloads) + * state - region state variable used for client-server coordination */ + typedef struct { + pthread_mutex_t sync; + volatile size_t meta_cnt; + volatile size_t bytes; + volatile shm_region_state_e state; +} shm_header_t; + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif /* UNIFYFS_META_H */ diff --git a/common/src/unifyfs_rpc_util.c b/common/src/unifyfs_rpc_util.c new file mode 100644 index 000000000..7a60725a9 --- /dev/null +++ b/common/src/unifyfs_rpc_util.c @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2017, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#include +#include +#include +#include +#include + +#include +#include "unifyfs_log.h" +#include "unifyfs_keyval.h" +#include "unifyfs_rpc_util.h" + +#define LOCAL_RPC_ADDR_FILE "/tmp/unifyfsd.margo-shm" + +/* publishes client-server RPC address */ +void rpc_publish_local_server_addr(const char* addr) +{ + LOGDBG("publishing client-server rpc address '%s'", addr); + + // publish client-server margo address + unifyfs_keyval_publish_local(key_unifyfsd_margo_shm, addr); + + /* write server address to local file for client to read */ + FILE* fp = fopen(LOCAL_RPC_ADDR_FILE, "w+"); + if (fp != NULL) { + fprintf(fp, "%s", addr); + fclose(fp); + } else { + LOGERR("Error writing server rpc addr file " LOCAL_RPC_ADDR_FILE); + } +} + +/* publishes server-server RPC address */ +void rpc_publish_remote_server_addr(const char* addr) +{ + LOGDBG("publishing server-server rpc address '%s'", addr); + + // publish client-server margo address + unifyfs_keyval_publish_remote(key_unifyfsd_margo_svr, addr); +} + +/* lookup address of server, returns NULL if server address is not found, + * otherwise returns server address in newly allocated string that caller + * must free */ +char* rpc_lookup_local_server_addr(void) +{ + /* returns NULL if we can't find server address */ + char* addr = NULL; + char* valstr = NULL; + + // lookup client-server margo address + if (0 == unifyfs_keyval_lookup_local(key_unifyfsd_margo_shm, &valstr)) { + addr = strdup(valstr); + free(valstr); + } + + if (NULL == addr) { + /* read server address from local file */ + FILE* fp = fopen(LOCAL_RPC_ADDR_FILE, "r"); + if (fp != NULL) { + char addr_string[256]; + memset(addr_string, 0, sizeof(addr_string)); + if (1 == fscanf(fp, "%255s", addr_string)) { + addr = strdup(addr_string); + } + fclose(fp); + } + } + + /* print server address (debugging) */ + if (NULL != addr) { + LOGDBG("found local server rpc address '%s'", addr); + } + return addr; +} + +/* lookup address of server, returns NULL if server address is not found, + * otherwise returns server address in newly allocated string that caller + * must free */ +char* rpc_lookup_remote_server_addr(int srv_rank) +{ + /* returns NULL if we can't find server address */ + char* addr = NULL; + char* valstr = NULL; + + // lookup server-server margo address + if (0 == unifyfs_keyval_lookup_remote(srv_rank, key_unifyfsd_margo_svr, + &valstr)) { + addr = strdup(valstr); + free(valstr); + } + + /* print sserver address (debugging) */ + if (NULL != addr) { + LOGDBG("found server %d rpc address '%s'", srv_rank, addr); + } + return addr; +} + +/* remove local server RPC address file */ +void rpc_clean_local_server_addr(void) +{ + int rc = unlink(LOCAL_RPC_ADDR_FILE); + if (rc != 0) { + int err = errno; + if (err != ENOENT) { + LOGERR("Error (%s) removing local server rpc addr file " + LOCAL_RPC_ADDR_FILE, strerror(err)); + } + } +} + diff --git a/common/src/unifyfs_rpc_util.h b/common/src/unifyfs_rpc_util.h new file mode 100644 index 000000000..8067ed5c6 --- /dev/null +++ b/common/src/unifyfs_rpc_util.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2018, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2018, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#ifndef UNIFYFS_UTIL_H +#define UNIFYFS_UTIL_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* publish the address of the server */ +void rpc_publish_local_server_addr(const char* addr); +void rpc_publish_remote_server_addr(const char* addr); + +/* lookup address of server */ +char* rpc_lookup_local_server_addr(void); +char* rpc_lookup_remote_server_addr(int srv_rank); + +/* remove server rpc address file */ +void rpc_clean_local_server_addr(void); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // UNIFYFS_UTIL_H + diff --git a/common/src/unifyfs_runstate.c b/common/src/unifyfs_runstate.c new file mode 100644 index 000000000..660b7e693 --- /dev/null +++ b/common/src/unifyfs_runstate.c @@ -0,0 +1,97 @@ +#include +#include +#include +#include +#include + +#include "unifyfs_keyval.h" +#include "unifyfs_log.h" +#include "unifyfs_runstate.h" + +const char* runstate_file = "unifyfs-runstate.conf"; + +int unifyfs_read_runstate(unifyfs_cfg_t* cfg, + const char* runstate_path) +{ + int rc = (int)UNIFYFS_SUCCESS; + int uid = (int)getuid(); + char runstate_fname[UNIFYFS_MAX_FILENAME] = {0}; + + if (cfg == NULL) { + LOGERR("NULL config"); + return (int)UNIFYFS_ERROR_INVAL; + } + + if (runstate_path == NULL) { + if (cfg->runstate_dir == NULL) { + LOGERR("bad runstate dir config setting"); + return (int)UNIFYFS_ERROR_APPCONFIG; + } + snprintf(runstate_fname, sizeof(runstate_fname), + "%s/%s.%d", cfg->runstate_dir, runstate_file, uid); + } else { + snprintf(runstate_fname, sizeof(runstate_fname), + "%s", runstate_path); + } + + if (unifyfs_config_process_ini_file(cfg, runstate_fname) != 0) { + LOGERR("failed to process runstate file %s", runstate_fname); + rc = (int)UNIFYFS_ERROR_APPCONFIG; + } + + return rc; +} + +int unifyfs_write_runstate(unifyfs_cfg_t* cfg) +{ + int rc = (int)UNIFYFS_SUCCESS; + int uid = (int)getuid(); + FILE* runstate_fp = NULL; + char runstate_fname[UNIFYFS_MAX_FILENAME] = {0}; + + if (cfg == NULL) { + LOGERR("NULL config"); + return (int)UNIFYFS_ERROR_INVAL; + } + + snprintf(runstate_fname, sizeof(runstate_fname), + "%s/%s.%d", cfg->runstate_dir, runstate_file, uid); + + runstate_fp = fopen(runstate_fname, "w"); + if (runstate_fp == NULL) { + LOGERR("failed to create file %s", runstate_fname); + rc = (int)UNIFYFS_ERROR_FILE; + } else { + if ((unifyfs_log_stream != NULL) && + (unifyfs_log_level >= LOG_INFO)) { + unifyfs_config_print(cfg, unifyfs_log_stream); + } + unifyfs_config_print_ini(cfg, runstate_fp); + fclose(runstate_fp); + } + + return rc; +} + +int unifyfs_clean_runstate(unifyfs_cfg_t* cfg) +{ + int rc = (int)UNIFYFS_SUCCESS; + int uid = (int)getuid(); + char runstate_fname[UNIFYFS_MAX_FILENAME] = {0}; + + if (cfg == NULL) { + LOGERR("invalid config arg"); + return (int)UNIFYFS_ERROR_INVAL; + } + + snprintf(runstate_fname, sizeof(runstate_fname), + "%s/%s.%d", cfg->runstate_dir, runstate_file, uid); + + rc = unlink(runstate_fname); + if (rc != 0) { + LOGERR("failed to remove file %s", runstate_fname); + rc = (int)UNIFYFS_ERROR_FILE; + } + + return rc; +} diff --git a/common/src/unifyfs_runstate.h b/common/src/unifyfs_runstate.h new file mode 100644 index 000000000..e32c67372 --- /dev/null +++ b/common/src/unifyfs_runstate.h @@ -0,0 +1,21 @@ +#ifndef _UNIFYFS_RUNSTATE_H_ +#define _UNIFYFS_RUNSTATE_H_ + +#include "unifyfs_configurator.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int unifyfs_read_runstate(unifyfs_cfg_t* cfg, + const char* runstate_path); + +int unifyfs_write_runstate(unifyfs_cfg_t* cfg); + +int unifyfs_clean_runstate(unifyfs_cfg_t* cfg); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // UNIFYFS_RUNSTATE_H diff --git a/common/src/unifyfs_server_rpcs.h b/common/src/unifyfs_server_rpcs.h new file mode 100644 index 000000000..fd86f674d --- /dev/null +++ b/common/src/unifyfs_server_rpcs.h @@ -0,0 +1,74 @@ +#ifndef __UNIFYFS_SERVER_RPCS_H +#define __UNIFYFS_SERVER_RPCS_H + +/* + * Declarations for server-server margo RPCs + */ + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* server_hello_rpc (server => server) + * + * say hello from one server to another */ +MERCURY_GEN_PROC(server_hello_in_t, + ((int32_t)(src_rank)) + ((hg_const_string_t)(message_str))) +MERCURY_GEN_PROC(server_hello_out_t, + ((int32_t)(ret))) +DECLARE_MARGO_RPC_HANDLER(server_hello_rpc) + +/* server_request_rpc (server => server) + * + * request from one server to another */ +MERCURY_GEN_PROC(server_request_in_t, + ((int32_t)(src_rank)) + ((int32_t)(req_id)) + ((int32_t)(req_tag)) + ((hg_size_t)(bulk_size)) + ((hg_bulk_t)(bulk_handle))) +MERCURY_GEN_PROC(server_request_out_t, + ((int32_t)(ret))) +DECLARE_MARGO_RPC_HANDLER(server_request_rpc) + +/* chunk_read_request_rpc (server => server) + * + * request for chunk reads from another server */ +MERCURY_GEN_PROC(chunk_read_request_in_t, + ((int32_t)(src_rank)) + ((int32_t)(app_id)) + ((int32_t)(client_id)) + ((int32_t)(req_id)) + ((int32_t)(num_chks)) + ((hg_size_t)(bulk_size)) + ((hg_bulk_t)(bulk_handle))) +MERCURY_GEN_PROC(chunk_read_request_out_t, + ((int32_t)(ret))) +DECLARE_MARGO_RPC_HANDLER(chunk_read_request_rpc) + +/* chunk_read_response_rpc (server => server) + * + * response to remote chunk reads request */ +MERCURY_GEN_PROC(chunk_read_response_in_t, + ((int32_t)(src_rank)) + ((int32_t)(app_id)) + ((int32_t)(client_id)) + ((int32_t)(req_id)) + ((int32_t)(num_chks)) + ((hg_size_t)(bulk_size)) + ((hg_bulk_t)(bulk_handle))) +MERCURY_GEN_PROC(chunk_read_response_out_t, + ((int32_t)(ret))) +DECLARE_MARGO_RPC_HANDLER(chunk_read_response_rpc) + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // UNIFYFS_SERVER_RPCS_H diff --git a/common/src/unifycr_shm.c b/common/src/unifyfs_shm.c similarity index 78% rename from common/src/unifycr_shm.c rename to common/src/unifyfs_shm.c index f42f32b7e..75d63a4c2 100644 --- a/common/src/unifycr_shm.c +++ b/common/src/unifyfs_shm.c @@ -7,10 +7,11 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ +#include #include #include @@ -21,20 +22,20 @@ #include #include -#include "unifycr_log.h" -#include "unifycr_const.h" +#include "unifyfs_log.h" +#include "unifyfs_const.h" /* TODO: same function exists in client code, move this to common */ /* creates a shared memory of given size under specified name, * returns address of new shared memory if successful, * returns NULL on error */ -void* unifycr_shm_alloc(const char* name, size_t size) +void* unifyfs_shm_alloc(const char* name, size_t size) { int ret; /* open shared memory file */ errno = 0; - int fd = shm_open(name, MMAP_OPEN_FLAG, MMAP_OPEN_MODE); + int fd = shm_open(name, O_RDWR | O_CREAT, 0770); if (fd == -1) { /* failed to open shared memory */ LOGERR("Failed to open shared memory %s errno=%d (%s)", @@ -93,14 +94,14 @@ void* unifycr_shm_alloc(const char* name, size_t size) } /* unmaps shared memory region from memory, and releases it, - * caller should povider the address of a pointer to the region + * caller should provide the address of a pointer to the region * in paddr, sets paddr to NULL on return, - * returns UNIFYCR_SUCCESS on success */ -int unifycr_shm_free(const char* name, size_t size, void** paddr) + * returns UNIFYFS_SUCCESS on success */ +int unifyfs_shm_free(const char* name, size_t size, void** paddr) { /* check that we got an address (to something) */ if (paddr == NULL) { - return UNIFYCR_FAILURE; + return UNIFYFS_FAILURE; } /* get address of shared memory region */ @@ -112,7 +113,7 @@ int unifycr_shm_free(const char* name, size_t size, void** paddr) errno = 0; int rc = munmap(addr, size); if (rc == -1) { - /* failed to open shared memory */ + /* failed to unmap shared memory */ LOGERR("Failed to unmap shared memory %s errno=%d (%s)", name, errno, strerror(errno)); @@ -123,10 +124,12 @@ int unifycr_shm_free(const char* name, size_t size, void** paddr) errno = 0; rc = shm_unlink(name); if (rc == -1) { - /* failed to open shared memory */ - LOGERR("Failed to unlink shared memory %s errno=%d (%s)", - name, errno, strerror(errno)); - + int err = errno; + if (ENOENT != err) { + /* failed to remove shared memory */ + LOGERR("Failed to unlink shared memory %s errno=%d (%s)", + name, err, strerror(err)); + } /* not fatal, so keep going */ } } @@ -134,5 +137,5 @@ int unifycr_shm_free(const char* name, size_t size, void** paddr) /* set caller's pointer to NULL */ *paddr = NULL; - return UNIFYCR_SUCCESS; + return UNIFYFS_SUCCESS; } diff --git a/common/src/unifycr_shm.h b/common/src/unifyfs_shm.h similarity index 54% rename from common/src/unifycr_shm.h rename to common/src/unifyfs_shm.h index 5ce3f14af..33afe6a19 100644 --- a/common/src/unifycr_shm.h +++ b/common/src/unifyfs_shm.h @@ -7,27 +7,31 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ -#ifndef UNIFYCR_SHM_H -#define UNIFYCR_SHM_H +#ifndef UNIFYFS_SHM_H +#define UNIFYFS_SHM_H -/* TODO: same functions exist in client code, move this to common */ +#ifdef __cplusplus +extern "C" { +#endif /* allocate and attach a named shared memory region of a particular size * and mmap into our memory, returns starting memory address on success, * returns NULL on failure */ -void* unifycr_shm_alloc(const char* name, size_t size); +void* unifyfs_shm_alloc(const char* name, size_t size); /* unmaps shared memory region from memory, and releases it, * caller should povider the address of a pointer to the region * in paddr, sets paddr to NULL on return, - * returns UNIFYCR_SUCCESS on success */ -int unifycr_shm_free(const char* name, size_t size, void** paddr); + * returns UNIFYFS_SUCCESS on success */ +int unifyfs_shm_free(const char* name, size_t size, void** paddr); -/* release a shared memory region mapping */ +#ifdef __cplusplus +} // extern "C" +#endif -#endif // UNIFYCR_SHM_H +#endif // UNIFYFS_SHM_H diff --git a/configure.ac b/configure.ac index b6f927d65..8d7b802c3 100755 --- a/configure.ac +++ b/configure.ac @@ -1,9 +1,9 @@ dnl -dnl This file is a part of UnifyCR. Please see LICENSE for the license +dnl This file is a part of UnifyFS. Please see LICENSE for the license dnl information. dnl Process this file with autoconf to produce a configure script. -AC_INIT([unifycr], +AC_INIT([unifyfs], m4_esyscmd([git describe --always | awk '/.*/{sub(/^v/,""); printf "%s",$1; exit}']), [unifycr@llnl.gov]) @@ -16,6 +16,8 @@ AM_SILENT_RULES([yes]) AM_MAINTAINER_MODE([disable]) +LT_INIT + AC_PROG_CC_STDC AC_PROG_AWK AC_PROG_CPP @@ -23,7 +25,23 @@ AC_PROG_INSTALL AC_PROG_LN_S AC_PROG_MAKE_SET AC_PROG_RANLIB -AC_PROG_LIBTOOL + + +# fortran support +AC_ARG_ENABLE([fortran], + AC_HELP_STRING([--enable-fortran], + [Enable fortran compatibility and features])) +AC_MSG_CHECKING(if fortran is wanted ) +if test "x$enable_fortran" = "xyes"; then +AC_MSG_RESULT(yes) +AC_PROG_FC +AC_FC_LIBRARY_LDFLAGS +AC_FC_DUMMY_MAIN +AM_CONDITIONAL([HAVE_FORTRAN], [true]) +else +AC_MSG_RESULT(no) +AM_CONDITIONAL([HAVE_FORTRAN], [false]) +fi # Checks for typedefs, structures, and compiler characteristics. AC_TYPE_MODE_T @@ -60,6 +78,16 @@ AC_CHECK_FUNCS([ftruncate getpagesize gettimeofday memset socket floor]) AC_CHECK_FUNCS([gethostbyname strcasecmp strdup strerror strncasecmp strrchr]) AC_CHECK_FUNCS([gethostname strstr strtoumax strtol uname posix_fallocate]) +# PMPI Init/Fini mount/unmount option +AC_ARG_ENABLE([mpi-mount], + AC_HELP_STRING([--enable-mpi-mount], + [Enable transparent mount/unmount at MPI_Init/Finalize.])) +if test "x$enable_mpi_mount" = "xyes"; then +AM_CONDITIONAL([USE_PMPI_WRAPPERS], [true]) +else +AM_CONDITIONAL([USE_PMPI_WRAPPERS], [false]) +fi + # PMIx support build option AC_ARG_ENABLE([pmix], AC_HELP_STRING([--enable-pmix], @@ -72,19 +100,17 @@ else AM_CONDITIONAL([USE_PMIX], [false]) fi -# Debug build -AC_ARG_ENABLE([debug], - AC_HELP_STRING([--enable-debug], - [Enable debug build options.])) - -BUILD_DEBUG=no -if test "x$enable_debug" = "xyes"; then - BUILD_DEBUG=yes - CFLAGS=`echo $CFLAGS | sed -e s/O2/O1/` +# PMI2 support build option +AC_ARG_ENABLE([pmi], + AC_HELP_STRING([--enable-pmi], + [Enable PMI2 build options.])) +if test "x$enable_pmi" = "xyes"; then +AC_CHECK_HEADERS([pmi2.h], + [AM_CONDITIONAL([USE_PMI2], [true])], + [AM_CONDITIONAL([USE_PMI2], [false])]) else - BUILD_DEBUG=no +AM_CONDITIONAL([USE_PMI2], [false]) fi -AC_SUBST(CFLAGS) CHECK_NUMA @@ -94,7 +120,7 @@ AC_ARG_WITH(pkgconfigdir, [pkgconfigdir='${libdir}/pkgconfig']) AC_SUBST(pkgconfigdir) -## unifycr options +## unifyfs options AC_ARG_ENABLE(cuserid, [ --disable-cuserid Disables attempted use of cuserid() at run time], [if test "x$enableval" = "xno" ; then @@ -142,23 +168,34 @@ AC_CHECK_HEADERS(mntent.h sys/mount.h) # look for MPI and set flags LX_FIND_MPI +if test "x$enable_fortran" = "xyes"; then +AC_LANG_PUSH([Fortran]) +LX_FIND_MPI +AC_LANG_POP +fi + +if test "$have_C_mpi" != "yes" ; then + AC_MSG_ERROR(["Couldn't find MPI"]) +fi # look for leveldb library, sets LEVELDB_CFLAGS/LDFLAGS/LIBS -UNIFYCR_AC_LEVELDB +UNIFYFS_AC_LEVELDB # look for gotcha library, sets GOTCHA_INCLUDE, GOTCHA_LIB -UNIFYCR_AC_GOTCHA +UNIFYFS_AC_GOTCHA -UNIFYCR_AC_MARGO -UNIFYCR_AC_FLATCC +UNIFYFS_AC_MARGO +UNIFYFS_AC_FLATCC # HDF found? AX_LIB_HDF5 AM_CONDITIONAL([HAVE_HDF5], [test x$with_hdf5 = xyes]) -# libc functions wrapped by unifycr +# libc functions wrapped by unifyfs CP_WRAPPERS+="-Wl,-wrap,access" +CP_WRAPPERS+=",-wrap,chmod" +CP_WRAPPERS+=",-wrap,fchmod" CP_WRAPPERS+=",-wrap,lio_listio" CP_WRAPPERS+=",-wrap,mkdir" CP_WRAPPERS+=",-wrap,rmdir" @@ -252,7 +289,7 @@ CP_WRAPPERS+=",-wrap,ungetwc" # ,-u,__wrap___fxstat64,-u,pthread_mutex_lock,-u,pthread_mutex_unlock # We need to know the value of the $libdir and $bindir variables so that -# we can reference the correct path in the unifycr compiler wrappers. +# we can reference the correct path in the unifyfs compiler wrappers. # Unfortunately, those two variables are not normally evaluated by autoconf. # They are evaluated at build time using Makefile variable substitutions. # @@ -265,8 +302,8 @@ savePrefix=$prefix saveExecprefix=$exec_prefix test "x$prefix" = xNONE && prefix=$ac_default_prefix test "x$exec_prefix" = xNONE && exec_prefix=$prefix -eval unifycr_lib_path=$libdir -eval unifycr_bin_path=$bindir +eval unifyfs_lib_path=$libdir +eval unifyfs_bin_path=$bindir prefix=$savePrefix exec_prefix=$saveExecprefix @@ -275,13 +312,12 @@ if test "x$runstatedir" = x; then AC_SUBST([runstatedir], ['${localstatedir}/run']) fi -AC_SUBST(unifycr_lib_path) -AC_SUBST(unifycr_bin_path) +AC_SUBST(unifyfs_lib_path) +AC_SUBST(unifyfs_bin_path) AC_SUBST(LDFLAGS) AC_SUBST(__CP_LOG_PATH) AC_SUBST(CP_WRAPPERS) AC_SUBST(DISABLE_LDPRELOAD) -AC_OUTPUT(client/unifycr-config) AC_CONFIG_FILES([Makefile common/Makefile @@ -292,31 +328,37 @@ AC_CONFIG_FILES([Makefile server/src/Makefile client/Makefile client/src/Makefile - client/unifycr.pc + client/unifyfs.pc examples/Makefile examples/src/Makefile extras/Makefile - extras/unifycr.conf + extras/unifyfs.conf t/Makefile t/lib/Makefile util/Makefile - util/unifycr/Makefile - util/unifycr/src/Makefile]) + util/scripts/Makefile + util/scripts/lsfcsm/Makefile + util/unifyfs/Makefile + util/unifyfs/src/Makefile]) + +AC_CONFIG_FILES([client/unifyfs-config], [chmod +x client/unifyfs-config]) +AC_CONFIG_FILES([util/scripts/lsfcsm/unifyfs_lsfcsm_prolog], [chmod +x util/scripts/lsfcsm/unifyfs_lsfcsm_prolog]) +AC_CONFIG_FILES([util/scripts/lsfcsm/unifyfs_lsfcsm_epilog], [chmod +x util/scripts/lsfcsm/unifyfs_lsfcsm_epilog]) -UNIFYCR_VERSION=${PACKAGE_VERSION} -AC_SUBST(UNIFYCR_VERSION) -AC_SUBST([LIBUNIFYCR_LT_VERSION], [1:0:0]) +UNIFYFS_VERSION=${PACKAGE_VERSION} +AC_SUBST(UNIFYFS_VERSION) +AC_SUBST([LIBUNIFYFS_LT_VERSION], [1:0:0]) # pkgconfig versioning -LIBUNIFYCR_API_VERSION="1.0.0" -AC_SUBST(LIBUNIFYCR_API_VERSION) +LIBUNIFYFS_API_VERSION="1.0.0" +AC_SUBST(LIBUNIFYFS_API_VERSION) AC_OUTPUT AC_MSG_RESULT([ ========================== - UNIFYCR Checkpoint-Restart + UNIFYFS Checkpoint-Restart ========================== prefix ${prefix} compiler ${CC} diff --git a/docs/Makefile b/docs/Makefile index a5f7ffca7..8abc83175 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -107,9 +107,9 @@ qthelp: @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/unifycr.qhcp" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/unifyfs.qhcp" @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/unifycr.qhc" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/unifyfs.qhc" .PHONY: applehelp applehelp: @@ -126,8 +126,8 @@ devhelp: @echo @echo "Build finished." @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/unifycr" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/unifycr" + @echo "# mkdir -p $$HOME/.local/share/devhelp/unifyfs" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/unifyfs" @echo "# devhelp" .PHONY: epub diff --git a/docs/add-rpcs.rst b/docs/add-rpcs.rst new file mode 100644 index 000000000..e4691e145 --- /dev/null +++ b/docs/add-rpcs.rst @@ -0,0 +1,151 @@ +======================================== +Adding RPC Functions With Margo Library +======================================== + +In this section, we describe how to add an RPC function using +the Margo library API. + +.. note:: + + This uses the `unifyfs_mount_rpc` as an example RPC + function to follow throughout. + +--------------------------- +Common +--------------------------- + +1. Define structs for the input and output parameters of your RPC handler. + + The struct definition macro `MERCURY_GEN_PROC()` is used to define + both input and output parameters. For client-server RPCs, the + definitions should be placed in `common/src/unifyfs_clientcalls_rpc.h`, + while server-server RPC structs are defined in + `common/src/unifyfs_servercalls_rpc.h`. + + The input parameters struct should contain all values the client needs + to pass to the server handler function. + The output parameters struct should contain all values the server needs + to pass back to the client upon completion of the handler function. + The following shows the input and output structs for `unifyfs_mount_rpc`. + +.. code-block:: C + MERCURY_GEN_PROC(unifyfs_mount_in_t, + ((int32_t)(app_id)) + ((int32_t)(local_rank_idx)) + ((int32_t)(dbg_rank)) + ((int32_t)(num_procs_per_node)) + ((hg_const_string_t)(client_addr_str)) + ((hg_size_t)(req_buf_sz)) + ((hg_size_t)(recv_buf_sz)) + ((hg_size_t)(superblock_sz)) + ((hg_size_t)(meta_offset)) + ((hg_size_t)(meta_size)) + ((hg_size_t)(fmeta_offset)) + ((hg_size_t)(fmeta_size)) + ((hg_size_t)(data_offset)) + ((hg_size_t)(data_size)) + ((hg_const_string_t)(external_spill_dir))) + MERCURY_GEN_PROC(unifyfs_mount_out_t, + ((hg_size_t)(max_recs_per_slice)) + ((int32_t)(ret))) + +.. note:: +Passing some types can be an issue. Refer to the Mercury documentation for +supported types: `mid, "unifyfs_mount_rpc", + unifyfs_mount_in_t, unifyfs_mount_out_t, unifyfs_mount_rpc); + + The last argument to `MARGO_REGISTER()` is the handler function name. The + prior two arguments are the input and output parameters structs. The input + struct is passed in by the client to the RPC handler function, and then + forwarded to the server. When the RPC handler function finishes, the output + struct will be passed back to the client, which is generally a return code. + +--------------------------- +Client +--------------------------- + +1. Add a Mercury id for the RPC handler to the client RPC context. + + In `client/src/margo_client.h`, update the `ClientRpcIds` structure + to add a new `hg_id_t` variable to hold the RPC handler id. +.. code-block:: C + typedef struct ClientRpcIds { + ... + hg_id_t mount_id; + } + +2. Register the RPC handler with Margo. + + In `client/src/margo_client.c`, update `register_client_rpcs()` to register + the RPC handler and store its Mercury id in the newly defined `ClientRpcIds` + variable. + .. code-block:: C + client_rpc_context->rpcs.mount_id = MARGO_REGISTER(client_rpc_context->mid, "unifyfs_mount_rpc", + unifyfs_mount_in_t, unifyfs_mount_out_t, NULL); + + When the client calls `MARGO_REGISTER()` the last parameter is `NULL`. This + is the RPC handler function that is only defined on the server. + +3. Define and implement an invocation function that will execute the RPC. + + The declaration should be placed in `client/src/margo_client.h`, and the + definition should go in `client/src/margo_client.c`. + .. code-block:: C + int invoke_client_mount_rpc(); + + A handle for the RPC is obtained using `margo_create()`, which takes the + server address and the id of the RPC as parameters. The RPC is actually + initiated using `margo_forward()`, where the RPC handle and input struct + are supplied. Use `margo_get_output()` to obtain the returned output + parameters struct, and release it with `margo_free_output()`. Finally, + `margo_destroy()` is used to release the RPC handle. See the existing + invocation functions for more info. + +.. note:: +The general workflow for creating new RPC functions is the same if you want to +invoke an RPC on the server, and execute it on the client. One difference is +that you will have to pass `NULL` to the last parameter of `MARGO_REGISTER()` on +the server, and on the client the last parameter to `MARGO_REGISTER()` will be +the name of the RPC handler function. To execute RPCs on the client it needs to +be started in Margo as a `SERVER`, and the server needs to know the address of +the client where the RPC will be executed. The client has already been +configured to do those two things, so the only change going forward is how +`MARGO_REGISTER()` is called depending on where the RPC is being executed +(client or server). diff --git a/docs/api-mount.rst b/docs/api-mount.rst index f1dc98bb5..5ce67a556 100644 --- a/docs/api-mount.rst +++ b/docs/api-mount.rst @@ -1,36 +1,80 @@ ================= -Mounting UnifyCR +Mounting UnifyFS ================= -In this section, we describe how to use the UnifyCR API in an application. +In this section, we describe how to use the UnifyFS API in an application. + +.. Attention:: **Fortran Compatibility** + + ``unifyfs_mount`` and ``unifyfs_unmount`` are now usable with GFortran. + There is a known ifort_issue_ with the Intel Fortran compiler as well as an + xlf_issue_ with the IBM Fortran compiler. Other Fortran compilers are + currently unknown. + + If using fortran, when :ref:`installing UnifyFS ` with Spack, + include the ``+fortran`` variant, or configure UnifyFS with the + ``--enable-fortran`` option if building manually. --------------------------- Mounting --------------------------- -To use the UnifyCR filesystem a user will have to provide a path prefix. All -file operations under the path prefix will be intercepted by the UnifyCR -filesystem. For instance, to use UnifyCR on all path prefixes that begin with +In ``C`` applications, include *unifyfs.h*. See writeread.c_ for a full +example. + +.. code-block:: C + + #include + +In ``Fortran`` applications, include *unifyfsf.h*. See writeread.f90_ for a +full example. + +.. code-block:: Fortran + + include 'unifyfsf.h' + +To use the UnifyFS filesystem a user will have to provide a path prefix. All +file operations under the path prefix will be intercepted by the UnifyFS +filesystem. For instance, to use UnifyFS on all path prefixes that begin with /tmp this would require a: .. code-block:: C + :caption: C + + unifyfs_mount('/tmp', rank, rank_num, 0); - unifycr_mount('/tmp', rank, rank_num, 0); +.. code-block:: Fortran + :caption: Fortran -Where /tmp is the path prefix you want UnifyCR to intercept. The rank and rank -number is the rank you are currently on, and the number of tasks you have + call UNIFYFS_MOUNT('/tmp', rank, size, 0, ierr); + +Where /tmp is the path prefix you want UnifyFS to intercept. The rank and rank +number is the rank you are currently on, and the number of tasks you have running in your job. Lastly, the zero corresponds to the app id. --------------------------- Unmounting --------------------------- -When you are finished using UnifyCR in your application, you should unmount. - +When you are finished using UnifyFS in your application, you should unmount. + .. code-block:: C + :caption: C - if (rank == 0) { - unifycr_unmount(); - } + if (rank == 0) { + unifyfs_unmount(); + } + +.. code-block:: Fortran + :caption: Fortran + + call UNIFYFS_UNMOUNT(ierr); It is only necessary to call unmount once on rank zero. + +.. explicit external hyperlink targets + +.. _ifort_issue: https://github.com/LLNL/UnifyFS/issues/300 +.. _writeread.c: https://github.com/LLNL/UnifyFS/blob/dev/examples/src/writeread.c +.. _writeread.f90: https://github.com/LLNL/UnifyFS/blob/dev/examples/src/writeread.f90 +.. _xlf_issue: https://github.com/LLNL/UnifyFS/issues/304 diff --git a/docs/assumptions.rst b/docs/assumptions.rst index 985294e52..0c099c2dc 100644 --- a/docs/assumptions.rst +++ b/docs/assumptions.rst @@ -3,7 +3,7 @@ Assumptions ================ In this section, we provide assumptions we make about the behavior of -applications that use UnifyCR, and about how UnifyCR currently functions. +applications that use UnifyFS, and about how UnifyFS currently functions. --------------------------- Application Behavior @@ -28,30 +28,33 @@ Application Behavior --------------------------- Consistency Model --------------------------- -In the first version of UnifyCR, lamination will be explicitly initiated -by a UnifyCR API call. In subsequent versions, we will support implicit -initiation of file lamination. Here, UnifyCR will determine a -file to be laminated based on conditions, e.g., \texttt{fsync} or -\texttt{ioctl} calls, or a time out on \texttt{close} operations. -As part of the UnifyCR project, we will -investigate these implicit lamination conditions to determine the -best way to enable lamination of files without explicit UnifyCR API calls -being made by the application. - -In the first version of UnifyCR, eventually, a process declares the file to be -laminated through a UnifyCR API call. -After a file has been laminated, the contents of the file cannot be changed. -The file becomes permanently read-only. -After lamination, any process may freely read any part of the file. -If the application process group fails before a file has been laminated, -UnifyCR may delete the file. -An application can delete a laminated file. +One key aspect of UnifyFS is the idea of "laminating" a file. After a file is +laminated, it becomes "set in stone" and its data is accessible across all the +nodes. Laminated files are permanently read-only, and cannot be modified in +any way (but can be deleted). If the application process group fails before a +file has been laminated, UnifyFS may delete the file. + +The typical use case is to laminate your checkpoint files after they've been +written. To laminate a file, first call fsync() to sync all your writes to the +server, then call chmod() to remove all the write bits. Removing the write +bits does the actual lamination. A typical checkpoint will look like this: + +.. code-block:: C + + fp = fopen("checkpoint1.chk") + write(fp, ) + fsync(fp) + fclose(fp) + chmod("checkpoint1.chk", 0444) + +Future versions of UnifyFS may support different laminate semantics, such as +laminate on close(), or laminate via an explicit API call. We define the laminated consistency model to enable certain optimizations while supporting the perceived requirements of application checkpoints. -Since remote processes are not permitted to read arbitrary bytes within the +Since remote processes are not permitted to read arbitrary bytes within the file until lamination, -global exchange of file data and/or data index information can be buffered +global exchange of file data and/or data index information can be buffered locally on each node until the point of lamination. Since file contents cannot change after lamination, aggressive caching may be used during the read-only phase with minimal locking. @@ -62,10 +65,10 @@ Behavior before lamination: - open/close: A process may open/close a file multiple times. - - write: A process may write to any part of a file. If two processes write + - write: A process may write to any part of a file. If two processes write to the same location, the value is undefined. - - read: A process may read bytes it has written. Reading other bytes is + - read: A process may read bytes it has written. Reading other bytes is invalid. - rename: A process may rename a file. @@ -93,31 +96,31 @@ File System Behavior --------------------------- - The file system exists on node local storage only and is not persisted to - stable storage like a parallel file system (PFS). Can be coupled with + stable storage like a parallel file system (PFS). Can be coupled with - SymphonyFS or high level I/O or checkpoint library (VeloC) to move data to PFS periodically, or data can be moved manually - - Can be used with checkpointing libraries (VeloC) or I/O libraries to + - Can be used with checkpointing libraries (VeloC) or I/O libraries to support shared files on burst buffers - - File system starts empty at job start. User job must populate the file + - File system starts empty at job start. User job must populate the file system. - - Shared file system namespace across all compute nodes in a job, even if + - Shared file system namespace across all compute nodes in a job, even if an application process is not running on all compute nodes - Survives application termination and/or relaunch within a job - - Will transparently intercept system level I/O calls of applications and + - Will transparently intercept system level I/O calls of applications and I/O libraries --------------------------- System Characteristics --------------------------- - - There is some storage available for storing file data on a compute node, - e.g. SSD or RAM disk + - There is some storage available for storing file data on a compute node, + e.g. SSD or RAM disk - - We can run user-level daemon processes on compute nodes concurrently with + - We can run user-level daemon processes on compute nodes concurrently with a user application diff --git a/docs/build-intercept.rst b/docs/build-intercept.rst index f4d809bc5..2989c4cb0 100644 --- a/docs/build-intercept.rst +++ b/docs/build-intercept.rst @@ -2,22 +2,22 @@ Build & I/O Interception ======================== -In this section, we describe how to build UnifyCR with I/O interception. +In this section, we describe how to build UnifyFS with I/O interception. .. note:: - The current version of UnifyCR adopts the mdhim key-value store, which strictly + The current version of UnifyFS adopts the mdhim key-value store, which strictly requires: "An MPI distribution that supports MPI_THREAD_MULTIPLE and per-object locking of critical sections (this excludes OpenMPI up to version 3.0.1, the current version as of this writing)" - as specified in the project `github `_ + as specified in the project `github `_. .. _build-label: --------------------------- -How to Build UnifyCR +How to Build UnifyFS --------------------------- To install all dependencies and set up your build environment, we recommend @@ -30,37 +30,50 @@ Building with Spack These instructions assume that you do not already have a module system installed such as LMod, Dotkit, or Environment Modules. If your system already has Dotkit -or LMod installed then installing the environment-modules package with spack +or LMod installed then installing the environment-modules package with Spack is unnecessary (so you can safely skip that step). If you use Dotkit then replace ``spack load`` with ``spack use``. +First, install Spack if you don't already have it: .. code-block:: Bash $ git clone https://github.com/spack/spack $ ./spack/bin/spack install environment-modules $ . spack/share/spack/setup-env.sh - $ spack install unifycr - $ spack load unifycr + +Make use of Spack's `shell support `_ +to automatically add Spack to your ``PATH`` and allow the use of the ``spack`` +command. + +Then install UnifyFS: + +.. code-block:: Bash + + $ spack install unifyfs + $ spack load unifyfs .. Edit the following admonition if the default of variants are changed or when new variants are added. -Include or remove variants with Spack when installing UnifyCR when a custom -build is desired. Type ``spack info unifycr`` for more info. +Include or remove variants with Spack when installing UnifyFS when a custom +build is desired. Type ``spack info unifyfs`` for more info. -.. table:: UnifyCR Build Variants +.. table:: UnifyFS Build Variants :widths: auto - ======= ======================================== ======================== + ======= ======================================== ========================= Variant Command Description - ======= ======================================== ======================== - Debug ``spack install unifycr+debug`` Enable debug build - HDF5 ``spack install unifycr+hdf5`` Build with parallel HDF5 + ======= ======================================== ========================= + HDF5 ``spack install unifyfs+hdf5`` Build with parallel HDF5 - ``spack install unifycr+hdf5 ^hdf5~mpi`` Build with serial HDF5 - NUMA ``spack install unifycr+numa`` Build with NUMA - ======= ======================================== ======================== + ``spack install unifyfs+hdf5 ^hdf5~mpi`` Build with serial HDF5 + Fortran ``spack install unifyfs+fortran`` Build with gfortran + NUMA ``spack install unifyfs+numa`` Build with NUMA + pmpi ``spack install unifyfs+pmpi`` Transparent mount/unmount + PMI ``spack install unifyfs+pmi`` Enable PMI2 build options + PMIx ``spack install unifyfs+pmix`` Enable PMIx build options + ======= ======================================== ========================= .. attention:: @@ -70,7 +83,7 @@ build is desired. Type ``spack info unifycr`` for more info. have these dependencies installed through Spack or haven't told Spack where they are locally installed on your system (i.e., through a custom `packages.yaml `_). - Type ``spack spec -I unifycr`` before installing to see what Spack is going + Type ``spack spec -I unifyfs`` before installing to see what Spack is going to do. --------------------------- @@ -78,47 +91,42 @@ build is desired. Type ``spack info unifycr`` for more info. Building with Autotools ************************ -Download the latest UnifyCR release from the `Releases -`_ page. +Download the latest UnifyFS release from the `Releases +`_ page. Building the Dependencies ^^^^^^^^^^^^^^^^^^^^^^^^^^ -UnifyCR requires MPI, LevelDB, and GOTCHA(version 0.0.2). +UnifyFS requires MPI, LevelDB, GOTCHA(version 0.0.2), FlatCC, and Margo. +References to these dependencies can be found :doc:`here `. + +.. _spack-build-label: Build the Dependencies with Spack """""""""""""""""""""""""""""""""" Once Spack is installed on your system (see :ref:`above `), you -can install just the dependencies for an easier manual installation of UnifyCR. +can install just the dependencies for an easier manual installation of UnifyFS. If you use Dotkit then replace ``spack load`` with ``spack use``. .. code-block:: Bash - $ spack install environment-modules - $ $ spack install leveldb $ spack install gotcha@0.0.2 $ spack install flatcc - $ - $ git clone https://xgitlab.cels.anl.gov/sds/sds-repo.git sds-repo.git - $ cd sds-repo.git - $ spack repo add . - $ cd .. - $ $ spack install margo .. tip:: - You can use ``spack install --only=dependencies unifycr`` to install all of - UnifyCR's dependencies without installing UnifyCR. + You can use ``spack install --only=dependencies unifyfs`` to install all of + UnifyFS's dependencies without installing UnifyFS. Keep in mind this will also install all the build dependencies and dependencies of dependencies if you haven't already installed them through Spack or told Spack where they are locally installed on your system. -Then to build UnifyCR: +Then to manually build UnifyFS: .. code-block:: Bash @@ -130,39 +138,50 @@ Then to build UnifyCR: $ spack load margo $ $ ./autogen.sh - $ ./configure --prefix=/path/to/install --enable-debug + $ ./configure --prefix=/path/to/install $ make $ make install +.. note:: **Fortran Compatibility** + + To build with gfortran compatibility, include the ``--enable-fortran`` + configure option: + + ``./configure --prefix=/path/to/install/ --enable-fortran`` + + There is a known `ifort_issue `_ + with the Intel Fortran compiler as well as an `xlf_issue <://github.com/LLNL/UnifyFS/issues/304>`_ + with the IBM Fortran compiler. Other Fortran compilers are currently + unknown. + +To see all available build configuration options, type ``./configure --help`` +after ``./autogen.sh`` has been run. + +.. TODO: Add a section in build docs that shows all the build config options + Build the Dependencies without Spack """"""""""""""""""""""""""""""""""""" -For users who cannot use Spack, you may fetch version 0.0.2 (compatibility with -latest release in progress) of `GOTCHA `_ +For users who cannot use Spack, a `bootstrap.sh `_ +script has been provided in order to make manual build and installation of +dependencies easier. Simply run the script in the top level directory of the source code. -And leveldb (if not already installed on your system): -`leveldb `_ - -To get flatcc `flatcc `_ +.. code-block:: Bash -To download and install Margo and its dependencies (Mercury and Argobots) -follow the instructions here: `Margo `_ + $ ./bootstrap.sh -.. important:: +References to the UnifyFS dependencies can be found :doc:`here `. - Margo uses pkg-config to ensure it compiles and links correctly with all of - its dependencies' libraries. When building without Spack, you'll need to - manually set the ``PKG_CONFIG_PATH`` environment variable and include in - that variable the paths for the ``.pc`` files for Mercury, Argobots, and - Margo separated by colons. -Then to build UnifyCR: +After bootstrap.sh is finished building the dependencies, it will print out the +commands you need to run to build UnifyFS. The commands look something like +this: .. code-block:: Bash $ export PKG_CONFIG_PATH=path/to/mercury/lib/pkgconfig:path/to/argobots/lib/pkgconfig:path/to/margo/lib/pkgconfig $ ./autogen.sh - $ ./configure --prefix=/path/to/install --enable-debug --with-gotcha=/path/to/gotcha --with-leveldb=/path/to/leveldb --with-flatcc=/path/to/flatcc + $ ./configure --prefix=/path/to/install --with-gotcha=/path/to/gotcha --with-leveldb=/path/to/leveldb --with-flatcc=/path/to/flatcc $ make $ make install @@ -174,8 +193,8 @@ Then to build UnifyCR: ``--with-numa=$PATH_TO_NUMA`` This is needed to enable NUMA-aware memory allocation on Linux machines. Set the - NUMA policy at runtime with ``UNIFYCR_NUMA_POLICY = local | interleaved``, or set - NUMA nodes explicitly with ``UNIFYCR_USE_NUMA_BANK = `` + NUMA policy at runtime with ``UNIFYFS_NUMA_POLICY = local | interleaved``, or set + NUMA nodes explicitly with ``UNIFYFS_USE_NUMA_BANK = `` --------------------------- @@ -191,14 +210,14 @@ Statically Steps for static linking using --wrap: To intercept I/O calls using a static link, you must add flags to your link -line. UnifyCR installs a unifycr-config script that returns those flags, e.g., +line. UnifyFS installs a unifyfs-config script that returns those flags, e.g., .. code-block:: Bash $ mpicc -o test_write \ - `/bin/unifycr-config --pre-ld-flags` \ + `/bin/unifyfs-config --pre-ld-flags` \ test_write.c \ - `/bin/unifycr-config --post-ld-flags` + `/bin/unifyfs-config --post-ld-flags` Dynamically ************** @@ -208,8 +227,19 @@ Steps for dynamic linking using gotcha: To intercept I/O calls using gotcha, use the following syntax to link an application. +C +^^^^^^^^^^^^^^ + .. code-block:: Bash $ mpicc -o test_write test_write.c \ - -I/include -L/lib -lunifycr_gotcha \ + -I/include -L/lib -lunifyfs_gotcha \ -L/lib64 -lgotcha + +Fortran +^^^^^^^^^^^^^^ + +.. code-block:: Bash + + $ mpif90 -o test_write test_write.F \ + -I/include -L/lib -lunifyfsf -lunifyfs_gotcha diff --git a/docs/conf.py b/docs/conf.py index 17768f57a..f1abca286 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# unifycr documentation build configuration file, created by +# unifyfs documentation build configuration file, created by # sphinx-quickstart on Sun Nov 26 20:15:18 2017. # # This file is execfile()d with the current directory set to its @@ -45,7 +45,7 @@ master_doc = 'index' # General information about the project. -project = u'UnifyCR' +project = u'UnifyFS' copyright = u'2017, Lawrence Livermore National Security LLC LLNL-CODE-741539, UT-Batelle LLC' author = u'Kathryn Mohror, Adam Moody, Oral Sarp, Feiyi Wang, Hyogi Sim, Danielle Sikich, Joseph Moore, Ned Bass' @@ -120,7 +120,7 @@ # The name for this set of Sphinx documents. # " v documentation" by default. -#html_title = u'unifycr v0.1' +#html_title = u'unifyfs v0.1' # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None @@ -204,7 +204,7 @@ def setup(app): app.add_stylesheet("theme_overrides.css" ) #html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. -htmlhelp_basename = 'unifycrdoc' +htmlhelp_basename = 'unifyfsdoc' # -- Options for LaTeX output --------------------------------------------- @@ -226,7 +226,7 @@ def setup(app): app.add_stylesheet("theme_overrides.css" ) # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'unifycr.tex', u'UnifyCR Documentation', author, 'manual'), + (master_doc, 'unifyfs.tex', u'UnifyFS Documentation', author, 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -255,7 +255,7 @@ def setup(app): app.add_stylesheet("theme_overrides.css" ) # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (master_doc, 'unifycr', u'UnifyCR Documentation', [author], 1) + (master_doc, 'unifyfs', u'UnifyFS Documentation', [author], 1) ] # If true, show URL addresses after external links. @@ -268,8 +268,8 @@ def setup(app): app.add_stylesheet("theme_overrides.css" ) # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'unifycr', u'UnifyCR Documentation', - 'unifycr', 'One line description of project.', + (master_doc, 'unifyfs', u'UnifyFS Documentation', + 'unifyfs', 'One line description of project.', 'Miscellaneous'), ] diff --git a/docs/configuration.rst b/docs/configuration.rst index b9c3cf519..2a4f55ec1 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1,18 +1,25 @@ ===================== -UnifyCR Configuration +UnifyFS Configuration ===================== -Here, we explain how users can customize the runtime behavior of UnifyCR. In -particular, UnifyCR provides the following ways to configure: +Here, we explain how users can customize the runtime behavior of UnifyFS. In +particular, UnifyFS provides the following ways to configure: -- System-wide configuration file: ``/etc/unifycr/unifycr.conf`` +- System-wide configuration file: ``/etc/unifyfs/unifyfs.conf`` - Environment variables -- Command line options to ``unifycrd`` +- Command line options to ``unifyfsd`` All configuration settings have corresponding environment variables, but only certain settings have command line options. When defined via multiple methods, the command line options have the highest priority, followed by environment -variables, and finally config file options from ``unifycr.conf``. +variables, and finally config file options from ``unifyfs.conf``. + +The config file is installed in /etc by default. However, one can +specify a custom location for the +unifyfs.conf file with the -f command-line option to unifyfsd (see below). +There is a sample unifyfs.conf file in the installation directory +under etc/unifyfs/. This file is also available in the "extras" directory +in the source repository. The unified method for providing configuration control is adapted from CONFIGURATOR_. Configuration settings are grouped within named sections, and @@ -25,10 +32,10 @@ each setting consists of a key-value pair with one of the following types: .. _CONFIGURATOR: https://github.com/MichaelBrim/tedium/tree/master/configurator -------------- - unifycr.conf + unifyfs.conf -------------- -``unifycr.conf`` specifies the system-wide configuration options. The file is +``unifyfs.conf`` specifies the system-wide configuration options. The file is written in INI_ language format, as supported by the inih_ parser. .. _INI: http://en.wikipedia.org/wiki/INI_file @@ -40,17 +47,17 @@ In this description, we use ``section.key`` as shorthand for the name of a given section and key. -.. table:: ``[unifycr]`` section - main configuration settings +.. table:: ``[unifyfs]`` section - main configuration settings :widths: auto ============= ====== ===================================================== Key Type Description ============= ====== ===================================================== + cleanup BOOL cleanup storage on server exit (default: off) configfile STRING path to custom configuration file consistency STRING consistency model [ LAMINATED | POSIX | NONE ] daemonize BOOL enable server daemonization (default: off) - debug BOOL enable debug output (default: off) - mountpoint STRING mountpoint path prefix (default: /unifycr) + mountpoint STRING mountpoint path prefix (default: /unifyfs) ============= ====== ===================================================== .. table:: ``[client]`` section - client settings @@ -82,7 +89,7 @@ a given section and key. db_name STRING metadata database file name db_path STRING path to directory to contain metadata database range_size INT metadata range size (B) (default: 1 MiB) - server_ratio INT # of UnifyCR servers per metadata server (default: 1) + server_ratio INT # of UnifyFS servers per metadata server (default: 1) ============= ====== ===================================================== .. table:: ``[runstate]`` section - server runstate settings @@ -94,6 +101,24 @@ a given section and key. dir STRING path to directory to contain server runstate file ============= ====== ===================================================== +.. table:: ``[server]`` section - server settings + :widths: auto + + ============= ====== ===================================================== + Key Type Description + ============= ====== ===================================================== + hostfile STRING path to server hostfile + ============= ====== ===================================================== + +.. table:: ``[sharedfs]`` section - server shared files settings + :widths: auto + + ============= ====== ===================================================== + Key Type Description + ============= ====== ===================================================== + dir STRING path to directory to contain server shared files + ============= ====== ===================================================== + .. table:: ``[shmem]`` section - shared memory segment usage settings :widths: auto @@ -124,11 +149,11 @@ a given section and key. Environment Variables ----------------------- -All environment variables take the form ``UNIFYCR_SECTION_KEY``, except for -the ``[unifycr]`` section, which uses ``UNIFYCR_KEY``. For example, +All environment variables take the form ``UNIFYFS_SECTION_KEY``, except for +the ``[unifyfs]`` section, which uses ``UNIFYFS_KEY``. For example, the setting ``log.verbosity`` has a corresponding environment variable -named ``UNIFYCR_LOG_VERBOSITY``, while ``unifycr.mountpoint`` corresponds to -``UNIFYCR_MOUNTPOINT``. +named ``UNIFYFS_LOG_VERBOSITY``, while ``unifyfs.mountpoint`` corresponds to +``UNIFYFS_MOUNTPOINT``. ---------------------- @@ -140,20 +165,22 @@ command line options have long and short forms. The long form uses ``--section-key=value``, while the short form ``- value``, where the short option character is given in the below table. -.. table:: ``unifycrd`` command line options +.. table:: ``unifyfsd`` command line options :widths: auto ====================== ======== LongOpt ShortOpt ====================== ======== - --unifycr-configfile -C - --unifycr-consistency -c - --unifycr-daemonize -D - --unifycr-debug -d - --unifycr-mountpoint -m + --unifyfs-cleanup -C + --unifyfs-configfile -f + --unifyfs-consistency -c + --unifyfs-daemonize -D + --unifyfs-mountpoint -m --log-dir -L --log-file -l --log-verbosity -v --runstate-dir -R + --server-hostfile -H + --sharedfs-dir -S ====================== ======== diff --git a/docs/contribute-ways.rst b/docs/contribute-ways.rst index 751020a12..1deac78f2 100644 --- a/docs/contribute-ways.rst +++ b/docs/contribute-ways.rst @@ -4,20 +4,20 @@ Ways to Contribute *First of all, thank you for taking the time to contribute!* -By using the following guidelines, you can help us make UnifyCR even better. +By using the following guidelines, you can help us make UnifyFS even better. Getting Started =============== -Get UnifyCR +Get UnifyFS ----------- -You can build and run UnifyCR by following :doc:`these instructions `. +You can build and run UnifyFS by following :doc:`these instructions `. Getting Help ------------ -To contact the UnifyCR team, send an email to the `mailing list`_. +To contact the UnifyFS team, send an email to the `mailing list`_. --------------- @@ -38,7 +38,7 @@ found anything similar to your issue. **When opening a new issue, please include the following information at the top of the issue:** - What operating system (with version) you are using - - The UnifyCR version you are using + - The UnifyFS version you are using - Describe the issue you are experiencing - Describe how to reproduce the issue - Include any warnings or errors @@ -84,7 +84,7 @@ Pull Requests - If your pull request improves performance, please include some benchmark results. - The pull request must pass all regression tests before being accepted. -- All proposed changes must be approved by a UnifyCR project member. +- All proposed changes must be approved by a UnifyFS project member. --------------- @@ -102,7 +102,7 @@ able to quickly identify and resolve issues. Documentation ============= -As UnifyCR is continually improved and updated, it is easy for documentation to +As UnifyFS is continually improved and updated, it is easy for documentation to become out-of-date. Any contributions to the documentation, no matter how small, is always greatly appreciated. If you are not in a position to update the documentation yourself, please notify us via the `mailing list`_ of @@ -111,4 +111,4 @@ anything you notice that needs to be changed. .. explicit external hyperlink targets .. _mailing list: ecp-unifycr@exascaleproject.org -.. _issue tracker: https://github.com/LLNL/UnifyCR/issues +.. _issue tracker: https://github.com/LLNL/UnifyFS/issues diff --git a/docs/dependencies.rst b/docs/dependencies.rst new file mode 100644 index 000000000..aebb50112 --- /dev/null +++ b/docs/dependencies.rst @@ -0,0 +1,24 @@ +==================== +UnifyFS Dependencies +==================== + +- `GOTCHA `_ version 0.0.2 (compatibility with latest release in progress) + +- `leveldb `_ version 1.22 + +- `flatcc `_ version 0.5.3 + +- `Margo `_ version 0.4.3 and its dependencies: + + - `Argobots `_ version 1.0rc1 + - `Mercury `_ version 1.0.1 + + - `bmi `_ + +.. important:: + + Margo uses pkg-config to ensure it compiles and links correctly with all of + its dependencies' libraries. When building manually, you'll need to set the + ``PKG_CONFIG_PATH`` environment variable and include in + that variable the paths for the ``.pc`` files for Mercury, Argobots, and + Margo separated by colons. diff --git a/docs/examples.rst b/docs/examples.rst index 3b62210da..1874d67a4 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -2,39 +2,39 @@ Examples ******** -There are several examples_ available on ways to use UnifyCR. These examples +There are several examples_ available on ways to use UnifyFS. These examples build into static and GOTCHA versions (pure POSIX versions coming soon) and are also used as a form of :doc:`intregraton testing `. Examples Locations ================== -The example programs can be found in two locations, where UnifyCR is built and -where UnifyCR is installed. +The example programs can be found in two locations, where UnifyFS is built and +where UnifyFS is installed. Install Location ---------------- -Upon installation of UnifyCR, the example programs are installed into the +Upon installation of UnifyFS, the example programs are installed into the *install/libexec* folder. Installed with Spack ^^^^^^^^^^^^^^^^^^^^ -The Spack installation location of UnifyCR can be found with the command -``spack location -i unifycr``. +The Spack installation location of UnifyFS can be found with the command +``spack location -i unifyfs``. To easily navigate to this location and find the examples, do: .. code-block:: Bash - $ spack cd -i unifycr + $ spack cd -i unifyfs $ cd libexec Installed without Spack ^^^^^^^^^^^^^^^^^^^^^^^ -The autotools installation of UnifyCR will place the example programs in the +The autotools installation of UnifyFS will place the example programs in the *libexec/* directory of the path provided to ``--prefix=/path/to/install`` during the configure step of :doc:`building and installing `. @@ -44,16 +44,16 @@ Build Location Built with Spack ^^^^^^^^^^^^^^^^ -The Spack build location of UnifyCR (on a successful install) only exists when +The Spack build location of UnifyFS (on a successful install) only exists when ``--keep-stage`` in included during installation or if the build fails. This -location can be found with the command ``spack location unifycr``. +location can be found with the command ``spack location unifyfs``. To navigate to the location of the static and POSIX examples, do: .. code-block:: Bash - $ spack install --keep-stage unifycr - $ spack cd unifycr + $ spack install --keep-stage unifyfs + $ spack cd unifyfs $ cd spack-build/examples/src The GOTCHA examples are one directory deeper in @@ -61,25 +61,27 @@ The GOTCHA examples are one directory deeper in .. note:: - If you installed UnifyCR with any variants, in order to navigate to the + If you installed UnifyFS with any variants, in order to navigate to the build directory you must include these variants in the ``spack cd`` command. E.g.: - ``spack cd unifycr+hdf5 ^hdf5~mpi`` + ``spack cd unifyfs+hdf5 ^hdf5~mpi`` Built without Spack ^^^^^^^^^^^^^^^^^^^ -The autotools build of UnifyCR will place the static and POSIX example programs +The autotools build of UnifyFS will place the static and POSIX example programs in the *examples/src* directory and the GOTCHA example programs in the *examples/src/.libs* directory of your build directory. ------------ +.. _run-ex-label: + Running the Examples ==================== -In order to run any of the example programs you first need to start the UnifyCR +In order to run any of the example programs you first need to start the UnifyFS server daemon on the nodes in the job allocation. To do this, see :doc:`start-stop`. @@ -88,42 +90,54 @@ to aid in this process. .. code-block:: none - [prompt]$ ./sysio-write-static --help + [prompt]$ ./write-static --help - Usage: sysio-write-static [options...] + Usage: write-static [options...] Available options: - -b, --blocksize= logical block size for the target file - (default 1048576, 1MB) - -n, --nblocks= count of blocks each process will write - (default 128) - -c, --chunksize= I/O chunk size for each write operation - (default 64436, 64KB) - -d, --debug pause before running test - (handy for attaching in debugger) - -f, --filename= target file name under mountpoint - (default: testfile) - -h, --help help message - -L, --lipsum generate contents to verify correctness - -m, --mount= use for unifycr - (default: /unifycr) - -P, --pwrite use pwrite(2) instead of write(2) - -p, --pattern= should be 'n1'(n to 1) or 'nn' (n to n) - (default: n1) - -S, --synchronous sync metadata on each write - -s, --standard do not use unifycr but run standard I/O - -u, --unmount unmount the filesystem after test - -Notice the mountpoint is defaulted to ``-mount=/unifycr``. If you chose a -different mountpoint during :doc:`start-stop`, the ``-m`` option for the -example will need to be provided to match. + -a, --appid= use given application id + (default: 0) + -A, --aio use asynchronous I/O instead of read|write + (default: off) + -b, --blocksize= I/O block size + (default: 16 MiB) + -c, --chunksize= I/O chunk size for each operation + (default: 1 MiB) + -d, --debug for debugging, wait for input (at rank 0) at start + (default: off) + -f, --file= target file name (or path) under mountpoint + (default: 'testfile') + -k, --check check data contents upon read + (default: off) + -L, --listio use lio_listio instead of read|write + (default: off) + -m, --mount= use for unifyfs + (default: /unifyfs) + -M, --mapio use mmap instead of read|write + (default: off) + -n, --nblocks= count of blocks each process will read|write + (default: 32) + -p, --pattern= 'n1' (N-to-1 shared file) or 'nn' (N-to-N file per process) + (default: 'n1') + -P, --prdwr use pread|pwrite instead of read|write + (default: off) + -S, --stdio use fread|fwrite instead of read|write + (default: off) + -U, --disable-unifyfs do not use UnifyFS + (default: enable UnifyFS) + -v, --verbose print verbose information + (default: off) + -V, --vecio use readv|writev instead of read|write + (default: off) + -x, --shuffle read different data than written + (default: off) One form of running this example could be: .. code-block:: Bash - $ srun -N4 -n4 sysio-write-static -m /myMountPoint -f myTestFile + $ srun -N4 -n4 write-static -m /myMountPoint -f myTestFile .. explicit external hyperlink targets -.. _examples: https://github.com/LLNL/UnifyCR/tree/dev/examples/src +.. _examples: https://github.com/LLNL/UnifyFS/tree/dev/examples/src diff --git a/docs/index.rst b/docs/index.rst index 518308f1e..64141a4c1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,10 +1,10 @@ -.. unifycr documentation master file, created by +.. unifyfs documentation master file, created by sphinx-quickstart on Sun Nov 26 20:15:18 2017. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. ======================================== -UnifyCR: A file system for burst buffers +UnifyFS: A file system for burst buffers ======================================== .. toctree:: @@ -28,6 +28,7 @@ UnifyCR: A file system for burst buffers style-guides testing wrappers + add-rpcs .. toctree:: :maxdepth: 2 @@ -37,6 +38,8 @@ UnifyCR: A file system for burst buffers :maxdepth: 2 :caption: Reference + dependencies + ================== Indices and tables ================== diff --git a/docs/make.bat b/docs/make.bat index fe3029135..a83da687c 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -128,9 +128,9 @@ if "%1" == "qthelp" ( echo. echo.Build finished; now you can run "qcollectiongenerator" with the ^ .qhcp project file in %BUILDDIR%/qthelp, like this: - echo.^> qcollectiongenerator %BUILDDIR%\qthelp\unifycr.qhcp + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\unifyfs.qhcp echo.To view the help file: - echo.^> assistant -collectionFile %BUILDDIR%\qthelp\unifycr.ghc + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\unifyfs.ghc goto end ) diff --git a/docs/overview.rst b/docs/overview.rst index 170ad6573..df735beb0 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -2,13 +2,13 @@ Overview ================ -UnifyCR is a user level file system currently under active development. An -application can use node-local storage as burst buffers for shared files. -UnifyCR is designed to support both checkpoint/restart which is the most -important I/O workload for HPC and other common I/O workloads as well. With -UnifyCR, applications can write to fast, scalable, node-local burst buffers as -easily as they do the parallel file system. This section will provide a high -level design of UnifyCR. It will describe the UnifyCR library and the UnifyCR +UnifyFS is a user level file system currently under active development. An +application can use node-local storage as burst buffers for shared files. +UnifyFS is designed to support both checkpoint/restart which is the most +important I/O workload for HPC and other common I/O workloads as well. With +UnifyFS, applications can write to fast, scalable, node-local burst buffers as +easily as they do the parallel file system. This section will provide a high +level design of UnifyFS. It will describe the UnifyFS library and the UnifyFS daemon. --------------------------- @@ -17,19 +17,19 @@ High Level Design .. image:: images/design-high-lvl.png -UnifyCR will present a shared namespace (e.g., /unifycr as a mount point) to -all compute nodes in a users job allocation. There are two main components of -UnifyCR: the UnifyCR library and the UnifyCR daemon. The UnifyCR library (also -referred to as the UnifyCR client library) is linked into the user application -and is responsible for intercepting I/O calls from the user application and -then sending the I/O requests on to a UnifyCR server to be handled. The UnifyCR -client library uses the ECP `GOTCHA `_ software -as its primary mechanism for intercepting I/O calls. Each UnifyCR daemon (also -referred to as a UnifyCR server daemon) runs as a daemon on a compute node in -the users allocation. The UnifyCR server is responsible for handling the I/O -requests from the UnifyCR library. On each compute node, there will be user -application processes running as well as tool daemon processes. The user -application is linked with the UnifyCR client library and a high-level I/O -library, e.g. HDF5, ADIOS, or PnetCDF. The UnifyCR server daemon also runs on -the compute node and is linked with the MDHIM library which is used for +UnifyFS will present a shared namespace (e.g., /unifyfs as a mount point) to +all compute nodes in a users job allocation. There are two main components of +UnifyFS: the UnifyFS library and the UnifyFS daemon. The UnifyFS library (also +referred to as the UnifyFS client library) is linked into the user application +and is responsible for intercepting I/O calls from the user application and +then sending the I/O requests on to a UnifyFS server to be handled. The UnifyFS +client library uses the ECP `GOTCHA `_ software +as its primary mechanism for intercepting I/O calls. Each UnifyFS daemon (also +referred to as a UnifyFS server daemon) runs as a daemon on a compute node in +the users allocation. The UnifyFS server is responsible for handling the I/O +requests from the UnifyFS library. On each compute node, there will be user +application processes running as well as tool daemon processes. The user +application is linked with the UnifyFS client library and a high-level I/O +library, e.g. HDF5, ADIOS, or PnetCDF. The UnifyFS server daemon also runs on +the compute node and is linked with the MDHIM library which is used for metadata services. diff --git a/docs/start-stop.rst b/docs/start-stop.rst index f79abcdb7..e7e5947e3 100644 --- a/docs/start-stop.rst +++ b/docs/start-stop.rst @@ -2,40 +2,40 @@ Starting & Stopping in a Job ================================ -In this section, we describe the mechanisms for starting and stopping UnifyCR in +In this section, we describe the mechanisms for starting and stopping UnifyFS in a user's job allocation. -Overall, the steps taken to run an application with UnifyCR include: +Overall, the steps taken to run an application with UnifyFS include: 1. Allocate nodes using the system resource manager (i.e., start a job) - 2. Update any desired UnifyCR server configuration settings + 2. Update any desired UnifyFS server configuration settings - 3. Start UnifyCR servers on each allocated node using ``unifycr`` + 3. Start UnifyFS servers on each allocated node using ``unifyfs`` - 4. Run one or more UnifyCR-enabled applications + 4. Run one or more UnifyFS-enabled applications - 5. Terminate the UnifyCR servers using ``unifycr`` + 5. Terminate the UnifyFS servers using ``unifyfs`` -------------------- - Starting UnifyCR + Starting UnifyFS -------------------- -First, we need to start the UnifyCR server daemon (``unifycrd``) on the nodes in -the job allocation. UnifyCR provides the ``unifycr`` command line utility to +First, we need to start the UnifyFS server daemon (``unifyfsd``) on the nodes in +the job allocation. UnifyFS provides the ``unifyfs`` command line utility to simplify this action on systems with supported resource managers. The easiest way to determine if you are using a supported system is to run -``unifycr start`` within an interactive job allocation. If no compatible +``unifyfs start`` within an interactive job allocation. If no compatible resource management system is detected, the utility will report an error message to that effect. -In ``start`` mode, the ``unifycr`` utility automatically detects the allocated +In ``start`` mode, the ``unifyfs`` utility automatically detects the allocated nodes and launches a server on each node. For example, the following script -could be used to launch the ``unifycrd`` servers with a customized +could be used to launch the ``unifyfsd`` servers with a customized configuration. On systems with resource managers that propagate environment settings to compute nodes, the environment variables will override any -settings in ``/etc/unifycr/unifycr.conf``. See :doc:`configuration` -for further details on customizing the UnifyCR runtime configuration. +settings in ``/etc/unifyfs/unifyfs.conf``. See :doc:`configuration` +for further details on customizing the UnifyFS runtime configuration. .. code-block:: Bash :linenos: @@ -43,55 +43,60 @@ for further details on customizing the UnifyCR runtime configuration. #!/bin/bash # spillover checkpoint data to node-local ssd storage - export UNIFYCR_SPILLOVER_DATA_DIR=/mnt/ssd/$USER/data - export UNIFYCR_SPILLOVER_META_DIR=/mnt/ssd/$USER/meta + export UNIFYFS_SPILLOVER_DATA_DIR=/mnt/ssd/$USER/data + export UNIFYFS_SPILLOVER_META_DIR=/mnt/ssd/$USER/meta # store server logs in job-specific scratch area - export UNIFYCR_LOG_DIR=$JOBSCRATCH/logs + export UNIFYFS_LOG_DIR=$JOBSCRATCH/logs - unifycr start --mount=/mnt/unifycr + unifyfs start --share-dir=/path/to/shared/file/system & -``unifycr`` provides command-line options to choose the client mountpoint, +``unifyfs`` provides command-line options to choose the client mountpoint, adjust the consistency model, and control stage-in and stage-out of files. -The full usage for ``unifycr`` is as follows: +The full usage for ``unifyfs`` is as follows: .. code-block:: Bash :linenos: - [prompt]$ unifycr --help + [prompt]$ unifyfs --help - Usage: unifycr [options...] + Usage: unifyfs [options...] should be one of the following: - start start the unifycr server daemon - terminate terminate the unifycr server daemon - - Available options for "start": - -C, --consistency= consistency model (NONE | LAMINATED | POSIX) - -e, --exe= where unifycrd is installed - -m, --mount= mount unifycr at - -s, --script= to custom launch script - -i, --stage-in= stage in file(s) at - -o, --stage-out= stage out file(s) to on termination - - Available options for "terminate": - -c, --cleanup clean up the unifycr storage on termination + start start the UnifyFS server daemons + terminate terminate the UnifyFS server daemons + + Common options: + -d, --debug enable debug output + -h, --help print usage + + Command options for "start": + -c, --cleanup [OPTIONAL] clean up the UnifyFS storage upon server exit + -C, --consistency= [OPTIONAL] consistency model (NONE | LAMINATED | POSIX) + -e, --exe= [OPTIONAL] where unifyfsd is installed + -m, --mount= [OPTIONAL] mount UnifyFS at + -s, --script= [OPTIONAL] to custom launch script + -S, --share-dir= [REQUIRED] shared file system for use by servers + -i, --stage-in= [OPTIONAL] stage in file(s) at + -o, --stage-out= [OPTIONAL] stage out file(s) to on termination + + Command options for "terminate": -s, --script= to custom termination script -After UnifyCR servers have been successfully started, you may run your -UnifyCR-enabled applications as you normally would (e.g., using mpirun). -Only applications that explcitly call ``unifycr_mount()`` and access files -with the specified mountpoint prefix will utils UnifyCR for their I/O. All +After UnifyFS servers have been successfully started, you may run your +UnifyFS-enabled applications as you normally would (e.g., using mpirun). +Only applications that explicitly call ``unifyfs_mount()`` and access files +with the specified mountpoint prefix will utilize UnifyFS for their I/O. All other applications will operate unchanged. -------------------- - Stopping UnifyCR + Stopping UnifyFS -------------------- -After all UnifyCR-enabled applications have completed running, you should -use ``unifycr terminate`` to terminate the servers. Typically, one would -also pass the ``--cleanup`` option to have the servers remove temporary data -locally stored on each node. +After all UnifyFS-enabled applications have completed running, you should +use ``unifyfs terminate`` to terminate the servers. Typically, one would pass +the ``--cleanup`` option to ``unifyfs start`` to have the servers remove +temporary data locally stored on each node after termination. diff --git a/docs/style-guides.rst b/docs/style-guides.rst index f90878fda..9461a61eb 100644 --- a/docs/style-guides.rst +++ b/docs/style-guides.rst @@ -5,7 +5,7 @@ Style Guides Coding Conventions ================== -UnifyCR follows the `Linux kernel coding style +UnifyFS follows the `Linux kernel coding style `_ except that code is indented using four spaces per level instead of tabs. Please run ``make checkstyle`` to check your patch for style problems before submitting it @@ -20,10 +20,10 @@ the project. .. code-block:: Bash :caption: To apply style to the source file foo.c: - astyle --options=scripts/unifycr.astyle foo.c + astyle --options=scripts/unifyfs.astyle foo.c -The `unifycr.astyle file -`_ specifies +The `unifyfs.astyle file +`_ specifies the options used for this project. For a full list of available astyle options, see http://astyle.sourceforge.net/astyle.html. diff --git a/docs/testing.rst b/docs/testing.rst index 5ed9b1af8..5622a8c1a 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -1,19 +1,27 @@ -************* +============= Testing Guide -************* - -Implementing Tests -================== +============= We can never have enough testing. Any additional tests you can write are always greatly appreciated. +---------- +Unit Tests +---------- + +Implementing Tests +****************** + +The UnifyFS Test Suite uses the `Test Anything Protocol`_ (TAP) and the +Automake test harness. This test suite has two types of TAP tests (shell scripts +and C) to allow for testing multiple aspects of UnifyFS. + Shell Script Tests ------------------- +^^^^^^^^^^^^^^^^^^ Test cases in shell scripts are implemented with sharness_, which is included -in the UnifyCR source distribution. See the file sharness.sh_ for all available -test interfaces. UnifyCR-specific sharness code is implemented in scripts in +in the UnifyFS source distribution. See the file sharness.sh_ for all available +test interfaces. UnifyFS-specific sharness code is implemented in scripts in the directory sharness.d_. Scripts in sharness.d_ are primarily used to set environment variables and define convenience functions. All scripts in sharness.d_ are automatically included when your script sources sharness.sh_. @@ -52,13 +60,13 @@ Here is an example of a sharness test: .. _C-tests-label: C Program Tests ---------------- +^^^^^^^^^^^^^^^ C programs use the `libtap library`_ to implement test cases. Convenience functions common to test cases written in C are implemented in the library `lib/testutil.c`_. If your C program needs to use environment variables set by sharness, it can be wrapped in a shell script that first sources -`sharness.d/00-test-env.sh`_ and `sharness.d/01-unifycr-settings.sh`_. Your +`sharness.d/00-test-env.sh`_ and `sharness.d/01-unifyfs-settings.sh`_. Your wrapper shouldn't normally source sharness.sh_ itself because the TAP output from sharness might conflict with that from libtap. @@ -94,9 +102,9 @@ Here is an example libtap test: ------------ Adding Tests -============ +************ -The UnifyCR Test Suite uses the `Test Anything Protocol`_ (TAP) and the +The UnifyFS Test Suite uses the `Test Anything Protocol`_ (TAP) and the Automake test harness. By convention, test scripts and programs that output TAP are named with a ".t" extension. @@ -106,7 +114,7 @@ To add a new test case to the test harness, follow the existing examples in so that it gets included in the source distribution tarball. Test Suites ------------ +^^^^^^^^^^^ If multiple tests fit within the same category (i.e., tests for creat and mkdir both fall under tests for sysio) then create a test suite to run those tests. @@ -122,7 +130,7 @@ sysio_suite in `t/Makefile.am`_ and `t/sys/sysio_suite.c`_: - Create a shell script, *<####-suite-name>.t* (the #### indicates the order in which they should be run by the tap-driver), that wraps your suite and sources `sharness.d/00-test-env.sh`_ and - `sharness.d/01-unifycr-settings.sh`_ + `sharness.d/01-unifyfs-settings.sh`_ - Add this file to `t/Makefile.am`_ in the ``TESTS`` and ``check_SCRIPTS`` variables and add the name of the file (but with a .t extension) this script runs to the ``libexec_PROGRAMS`` variable @@ -133,7 +141,7 @@ sysio_suite in `t/Makefile.am`_ and `t/sys/sysio_suite.c`_: - Create a .c file (i.e., *sysio_suite.c*) that will contain the main function and mpi job that drives your suite - - Mount unifycr from this file + - Mount unifyfs from this file - Call testing functions that contain the test cases (created in other files) in the order desired for testing, passing the mount point to those functions @@ -141,7 +149,7 @@ sysio_suite in `t/Makefile.am`_ and `t/sys/sysio_suite.c`_: functions to be run by this suite and ``include`` this in the .c file - Create .c files (i.e., *open.c*) that contains the testing - function (i.e., ``open_test(char* unifycr_root)``) that houses the + function (i.e., ``open_test(char* unifyfs_root)``) that houses the variables and libtap tests needed to test that individual function - Add the function name to the .h file @@ -158,7 +166,7 @@ sysio_suite in `t/Makefile.am`_ and `t/sys/sysio_suite.c`_: wrappers, add a suite and flags for both a gotcha and a static build) Test Cases ----------- +^^^^^^^^^^ For testing C code, test cases are written using the `libtap library`_. See the :ref:`C Program Tests ` section above on how to write these @@ -173,7 +181,7 @@ If the test cases needing to be written don't already have a file they belong in (i.e., testing a wrapper that doesn't have any tests yet): 1. Creata a .c file with a function called - _test(char* unifycr_root) that contains the desired + _test(char* unifyfs_root) that contains the desired libtap test cases 2. Add the _test to the corresponding .h file @@ -185,40 +193,40 @@ in (i.e., testing a wrapper that doesn't have any tests yet): ------------ Running the Tests -================= +***************** -To manually run the UnifyCR test suite, simply run ``make check`` from your +To manually run the UnifyFS test suite, simply run ``make check`` from your build/t directory. If changes are made to existing files in the test suite, the tests can be run again by simply doing ``make clean`` followed by ``make check``. Individual tests may be run by hand. The test ``0001-setup.t`` should -normally be run first to start the UnifyCR daemon. +normally be run first to start the UnifyFS daemon. .. note:: - If you are using Spack to install UnifyCR then there are two ways to + If you are using Spack to install UnifyFS then there are two ways to manually run these tests: 1. Upon your installation with Spack - ``spack install -v --test=root unifycr`` + ``spack install -v --test=root unifyfs`` 2. Manually from Spack's build directory - ``spack install --keep-stage unifycr`` + ``spack install --keep-stage unifyfs`` - ``spack cd unifycr`` + ``spack cd unifyfs`` ``cd spack-build/t`` ``make check`` -The tests in https://github.com/LLNL/UnifyCR/tree/dev/t are run automatically +The tests in https://github.com/LLNL/UnifyFS/tree/dev/t are run automatically by `Travis CI`_ along with the :ref:`style checks ` when a pull request is created or updated. All pull requests must pass these tests before they will be accepted. Interpreting the Results ------------------------- +^^^^^^^^^^^^^^^^^^^^^^^^ .. sidebar:: TAP Output @@ -282,27 +290,526 @@ ERROR ERROR. Running the Examples --------------------- +^^^^^^^^^^^^^^^^^^^^ + +To run any of these examples manually, refer to the :doc:`examples` +documentation. -The UnifyCR examples_ are also being used as integration tests with +The UnifyFS examples_ are also being used as integration tests with continuation integration tools such as Bamboo_ or GitLab_. +------------ + +----------------- +Integration Tests +----------------- + +The UnifyFS examples_ are being used as integration tests with continuation +integration tools such as Bamboo_ or GitLab_. + To run any of these examples manually, refer to the :doc:`examples` documentation. +------------ + +Running the Tests +***************** + +.. attention:: + + UnifyFS's integration test suite requires MPI and currently only supports + ``srun`` and ``jsrun`` MPI launch commands. Changes are coming to support + ``mpirun``. + +UnifyFS's integration tests are primarly set up to be run all as one suite. +However, they can be run individually if desired. + +The testing scripts in `t/ci`_ depend on sharness_, which is set up in the +containing *t/* directory. These tests will not function properly if moved or if +they cannot find the sharness files. + +.. important:: + + Whether running all tests or individual tests, first make sure you have + either interactively allocated nodes or are submitting a batch job to run + them. + + Also make sure all :ref:`dependencies ` are installed and + loaded. + +By default, the integration tests will use the number of processes-per-node as +there are nodes allocated for the job (i.e., if 4 nodes were allocated, then 4 +processes will be run per node). This can be changed by setting the +:ref:`$CI_NPROCS ` environment variable. + +.. note:: + + In order to run the the integration tests from a Spack_ installation of + UnifyFS, you'll need to tell Spack to use a different location for staging + builds in order to have the source files available from inside an allocation. + + Open your Spack config file + + ``spack config edit config`` + + and provide a path that is visible during job allocations: + + .. code-block:: yaml + + config: + build_stage: + - /visible/path/from/all/allocated/nodes + # or build directly inside Spack's install directory + - $spack/var/spack/stage + + Then make sure to include the ``--keep-stage`` option when installing: + + ``spack install --keep-stage unifyfs`` + +Running All Tests +^^^^^^^^^^^^^^^^^ + +To run all of the tests, simply run ``./RUN_CI_TESTS.sh``. + +.. code-block:: BASH + + $ ./RUN_CI_TESTS.sh + +or + +.. code-block:: BASH + + $ prove -v RUN_CI_TESTS.sh + +Running Individual Tests +^^^^^^^^^^^^^^^^^^^^^^^^ + +In order to run individual tests, testing functions and variables need to be set +up first, and the UnifyFS server needs to be started. To do this, first source +the *t/ci/001-setup.sh* script followed by *002-start-server.sh*. Then source +each desired test script after that preceded by ``$CI_DIR/``. When finished, +source the *990-stop-server.sh* script last to stop the server and clean up. + +.. code-block:: BASH + + $ . full/path/to/001-setup.sh + $ . $CI_DIR/002-start-server.sh + $ . $CI_DIR/100-writeread-tests.sh + $ . $CI_DIR/990-stop-server.sh + +Configuration Variables +^^^^^^^^^^^^^^^^^^^^^^^ + +Along with the already provided :doc:`configuration` options/environment +variables, there are available environment variables used by the integration +testing suite that can be set in order to change the default behavior. They are +listed below in the order they are set up. + +``CI_PROJDIR`` +"""""""""""""" + +USAGE: ``CI_PROJDIR=/base/location/to/search/for/UnifyFS/source/files`` + +During setup, the integration tests will search for the ``unifyfsd`` executable +and installed example scripts if the UnifyFS install directory is not provided by +the user with the ``UNIFYFS_INSTALL`` envar. ``CI_PROJDIR`` is the base location +where this search will start and defaults to ``CI_PROJDIR=$HOME``. + + +``UNIFYFS_INSTALL`` +""""""""""""""""""" + +USAGE: ``UNIFYFS_INSTALL=/path/to/dir/containing/UnifyFS/bin/directory`` + +The full path to the directory containing the *bin/* and *libexec/* directories +for a UnifyFS installation. Set this envar to prevent the integration tests from +searching for a UnifyFS install directory automatically. + +.. _ci-nprocs-label: + +``CI_NPROCS`` +""""""""""""" + +USAGE: ``CI_NPROCS=`` + +The number of processes to use per node inside a job allocation. This defaults +to the number of processes per node as there are nodes in the allocation (i.e., +if 4 nodes were allocated, then 4 processes will be run per node). This should +be adjusted if fewer processes are desired on multiple nodes, multiple processes +are desired on a single node, or a large number of nodes have been allocated. + +``CI_LOG_CLEANUP`` +"""""""""""""""""" + +USAGE: ``CI_LOG_CLEANUP=yes|YES|no|NO`` + +In the event ``$UNIFYFS_LOG_DIR`` has **not** been set, the logs will be put in +``$SHARNESS_TRASH_DIRECTORY``, as set up by sharness.sh_, and cleaned up +automatically after the tests have run. The logs will be in a +*_/* subdirectory. Should any tests fail, the trash +directory will not be cleaned up for debugging purposes. Setting +``CI_LOG_CLEANUP=no|NO`` will move the *_/* logs directory +to ``$CI_DIR`` (the directory containing the integration tests) to +allow them to persist even when all tests pass. This envar defauls to ``yes``. + +.. note:: + + Setting ``$UNIFYFS_LOG_DIR`` will put all created logs in the designated path + and will not clean them up. + +``CI_HOST_CLEANUP`` +""""""""""""""""""" + +USAGE: ``CI_HOST_CLEANUP=yes|YES|no|NO`` + +After all tests have run, the nodes on which the tests were ran will +automatically be cleaned up. This cleanup includes ensuring ``unifyfsd`` has +stopped and deleting any files created by UnifyFS or its dependencies. Set +``CI_HOST_CLEANUP=no|NO`` to skip cleaning up. This envar defaults to ``yes``. + +.. note:: + + PDSH_ is required for cleanup and cleaning up is simply skipped if not + found. + +``CI_CLEANUP`` +"""""""""""""" + +USAGE: ``CI_CLEANUP=yes|YES|no|NO`` + +Setting this to ``no|NO`` sets both ``$CI_LOG_CLEANUP`` and ``$CI_HOST_CLEANUP`` +to ``no|NO``. + +``CI_TEMP_DIR`` +"""""""""""""""" + +USAGE: ``CI_TEMP_DIR=/path/for/temporary/files/created/by/UnifyFS`` + +Can be used as a shortcut to set ``UNIFYFS_RUNSTATE_DIR`` and +``UNIFYFS_META_DB_PATH`` to the same path. This envar defaults to +``CI_TEMP_DIR=${TMPDIR}/unifyfs.${USER}.${JOB_ID}``. + +``CI_STORAGE_DIR`` +""""""""""""""""""" + +USAGE: ``CI_STORAGE_DIR=/path/for/storage/files/`` + +Can be used as a shortcut to set ``UNIFYFS_SPILLOVER_DATA_DIR`` and +``UNIFYFS_SPILLOVER_META_DIR`` to the same path. This envar defaults to +``CI_STORAGE_DIR=${TMPDIR}/unifyfs.${USER}.${JOB_ID}``. + +``CI_TEST_POSIX`` +""""""""""""""""" + +USAGE: ``CI_TEST_POSIX=yes|YES|no|NO`` + +Determines whether any ``-posix`` tests should be run since they +require a real mountpoint to exist. + +This envar defaults to ``yes``. However, when ``$UNIFYFS_MOUNTPOINT`` is set to a +real directory, this envar is switched to ``no``. The idea behind this is that +the tests can be run a first time with a fake mountpoint (which will also run +the posix tests), and then the tests can be run again with a real mountpoint and +the posix tests wont be run twice. This behavior can be overridden by setting +``CI_TEST_POSIX=yes|YES`` before running the integration tests when +``$UNIFYFS_MOUNTPOINT`` is set to an existing directory. + +An example of testing a posix example can be see :ref:`below `. + +.. note:: + + The the posix mountpoint envar, ``CI_POSIX_MP``, is set up inside + ``$SHARNESS_TRASH_DIRECTORY`` automatically and cleaned up afterwards. + However, this envar can be set before running the integration tests as well. + If setting this, ensure that it is a shared file system that all allocated + nodes can see. + +------------ + +Adding New Tests +**************** + +In order to add additional tests, create a script after the fashion of +`t/ci/100-writeread-tests.sh`_ where the prefixed number indicates the desired +order for running the tests. Then source that script in `t/ci/RUN_CI_TESTS.sh`_ +in the desired order. + +Just like the helpers functions found in sharness.d_, there are continuous +integration helper functions (see :ref:`below ` for more details) +available in `t/ci/ci-functions.sh`_. These exist to help make adding new tests +as simple as possible. + +One particularly useful function is ``unify_run_test()``. Currently, this +function is set up to work for the *write*, *read*, *writeread*, and +*checkpoint-restart* examples. This function sets up the MPI job run command and +default arguments as well as any default arguments wanted by all examples. See +:ref:`below ` for details. + +.. _helper-label: + +Example Helper Functions +^^^^^^^^^^^^^^^^^^^^^^^^ + +There are helper functions available in `t/ci/ci-functions.sh`_ that can make +running and testing the examples much easier. These may get adjusted over time +to accommodate other examples, or additional functions may need to be written. +Some of the main helper functions that might be useful for running examples are: + +.. _unify-run-test-label: + +``unify_run_test()`` +"""""""""""""""""""" + +USAGE: ``unify_run_test app_name "app_args" [output_variable_name]`` + +Given a example application name and application args, this function runs the +example with the appropriate MPI runner and args. This function is meant to make +running the cr, write, read, and writeread examples as easy as possible. + +The ``build_test_command()`` function is called by this function which +automatically sets any options that are always wanted (-vkf as well as -U and +the appropriate -m if posix test or not). The stderr output file is also created +(based on the filename that is autogenerated) and the appropriate option is set +for the MPI job run command. + +Args that can be passed in are ([-pncbx][-A|-M|-P|-S|-V]). All other args (see +:ref:`Running the Examples `) are set automatically, including the +filename (which is generated based on the input ``$app_name`` and ``$app_args``). + +The third parameter is an optional "pass-by-reference" parameter that can +contain the variable name for the resulting output to be stored in, allowing +this function to be used in one of two ways: + +.. code-block:: BASH + :caption: Using command substitution + + app_output=$(unify_run_test $app_name "$app_args") + +or + +.. code-block:: BASH + :caption: Using a "pass-by-reference" variable + + unifyfs_run_test $app_name "$app_args" app_output + +This function returns the return code of the executed example as well as the +output produced by running the example. + +.. note:: + + If ``unify_run_test()`` is simply called with only two arguments and without + using command substitution, the resulting output will be sent to the standard + output. + +The results can then be tested with sharness_: + +.. code-block:: BASH + :emphasize-lines: 7,11-14 + + basetest=writeread + runmode=static + + app_name=${basetest}-${runmode} + app_args="-p n1 -n32 -c $((16 * $KB)) -b $MB + + unify_run_test $app_name "$app_args" app_output + rc=$? + line_count=$(echo "$app_output" | wc -l) + + test_expect_success "$app_name $app_args: (line_count=$line_count, rc=$rc)" ' + test $rc = 0 && + test $line_count = 8 + ' + +``get_filename()`` +"""""""""""""""""" + +USAGE: ``get_filename app_name app_args [app_suffix]`` + +Builds and returns the filename for an example so that if it shows up in the +``$UNIFYFS_MOUNTPOINT`` (when using an existing mountpoint), it can be tracked +to its originating test for debugging. Error files are created with this +filename and a ``.err`` suffix and placed in the logs directory for debugging. + +Also allows testers to get what the filename will be in advance if called +from a test suite. This can be used for posix tests to ensure the file showed +up in the mount point, as well as for cp/stat tests that potentially need the +filename from a previous test. + +Note that the filename created by ``unify_run_test()`` will have a ``.app`` +suffix. + +Returns a string with the spaces removed and hyphens replaced by underscores. + +.. code-block:: BASH + + get_filename write-static "-p n1 -n 32 -c 1024 -b 1048576" ".app" + write-static_pn1_n32_c1KB_b1MB.app + +Some uses cases may be: + +- posix tests where the file existence is checked for after a test was run +- cp/stat tests where an already existing filename from a prior test is needed + +For example: + +.. _posix-ex-label: + +.. code-block:: BASH + :emphasize-lines: 10,15 + + basetest=writeread + runmode=posix + + app_name=${basetest}-${runmode} + app_args="-p nn -n32 -c $((16 * $KB)) -b $MB + + unify_run_test $app_name "$app_args" app_output + rc=$? + line_count=$(echo "$app_output" | wc -l) + filename=$(get_filename $app_name "$app_args" ".app") + + test_expect_success POSIX "$app_name $app_args: (line_count=$line_count, rc=$rc)" ' + test $rc = 0 && + test $line_count = 8 && + test_path_has_file_per_process $CI_POSIX_MP $filename + ' + +Sharness Helper Functions +^^^^^^^^^^^^^^^^^^^^^^^^^ + +There are also additional sharness functions for testing the examples available +when `t/ci/ci-functions.sh`_ is sourced. These are to be used with sharness_ for +testing the results of running the examples with or without using the +:ref:`Example Helper Functions `. + +``process_is_running()`` +"""""""""""""""""""""""" + +USAGE: ``process_is_running process_name seconds_before_giving_up`` + +Checks if a process with the given name is running on every host, retrying up to +a given number of seconds before giving up. This function overrides the +``process_is_running()`` function used by the UnifyFS unit tests. The primary +difference being that this function checks for the process on every host. + +Expects two arguments: + +- $1 - Name of a process to check for +- $2 - Number of seconds to wait before giving up + +.. code-block:: BASH + :emphasize-lines: + + test_expect_success "unifyfsd is running" ' + process_is_running unifyfsd 5 + ' + +``process_is_not_running()`` +"""""""""""""""""""""""""""" + +USAGE: ``process_is_not_running process_name seconds_before_giving_up`` + +Checks if a process with the given name is not running on every host, retrying +up to a given number of seconds before giving up. This function overrides the +``process_is_not_running()`` function used by the UnifyFS unit tests. The primary +difference being that this function checks that the process is not running on +every host. + +Expects two arguments: + +- $1 - Name of a process to check for +- $2 - Number of seconds to wait before giving up + +.. code-block:: BASH + + test_expect_success "unifyfsd is not running" ' + process_is_not_running unifyfsd 5 + ' + +``test_path_is_dir()`` +"""""""""""""""""""""" + +USAGE: ``test_path_is_dir dir_name [optional]`` + +Checks that a directory with the given name exists and is accessible from each +host. Does NOT need to be a shared directory. This function overrides the +``test_path_is_dir()`` function in sharness.sh_, the primary difference being +that this function checks for the dir on every host in the allocation. + +Takes once argument with an optional second: + +- $1 - Path of the directory to check for +- $2 - Can be given to provide a more precise diagnosis + +.. code-block:: BASH + + test_expect_success "$dir_name is an existing directory" ' + test_path_is_dir $dir_name + ' + +``test_path_is_shared_dir()`` +""""""""""""""""""""""""""""" + +USAGE: ``test_path_is_shared_dir dir_name [optional]`` + +Check if same directory (actual directory, not just name) exists and is +accessible from each host. + +Takes once argument with an optional second: + +- $1 - Path of the directory to check for +- $2 - Can be given to provide a more precise diagnosis + +.. code-block:: BASH + + test_expect_success "$dir_name is a shared directory" ' + test_path_is_shared_dir $dir_name + ' + +``test_path_has_file_per_process()`` +"""""""""""""""""""""""""""""""""""" + +USAGE: ``test_path_has_file_per_process dir_path file_name [optional]`` + +Check if the provided directory path contains a file-per-process of the provided +file name. Assumes the directory is a shared directory. + +Takes two arguments with an optional third: + +- $1 - Path of the shared directory to check for the files +- $2 - File name without the appended process number +- $3 - Can be given to provided a more precise diagnosis + +.. code-block:: BASH + + test_expect_success "$dir_name has file-per-process of $file_name" ' + test_path_has_file_per_process $dir_name $file_name + ' + +There are other helper functions available as well, most of which are being used +by the test suite itself. Details on these functions can be found in their +comments in `t/ci/ci-functions.sh`_. + .. explicit external hyperlink targets .. _Bamboo: https://www.atlassian.com/software/bamboo .. _GitLab: https://about.gitlab.com -.. _examples: https://github.com/LLNL/UnifyCR/tree/dev/examples/src +.. _examples: https://github.com/LLNL/UnifyFS/tree/dev/examples/src .. _libtap library: https://github.com/zorgnax/libtap -.. _lib/testutil.c: https://github.com/LLNL/UnifyCR/blob/dev/t/lib/testutil.c -.. _t/Makefile.am: https://github.com/LLNL/UnifyCR/blob/dev/t/Makefile.am -.. _t/sys/sysio_suite.c: https://github.com/LLNL/UnifyCR/blob/dev/t/sys/sysio_suite.c +.. _lib/testutil.c: https://github.com/LLNL/UnifyFS/blob/dev/t/lib/testutil.c +.. _PDSH: https://github.com/chaos/pdsh +.. _sharness: https://github.com/chriscool/sharness +.. _sharness.d: https://github.com/LLNL/UnifyFS/tree/dev/t/sharness.d +.. _sharness.d/00-test-env.sh: https://github.com/LLNL/UnifyFS/blob/dev/t/sharness.d/00-test-env.sh +.. _sharness.d/01-unifyfs-settings.sh: https://github.com/LLNL/UnifyFS/blob/dev/t/sharness.d/01-unifyfs-settings.sh +.. _sharness.sh: https://github.com/LLNL/UnifyFS/blob/dev/t/sharness.sh +.. _Spack: https://github.com/spack/spack +.. _t/ci: https://github.com/LLNL/UnifyFS/blob/dev/t/ci +.. _t/Makefile.am: https://github.com/LLNL/UnifyFS/blob/dev/t/Makefile.am +.. _t/sys/sysio_suite.c: https://github.com/LLNL/UnifyFS/blob/dev/t/sys/sysio_suite.c +.. _t/ci/100-writeread-tests.sh: https://github.com/LLNL/UnifyFS/blob/dev/t/ci/100-writeread-tests.sh +.. _t/ci/ci-functions.sh: https://github.com/LLNL/UnifyFS/blob/dev/t/ci/ci-functions.sh +.. _t/ci/RUN_CI_TESTS.sh: https://github.com/LLNL/UnifyFS/blob/dev/t/ci/RUN_CI_TESTS.sh .. _Test Anything Protocol: https://testanything.org .. _Travis CI: https://docs.travis-ci.com -.. _sharness: https://github.com/chriscool/sharness -.. _sharness.d: https://github.com/LLNL/UnifyCR/tree/dev/t/sharness.d -.. _sharness.d/00-test-env.sh: https://github.com/LLNL/UnifyCR/blob/dev/t/sharness.d/00-test-env.sh -.. _sharness.d/01-unifycr-settings.sh: https://github.com/LLNL/UnifyCR/blob/dev/t/sharness.d/01-unifycr-settings.sh -.. _sharness.sh: https://github.com/LLNL/UnifyCR/blob/dev/t/sharness.sh diff --git a/docs/wrappers.rst b/docs/wrappers.rst index 8e7552a32..a06695791 100644 --- a/docs/wrappers.rst +++ b/docs/wrappers.rst @@ -5,45 +5,45 @@ Wrapper Guide .. warning:: This document is out-of-date as the process for generating - :ref:`unifycr_list.txt ` has bugs which causes the - generation of :ref:`gotcha_map_unifycr_list.h ` to have + :ref:`unifyfs_list.txt ` has bugs which causes the + generation of :ref:`gotcha_map_unifyfs_list.h ` to have bugs as well. More information on this can be found in `issue #172 - `_. + `_. An updated guide and scripts needs to be created for writing and adding - new wrappers to UnifyCR. + new wrappers to UnifyFS. The files in `client/check_fns/ -`_ folder help +`_ folder help manage the set of wrappers that are implemented. In particular, they are used to enable a tool that detects I/O routines used by an application that are not -yet supported in UnifyCR. They are also used to generate the code required for +yet supported in UnifyFS. They are also used to generate the code required for GOTCHA. - fakechroot_list.txt_ - lists I/O routines from fakechroot - gnulibc_list.txt_ - I/O routines from libc - cstdio_list.txt_ - I/O routines from stdio - posix_list.txt_ - I/O routines in POSIX -- unifycr_list.txt_ - list of wrappers in UnifyCR -- unifycr_unsupported_list.txt_ - list of wrappers in UnifyCR that are +- unifyfs_list.txt_ - list of wrappers in UnifyFS +- unifyfs_unsupported_list.txt_ - list of wrappers in UnifyFS that are implemented, but not supported ------------ -unifycr_check_fns Tool +unifyfs_check_fns Tool ====================== This tool identifies the set of I/O calls used in an application by running nm on the executable. It reports any I/O routines used by the app, which are not -supported by UnifyCR. If an application uses an I/O routine that is not -supported, it likely cannot use UnifyCR. If the tool does not report -unsupported wrappers, the app may work with UnifyCR but it is not guaranteed to +supported by UnifyFS. If an application uses an I/O routine that is not +supported, it likely cannot use UnifyFS. If the tool does not report +unsupported wrappers, the app may work with UnifyFS but it is not guaranteed to work. .. code-block:: Bash - unifycr_check_fns + unifyfs_check_fns ------------ @@ -52,14 +52,14 @@ work. Building the GOTCHA List ======================== -The gotcha_map_unifycr_list.h_ file contains the code necessary to wrap I/O +The gotcha_map_unifyfs_list.h_ file contains the code necessary to wrap I/O functions with GOTCHA. This is generated from the -:ref:`unifycr_list.txt ` file by running the following +:ref:`unifyfs_list.txt ` file by running the following command: .. code-block:: Bash - python unifycr_translate.py unifycr_list + python unifyfs_translate.py unifyfs_list ------------ @@ -92,30 +92,30 @@ The cstdio_list.txt_ file lists I/O routines available in libstdio. This list was written by hand using information from http://www.open-std.org/jtc1/sc22/wg14/www/docs/n1124.pdf. -.. _unifycr-list-label: +.. _unifyfs-list-label: -unifycr_list.txt +unifyfs_list.txt ---------------- -The unifycr_list.txt_ file specifies the set of wrappers in UnifyCR. Most but -not all such wrappers are supported. The command to build unifycr list: +The unifyfs_list.txt_ file specifies the set of wrappers in UnifyFS. Most but +not all such wrappers are supported. The command to build unifyfs list: .. code-block:: - grep UNIFYCR_WRAP ../src/\*.c > unifycr_list.txt + grep UNIFYFS_WRAP ../src/\*.c > unifyfs_list.txt -unifycr_unsupported_list.txt +unifyfs_unsupported_list.txt ---------------------------- -The unifycr_unsupported_list.txt_ file specifies wrappers that are in UnifyCR, +The unifyfs_unsupported_list.txt_ file specifies wrappers that are in UnifyFS, but are known to not actually be supported. This list is written by hand. .. explicit external hyperlink targets -.. _cstdio_list.txt: https://github.com/LLNL/UnifyCR/blob/dev/client/check_fns/cstdio_list.txt -.. _fakechroot_list.txt: https://github.com/LLNL/UnifyCR/blob/dev/client/check_fns/fakechroot_list.txt -.. _gotcha_map_unifycr_list.h: https://github.com/LLNL/UnifyCR/blob/dev/client/src/gotcha_map_unifycr_list.h -.. _gnulibc_list.txt: https://github.com/LLNL/UnifyCR/blob/dev/client/check_fns/gnulibc_list.txt -.. _posix_list.txt: https://github.com/LLNL/UnifyCR/blob/dev/client/check_fns/posix_list.txt -.. _unifycr_list.txt: https://github.com/LLNL/UnifyCR/blob/dev/client/check_fns/unifycr_list.txt -.. _unifycr_unsupported_list.txt: https://github.com/LLNL/UnifyCR/blob/dev/client/check_fns/unifycr_unsupported_list.txt +.. _cstdio_list.txt: https://github.com/LLNL/UnifyFS/blob/dev/client/check_fns/cstdio_list.txt +.. _fakechroot_list.txt: https://github.com/LLNL/UnifyFS/blob/dev/client/check_fns/fakechroot_list.txt +.. _gotcha_map_unifyfs_list.h: https://github.com/LLNL/UnifyFS/blob/dev/client/src/gotcha_map_unifyfs_list.h +.. _gnulibc_list.txt: https://github.com/LLNL/UnifyFS/blob/dev/client/check_fns/gnulibc_list.txt +.. _posix_list.txt: https://github.com/LLNL/UnifyFS/blob/dev/client/check_fns/posix_list.txt +.. _unifyfs_list.txt: https://github.com/LLNL/UnifyFS/blob/dev/client/check_fns/unifyfs_list.txt +.. _unifyfs_unsupported_list.txt: https://github.com/LLNL/UnifyFS/blob/dev/client/check_fns/unifyfs_unsupported_list.txt diff --git a/examples/README.md b/examples/README.md index c615f9cd3..d61b7203f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,5 +1,5 @@ # Examples -Our [Examples](https://unifycr.readthedocs.io/en/dev/examples.html) -documentation has information on how to run the UnifyCR example programs in +Our [Examples](https://unifyfs.readthedocs.io/en/dev/examples.html) +documentation has information on how to run the UnifyFS example programs in `/src`. diff --git a/examples/src/Makefile.am b/examples/src/Makefile.am index f9462059f..6835decac 100644 --- a/examples/src/Makefile.am +++ b/examples/src/Makefile.am @@ -1,139 +1,282 @@ -libexec_PROGRAMS = sysio-write-gotcha sysio-write-static \ - sysio-read-gotcha sysio-read-static \ - sysio-writeread-gotcha sysio-writeread-static sysio-writeread-posix \ - sysio-writeread2-gotcha sysio-writeread2-static \ - sysio-dir-gotcha sysio-dir-static \ - sysio-stat-gotcha sysio-stat-static \ - sysio-cp-gotcha sysio-cp-static \ - app-mpiio-gotcha app-mpiio-static \ - app-btio-gotcha app-btio-static \ - app-tileio-gotcha app-tileio-static - -noinst_HEADERS = testlib.h - -test_gotcha_ldadd = $(top_builddir)/client/src/libunifycr_gotcha.la -lrt -lm $(MPI_CLDFLAGS) $(FLATCC_LDFLAGS) $(FLATCC_LIBS) -test_static_ldadd = $(top_builddir)/client/src/libunifycr.la -lrt -lm -test_static_ldflags = -static $(CP_WRAPPERS) $(AM_LDFLAGS) $(MPI_CLDFLAGS) $(FLATCC_LDFLAGS) $(FLATCC_LIBS) -test_cppflags = $(AM_CPPFLAGS) -I$(top_srcdir)/client/src -I$(top_srcdir)/common/src $(MPI_CFLAGS) - -sysio_write_gotcha_SOURCES = sysio-write.c +libexec_PROGRAMS = \ + cr-posix cr-gotcha cr-static \ + read-posix read-gotcha read-static \ + write-posix write-gotcha write-static \ + writeread-posix writeread-gotcha writeread-static \ + sysio-write-gotcha sysio-write-static \ + sysio-read-gotcha sysio-read-static \ + sysio-writeread-gotcha sysio-writeread-static sysio-writeread-posix \ + sysio-writeread2-gotcha sysio-writeread2-static \ + sysio-dir-gotcha sysio-dir-static \ + sysio-stat-gotcha sysio-stat-static \ + sysio-cp-gotcha sysio-cp-static \ + app-mpiio-gotcha app-mpiio-static \ + app-btio-gotcha app-btio-static \ + app-tileio-gotcha app-tileio-static \ + transfer-gotcha transfer-static \ + size-gotcha size-static \ + chmod-gotcha chmod-static + +if HAVE_FORTRAN +libexec_PROGRAMS += \ + writeread-ftn +endif + +if HAVE_HDF5 +libexec_PROGRAMS += \ + app-hdf5-create-gotcha \ + app-hdf5-writeread-gotcha +endif + +CLEANFILES = $(libexec_PROGRAMS) + +noinst_HEADERS = \ + testlib.h \ + testutil.h \ + testutil_rdwr.h + +# Common compile/link flag definitions + +test_cppflags = $(AM_CPPFLAGS) $(MPI_CFLAGS) \ + -I$(top_srcdir)/client/src -I$(top_srcdir)/common/src + +if HAVE_FORTRAN +test_ftn_flags = $(AM_FCFLAGS) $(MPI_FFLAGS) \ + -I$(top_srcdir)/client/src -I$(top_srcdir)/common/src +test_ftn_ldadd = $(top_builddir)/client/src/libunifyfsf.la -lrt -lm $(FCLIBS) +test_ftn_ldflags = $(AM_LDFLAGS) $(MPI_FLDFLAGS) \ + $(FLATCC_LDFLAGS) $(FLATCC_LIBS) +endif + +test_gotcha_ldadd = $(top_builddir)/client/src/libunifyfs_gotcha.la -lrt -lm +test_gotcha_ldflags = $(AM_LDFLAGS) $(MPI_CLDFLAGS) \ + $(FLATCC_LDFLAGS) $(FLATCC_LIBS) + +test_posix_cppflags = $(AM_CPPFLAGS) $(MPI_CFLAGS) -DDISABLE_UNIFYFS +test_posix_ldadd = -lrt -lm +test_posix_ldflags = $(AM_LDFLAGS) $(MPI_CLDFLAGS) + +test_static_ldadd = $(top_builddir)/client/src/libunifyfs.la -lrt -lm +test_static_ldflags = -static $(CP_WRAPPERS) $(AM_LDFLAGS) $(MPI_CLDFLAGS) \ + $(FLATCC_LDFLAGS) $(FLATCC_LIBS) + +# Per-target flags begin here + +sysio_write_gotcha_SOURCES = sysio-write.c sysio_write_gotcha_CPPFLAGS = $(test_cppflags) -sysio_write_gotcha_LDADD = $(test_gotcha_ldadd) -sysio_write_gotcha_LDFLAGS = $(AM_LDFLAGS) +sysio_write_gotcha_LDADD = $(test_gotcha_ldadd) +sysio_write_gotcha_LDFLAGS = $(test_gotcha_ldflags) -sysio_write_static_SOURCES = sysio-write.c +sysio_write_static_SOURCES = sysio-write.c sysio_write_static_CPPFLAGS = $(test_cppflags) -sysio_write_static_LDADD = $(test_static_ldadd) -sysio_write_static_LDFLAGS = $(test_static_ldflags) +sysio_write_static_LDADD = $(test_static_ldadd) +sysio_write_static_LDFLAGS = $(test_static_ldflags) -sysio_read_gotcha_SOURCES = sysio-read.c +sysio_read_gotcha_SOURCES = sysio-read.c sysio_read_gotcha_CPPFLAGS = $(test_cppflags) -sysio_read_gotcha_LDADD = $(test_gotcha_ldadd) -sysio_read_gotcha_LDFLAGS = $(AM_LDFLAGS) +sysio_read_gotcha_LDADD = $(test_gotcha_ldadd) +sysio_read_gotcha_LDFLAGS = $(test_gotcha_ldflags) -sysio_read_static_SOURCES = sysio-read.c +sysio_read_static_SOURCES = sysio-read.c sysio_read_static_CPPFLAGS = $(test_cppflags) -sysio_read_static_LDADD = $(test_static_ldadd) -sysio_read_static_LDFLAGS = $(test_static_ldflags) +sysio_read_static_LDADD = $(test_static_ldadd) +sysio_read_static_LDFLAGS = $(test_static_ldflags) -sysio_writeread_posix_SOURCES = sysio-writeread.c -sysio_writeread_posix_CPPFLAGS = $(AM_CPPFLAGS) -DNO_UNIFYCR $(MPI_CFLAGS) -sysio_writeread_posix_LDADD = -lrt -lm -sysio_writeread_posix_LDFLAGS = $(AM_LDFLAGS) $(MPI_CLDFLAGS) +sysio_writeread_posix_SOURCES = sysio-writeread.c +sysio_writeread_posix_CPPFLAGS = $(test_posix_cppflags) +sysio_writeread_posix_LDADD = $(test_posix_ldadd) +sysio_writeread_posix_LDFLAGS = $(test_posix_ldflags) -sysio_writeread_gotcha_SOURCES = sysio-writeread.c +sysio_writeread_gotcha_SOURCES = sysio-writeread.c sysio_writeread_gotcha_CPPFLAGS = $(test_cppflags) -sysio_writeread_gotcha_LDADD = $(test_gotcha_ldadd) -sysio_writeread_gotcha_LDFLAGS = $(AM_LDFLAGS) +sysio_writeread_gotcha_LDADD = $(test_gotcha_ldadd) +sysio_writeread_gotcha_LDFLAGS = $(test_gotcha_ldflags) -sysio_writeread_static_SOURCES = sysio-writeread.c +sysio_writeread_static_SOURCES = sysio-writeread.c sysio_writeread_static_CPPFLAGS = $(test_cppflags) -sysio_writeread_static_LDADD = $(test_static_ldadd) -sysio_writeread_static_LDFLAGS = $(test_static_ldflags) +sysio_writeread_static_LDADD = $(test_static_ldadd) +sysio_writeread_static_LDFLAGS = $(test_static_ldflags) -sysio_writeread2_gotcha_SOURCES = sysio-writeread2.c +sysio_writeread2_gotcha_SOURCES = sysio-writeread2.c sysio_writeread2_gotcha_CPPFLAGS = $(test_cppflags) -sysio_writeread2_gotcha_LDADD = $(test_gotcha_ldadd) -sysio_writeread2_gotcha_LDFLAGS = $(AM_LDFLAGS) +sysio_writeread2_gotcha_LDADD = $(test_gotcha_ldadd) +sysio_writeread2_gotcha_LDFLAGS = $(test_gotcha_ldflags) -sysio_writeread2_static_SOURCES = sysio-writeread2.c +sysio_writeread2_static_SOURCES = sysio-writeread2.c sysio_writeread2_static_CPPFLAGS = $(test_cppflags) -sysio_writeread2_static_LDADD = $(test_static_ldadd) -sysio_writeread2_static_LDFLAGS = $(test_static_ldflags) +sysio_writeread2_static_LDADD = $(test_static_ldadd) +sysio_writeread2_static_LDFLAGS = $(test_static_ldflags) -sysio_dir_gotcha_SOURCES = sysio-dir.c +sysio_dir_gotcha_SOURCES = sysio-dir.c sysio_dir_gotcha_CPPFLAGS = $(test_cppflags) -sysio_dir_gotcha_LDADD = $(test_gotcha_ldadd) -sysio_dir_gotcha_LDFLAGS = $(AM_LDFLAGS) +sysio_dir_gotcha_LDADD = $(test_gotcha_ldadd) +sysio_dir_gotcha_LDFLAGS = $(test_gotcha_ldflags) -sysio_dir_static_SOURCES = sysio-dir.c +sysio_dir_static_SOURCES = sysio-dir.c sysio_dir_static_CPPFLAGS = $(test_cppflags) -sysio_dir_static_LDADD = $(test_static_ldadd) -sysio_dir_static_LDFLAGS = $(test_static_ldflags) +sysio_dir_static_LDADD = $(test_static_ldadd) +sysio_dir_static_LDFLAGS = $(test_static_ldflags) -sysio_stat_gotcha_SOURCES = sysio-stat.c +sysio_stat_gotcha_SOURCES = sysio-stat.c sysio_stat_gotcha_CPPFLAGS = $(test_cppflags) -sysio_stat_gotcha_LDADD = $(test_gotcha_ldadd) -sysio_stat_gotcha_LDFLAGS = $(AM_LDFLAGS) +sysio_stat_gotcha_LDADD = $(test_gotcha_ldadd) +sysio_stat_gotcha_LDFLAGS = $(test_gotcha_ldflags) -sysio_stat_static_SOURCES = sysio-stat.c +sysio_stat_static_SOURCES = sysio-stat.c sysio_stat_static_CPPFLAGS = $(test_cppflags) -sysio_stat_static_LDADD = $(test_static_ldadd) -sysio_stat_static_LDFLAGS = $(test_static_ldflags) +sysio_stat_static_LDADD = $(test_static_ldadd) +sysio_stat_static_LDFLAGS = $(test_static_ldflags) -sysio_cp_gotcha_SOURCES = sysio-cp.c +sysio_cp_gotcha_SOURCES = sysio-cp.c sysio_cp_gotcha_CPPFLAGS = $(test_cppflags) -sysio_cp_gotcha_LDADD = $(test_gotcha_ldadd) -sysio_cp_gotcha_LDFLAGS = $(AM_LDFLAGS) +sysio_cp_gotcha_LDADD = $(test_gotcha_ldadd) +sysio_cp_gotcha_LDFLAGS = $(test_gotcha_ldflags) -sysio_cp_static_SOURCES = sysio-cp.c +sysio_cp_static_SOURCES = sysio-cp.c sysio_cp_static_CPPFLAGS = $(test_cppflags) -sysio_cp_static_LDADD = $(test_static_ldadd) -sysio_cp_static_LDFLAGS = $(test_static_ldflags) +sysio_cp_static_LDADD = $(test_static_ldadd) +sysio_cp_static_LDFLAGS = $(test_static_ldflags) + +cr_posix_SOURCES = checkpoint-restart.c +cr_posix_CPPFLAGS = $(test_posix_cppflags) +cr_posix_LDADD = $(test_posix_ldadd) +cr_posix_LDFLAGS = $(test_posix_ldflags) + +cr_gotcha_SOURCES = checkpoint-restart.c +cr_gotcha_CPPFLAGS = $(test_cppflags) +cr_gotcha_LDADD = $(test_gotcha_ldadd) +cr_gotcha_LDFLAGS = $(test_gotcha_ldflags) + +cr_static_SOURCES = checkpoint-restart.c +cr_static_CPPFLAGS = $(test_cppflags) +cr_static_LDADD = $(test_static_ldadd) +cr_static_LDFLAGS = $(test_static_ldflags) + +read_posix_SOURCES = read.c +read_posix_CPPFLAGS = $(test_posix_cppflags) +read_posix_LDADD = $(test_posix_ldadd) +read_posix_LDFLAGS = $(test_posix_ldflags) -app_mpiio_gotcha_SOURCES = app-mpiio.c +read_gotcha_SOURCES = read.c +read_gotcha_CPPFLAGS = $(test_cppflags) +read_gotcha_LDADD = $(test_gotcha_ldadd) +read_gotcha_LDFLAGS = $(test_gotcha_ldflags) + +read_static_SOURCES = read.c +read_static_CPPFLAGS = $(test_cppflags) +read_static_LDADD = $(test_static_ldadd) +read_static_LDFLAGS = $(test_static_ldflags) + +write_posix_SOURCES = write.c +write_posix_CPPFLAGS = $(test_posix_cppflags) +write_posix_LDADD = $(test_posix_ldadd) +write_posix_LDFLAGS = $(test_posix_ldflags) + +write_gotcha_SOURCES = write.c +write_gotcha_CPPFLAGS = $(test_cppflags) +write_gotcha_LDADD = $(test_gotcha_ldadd) +write_gotcha_LDFLAGS = $(test_gotcha_ldflags) + +write_static_SOURCES = write.c +write_static_CPPFLAGS = $(test_cppflags) +write_static_LDADD = $(test_static_ldadd) +write_static_LDFLAGS = $(test_static_ldflags) + +writeread_posix_SOURCES = writeread.c +writeread_posix_CPPFLAGS = $(test_posix_cppflags) +writeread_posix_LDADD = $(test_posix_ldadd) +writeread_posix_LDFLAGS = $(test_posix_ldflags) + +writeread_gotcha_SOURCES = writeread.c +writeread_gotcha_CPPFLAGS = $(test_cppflags) +writeread_gotcha_LDADD = $(test_gotcha_ldadd) +writeread_gotcha_LDFLAGS = $(test_gotcha_ldflags) + +writeread_static_SOURCES = writeread.c +writeread_static_CPPFLAGS = $(test_cppflags) +writeread_static_LDADD = $(test_static_ldadd) +writeread_static_LDFLAGS = $(test_static_ldflags) + +app_mpiio_gotcha_SOURCES = app-mpiio.c app_mpiio_gotcha_CPPFLAGS = $(test_cppflags) -app_mpiio_gotcha_LDADD = $(test_gotcha_ldadd) -app_mpiio_gotcha_LDFLAGS = $(AM_LDFLAGS) +app_mpiio_gotcha_LDADD = $(test_gotcha_ldadd) +app_mpiio_gotcha_LDFLAGS = $(test_gotcha_ldflags) -app_mpiio_static_SOURCES = app-mpiio.c +app_mpiio_static_SOURCES = app-mpiio.c app_mpiio_static_CPPFLAGS = $(test_cppflags) -app_mpiio_static_LDADD = $(test_static_ldadd) -app_mpiio_static_LDFLAGS = $(test_static_ldflags) +app_mpiio_static_LDADD = $(test_static_ldadd) +app_mpiio_static_LDFLAGS = $(test_static_ldflags) -app_btio_gotcha_SOURCES = app-btio.c +app_btio_gotcha_SOURCES = app-btio.c app_btio_gotcha_CPPFLAGS = $(test_cppflags) -app_btio_gotcha_LDADD = $(test_gotcha_ldadd) -app_btio_gotcha_LDFLAGS = $(AM_LDFLAGS) +app_btio_gotcha_LDADD = $(test_gotcha_ldadd) +app_btio_gotcha_LDFLAGS = $(test_gotcha_ldflags) -app_btio_static_SOURCES = app-btio.c +app_btio_static_SOURCES = app-btio.c app_btio_static_CPPFLAGS = $(test_cppflags) -app_btio_static_LDADD = $(test_static_ldadd) -app_btio_static_LDFLAGS = $(test_static_ldflags) +app_btio_static_LDADD = $(test_static_ldadd) +app_btio_static_LDFLAGS = $(test_static_ldflags) -app_tileio_gotcha_SOURCES = app-tileio.c +app_tileio_gotcha_SOURCES = app-tileio.c app_tileio_gotcha_CPPFLAGS = $(test_cppflags) -app_tileio_gotcha_LDADD = $(test_gotcha_ldadd) -app_tileio_gotcha_LDFLAGS = $(AM_LDFLAGS) +app_tileio_gotcha_LDADD = $(test_gotcha_ldadd) +app_tileio_gotcha_LDFLAGS = $(test_gotcha_ldflags) -app_tileio_static_SOURCES = app-tileio.c +app_tileio_static_SOURCES = app-tileio.c app_tileio_static_CPPFLAGS = $(test_cppflags) -app_tileio_static_LDADD = $(test_static_ldadd) -app_tileio_static_LDFLAGS = $(test_static_ldflags) +app_tileio_static_LDADD = $(test_static_ldadd) +app_tileio_static_LDFLAGS = $(test_static_ldflags) + +if HAVE_FORTRAN + +writeread_ftn_SOURCES = writeread.f90 +writeread_ftn_FCFLAGS = $(test_ftn_flags) +writeread_ftn_LDADD = $(test_ftn_ldadd) +writeread_ftn_LDFLAGS = $(test_ftn_ldflags) + +endif if HAVE_HDF5 -libexec_PROGRAMS += app-hdf5-create-gotcha \ - app-hdf5-writeread-gotcha -app_hdf5_create_gotcha_SOURCES = app-hdf5-create.c +app_hdf5_create_gotcha_SOURCES = app-hdf5-create.c app_hdf5_create_gotcha_CPPFLAGS = $(test_cppflags) $(HDF5_CPPFLAGS) -app_hdf5_create_gotcha_LDADD = $(test_gotcha_ldadd) $(HDF5_LIBS) -app_hdf5_create_gotcha_LDFLAGS = $(AM_LDFLAGS) $(HDF5_LDFLAGS) +app_hdf5_create_gotcha_LDADD = $(test_gotcha_ldadd) $(HDF5_LIBS) +app_hdf5_create_gotcha_LDFLAGS = $(test_gotcha_ldflags) $(HDF5_LDFLAGS) -app_hdf5_writeread_gotcha_SOURCES = app-hdf5-writeread.c +app_hdf5_writeread_gotcha_SOURCES = app-hdf5-writeread.c app_hdf5_writeread_gotcha_CPPFLAGS = $(test_cppflags) $(HDF5_CPPFLAGS) -app_hdf5_writeread_gotcha_LDADD = $(test_gotcha_ldadd) $(HDF5_LIBS) -app_hdf5_writeread_gotcha_LDFLAGS = $(AM_LDFLAGS) $(HDF5_LDFLAGS) +app_hdf5_writeread_gotcha_LDADD = $(test_gotcha_ldadd) $(HDF5_LIBS) +app_hdf5_writeread_gotcha_LDFLAGS = $(test_gotcha_ldflags) $(HDF5_LDFLAGS) + endif -CLEANFILES = $(noinst_PROGRAMS) +transfer_gotcha_SOURCES = transfer.c +transfer_gotcha_CPPFLAGS = $(test_cppflags) +transfer_gotcha_LDADD = $(test_gotcha_ldadd) +transfer_gotcha_LDFLAGS = $(test_gotcha_ldflags) + +transfer_static_SOURCES = transfer.c +transfer_static_CPPFLAGS = $(test_cppflags) +transfer_static_LDADD = $(test_static_ldadd) +transfer_static_LDFLAGS = $(test_static_ldflags) + +size_gotcha_SOURCES = size.c testutil.c +size_gotcha_CPPFLAGS = $(test_cppflags) +size_gotcha_LDADD = $(test_gotcha_ldadd) +size_gotcha_LDFLAGS = $(test_gotcha_ldflags) + +size_static_SOURCES = size.c testutil.c +size_static_CPPFLAGS = $(test_cppflags) +size_static_LDADD = $(test_static_ldadd) +size_static_LDFLAGS = $(test_static_ldflags) + +chmod_gotcha_SOURCES = chmod.c testutil.c +chmod_gotcha_CPPFLAGS = $(test_cppflags) +chmod_gotcha_LDADD = $(test_gotcha_ldadd) +chmod_gotcha_LDFLAGS = $(test_gotcha_ldflags) + +chmod_static_SOURCES = chmod.c testutil.c +chmod_static_CPPFLAGS = $(test_cppflags) +chmod_static_LDADD = $(test_static_ldadd) +chmod_static_LDFLAGS = $(test_static_ldflags) + diff --git a/examples/src/app-btio.c b/examples/src/app-btio.c index 26fa28271..57a3a39da 100644 --- a/examples/src/app-btio.c +++ b/examples/src/app-btio.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -54,7 +54,7 @@ #include #include #include -#include +#include #define TEST_STR_LEN 1024 #define SZ_PER_ELEM 40 /*size of each data point in BTIO*/ @@ -197,7 +197,7 @@ int main(int argc, char* argv[]) memset(buf, 0, elems_per_tile * SZ_PER_ELEM); MPI_Barrier(MPI_COMM_WORLD); - unifycr_mount("/unifycr", rank, ranknum, 0); + unifyfs_mount("/unifyfs", rank, ranknum, 0); MPI_Barrier(MPI_COMM_WORLD); int fd = open(fname, O_RDWR | O_CREAT | O_TRUNC, 0600); @@ -247,7 +247,7 @@ int main(int argc, char* argv[]) if (direction != 0) {/*only write without reading*/ close(fd); - unifycr_unmount(); + unifyfs_unmount(); if (rank == 0) { printf("Aggregated Write BW is %lf, Min Write BW is %lf\n", aggwrbw, min_wr_bw); @@ -291,7 +291,7 @@ int main(int argc, char* argv[]) close(fd); MPI_Barrier(MPI_COMM_WORLD); - unifycr_unmount(); + unifyfs_unmount(); free(aiocb_list); free(cb_list); diff --git a/examples/src/app-hdf5-create.c b/examples/src/app-hdf5-create.c index 1961f08cc..bf6d41333 100644 --- a/examples/src/app-hdf5-create.c +++ b/examples/src/app-hdf5-create.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* * Copyright by The HDF Group. @@ -29,7 +29,7 @@ * array. It is used in the HDF5 Tutorial. */ /* - * The example is modified to test unifycr userspace file system. + * The example is modified to test unifyfs userspace file system. */ #include @@ -41,7 +41,7 @@ #include #include #include -#include +#include #include #include "testlib.h" @@ -51,10 +51,10 @@ static int total_ranks; static int write_data; static int debug; /* pause for attaching debugger */ -static int standard; /* not mounting unifycr when set */ -static int unmount; /* unmount unifycr after running the test */ -static char *mountpoint = "/unifycr"; /* unifycr mountpoint */ -static char *filename = "test.h5"; /* testfile name under mountpoint */ +static int standard; /* not mounting unifyfs when set */ +static int unmount; /* unmount unifyfs after running the test */ +static char* mountpoint = "/unifyfs"; /* unifyfs mountpoint */ +static char* filename = "test.h5"; /* testfile name under mountpoint */ static char targetfile[NAME_MAX]; /* target file name */ static struct option const long_opts[] = { @@ -68,9 +68,9 @@ static struct option const long_opts[] = { { 0, 0, 0, 0}, }; -static char *short_opts = "df:hm:suw"; +static char* short_opts = "df:hm:suw"; -static const char *usage_str = +static const char* usage_str = "\n" "Usage: %s [options...]\n" "\n" @@ -80,14 +80,14 @@ static const char *usage_str = " -f, --filename= target file name under mountpoint\n" " (default: test.h5)\n" " -h, --help help message\n" -" -m, --mount= use for unifycr\n" -" (default: /unifycr)\n" -" -s, --standard do not use unifycr but run standard I/O\n" +" -m, --mount= use for unifyfs\n" +" (default: /unifyfs)\n" +" -s, --standard do not use unifyfs but run standard I/O\n" " -u, --unmount unmount the filesystem after test\n" " -w, --write populate the dataset\n" "\n"; -static char *program; +static char* program; static void print_usage(void) { @@ -95,7 +95,7 @@ static void print_usage(void) exit(0); } -int main(int argc, char **argv) +int main(int argc, char** argv) { int ret = 0; int ch = 0; @@ -151,9 +151,9 @@ int main(int argc, char **argv) test_pause(rank, "Attempting to mount"); if (!standard) { - ret = unifycr_mount(mountpoint, rank, total_ranks, 0); + ret = unifyfs_mount(mountpoint, rank, total_ranks, 0); if (ret) { - test_print(rank, "unifycr_mount failed (return = %d)", ret); + test_print(rank, "unifyfs_mount failed (return = %d)", ret); exit(-1); } } @@ -205,7 +205,7 @@ int main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); if (!standard && unmount) { - unifycr_unmount(); + unifyfs_unmount(); } return 0; diff --git a/examples/src/app-hdf5-writeread.c b/examples/src/app-hdf5-writeread.c index a298a04bd..405888896 100644 --- a/examples/src/app-hdf5-writeread.c +++ b/examples/src/app-hdf5-writeread.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* * Copyright by The HDF Group. @@ -29,7 +29,7 @@ * dataset. It is used in the HDF5 Tutorial. */ /* - * The example is modified to test unifycr userspace file system. + * The example is modified to test unifyfs userspace file system. */ #include @@ -41,7 +41,7 @@ #include #include #include -#include +#include #include #include "testlib.h" @@ -57,10 +57,10 @@ static int readonly; static int writeonly; static int debug; /* pause for attaching debugger */ -static int standard; /* not mounting unifycr when set */ -static int unmount; /* unmount unifycr after running the test */ -static char *mountpoint = "/unifycr"; /* unifycr mountpoint */ -static char *filename = "test.h5"; /* testfile name under mountpoint */ +static int standard; /* not mounting unifyfs when set */ +static int unmount; /* unmount unifyfs after running the test */ +static char* mountpoint = "/unifyfs"; /* unifyfs mountpoint */ +static char* filename = "test.h5"; /* testfile name under mountpoint */ static char targetfile[NAME_MAX]; /* target file name */ static struct option const long_opts[] = { @@ -75,9 +75,9 @@ static struct option const long_opts[] = { { 0, 0, 0, 0}, }; -static char *short_opts = "df:hm:rsuw"; +static char* short_opts = "df:hm:rsuw"; -static const char *usage_str = +static const char* usage_str = "\n" "Usage: %s [options...]\n" "\n" @@ -87,17 +87,17 @@ static const char *usage_str = " -f, --filename= target file name under mountpoint\n" " (default: test.h5)\n" " -h, --help help message\n" -" -m, --mount= use for unifycr\n" -" (default: /unifycr)\n" +" -m, --mount= use for unifyfs\n" +" (default: /unifyfs)\n" " -r, --readonly only read the dataset\n" " (default: write then read)\n" -" -s, --standard do not use unifycr but run standard I/O\n" +" -s, --standard do not use unifyfs but run standard I/O\n" " -u, --unmount unmount the filesystem after test\n" " -w, --writeonly only write the dataset\n" " (default: write then read)\n" "\n"; -static char *program; +static char* program; static void print_usage(void) { @@ -105,7 +105,7 @@ static void print_usage(void) exit(0); } -int main(int argc, char **argv) +int main(int argc, char** argv) { int ret = 0; int ch = 0; @@ -171,9 +171,9 @@ int main(int argc, char **argv) test_pause(rank, "Attempting to mount"); if (!standard) { - ret = unifycr_mount(mountpoint, rank, total_ranks, 0); + ret = unifyfs_mount(mountpoint, rank, total_ranks, 0); if (ret) { - test_print(rank, "unifycr_mount failed (return = %d)", ret); + test_print(rank, "unifyfs_mount failed (return = %d)", ret); exit(-1); } } @@ -218,7 +218,7 @@ int main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); if (!standard && unmount) { - unifycr_unmount(); + unifyfs_unmount(); } return 0; diff --git a/examples/src/app-mpiio.c b/examples/src/app-mpiio.c index 913a2dab1..33efbd8c1 100644 --- a/examples/src/app-mpiio.c +++ b/examples/src/app-mpiio.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ #include @@ -26,34 +26,43 @@ #include #include #include -#include +#include #include "testlib.h" -static uint64_t blocksize = 1 << 20; /* 1MB */ -static uint64_t nblocks = 128; /* Each process writes 128MB */ -static uint64_t chunksize = 64 * (1 << 10); /* 64KB for each write(2) call */ +#ifndef KIB +# define KIB (1 << 10) +#endif + +#ifndef MIB +# define MIB (1 << 20) +#endif + +static size_t blocksize = MIB; /* 1 MiB */ +static size_t nblocks = 128; /* Each process writes 128 MiB */ +static size_t chunksize = 64 * KIB; /* 64 KiB for each write() call */ + +static int standard; /* do not use UnifyFS when set */ +static int type; /* 0 for write (default), 1 for read */ +static int iomode = MPI_MODE_CREATE | MPI_MODE_RDWR; +static int pattern = IO_PATTERN_N1; -static int type; /* 0 for write (default), 1 for read */ -static int pattern; /* N to 1 (N1, default) or N to N (NN) */ static MPI_File fh; /* MPI file handle */ static MPI_Status status; /* MPI I/O status */ -static int iomode = MPI_MODE_CREATE | MPI_MODE_RDWR; -static int standard; /* not mounting unifycr when set */ /* time statistics */ static struct timeval iostart, ioend; -#define BUFSIZE 128 -static char hostname[BUFSIZE]; +#define HOSTBUFSZ 128 +static char hostname[HOSTBUFSZ]; static int rank; static int total_ranks; static int debug; /* pause for attaching debugger */ -static int unmount; /* unmount unifycr after test */ +static int unmount; /* unmount unifyfs after test */ static char* buf; /* I/O buffer */ -static char* mountpoint = "/unifycr"; /* unifycr mountpoint */ +static char* mountpoint = "/unifyfs"; /* unifyfs mountpoint */ static char* filename = "testfile"; /* testfile name under mountpoint */ static char targetfile[NAME_MAX]; /* target file name */ @@ -69,15 +78,17 @@ static inline void mpi_handle_error(int err, char* str) MPI_Error_string(err, msg, &len); fprintf(stderr, "[%s:%d] %s: %s\n", hostname, rank, str, msg); + fflush(NULL); mpierror = 1; } -#define MPI_CHECK(fn) do { \ - int err = (fn); \ - if (err != MPI_SUCCESS) \ - mpi_handle_error(err, #fn); \ - } while (0) +#define MPI_CHECK(fn) \ + do { \ + int err = (fn); \ + if (err != MPI_SUCCESS) \ + mpi_handle_error(err, #fn); \ + } while (0) static int read_test_type(const char* str) { @@ -166,17 +177,17 @@ static void report_result(void) double max_io_time = .0F; double min_io_bw = .0F; double io_time = .0F; + double per_rank_mib = (1.0 * blocksize * nblocks) / MIB; io_time = timediff_sec(&iostart, &ioend); - io_bw = 1.0 * blocksize * nblocks / io_time / (1 << 20); + io_bw = per_rank_mib / io_time; MPI_Reduce(&io_bw, &agg_io_bw, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD); MPI_Reduce(&io_time, &max_io_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); - min_io_bw = 1.0 * blocksize * nblocks * total_ranks / - max_io_time / (1 << 20); + min_io_bw = (per_rank_mib * total_ranks) / max_io_time; errno = 0; @@ -184,17 +195,17 @@ static void report_result(void) "\n" "Number of processes: %d\n" "I/O type: %s\n" - "Each process performed: %lf MB\n" - "Total I/O size: %lf MB\n" + "Each process performed: %zu MiB\n" + "Total I/O size: %zu MiB\n" "I/O pattern: %s\n" "I/O request size: %llu B\n" - "Aggregate I/O bandwidth: %lf MB/s\n" - "Min. I/O bandwidth: %lf MB/s\n" + "Aggregate I/O bandwidth: %lf MiB/s\n" + "Min. I/O bandwidth: %lf MiB/s\n" "Total I/O time: %lf sec.\n", total_ranks, type ? "read" : "write", - 1.0 * blocksize * nblocks / (1 << 20), - 1.0 * total_ranks * blocksize * nblocks / (1 << 20), + blocksize * nblocks / MIB, + total_ranks * blocksize * nblocks / MIB, io_pattern_string(pattern), chunksize, agg_io_bw, @@ -210,14 +221,13 @@ static struct option const long_opts[] = { { "filename", 1, 0, 'f' }, { "help", 0, 0, 'h' }, { "mount", 1, 0, 'm' }, - { "pattern", 1, 0, 'p' }, { "standard", 0, 0, 's' }, { "type", 1, 0, 't' }, { "unmount", 0, 0, 'u' }, { 0, 0, 0, 0}, }; -static char* short_opts = "b:n:c:df:hm:p:Pst:u"; +static char* short_opts = "b:n:c:df:hm:Pst:u"; static const char* usage_str = "\n" @@ -225,19 +235,18 @@ static const char* usage_str = "\n" "Available options:\n" " -b, --blocksize= logical block size for the target file\n" - " (default 1048576, 1MB)\n" + " (default 1 MiB)\n" " -n, --nblocks= count of blocks each process will write\n" " (default 128)\n" " -c, --chunksize= I/O chunk size for each write operation\n" - " (default 64436, 64KB)\n" + " (default 64 KiB)\n" " -d, --debug pause before running test\n" " (handy for attaching in debugger)\n" " -f, --filename= target file name (default: testfile)\n" " -h, --help help message\n" - " -m, --mount= use for unifycr\n" - " (default: /unifycr)\n" - " -p, --pattern= should be 'n1'(n to 1) or 'nn' (n to n)\n" - " -s, --standard do not use unifycr but run standard I/O\n" + " -m, --mount= use for unifyfs\n" + " (default: /unifyfs)\n" + " -s, --standard do not use unifyfs but run standard I/O\n" " -t, --type= I/O type\n" " should be 'write' (default) or 'read'\n" " -u, --unmount unmount the filesystem after test\n" @@ -263,7 +272,7 @@ int main(int argc, char* argv[]) MPI_Comm_size(MPI_COMM_WORLD, &total_ranks); MPI_Comm_rank(MPI_COMM_WORLD, &rank); - gethostname(hostname, BUFSIZE); + gethostname(hostname, HOSTBUFSZ); while ((ch = getopt_long(argc, argv, short_opts, long_opts, &optidx)) >= 0) { @@ -288,10 +297,6 @@ int main(int argc, char* argv[]) debug = 1; break; - case 'p': - pattern = read_io_pattern(optarg); - break; - case 'm': mountpoint = strdup(optarg); break; @@ -326,59 +331,55 @@ int main(int argc, char* argv[]) exit(-1); } - if (chunksize % (1 << 10) > 0) { + if ((chunksize % KIB) > 0) { test_print_once(rank, "chunksize and blocksize should be divisible " "by 1024."); exit(-1); } - if (static_linked(program) && standard) { - test_print_once(rank, "--standard, -s option only works when " - "dynamically linked."); - exit(-1); - } - - sprintf(targetfile, "%s/%s", mountpoint, filename); - - if (debug) { - test_pause(rank, "Attempting to mount"); - } - if (!standard) { - ret = unifycr_mount(mountpoint, rank, total_ranks, 0); + if (debug) { + test_pause(rank, "Attempting to mount"); + } + ret = unifyfs_mount(mountpoint, rank, total_ranks, 0); if (ret) { - test_print(rank, "unifycr_mount failed (return = %d)", ret); + test_print(rank, "unifyfs_mount failed (return = %d)", ret); exit(-1); } + if (!standard) { + printf("[%d]: unifyfs mounted at %s\n", rank, mountpoint); + } } - buf = calloc(1, chunksize); - if (!buf) { - test_print(rank, "calloc failed"); - exit(-1); - } - + sprintf(targetfile, "%s/%s", mountpoint, filename); if (pattern == IO_PATTERN_NN) { - sprintf(&targetfile[strlen(targetfile)], "-%d", rank); - } - - if (!standard) { - printf("[%d]: unifycr mounted at %s\n", rank, mountpoint); + sprintf((targetfile + strlen(targetfile)), "-%d", rank); } + printf("[%d]: opening %s\n", rank, targetfile); + fflush(NULL); MPI_Barrier(MPI_COMM_WORLD); - printf("[%d]: opening %s\n", rank, targetfile); - MPI_CHECK(MPI_File_open(MPI_COMM_WORLD, targetfile, iomode, MPI_INFO_NULL, &fh)); + if (mpierror) { + MPI_Abort(MPI_COMM_WORLD, 1); + } if (debug) { test_pause(rank, "Attempting to perform I/O"); } + buf = calloc(1, chunksize); + if (NULL == buf) { + test_print(rank, "calloc failed"); + MPI_Abort(MPI_COMM_WORLD, 1); + } + int printable = (int)'0' + (rank % 10); + memset(buf, printable, chunksize); + if (type == 0) { ret = do_write(&fh); } else { @@ -389,17 +390,15 @@ int main(int argc, char* argv[]) MPI_Barrier(MPI_COMM_WORLD); - if (!standard && unmount) { - unifycr_unmount(); - } + free(buf); if (ret == 0) { report_result(); - } else { - MPI_Abort(MPI_COMM_WORLD, 1); } - free(buf); + if (!standard && unmount) { + unifyfs_unmount(); + } MPI_Finalize(); diff --git a/examples/src/app-tileio.c b/examples/src/app-tileio.c index 32188ce5f..681624be1 100644 --- a/examples/src/app-tileio.c +++ b/examples/src/app-tileio.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -59,7 +59,7 @@ #include #include #include -#include +#include #define TEST_STR_LEN 1024 @@ -175,7 +175,7 @@ int main(int argc, char* argv[]) memset(buf, 0, x_size * sz_per_elem); MPI_Barrier(MPI_COMM_WORLD); - unifycr_mount("/unifycr", rank, ranknum, 0); + unifyfs_mount("/unifyfs", rank, ranknum, 0); MPI_Barrier(MPI_COMM_WORLD); int fd; @@ -215,7 +215,7 @@ int main(int argc, char* argv[]) writetime = writetime / 1000000; if (direction == 1) { - unifycr_unmount(); + unifyfs_unmount(); close(fd); } @@ -280,7 +280,7 @@ int main(int argc, char* argv[]) MPI_Barrier(MPI_COMM_WORLD); close(fd); - unifycr_unmount(); + unifyfs_unmount(); double rd_bw = (double)tot_sz / ranknum / 1048576 / readtime; double max_rd_time; diff --git a/examples/src/checkpoint-restart.c b/examples/src/checkpoint-restart.c new file mode 100644 index 000000000..101d89c7f --- /dev/null +++ b/examples/src/checkpoint-restart.c @@ -0,0 +1,363 @@ +/* + * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2019, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#include "testutil.h" +#include "testutil_rdwr.h" + +// generate N-to-1 or N-to-N checkpoints according to test config +size_t generate_checkpoint(test_cfg* cfg, char* data, uint64_t ckpt_id, + struct aiocb** req_out) +{ + off_t blk_off; + size_t blk_sz = cfg->block_sz; + + size_t num_reqs = 1; + struct aiocb* req = calloc(num_reqs, sizeof(struct aiocb)); + if (NULL == req) { + *req_out = NULL; + return 0; + } + + // use beginning of data to store rank and checkpoint id + uint64_t* u64p = (uint64_t*) data; + *u64p = (uint64_t) cfg->rank; + u64p++; + *u64p = ckpt_id; + u64p++; + void* chkpt_data = (void*) u64p; + + if (IO_PATTERN_N1 == cfg->io_pattern) { + // interleaved checkpoint blocks + blk_off = blk_sz * cfg->rank; + } else { // IO_PATTERN_NN + blk_off = 0; + } + + uint64_t data_off = blk_off + (2 * sizeof(uint64_t)); + uint64_t data_sz = blk_sz - (2 * sizeof(uint64_t)); + if (cfg->io_check) { + // generate lipsum in source block + lipsum_generate(chkpt_data, data_sz, data_off); + } else { + // fill data with unique character per rank + int byte = (int)'0' + cfg->rank; + memset(chkpt_data, byte, (size_t)data_sz); + } + + req->aio_fildes = cfg->fd; + req->aio_buf = (void*)data; + req->aio_nbytes = blk_sz; + req->aio_offset = blk_off; + req->aio_lio_opcode = LIO_WRITE; + + *req_out = req; + return num_reqs; +} + +// generate N-to-1 or N-to-N restart according to test config +size_t generate_restart(test_cfg* cfg, char* data, + struct aiocb** req_out) +{ + off_t blk_off; + size_t blk_sz = cfg->block_sz; + + size_t num_reqs = 1; + struct aiocb* req = calloc(num_reqs, sizeof(struct aiocb)); + if (NULL == req) { + *req_out = NULL; + return 0; + } + + if (IO_PATTERN_N1 == cfg->io_pattern) { + // interleaved block writes + blk_off = blk_sz * cfg->rank; + } else { // IO_PATTERN_NN + blk_off = 0; + } + + req->aio_fildes = cfg->fd; + req->aio_buf = (void*)data; + req->aio_nbytes = blk_sz; + req->aio_offset = blk_off; + req->aio_lio_opcode = LIO_READ; + + *req_out = req; + return num_reqs; +} + +int verify_restart_data(test_cfg* cfg, char* data, uint64_t last_ckpt_id) +{ + int ret = 0; + + // check beginning of data for correct rank and checkpoint id + uint64_t rank, ckptid; + uint64_t* u64p = (uint64_t*) data; + rank = *u64p; + u64p++; + ckptid = *u64p; + u64p++; + + if (rank != (uint64_t)cfg->rank) { + ret = -1; + test_print(cfg, "ERROR - Failed to verify restart rank\n"); + } + + if (ckptid != last_ckpt_id) { + ret = -1; + test_print(cfg, "ERROR - Failed to verify restart checkpoint id\n"); + } + + if (cfg->io_check) { + // check lipsum in restart block + uint64_t blk_off; + uint64_t blk_sz = cfg->block_sz; + if (IO_PATTERN_N1 == cfg->io_pattern) { + // interleaved block writes + blk_off = blk_sz * cfg->rank; + } else { // IO_PATTERN_NN + blk_off = 0; + } + + uint64_t err_off; + uint64_t data_off = blk_off + (2 * sizeof(uint64_t)); + uint64_t data_sz = blk_sz - (2 * sizeof(uint64_t)); + const char* chkdata = (const char*) u64p; + int rc = lipsum_check(chkdata, data_sz, data_off, &err_off); + if (rc) { + ret = -1; + test_print(cfg, "ERROR - Failed to verify restart checkpoint data " + "@ offset %" PRIu64 "\n", err_off); + } + } + + return ret; +} + + +/* -------- Main Program -------- */ + +/* Description: + * + * [ Mode 1: N-to-1 shared-file checkpoint ] + * Each rank writes cfg.n_blocks checkpoints of size cfg.block_sz to + * the shared file. Checkpoint blocks are rank-interleaved (i.e., the + * block at offset 0 is written by rank 0, the block at offset + * cfg.block_sz is written by rank 1, and so on). After writing all + * checkpoint blocks, the data is synced (laminated) and the shared + * file is closed. Each rank then re-opens the shared file, reads + * back the last data written, and optionally checks the contents if + * cfg.io_check is set. + * + * [ Mode 2: N-to-N file-per-process checkpoint ] + * Each rank writes cfg.n_blocks checkpoints of size cfg.block_sz to + * its own file. After writing all checkpoint blocks, the data is + * synced (laminated) and the file is closed. Each rank then re-opens + * its file, reads back the last data written, and optionally checks + * the contents if cfg.io_check is set. + * + * [ Options for Both Modes ] + * cfg.use_aio - when enabled, aio(7) will be used to issue and + * deterimine completion of reads and writes. + * + * cfg.use_lio - when enabled, lio_listio(3) will be used to batch + * reads and writes. When cfg.use_aio is also enabled, the mode will + * be LIO_NOWAIT. + * + * cfg.use_mapio - support is not yet implemented. When enabled, + * direct memory loads and stores will be used for reads and writes. + * + * cfg.use_prdwr - when enabled, pread(2) and pwrite(2) will be used. + * + * cfg.use_stdio - when enabled, fread(2) and fwrite(2) will be used. + * + * cfg.use_vecio - support is not yet implemented. When enabled, + * readv(2) and writev(2) will be used for batching reads and writes. + * + * cfg.io_check - when enabled, ranks will verify the last checkpoint + * written matches the data read on restart. + * + * cfg.io_shuffle - ignored. + * + * cfg.chunk_sz - ignored. + */ + +int main(int argc, char* argv[]) +{ + char* chkpt_data; + char* restart_data; + char* target_file; + struct aiocb* req; + size_t num_reqs = 0; + int rc; + + double min_write_time, max_write_time; + double min_read_time, max_read_time; + double min_sync_time, max_sync_time; + + test_cfg test_config; + test_cfg* cfg = &test_config; + test_timer time_ckpt; + test_timer time_restart; + test_timer time_sync; + + timer_init(&time_ckpt, "checkpoint"); + timer_init(&time_restart, "restart"); + timer_init(&time_sync, "sync"); + + rc = test_init(argc, argv, cfg); + if (rc) { + fprintf(stderr, "ERROR - Test %s initialization failed!", + argv[0]); + fflush(NULL); + return rc; + } + + if (!test_config.use_mpi) { + fprintf(stderr, "ERROR - Test %s requires MPI!", + argv[0]); + fflush(NULL); + return -1; + } + + target_file = test_target_filename(cfg); + test_print_verbose_once(cfg, "DEBUG: creating target file %s", + target_file); + rc = test_create_file(cfg, target_file, O_RDWR); + if (rc) { + test_abort(cfg, rc); + } + + chkpt_data = malloc(test_config.block_sz); + if (NULL == chkpt_data) { + test_abort(cfg, ENOMEM); + } + + uint64_t chkpt_id; + for (chkpt_id = 1; chkpt_id <= test_config.n_blocks; chkpt_id++) { + // generate checkpoint data and write request + test_print_verbose_once(cfg, "DEBUG: generating checkpoint %" PRIu64, + chkpt_id); + num_reqs = generate_checkpoint(cfg, chkpt_data, chkpt_id, &req); + if (0 == num_reqs) { + test_abort(cfg, ENOMEM); + } + + // do checkpoint + test_print_verbose_once(cfg, "DEBUG: starting checkpoint %" PRIu64, + chkpt_id); + test_barrier(cfg); + timer_start(&time_ckpt); + rc = issue_write_req(cfg, req); + if (rc) { + test_abort(cfg, rc); + } + rc = wait_write_req(cfg, req); + if (rc) { + test_abort(cfg, rc); + } + timer_stop(&time_ckpt); + test_print_verbose_once(cfg, "DEBUG: finished checkpoint %" PRIu64, + chkpt_id); + + min_write_time = test_reduce_double_min(cfg, time_ckpt.elapsed_sec); + max_write_time = test_reduce_double_max(cfg, time_ckpt.elapsed_sec); + test_print_once(cfg, "Checkpoint %" PRIu64 " - Minimum Time is %.6lf s," + " Maximum Time is %.6lf s\n\n", + chkpt_id, min_write_time, max_write_time); + + sleep(15); // fake compute time + test_barrier(cfg); + } + + // sync/laminate + timer_start(&time_sync); + rc = test_close_file(cfg); + if (rc) { + test_abort(cfg, rc); + } + timer_stop(&time_sync); + test_barrier(cfg); + test_print_verbose_once(cfg, "DEBUG: finished sync"); + + min_sync_time = test_reduce_double_min(cfg, time_sync.elapsed_sec); + max_sync_time = test_reduce_double_max(cfg, time_sync.elapsed_sec); + test_print_once(cfg, "Minimum Sync Time is %.6lf s, " + "Maximum Sync Time is %.6lf s\n\n", + min_sync_time, max_sync_time); + + // post-checkpoint cleanup + free(chkpt_data); + free(req); + req = NULL; + + // re-open file + rc = test_open_file(cfg, target_file, O_RDONLY); + if (rc) { + test_abort(cfg, rc); + } + // generate restart read request + test_print_verbose_once(cfg, "DEBUG: generating restart read request"); + restart_data = malloc(test_config.block_sz); + if (NULL == restart_data) { + test_abort(cfg, ENOMEM); + } + num_reqs = generate_restart(cfg, restart_data, &req); + if (0 == num_reqs) { + test_abort(cfg, ENOMEM); + } + + // do restart read + test_print_verbose_once(cfg, "DEBUG: starting restart read"); + test_barrier(cfg); + timer_start(&time_restart); + rc = issue_read_req(cfg, req); + if (rc) { + test_abort(cfg, rc); + } + rc = wait_read_req(cfg, req); + if (rc) { + test_abort(cfg, rc); + } + timer_stop(&time_restart); + test_barrier(cfg); + test_print_verbose_once(cfg, "DEBUG: finished restart read"); + + min_read_time = test_reduce_double_min(cfg, time_restart.elapsed_sec); + max_read_time = test_reduce_double_max(cfg, time_restart.elapsed_sec); + test_print_once(cfg, "Minimum Restart Time is %.6lf s, " + "Maximum Restart Time is %.6lf s\n\n", + min_read_time, max_read_time); + + test_print_verbose_once(cfg, "DEBUG: verifying data"); + rc = verify_restart_data(cfg, restart_data, test_config.n_blocks); + if (rc) { + test_print(cfg, "ERROR - Restart data verification failed!"); + } + + // post-restart cleanup + free(restart_data); + free(req); + req = NULL; + + // cleanup + free(target_file); + + timer_fini(&time_ckpt); + timer_fini(&time_restart); + timer_fini(&time_sync); + + test_fini(cfg); + + return 0; +} diff --git a/examples/src/chmod.c b/examples/src/chmod.c new file mode 100644 index 000000000..af6ad85d4 --- /dev/null +++ b/examples/src/chmod.c @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2019, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + * + * Test chmod() and fchmod() + * + * Test description: + * 1. Make an empty file. + * 2. Try setting all the permission bits + * 3. Verify you see the bits in stat() + * 4. Use chmod() to remove the write bits to laminate it + * 5. Check file is laminated after + * 6. Make another empty file + * 7. Laminate it using fchmod() + * 8. Check file is laminated after + */ +#include "testutil.h" + +int do_test(test_cfg* cfg) +{ + char* file; + int rank = cfg->rank; + int fd; + + file = mktemp_cmd(cfg, "/unifyfs"); + assert(file); + + test_print(cfg, "Create empty file %s", file); + test_print(cfg, "Before lamination stat() is:"); + stat_cmd(cfg, file); + + test_print(cfg, "Try setting all the bits. stat() is:"); + chmod(file, 0777); + stat_cmd(cfg, file); + + if (rank == cfg->n_ranks - 1) { + test_print(cfg, "I'm the last rank, so I'll laminate the file"); + + sync_cmd(cfg, file); + /* laminate by setting permissions to read-only */ + chmod(file, 0444); + + test_print(cfg, "After lamination stat() is:"); + stat_cmd(cfg, file); + } + free(file); + + /* Do the same thing as before, but use fchmod() */ + file = mktemp_cmd(cfg, "/unifyfs"); + test_print(cfg, "Create empty file %s", file); + test_print(cfg, "Before lamination stat() is:"); + stat_cmd(cfg, file); + if (rank == cfg->n_ranks - 1) { + test_print(cfg, "I'm the last rank, so I'll fchmod laminate the file"); + + fd = open(file, O_RDWR); + /* laminate by removing write bits */ + fchmod(fd, 0444); /* set to read-only */ + close(fd); + + test_print(cfg, "After lamination stat() is:"); + stat_cmd(cfg, file); + test_print(cfg, "Verifying that writes to the laminated file fail"); + dd_cmd(cfg, "/dev/zero", file, 1024, 1, 0); + } + free(file); +} + +int main(int argc, char* argv[]) +{ + test_cfg test_config; + test_cfg* cfg = &test_config; + int rc; + + rc = test_init(argc, argv, cfg); + if (rc) { + test_print(cfg, "ERROR - Test %s initialization failed!", argv[0]); + fflush(NULL); + return rc; + } + do_test(cfg); + + test_fini(cfg); + + return 0; +} diff --git a/examples/src/read.c b/examples/src/read.c new file mode 100644 index 000000000..c69d2b749 --- /dev/null +++ b/examples/src/read.c @@ -0,0 +1,259 @@ +/* + * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2019, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#include "testutil.h" +#include "testutil_rdwr.h" + +// generate N-to-1 or N-to-N reads according to test config +size_t generate_read_reqs(test_cfg* cfg, char* dstbuf, + struct aiocb** reqs_out) +{ + off_t blk_off, chk_off; + size_t i, j, ndx = 0; + size_t blk_sz = cfg->block_sz; + size_t tran_sz = cfg->chunk_sz; + size_t n_tran_per_blk = blk_sz / tran_sz; + + size_t num_reqs = cfg->n_blocks * n_tran_per_blk; + struct aiocb* req; + struct aiocb* reqs = calloc(num_reqs, sizeof(struct aiocb)); + if (NULL == reqs) { + *reqs_out = NULL; + return 0; + } + + req = reqs; + for (i = 0; i < cfg->n_blocks; i++) { + if (IO_PATTERN_N1 == cfg->io_pattern) { + // interleaved block reads + blk_off = (i * blk_sz * cfg->n_ranks) + + (cfg->rank * blk_sz); + } else { // IO_PATTERN_NN + // blocked reads + blk_off = (i * blk_sz); + } + + for (j = 0; j < n_tran_per_blk; j++) { + chk_off = blk_off + (j * tran_sz); + + req->aio_fildes = cfg->fd; + req->aio_buf = (void*)(dstbuf + ndx); + req->aio_nbytes = tran_sz; + req->aio_offset = chk_off; + req->aio_lio_opcode = LIO_READ; + + req++; + ndx += tran_sz; + } + } + + *reqs_out = reqs; + return num_reqs; +} + +/* -------- Main Program -------- */ + +/* Description: + * + * [ Mode 1: N-to-1 shared-file ] + * Rank 0 first checks that the file size is as expected. Then, each + * rank reads cfg.n_blocks blocks of size cfg.block_sz from the + * shared file, using I/O operation sizes of cfg.chunk_sz. Blocks are + * rank-interleaved (i.e., the block at offset 0 is written by rank 0, + * the block at offset cfg.block_sz is written by rank 1, and so on). + * + * [ Mode 2: N-to-N file-per-process ] + * Each rank checks that its file size is as expected. Then, each + * rank reads cfg.n_blocks blocks of size cfg.block_sz using + * I/O operation sizes of cfg.chunk_sz. + * + * [ Options for Both Modes ] + * cfg.use_aio - when enabled, aio(7) will be used for issuing and + * completion of reads. + * + * cfg.use_lio - when enabled, lio_listio(3) will be used for batching + * reads. When cfg.use_aio is also enabled, the mode will be + * LIO_NOWAIT. + * + * cfg.use_mapio - support is not yet implemented. When enabled, + * direct memory loads will be used for reads. + * + * cfg.use_prdwr - when enabled, pread(2) will be used. + * + * cfg.use_stdio - when enabled, fread(3) will be used. + * + * cfg.use_vecio - support is not yet implemented. When enabled, + * readv(2) will be used for batching reads. + * + * cfg.io_check - when enabled, lipsum data is checked when reading + * the file. + * + * cfg.io_shuffle - ignored. + */ + +int main(int argc, char* argv[]) +{ + char* rd_buf; + char* target_file; + struct aiocb* reqs; + size_t num_reqs = 0; + int rc; + + test_cfg test_config; + test_cfg* cfg = &test_config; + test_timer time_rd; + test_timer time_check; + + timer_init(&time_rd, "read"); + timer_init(&time_check, "check"); + + rc = test_init(argc, argv, cfg); + if (rc) { + fprintf(stderr, "ERROR - Test %s initialization failed!", + argv[0]); + fflush(NULL); + return rc; + } + + if (!test_config.use_mpi) { + fprintf(stderr, "ERROR - Test %s requires MPI!", + argv[0]); + fflush(NULL); + return -1; + } + + target_file = test_target_filename(cfg); + test_print_verbose_once(cfg, "DEBUG: opening target file %s", + target_file); + + // file size check + size_t rank_bytes = test_config.n_blocks * test_config.block_sz; + size_t total_bytes = rank_bytes * test_config.n_ranks; + size_t expected = total_bytes; + if (IO_PATTERN_NN == test_config.io_pattern) { + expected = rank_bytes; + } + if ((test_config.rank == 0) || + (IO_PATTERN_NN == test_config.io_pattern)) { + struct stat s; + rc = stat(target_file, &s); + if (-1 == rc) { + test_print(cfg, "ERROR - stat(%s) failed", target_file); + } else { + if (s.st_size != expected) { + test_print(cfg, "ERROR - file size check failed - " + "actual size is %zu B, expected %zu B", + s.st_size, expected); + } + } + } + + // open file + rc = test_open_file(cfg, target_file, O_RDONLY); + if (rc) { + test_abort(cfg, rc); + } + + // generate read requests + test_print_verbose_once(cfg, "DEBUG: generating read requests"); + rd_buf = calloc(test_config.n_blocks, test_config.block_sz); + if (NULL == rd_buf) { + test_abort(cfg, ENOMEM); + } + num_reqs = generate_read_reqs(cfg, rd_buf, &reqs); + if (0 == num_reqs) { + test_abort(cfg, ENOMEM); + } + + // do reads + test_print_verbose_once(cfg, "DEBUG: starting read requests"); + test_barrier(cfg); + timer_start(&time_rd); + rc = issue_read_req_batch(cfg, num_reqs, reqs); + if (rc) { + test_abort(cfg, rc); + } + rc = wait_read_req_batch(cfg, num_reqs, reqs); + if (rc) { + test_abort(cfg, rc); + } + timer_stop(&time_rd); + test_print_verbose_once(cfg, "DEBUG: finished read requests"); + + // check file data + test_barrier(cfg); + test_print_verbose_once(cfg, "DEBUG: starting data check"); + timer_start(&time_check); + rc = check_read_req_batch(cfg, num_reqs, reqs); + if (rc) { + test_abort(cfg, rc); + } + timer_stop(&time_check); + test_print_verbose_once(cfg, "DEBUG: finished data check"); + + // post-read cleanup + free(rd_buf); + free(reqs); + reqs = NULL; + + // close file + rc = test_close_file(cfg); + if (rc) { + test_abort(cfg, rc); + } + + // calculate achieved bandwidth rates + double max_read_time, max_check_time; + double read_bw, aggr_read_bw, eff_read_bw; + + max_read_time = test_reduce_double_max(cfg, time_rd.elapsed_sec); + max_check_time = test_reduce_double_max(cfg, time_check.elapsed_sec); + + read_bw = bandwidth_mib(rank_bytes, time_rd.elapsed_sec); + aggr_read_bw = test_reduce_double_sum(cfg, read_bw); + eff_read_bw = bandwidth_mib(total_bytes, max_read_time); + + test_print_once(cfg, + "\n" + "I/O pattern: %s\n" + "I/O block size: %.2lf KiB\n" + "I/O request size: %.2lf KiB\n" + "Number of processes: %d\n" + "Each process wrote: %.2lf MiB\n" + "Total data written: %.2lf MiB\n" + "Maximum read time: %.6lf sec\n" + "Maximum check time: %.6lf sec\n" + "Aggregate read bandwidth: %.3lf MiB/s\n" + "Effective read bandwidth: %.3lf MiB/s\n", + io_pattern_str(test_config.io_pattern), + bytes_to_kib(test_config.block_sz), + bytes_to_kib(test_config.chunk_sz), + test_config.n_ranks, + bytes_to_mib(rank_bytes), + bytes_to_mib(total_bytes), + max_read_time, + max_check_time, + aggr_read_bw, + eff_read_bw); + + // cleanup + free(target_file); + + timer_fini(&time_rd); + timer_fini(&time_check); + + test_fini(cfg); + + return 0; +} diff --git a/examples/src/size.c b/examples/src/size.c new file mode 100644 index 000000000..e57db50aa --- /dev/null +++ b/examples/src/size.c @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2019, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + * + * Test file size functions + * + * Test description: + * 1. For each rank, write 1KB at a 1KB * rank offset. + * 2. Check the file size. It should be 0 since it hasn't been laminated. + * 3. Have the last rank laminate the file. + * 4. Check the file size again. It should be the real, laminated, file size. + */ +#include "testutil.h" + +int do_test(test_cfg* cfg) +{ + char* file; + int rank = cfg->rank; + + file = mktemp_cmd(cfg, "/unifyfs"); + assert(file); + + test_print(cfg, "I'm writing 1KB to %s at my offset at %ld", + file, rank * 1024); + + dd_cmd(cfg, "/dev/zero", file, 1024, 1, rank); + + test_print(cfg, "Stating the file"); + stat_cmd(cfg, file); + test_print(cfg, "After writing, file size is %lu, apparent-size %lu", + du_cmd(cfg, file, 0), + du_cmd(cfg, file, 1)); + + /* sync our extents */ + sync_cmd(cfg, file); + + /* Wait for everyone to finish writing */ + test_barrier(cfg); + + if (rank == cfg->n_ranks - 1) { + test_print(cfg, "I'm the last rank, so I'll laminate the file"); + + /* laminate by removing write bits */ + chmod(file, 0444); /* set to read-only */ + + test_print(cfg, "After lamination, file size is %lu", + du_cmd(cfg, file, 0)); + test_print(cfg, "Stating the file"); + stat_cmd(cfg, file); + } + free(file); +} + +int main(int argc, char* argv[]) +{ + test_cfg test_config; + test_cfg* cfg = &test_config; + int rc; + + rc = test_init(argc, argv, cfg); + if (rc) { + test_print(cfg, "ERROR - Test %s initialization failed!", argv[0]); + fflush(NULL); + return rc; + } + do_test(cfg); + + test_fini(cfg); + + return 0; +} diff --git a/examples/src/sysio-cp.c b/examples/src/sysio-cp.c index 4cb1c6bd7..62693fed1 100644 --- a/examples/src/sysio-cp.c +++ b/examples/src/sysio-cp.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ #include @@ -29,7 +29,7 @@ #include #include #include -#include +#include #include "testlib.h" @@ -38,8 +38,8 @@ static int total_ranks; static int rank_worker; static int debug; -static char* mountpoint = "/unifycr"; /* unifycr mountpoint */ -static int unmount; /* unmount unifycr after running the test */ +static char* mountpoint = "/unifyfs"; /* unifyfs mountpoint */ +static int unmount; /* unmount unifyfs after running the test */ static char* srcpath; static char* dstpath; @@ -169,8 +169,8 @@ static const char* usage_str = " -d, --debug pause before running test\n" " (handy for attaching in debugger)\n" " -h, --help help message\n" - " -m, --mount= use for unifycr\n" - " (default: /unifycr)\n" + " -m, --mount= use for unifyfs\n" + " (default: /unifyfs)\n" " -r, --rank= use to copy the file (default: 0)\n" " -u, --unmount unmount the filesystem after test\n" "\n"; @@ -240,9 +240,9 @@ int main(int argc, char** argv) test_pause(rank, "Attempting to mount"); } - ret = unifycr_mount(mountpoint, rank, total_ranks, 0); + ret = unifyfs_mount(mountpoint, rank, total_ranks, 0); if (ret) { - test_print(rank, "unifycr_mount failed (return = %d)", ret); + test_print(rank, "unifyfs_mount failed (return = %d)", ret); goto out; } @@ -282,7 +282,7 @@ int main(int argc, char** argv) MPI_Barrier(MPI_COMM_WORLD); if (unmount) { - unifycr_unmount(); + unifyfs_unmount(); } out: diff --git a/examples/src/sysio-dir.c b/examples/src/sysio-dir.c index 741078235..868870821 100644 --- a/examples/src/sysio-dir.c +++ b/examples/src/sysio-dir.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ #include @@ -28,20 +28,20 @@ #include #include #include -#include +#include #include "testlib.h" -static int standard; /* not mounting unifycr when set */ +static int standard; /* not mounting unifyfs when set */ static int synchronous; /* sync metadata for each op? (default: no)*/ static int rank; static int total_ranks; static int debug; /* pause for attaching debugger */ -static int unmount; /* unmount unifycr after running the test */ +static int unmount; /* unmount unifyfs after running the test */ static uint64_t count = 10; /* number of directories each rank creates */ -static char* mountpoint = "/unifycr"; /* unifycr mountpoint */ +static char* mountpoint = "/unifyfs"; /* unifyfs mountpoint */ static char* testdir = "testdir"; /* test directory under mountpoint */ static char targetdir[NAME_MAX]; /* target file name */ @@ -205,12 +205,12 @@ static const char* usage_str = " -D, --dirname= test directory name under mountpoint\n" " (default: testdir)\n" " -h, --help help message\n" - " -m, --mount= use for unifycr\n" - " (default: /unifycr)\n" + " -m, --mount= use for unifyfs\n" + " (default: /unifyfs)\n" " -n, --count= number of directories that each rank will\n" " create (default: 10)\n" " -S, --synchronous sync metadata on each write\n" - " -s, --standard do not use unifycr but run standard I/O\n" + " -s, --standard do not use unifyfs but run standard I/O\n" " -t, --singletest= only test a single operation\n" " (operations: mkdir, stat, readdir, rmdir)\n" " -u, --unmount unmount the filesystem after test\n" @@ -291,9 +291,9 @@ int main(int argc, char** argv) } if (!standard) { - ret = unifycr_mount(mountpoint, rank, total_ranks, 0); + ret = unifyfs_mount(mountpoint, rank, total_ranks, 0); if (ret) { - test_print(rank, "unifycr_mount failed (return = %d)", ret); + test_print(rank, "unifyfs_mount failed (return = %d)", ret); exit(-1); } } @@ -339,7 +339,7 @@ int main(int argc, char** argv) out_unmount: if (!standard && unmount) { - unifycr_unmount(); + unifyfs_unmount(); } out: MPI_Finalize(); diff --git a/examples/src/sysio-read.c b/examples/src/sysio-read.c index ceb8e9c3f..1cacbf6f3 100644 --- a/examples/src/sysio-read.c +++ b/examples/src/sysio-read.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ #include @@ -27,7 +27,7 @@ #include #include #include -#include +#include #include "testlib.h" @@ -41,7 +41,7 @@ static int pattern; /* N to 1 (N1, default) or N to N (NN) */ static int fd; /* target file descriptor */ static int lipsum; /* check contents written by the write test. */ -static int standard; /* not mounting unifycr when set */ +static int standard; /* not mounting unifyfs when set */ /* time statistics */ static struct timeval read_start, read_end; @@ -50,8 +50,8 @@ static int rank; static int total_ranks; static int debug; /* pause for attaching debugger */ -static int unmount; /* unmount unifycr after running the test */ -static char* mountpoint = "/unifycr"; /* unifycr mountpoint */ +static int unmount; /* unmount unifyfs after running the test */ +static char* mountpoint = "/unifyfs"; /* unifyfs mountpoint */ static char* filename = "testfile"; /* testfile name under mountpoint */ static char targetfile[NAME_MAX]; /* target file name */ @@ -242,12 +242,12 @@ static const char* usage_str = " -h, --help help message\n" " -L, --lipsum check contents written by write test\n" " -l, --listio use lio_listio(2) instead of read(2)\n" - " -m, --mount= use for unifycr\n" - " (default: /unifycr)\n" + " -m, --mount= use for unifyfs\n" + " (default: /unifyfs)\n" " -P, --pread use pread(2) instead of read(2)\n" " -p, --pattern= should be 'n1'(n to 1) or 'nn' (n to n)\n" " (default: n1)\n" - " -s, --standard do not use unifycr but run standard I/O\n" + " -s, --standard do not use unifyfs but run standard I/O\n" " -u, --unmount unmount the filesystem after test\n" "\n"; @@ -373,9 +373,9 @@ int main(int argc, char** argv) } if (!standard) { - ret = unifycr_mount(mountpoint, rank, total_ranks, 0); + ret = unifyfs_mount(mountpoint, rank, total_ranks, 0); if (ret) { - test_print(rank, "unifycr_mount failed (return = %d)", ret); + test_print(rank, "unifyfs_mount failed (return = %d)", ret); exit(-1); } } @@ -417,7 +417,7 @@ int main(int argc, char** argv) MPI_Barrier(MPI_COMM_WORLD); if (!standard && unmount) { - unifycr_unmount(); + unifyfs_unmount(); } if (ret == 0) { diff --git a/examples/src/sysio-stat.c b/examples/src/sysio-stat.c index de7baffb4..7f0159c6f 100644 --- a/examples/src/sysio-stat.c +++ b/examples/src/sysio-stat.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ #include @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -29,7 +30,7 @@ #include #include #include -#include +#include #include "testlib.h" @@ -37,9 +38,9 @@ static int rank; static int total_ranks; static int debug; -static char* mountpoint = "/unifycr"; /* unifycr mountpoint */ -static char* filename = "/unifycr"; -static int unmount; /* unmount unifycr after running the test */ +static char* mountpoint = "/unifyfs"; /* unifyfs mountpoint */ +static char* filename = "/unifyfs"; +static int unmount; /* unmount unifyfs after running the test */ #define FP_SPECIAL 1 @@ -125,8 +126,8 @@ static const char* usage_str = " -d, --debug pause before running test\n" " (handy for attaching in debugger)\n" " -h, --help help message\n" - " -m, --mount= use for unifycr\n" - " (default: /unifycr)\n" + " -m, --mount= use for unifyfs\n" + " (default: /unifyfs)\n" " -u, --unmount unmount the filesystem after test\n" "\n"; @@ -183,9 +184,9 @@ int main(int argc, char** argv) test_pause(rank, "Attempting to mount"); } - ret = unifycr_mount(mountpoint, rank, total_ranks, 0); + ret = unifyfs_mount(mountpoint, rank, total_ranks, 0); if (ret) { - test_print(rank, "unifycr_mount failed (return = %d)", ret); + test_print(rank, "unifyfs_mount failed (return = %d)", ret); exit(-1); } @@ -201,7 +202,7 @@ int main(int argc, char** argv) MPI_Barrier(MPI_COMM_WORLD); if (unmount) { - unifycr_unmount(); + unifyfs_unmount(); } MPI_Finalize(); diff --git a/examples/src/sysio-write.c b/examples/src/sysio-write.c index 689a466e4..2b5dd31b1 100644 --- a/examples/src/sysio-write.c +++ b/examples/src/sysio-write.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ #include @@ -26,7 +26,7 @@ #include #include #include -#include +#include #include "testlib.h" @@ -48,7 +48,7 @@ static int fd; /* target file descriptor */ static int synchronous; /* sync metadata for each write? (default: no)*/ static int lipsum; /* generate contents to verify correctness */ -static int standard; /* not mounting unifycr when set */ +static int standard; /* not mounting unifyfs when set */ /* time statistics */ static struct timeval write_start, meta_start, write_end; @@ -57,9 +57,9 @@ static int rank; static int total_ranks; static int debug; /* pause for attaching debugger */ -static int unmount; /* unmount unifycr after running the test */ +static int unmount; /* unmount unifyfs after running the test */ static char* buf; /* I/O buffer */ -static char* mountpoint = "/unifycr"; /* unifycr mountpoint */ +static char* mountpoint = "/unifyfs"; /* unifyfs mountpoint */ static char* filename = "testfile"; /* testfile name under mountpoint */ static char targetfile[NAME_MAX]; /* target file name */ @@ -194,13 +194,13 @@ static const char* usage_str = " (default: testfile)\n" " -h, --help help message\n" " -L, --lipsum generate contents to verify correctness\n" - " -m, --mount= use for unifycr\n" - " (default: /unifycr)\n" + " -m, --mount= use for unifyfs\n" + " (default: /unifyfs)\n" " -P, --pwrite use pwrite(2) instead of write(2)\n" " -p, --pattern= should be 'n1'(n to 1) or 'nn' (n to n)\n" " (default: n1)\n" " -S, --synchronous sync metadata on each write\n" - " -s, --standard do not use unifycr but run standard I/O\n" + " -s, --standard do not use unifyfs but run standard I/O\n" " -u, --unmount unmount the filesystem after test\n" "\n"; @@ -312,9 +312,9 @@ int main(int argc, char** argv) } if (!standard) { - ret = unifycr_mount(mountpoint, rank, total_ranks, 0); + ret = unifyfs_mount(mountpoint, rank, total_ranks, 0); if (ret) { - test_print(rank, "unifycr_mount failed (return = %d)", ret); + test_print(rank, "unifyfs_mount failed (return = %d)", ret); exit(-1); } } @@ -375,7 +375,7 @@ int main(int argc, char** argv) } if (!standard && unmount) { - unifycr_unmount(); + unifyfs_unmount(); } if (ret == 0) { diff --git a/examples/src/sysio-writeread.c b/examples/src/sysio-writeread.c index 4ada3bce3..899cbab50 100644 --- a/examples/src/sysio-writeread.c +++ b/examples/src/sysio-writeread.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -50,8 +50,8 @@ #include #include -#ifndef NO_UNIFYCR -# include +#ifndef DISABLE_UNIFYFS +# include #endif #define TEST_STR_LEN 1024 @@ -80,7 +80,7 @@ int main(int argc, char* argv[]) size_t index, i, j, offset = 0; ssize_t rc; int ret; - int pat = 0, c, num_rank, rank, fd, use_unifycr = 0; + int pat = 0, c, num_rank, rank, fd, use_unifyfs = 0; MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &num_rank); @@ -106,14 +106,14 @@ int main(int argc, char* argv[]) case 't': /*size of each write */ tran_sz = atol(optarg); break; - case 'u': /* use unifycr */ - use_unifycr = atoi(optarg); + case 'u': /* use unifyfs */ + use_unifyfs = atoi(optarg); break; } } - if (use_unifycr) { - strcpy(mntpt, "/unifycr"); + if (use_unifyfs) { + strcpy(mntpt, "/unifyfs"); } else { strcpy(mntpt, "/tmp"); } @@ -137,21 +137,21 @@ int main(int argc, char* argv[]) } size_t n_tran_per_blk = blk_sz / tran_sz; + double rank_mib = (double)(blk_sz * num_blk) / 1048576; + double total_mib = rank_mib * num_rank; char* buf = malloc(tran_sz); - if (buf == NULL) { return -1; } int byte = (int)'0' + rank; - memset(buf, byte, tran_sz); -#ifndef NO_UNIFYCR - if (use_unifycr) { - ret = unifycr_mount(mntpt, rank, num_rank, 0); - if (UNIFYCR_SUCCESS != ret) { +#ifndef DISABLE_UNIFYFS + if (use_unifyfs) { + ret = unifyfs_mount(mntpt, rank, num_rank, 0); + if (UNIFYFS_SUCCESS != ret) { MPI_Abort(MPI_COMM_WORLD, ret); } MPI_Barrier(MPI_COMM_WORLD); @@ -165,15 +165,14 @@ int main(int argc, char* argv[]) } int open_flags = O_CREAT | O_RDWR; - fd = open(tmpfname, open_flags, 0644); if (fd < 0) { printf("open file failure\n"); fflush(stdout); return -1; } - MPI_Barrier(MPI_COMM_WORLD); + MPI_Barrier(MPI_COMM_WORLD); gettimeofday(&write_start, NULL); for (i = 0; i < num_blk; i++) { @@ -196,6 +195,7 @@ int main(int argc, char* argv[]) fsync(fd); gettimeofday(&write_end, NULL); + free(buf); meta_time += 1000000 * (write_end.tv_sec - meta_start.tv_sec) + write_end.tv_usec - meta_start.tv_usec; @@ -205,9 +205,20 @@ int main(int argc, char* argv[]) + write_end.tv_usec - write_start.tv_usec; write_time = write_time / 1000000; - MPI_Barrier(MPI_COMM_WORLD); + double agg_write_bw, max_write_time; + double write_bw = rank_mib / write_time; + MPI_Reduce(&write_bw, &agg_write_bw, 1, MPI_DOUBLE, MPI_SUM, + 0, MPI_COMM_WORLD); + MPI_Reduce(&write_time, &max_write_time, 1, MPI_DOUBLE, MPI_MAX, + 0, MPI_COMM_WORLD); + double min_write_bw = total_mib / max_write_time; - free(buf); + if (rank == 0) { + printf("Aggregate Write BW is %.3lf MiB/s\n" + " Minimum Write BW is %.3lf MiB/s\n\n", + agg_write_bw, min_write_bw); + fflush(stdout); + } /* read buffer */ char* read_buf = calloc(num_blk, blk_sz); @@ -255,68 +266,44 @@ int main(int argc, char* argv[]) } } + MPI_Barrier(MPI_COMM_WORLD); gettimeofday(&read_start, NULL); ret = lio_listio(LIO_WAIT, cb_list, num_reqs, NULL); - if (ret < 0) { perror("lio_listio failed"); } gettimeofday(&read_end, NULL); - read_time = (read_end.tv_sec - read_start.tv_sec) * 1000000 - + read_end.tv_usec - read_start.tv_usec; - read_time = read_time / 1000000; - close(fd); free(read_buf); - MPI_Barrier(MPI_COMM_WORLD); - -#ifndef NO_UNIFYCR - if (use_unifycr) { - unifycr_unmount(); - } -#endif - - double rank_mib = (double)(blk_sz * num_blk) / 1048576; - - double total_mib = rank_mib * num_rank; - - double write_bw = rank_mib / write_time; + read_time = (read_end.tv_sec - read_start.tv_sec) * 1000000 + + read_end.tv_usec - read_start.tv_usec; + read_time = read_time / 1000000; + double agg_read_bw, max_read_time; double read_bw = rank_mib / read_time; - - double agg_write_bw, agg_read_bw; - double max_write_time, max_read_time; - - MPI_Reduce(&write_bw, &agg_write_bw, 1, MPI_DOUBLE, MPI_SUM, - 0, MPI_COMM_WORLD); - - MPI_Reduce(&write_time, &max_write_time, 1, MPI_DOUBLE, MPI_MAX, - 0, MPI_COMM_WORLD); - - double min_write_bw = total_mib / max_write_time; - MPI_Reduce(&read_bw, &agg_read_bw, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD); - MPI_Reduce(&read_time, &max_read_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); - double min_read_bw = total_mib / max_read_time; if (rank == 0) { - printf("Aggregate Write BW is %.3lf MiB/s\n" - " Minimum Write BW is %.3lf MiB/s\n\n", - agg_write_bw, min_write_bw); printf("Aggregate Read BW is %.3lf MiB/s\n" " Minimum Read BW is %.3lf MiB/s\n\n", agg_read_bw, min_read_bw); fflush(stdout); } +#ifndef DISABLE_UNIFYFS + if (use_unifyfs) { + unifyfs_unmount(); + } +#endif + MPI_Finalize(); return 0; diff --git a/examples/src/sysio-writeread2.c b/examples/src/sysio-writeread2.c index 9e373ace4..cf03fce86 100644 --- a/examples/src/sysio-writeread2.c +++ b/examples/src/sysio-writeread2.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ // build: mpigcc -g -O3 -o test_ramdisk test_ramdisk.c @@ -28,7 +28,7 @@ #include #include #include -#include +#include //size_t filesize = 100*1024*1024; size_t filesize = 1024*1024; @@ -392,11 +392,11 @@ int main(int argc, char *argv[]) MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &ranks); - unifycr_mount("/unifycr", rank, ranks, 0); + unifyfs_mount("/unifyfs", rank, ranks, 0); char name[256]; - sprintf(name, "/unifycr/rank.%d", rank); + sprintf(name, "/unifyfs/rank.%d", rank); /* allocate space for the checkpoint data (make filesize a function of rank * for some variation) @@ -406,7 +406,7 @@ int main(int argc, char *argv[]) /* verify data integrity in file */ checkdata(name, filesize, times); - unifycr_unmount(); + unifyfs_unmount(); MPI_Finalize(); diff --git a/examples/src/testlib.h b/examples/src/testlib.h index 3b8599f02..06c87c6ae 100644 --- a/examples/src/testlib.h +++ b/examples/src/testlib.h @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ #ifndef __TESTLIB_H #define __TESTLIB_H diff --git a/examples/src/testutil.c b/examples/src/testutil.c new file mode 100644 index 000000000..50b21219b --- /dev/null +++ b/examples/src/testutil.c @@ -0,0 +1,348 @@ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "testutil.h" + +/* Clone of the apsrintf(). See the standard asprintf() man page for details */ +static int asprintf(char** strp, const char* fmt, ...) +{ + /* + * This code is taken from the vmalloc(3) man page and modified slightly. + */ + int n; + int size = 100; /* Guess we need no more than 100 bytes */ + char* p; + char* np; + va_list ap; + + p = malloc(size); + if (p == NULL) { + *strp = NULL; + return -ENOMEM; + } + + while (1) { + /* Try to print in the allocated space */ + + va_start(ap, fmt); + n = vsnprintf(p, size, fmt, ap); + va_end(ap); + + /* Check error code */ + + if (n < 0) { + *strp = NULL; + return -1; + } + + /* If that worked, return the string */ + if (n < size) { + *strp = p; + return n; + } + + /* Else try again with more space */ + + size = n + 1; /* Precisely what is needed */ + + np = realloc(p, size); + if (np == NULL) { + *strp = NULL; + free(p); + return -ENOMEM; + } else { + p = np; + } + } +} + + +/****************************************************************************** + * Below are some standard unix commands replicated in C. These were included + * to make testing easier, since you can really do a lot with just a handful + * of commands. All these are named 'unix name'_cmd() to differentiate the + * command from the system calls by the same name (like "stat"). + *****************************************************************************/ + +/* + * C clone of 'dd'. All semantics are the same. + * Return 0 on success, errno otherwise. + */ +int dd_cmd(test_cfg* cfg, char* infile, char* outfile, unsigned long bs, + unsigned long count, unsigned long seek) +{ + FILE* infp; + FILE* outfp; + char* buf; + size_t rc; + int i; + infp = fopen(infile, "r"); + if (!infp) { + test_print(cfg, "Error opening infile"); + return errno; + } + outfp = fopen(outfile, "w"); + if (!outfp) { + test_print(cfg, "Error opening outfile"); + return errno; + } + buf = calloc(sizeof(*buf), bs); + assert(buf); + + rc = fseek(outfp, seek * bs, SEEK_SET); + if (rc) { + test_print(cfg, "Error seeking outfile"); + return errno; + } + + for (i = 0; i < count; i++) { + if (feof(infp)) { + break; + } + rc = fread(buf, bs, 1, infp); + if (ferror(infp)) { + test_print(cfg, "Error reading infile"); + return errno; + } + rc = fwrite(buf, rc * bs, 1, outfp); + if (ferror(outfp)) { + test_print(cfg, "Error writing outfile"); + return errno; + } + } + free(buf); + fclose(infp); + fclose(outfp); + return 0; +} + +/* + * C clone of the 'test' command. Only supports file checks like + * "test -f $file" or "test -d $dir" for now. + */ +int test_cmd(char* expression, char* path) +{ + struct stat sb; + int ret; + + ret = stat(path, &sb); + /* + * We're stat'ing a file/dir which may or may not exist. Clear errno so + * that we don't pollute future test_print() calls with bogus errors. + */ + errno = 0; + if (ret) { + return 0; + } + + ret = 0; + switch (sb.st_mode & S_IFMT) { + case S_IFREG: + if (strcmp(expression, "-f") == 0) { + ret = 1; + } + break; + case S_IFDIR: + if (strcmp(expression, "-d") == 0) { + ret = 1; + } + break; + default: + break; + } + return ret; +} + +/* + * C clone of 'mktemp'. tmpdir is the directory to create the file in. + * + * Makes the file with zero length and returns the filename. The filename + * is malloc'd so it must be freed when finished. + */ +char* mktemp_cmd(test_cfg* cfg, char* tmpdir) +{ + char* tmpfile = NULL; + int i; + char letters[11]; + char r; + char* job_id; + if (!tmpdir) { + tmpdir = "/tmp"; + } + + if (!test_cmd("-d", tmpdir)) { + test_print(cfg, "%s dir does not exists", tmpdir); + return NULL; + } + + /* If we can get our job ID, use it to seed our RNG. Else use our app_id */ + job_id = getenv("LSB_JOBID"); + if (job_id) { + srand(atol(job_id)); + } else { + srand(cfg->app_id); + } + + do { + /* + * The normal mktemp creates filenames like /tmp/tmp.qTHxL7SLGx. Let's + * try to replicate that. + */ + i = 0; + do { + /* + * First make our qTHxL7SLGx string. Pick random characters that + * are alphanumeric. + */ + r = rand() % 123; /* Ignore anything past char 122 ('z') */ + if (isalnum(r)) { + letters[i] = r; + i++; + } + } while (i < 10); + letters[10] = '\0'; + + asprintf(&tmpfile, "%s/tmp.%s", tmpdir, letters); + if (!tmpfile) { + return NULL; + } + + /* Loop until we've generated a filename that doesn't already exist */ + } while (test_cmd("-f", tmpfile)); + + /* + * Success, we've generated a tmpfile name that doesn't already exist. + * Make an empty file (use bs=1 to follow normal dd semantics). + */ + test_print(cfg, "File doesn't exist, create it"); + if (dd_cmd(cfg, "/dev/zero", tmpfile, 1, 0, 0) != 0) { + return NULL; + } + + return tmpfile; +} + +/* C clone of 'du'. Returns the logical file size unless apparent_size = 1, + * in which case it returns the apparent size (aka the "log size" in UnifyFS + * speak). Otherwise, return a negative number on error. + */ +long du_cmd(test_cfg* cfg, char* filename, int apparent_size) +{ + struct stat sb; + if (stat(filename, &sb) == -1) { + test_print(cfg, "Error stat"); + return -1; + } + + if (apparent_size) { + return sb.st_blksize * sb.st_blocks; + } else { + return sb.st_size; + } +} + +/* C clone of 'sync [filename]' */ +int sync_cmd(test_cfg* cfg, char* filename) +{ + int fd; + fd = open(filename, O_RDWR); + if (fd < 0) { + test_print(cfg, "Error opening file"); + return errno; + } + if (fsync(fd) != 0) { + test_print(cfg, "Error syncing file"); + return errno; + } + close(fd); +} + +/* C clone of 'stat' */ +int stat_cmd(test_cfg* cfg, char* filename) +{ + struct stat sb; + int rc; + const char* typestr; + char* tmp; + + rc = stat(filename, &sb); + if (rc) { + test_print(cfg, "Error stating %s: %s", filename, strerror(rc)); + return rc; + } + + switch (sb.st_mode & S_IFMT) { + case S_IFREG: + typestr = "regular file"; + break; + case S_IFDIR: + typestr = "directory"; + break; + case S_IFCHR: + typestr = "character device"; + break; + case S_IFBLK: + typestr = "block device"; + break; + case S_IFLNK: + typestr = "symbolic (soft) link"; + break; + case S_IFIFO: + typestr = "FIFO or pipe"; + break; + case S_IFSOCK: + typestr = "socket"; + break; + default: + typestr = "unknown file type?"; + break; + } + + /* Do some work to get the ':' in the right place */ + asprintf(&tmp, "%s:", filename); + test_print(cfg, "%-26s%s", tmp, typestr); + free(tmp); + + test_print(cfg, "Device containing i-node: major=%ld minor=%ld", + (long) major(sb.st_dev), (long) minor(sb.st_dev)); + + test_print(cfg, "I-node number: %ld", (long) sb.st_ino); + + test_print(cfg, "Mode: %lo", + (unsigned long) sb.st_mode); + + if (sb.st_mode & (S_ISUID | S_ISGID | S_ISVTX)) { + test_print(cfg, " special bits set: %s%s%s", + (sb.st_mode & S_ISUID) ? "set-UID " : "", + (sb.st_mode & S_ISGID) ? "set-GID " : "", + (sb.st_mode & S_ISVTX) ? "sticky " : ""); + } + + test_print(cfg, "Number of (hard) links: %ld", (long) sb.st_nlink); + + test_print(cfg, "Ownership: UID=%ld GID=%ld", + (long) sb.st_uid, (long) sb.st_gid); + + if (S_ISCHR(sb.st_mode) || S_ISBLK(sb.st_mode)) { + test_print(cfg, "Device number (st_rdev): major=%ld; minor=%ld", + (long) major(sb.st_rdev), (long) minor(sb.st_rdev)); + } + + test_print(cfg, "File size: %lld bytes", + (long long) sb.st_size); + test_print(cfg, "Optimal I/O block size: %ld bytes", + (long) sb.st_blksize); + test_print(cfg, "Blocks allocated: %lld", (long long) sb.st_blocks); + test_print(cfg, "Last file access: %s", ctime(&sb.st_atime)); + test_print(cfg, "Last file modification: %s", ctime(&sb.st_mtime)); + test_print(cfg, "Last status change: %s", ctime(&sb.st_ctime)); +} diff --git a/examples/src/testutil.h b/examples/src/testutil.h new file mode 100644 index 000000000..7ee02ad89 --- /dev/null +++ b/examples/src/testutil.h @@ -0,0 +1,1085 @@ +/* + * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2019, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#ifndef UNIFYFS_TEST_UTIL_H +#define UNIFYFS_TEST_UTIL_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef DISABLE_UNIFYFS +# include +#endif + +/* ---------- Common Types and Definitions ---------- */ + +#define TEST_STR_LEN 1024 + +#ifndef KIB +# define KIB (1024) +#endif + +#ifndef MIB +# define MIB (1048576) +#endif + +#ifndef GIB +# define GIB (1073741824) +#endif + +static inline +double bytes_to_kib(size_t bytes) +{ + return (double)bytes / KIB; +} + +static inline +double bytes_to_mib(size_t bytes) +{ + return (double)bytes / MIB; +} + +static inline +double bytes_to_gib(size_t bytes) +{ + return (double)bytes / GIB; +} + +static inline +double bandwidth_kib(size_t bytes, double seconds) +{ + return bytes_to_kib(bytes) / seconds; +} + +static inline +double bandwidth_mib(size_t bytes, double seconds) +{ + return bytes_to_mib(bytes) / seconds; +} + +static inline +double bandwidth_gib(size_t bytes, double seconds) +{ + return bytes_to_gib(bytes) / seconds; +} + +#define IO_PATTERN_N1 (0) +#define IO_PATTERN_NN (1) + +static inline +const char* io_pattern_str(int pattern) +{ + switch (pattern) { + case IO_PATTERN_N1: + return "N-to-1"; + case IO_PATTERN_NN: + return "N-to-N"; + default: + break; + } + return "Unknown I/O Pattern"; +} + +#define DEFAULT_IO_CHUNK_SIZE (MIB) // 1 MiB +#define DEFAULT_IO_BLOCK_SIZE (16 * DEFAULT_IO_CHUNK_SIZE) // 16 MiB +#define DEFAULT_IO_NUM_BLOCKS (32) // 32 blocks x 16 MiB = 512 MiB + +typedef struct { + /* program behavior options */ + int debug; /* during startup, wait for input at rank 0 */ + int verbose; /* print verbose information to stderr */ + int use_mpi; + int use_unifyfs; + + /* I/O behavior options */ + int io_pattern; /* N1 or NN */ + int io_check; /* use lipsum to verify data */ + int io_shuffle; /* read and write different extents */ + int use_aio; /* use asynchronous IO */ + int use_lio; /* use lio_listio instead of read/write */ + int use_mapio; /* use mmap instead of read/write */ + int use_prdwr; /* use pread/pwrite instead of read/write */ + int use_stdio; /* use fread/fwrite instead of read/write */ + int use_vecio; /* use readv/writev instead of read/write */ + + /* I/O size options */ + uint64_t n_blocks; /* number of I/O blocks */ + uint64_t block_sz; /* IO block size (multiple of chunk_sz) */ + uint64_t chunk_sz; /* per-IO-op size */ + + /* target file info */ + char* filename; + char* mountpt; + FILE* fp; + void* mapped; /* address of mapped extent of cfg.fd */ + off_t mapped_off; /* start offset for mapped extent */ + size_t mapped_sz; /* size of mapped extent */ + int fd; + int fd_access; /* access flags for cfg.fd */ + + /* MPI info */ + int app_id; + int rank; + int n_ranks; +} test_cfg; + +static inline +void test_config_init(test_cfg* cfg) +{ + if (NULL == cfg) { + fprintf(stderr, "INTERNAL ERROR: %s() - cfg is NULL\n", __func__); + fflush(stderr); + return; + } + + // set everything to 0/NULL + memset(cfg, 0, sizeof(test_cfg)); + + // N-to-1 UnifyFS by default + cfg->use_mpi = 1; + cfg->use_unifyfs = 1; + cfg->io_pattern = IO_PATTERN_N1; + + // invalidate file descriptor + cfg->fd = -1; + + // use default I/O sizes + cfg->n_blocks = DEFAULT_IO_NUM_BLOCKS; + cfg->block_sz = DEFAULT_IO_BLOCK_SIZE; + cfg->chunk_sz = DEFAULT_IO_CHUNK_SIZE; +} + +static inline +void test_config_print(test_cfg* cfg) +{ + assert(NULL != cfg); + + fprintf(stderr, " Test Configuration\n"); + fprintf(stderr, "==========================\n"); + + fprintf(stderr, "\n-- Program Behavior --\n"); + fprintf(stderr, "\t debug = %d\n", cfg->debug); + fprintf(stderr, "\t verbose = %d\n", cfg->verbose); + fprintf(stderr, "\t use_mpi = %d\n", cfg->use_mpi); + fprintf(stderr, "\t use_unifyfs = %d\n", cfg->use_unifyfs); + + fprintf(stderr, "\n-- IO Behavior --\n"); + fprintf(stderr, "\t io_pattern = %s\n", io_pattern_str(cfg->io_pattern)); + fprintf(stderr, "\t io_check = %d\n", cfg->io_check); + fprintf(stderr, "\t io_shuffle = %d\n", cfg->io_shuffle); + fprintf(stderr, "\t use_aio = %d\n", cfg->use_aio); + fprintf(stderr, "\t use_lio = %d\n", cfg->use_lio); + fprintf(stderr, "\t use_mapio = %d\n", cfg->use_mapio); + fprintf(stderr, "\t use_prdwr = %d\n", cfg->use_prdwr); + fprintf(stderr, "\t use_stdio = %d\n", cfg->use_stdio); + fprintf(stderr, "\t use_vecio = %d\n", cfg->use_vecio); + + fprintf(stderr, "\n-- IO Size Config --\n"); + fprintf(stderr, "\t n_blocks = %" PRIu64 "\n", cfg->n_blocks); + fprintf(stderr, "\t block_sz = %" PRIu64 "\n", cfg->block_sz); + fprintf(stderr, "\t chunk_sz = %" PRIu64 "\n", cfg->chunk_sz); + + fprintf(stderr, "\n-- Target File --\n"); + fprintf(stderr, "\t filename = %s\n", cfg->filename); + fprintf(stderr, "\t mountpt = %s\n", cfg->mountpt); + + fprintf(stderr, "\n-- MPI Info --\n"); + fprintf(stderr, "\t app_id = %d\n", cfg->app_id); + fprintf(stderr, "\t rank = %d\n", cfg->rank); + fprintf(stderr, "\t n_ranks = %d\n", cfg->n_ranks); +} + +static inline +char* test_target_filename(test_cfg* cfg) +{ + char fname[TEST_STR_LEN]; + + assert(NULL != cfg); + + if (IO_PATTERN_N1 == cfg->io_pattern) { + snprintf(fname, sizeof(fname), "%s/%s", + cfg->mountpt, cfg->filename); + } else { + snprintf(fname, sizeof(fname), "%s/%s-%d", + cfg->mountpt, cfg->filename, cfg->rank); + } + return strdup(fname); +} + +/* ---------- Print Utilities ---------- */ + +static inline +void test_print(test_cfg* cfg, const char* fmt, ...) +{ + int err = errno; + char buf[1024]; + + assert(NULL != cfg); + + printf("[%d] ", cfg->rank); + + va_list args; + va_start(args, fmt); + vsprintf(buf, fmt, args); + printf("%s", buf); + va_end(args); + + if (err) { + printf(" (errno=%d, %s)", err, strerror(err)); + } + + /* Add in a '\n' if the line didn't end with one */ + if (buf[strlen(buf) - 1] != '\n') { + printf("\n"); + } + + fflush(stdout); +} + +static inline +void test_print_once(test_cfg* cfg, const char* fmt, ...) +{ + int err = errno; + + assert(NULL != cfg); + + if (cfg->rank != 0) { + return; + } + + va_list args; + va_start(args, fmt); + vfprintf(stdout, fmt, args); + va_end(args); + + if (err) { + printf(" (errno=%d, %s)\n", err, strerror(err)); + } + + printf("\n"); + fflush(stdout); +} + +static inline +void test_print_verbose(test_cfg* cfg, const char* fmt, ...) +{ + assert(NULL != cfg); + + if (cfg->verbose == 0) { + return; + } + + va_list args; + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); + + fprintf(stderr, "\n"); + fflush(stderr); +} + +static inline +void test_print_verbose_once(test_cfg* cfg, const char* fmt, ...) +{ + assert(NULL != cfg); + + if ((cfg->verbose == 0) || (cfg->rank != 0)) { + return; + } + + va_list args; + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); + + fprintf(stderr, "\n"); + fflush(stderr); +} + +/* ---------- Timer Utilities ---------- */ + +typedef struct { + struct timeval start; + struct timeval stop; + char* name; + double elapsed_sec; +} test_timer; + +static inline +double timediff_sec(struct timeval* before, struct timeval* after) +{ + double diff; + if (!before || !after) { + return -1.0F; + } + diff = (double)(after->tv_sec - before->tv_sec); + diff += 0.000001 * ((double)(after->tv_usec) - (double)(before->tv_usec)); + return diff; +} + +static inline +double timediff_usec(struct timeval* before, struct timeval* after) +{ + return 1000000.0 * timediff_sec(before, after); +} + +static inline +void timer_init(test_timer* timer, + const char* name) +{ + memset(timer, 0, sizeof(test_timer)); + if (NULL != name) { + timer->name = strdup(name); + } +} + +static inline +void timer_fini(test_timer* timer) +{ + if (NULL != timer->name) { + free(timer->name); + } + memset(timer, 0, sizeof(test_timer)); +} + +static inline +void timer_start(test_timer* timer) +{ + gettimeofday(&(timer->start), NULL); +} + +static inline +void timer_stop(test_timer* timer) +{ + gettimeofday(&(timer->stop), NULL); + timer->elapsed_sec = timediff_sec(&(timer->start), + &(timer->stop)); +} + +/* ---------- Option Parsing Utilities ---------- */ + +static const char* unifyfs_mntpt = "/unifyfs"; +static const char* tmp_mntpt = "/tmp"; + +static inline +int check_io_pattern(const char* pstr) +{ + int pattern = -1; + + if (strcmp(pstr, "n1") == 0) { + pattern = IO_PATTERN_N1; + } else if (strcmp(pstr, "nn") == 0) { + pattern = IO_PATTERN_NN; + } + + return pattern; +} + +static inline +int test_is_static(const char* program) +{ + char* pos = strstr(program, "-static"); + return (pos != NULL); +} + +// common options for all tests + +static const char* test_short_opts = "a:Ab:c:df:hkLm:Mn:p:PSUvVx"; + +static const struct option test_long_opts[] = { + { "appid", 1, 0, 'a' }, + { "aio", 0, 0, 'A' }, + { "blocksize", 1, 0, 'b' }, + { "chunksize", 1, 0, 'c' }, + { "debug", 0, 0, 'd' }, + { "file", 1, 0, 'f' }, + { "help", 0, 0, 'h' }, + { "check", 0, 0, 'k' }, + { "listio", 0, 0, 'L' }, + { "mount", 1, 0, 'm' }, + { "mapio", 0, 0, 'M' }, + { "nblocks", 1, 0, 'n' }, + { "pattern", 1, 0, 'p' }, + { "prdwr", 0, 0, 'P' }, + { "stdio", 0, 0, 'S' }, + { "disable-unifyfs", 0, 0, 'U' }, + { "verbose", 0, 0, 'v' }, + { "vecio", 0, 0, 'V' }, + { "shuffle", 0, 0, 'x' }, + { 0, 0, 0, 0}, +}; + +static const char* test_usage_str = + "\n" + "Usage: %s [options...]\n" + "\n" + "Available options:\n" + " -a, --appid= use given application id\n" + " (default: 0)\n" + " -A, --aio use asynchronous I/O instead of read|write\n" + " (default: off)\n" + " -b, --blocksize= I/O block size\n" + " (default: 16 MiB)\n" + " -c, --chunksize= I/O chunk size for each operation\n" + " (default: 1 MiB)\n" + " -d, --debug for debugging, wait for input (at rank 0) at start\n" + " (default: off)\n" + " -f, --file= target file name (or path) under mountpoint\n" + " (default: 'testfile')\n" + " -k, --check check data contents upon read\n" + " (default: off)\n" + " -L, --listio use lio_listio instead of read|write\n" + " (default: off)\n" + " -m, --mount= use for unifyfs\n" + " (default: /unifyfs)\n" + " -M, --mapio use mmap instead of read|write\n" + " (default: off)\n" + " -n, --nblocks= count of blocks each process will read|write\n" + " (default: 32)\n" + " -p, --pattern= 'n1' (N-to-1 shared file) or 'nn' (N-to-N file per process)\n" + " (default: 'n1')\n" + " -P, --prdwr use pread|pwrite instead of read|write\n" + " (default: off)\n" + " -S, --stdio use fread|fwrite instead of read|write\n" + " (default: off)\n" + " -U, --disable-unifyfs do not use UnifyFS\n" + " (default: enable UnifyFS)\n" + " -v, --verbose print verbose information\n" + " (default: off)\n" + " -V, --vecio use readv|writev instead of read|write\n" + " (default: off)\n" + " -x, --shuffle read different data than written\n" + " (default: off)\n" + "\n"; + +static inline +void test_print_usage(test_cfg* cfg, const char* program) +{ + test_print_once(cfg, test_usage_str, program); + exit(0); +} + +static inline +int test_process_argv(test_cfg* cfg, + int argc, char** argv) +{ + const char* program; + int ch; + + assert(NULL != cfg); + + program = basename(strdup(argv[0])); + + while ((ch = getopt_long(argc, argv, test_short_opts, + test_long_opts, NULL)) != -1) { + switch (ch) { + case 'a': + cfg->app_id = atoi(optarg); + break; + + case 'A': + cfg->use_aio = 1; + break; + + case 'b': + cfg->block_sz = (uint64_t) strtoul(optarg, NULL, 0); + break; + + case 'c': + cfg->chunk_sz = (uint64_t) strtoul(optarg, NULL, 0); + break; + + case 'd': + cfg->debug = 1; + break; + + case 'f': + cfg->filename = strdup(optarg); + break; + + case 'k': + cfg->io_check = 1; + break; + + case 'L': + cfg->use_lio = 1; + break; + + case 'm': + cfg->mountpt = strdup(optarg); + break; + + case 'M': + cfg->use_mapio = 1; + break; + + case 'n': + cfg->n_blocks = (uint64_t) strtoul(optarg, NULL, 0); + break; + + case 'p': + cfg->io_pattern = check_io_pattern(optarg); + break; + + case 'P': + cfg->use_prdwr = 1; + break; + + case 'S': + cfg->use_stdio = 1; + break; + + case 'U': + cfg->use_unifyfs = 0; + break; + + case 'v': + cfg->verbose = 1; + break; + + case 'V': + cfg->use_vecio = 1; + break; + + case 'x': + cfg->io_shuffle = 1; + break; + + case 'h': + default: + if (ch != 'h') { + test_print_once(cfg, "USAGE ERROR: unknown flag '-%c'", ch); + } + test_print_usage(cfg, program); + break; + } + } + + if (cfg->io_pattern < 0) { + test_print_once(cfg, "USAGE ERROR: pattern should be 'n1' or 'nn'"); + exit(-1); + } + + if ((cfg->block_sz < cfg->chunk_sz) || + (cfg->block_sz % cfg->chunk_sz)) { + test_print_once(cfg, "USAGE ERROR: blocksize should be larger than " + "and evenly divisible by chunksize."); + exit(-1); + } + + if (cfg->chunk_sz % 4096 > 0) { + test_print_once(cfg, "USAGE ERROR: chunksize and blocksize should be " + "divisible by 4096."); + exit(-1); + } + +#ifdef DISABLE_UNIFYFS + if (cfg->use_unifyfs) { + test_print_once(cfg, "USAGE ERROR: UnifyFS enabled, " + "but not compiled/linked"); + exit(-1); + } +#endif + + if (test_is_static(program) && !cfg->use_unifyfs) { + test_print_once(cfg, "USAGE ERROR: --disable-unifyfs only valid when " + "dynamically linked."); + exit(-1); + } + + // exhaustive check of incompatible I/O modes + if (cfg->use_aio && + (cfg->use_mapio || cfg->use_prdwr || cfg->use_stdio || cfg->use_vecio)) { + test_print_once(cfg, "USAGE ERROR: --aio incompatible with " + "[--mapio, --prdwr, --stdio, --vecio]"); + exit(-1); + } + + if (cfg->use_lio && + (cfg->use_mapio || cfg->use_prdwr || cfg->use_stdio || cfg->use_vecio)) { + test_print_once(cfg, "USAGE ERROR: --listio incompatible with " + "[--mapio, --prdwr, --stdio, --vecio]"); + exit(-1); + } + + if (cfg->use_mapio && + (cfg->use_prdwr || cfg->use_stdio || cfg->use_vecio)) { + test_print_once(cfg, "USAGE ERROR: --mapio incompatible with " + "[--aio, --listio, --prdwr, --stdio, --vecio]"); + exit(-1); + } + + if (cfg->use_prdwr && + (cfg->use_stdio || cfg->use_vecio)) { + test_print_once(cfg, "USAGE ERROR: --prdwr incompatible with " + "[--aio, --listio, --mapio, --stdio, --vecio]"); + exit(-1); + } + + if (cfg->use_stdio && cfg->use_vecio) { + test_print_once(cfg, "USAGE ERROR: --stdio incompatible with " + "[--aio, --listio, --mapio, --prdwr, --vecio]"); + exit(-1); + } + + + if (NULL == cfg->filename) { + // set filename default + cfg->filename = strdup("testfile"); + } + + if (NULL == cfg->mountpt) { + // set mountpoint default + if (cfg->use_unifyfs) { + cfg->mountpt = strdup(unifyfs_mntpt); + } else { + cfg->mountpt = strdup(tmp_mntpt); + } + } + + return 0; +} + +/* ---------- Data Utilities ---------- */ + +/* + * sequentially number every 8 bytes (uint64_t) + */ +static inline +void lipsum_generate(char* buf, uint64_t len, uint64_t offset) +{ + uint64_t i; + uint64_t start = offset / sizeof(uint64_t); + uint64_t count = len / sizeof(uint64_t); + uint64_t* ibuf = (uint64_t*) buf; + + for (i = 0; i < count; i++) { + ibuf[i] = start + i; + } +} + +/* + * check buffer contains lipsum generated data + * returns 0 on success, -1 otherwise with @error_offset set. + */ +static inline +int lipsum_check(const char* buf, uint64_t len, uint64_t offset, + uint64_t* error_offset) +{ + uint64_t i, val; + uint64_t start = offset / sizeof(uint64_t); + uint64_t count = len / sizeof(uint64_t); + const uint64_t* ibuf = (uint64_t*) buf; + + for (i = 0; i < count; i++) { + val = start + i; + if (ibuf[i] != val) { + *error_offset = offset + (i * sizeof(uint64_t)); + fprintf(stderr, "DEBUG: [%" PRIu64 "] @ offset %" PRIu64 + ", expected=%" PRIu64 " found=%" PRIu64 "\n", + i, *error_offset, val, ibuf[i]); + return -1; + } + } + + return 0; +} + +/* ---------- MPI Utilities ---------- */ + +static inline +void test_barrier(test_cfg* cfg) +{ + assert(NULL != cfg); + + if (cfg->use_mpi) { + MPI_Barrier(MPI_COMM_WORLD); + } +} + +static inline +double test_reduce_double_sum(test_cfg* cfg, double local_val) +{ + double aggr_val = 0.0; + + assert(NULL != cfg); + + if (cfg->use_mpi) { + MPI_Reduce(&local_val, &aggr_val, + 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD); + } else { + aggr_val = local_val; + } + return aggr_val; +} + +static inline +double test_reduce_double_max(test_cfg* cfg, double local_val) +{ + double aggr_val = 0.0; + + assert(NULL != cfg); + + if (cfg->use_mpi) { + MPI_Reduce(&local_val, &aggr_val, + 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); + } else { + aggr_val = local_val; + } + return aggr_val; +} + +static inline +double test_reduce_double_min(test_cfg* cfg, double local_val) +{ + double aggr_val = 0.0; + + assert(NULL != cfg); + + if (cfg->use_mpi) { + MPI_Reduce(&local_val, &aggr_val, + 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD); + } else { + aggr_val = local_val; + } + return aggr_val; +} + +/* ---------- Program Utilities ---------- */ +static +int test_access_to_mmap_prot(int access) +{ + switch (access) { + case O_RDWR: + return (PROT_READ | PROT_WRITE); + case O_RDONLY: + return PROT_READ; + case O_WRONLY: + return PROT_WRITE; + default: + break; + } + return PROT_NONE; +} + +static +const char* test_access_to_stdio_mode(int access) +{ + switch (access) { + case O_RDWR: + return "w+"; + case O_RDONLY: + return "r"; + case O_WRONLY: + return "w"; + default: + break; + } + return NULL; +} + +/* + * open the given file + */ +static inline +int test_open_file(test_cfg* cfg, const char* filepath, int access) +{ + FILE* fp = NULL; + const char* fmode; + int fd = -1; + + assert(NULL != cfg); + + if (cfg->use_stdio) { + fmode = test_access_to_stdio_mode(access); + fp = fopen(filepath, fmode); + if (NULL == fp) { + test_print(cfg, "ERROR: fopen(%s) failed", filepath); + return -1; + } + cfg->fp = fp; + return 0; + } + + fd = open(filepath, access); + if (-1 == fd) { + test_print(cfg, "ERROR: open(%s) failed", filepath); + return -1; + } + cfg->fd = fd; + cfg->fd_access = access; + return 0; +} + +/* + * close the given file + */ +static inline +int test_close_file(test_cfg* cfg) +{ + assert(NULL != cfg); + + if (NULL != cfg->fp) { + fclose(cfg->fp); + } + + if (NULL != cfg->mapped) { + munmap(cfg->mapped, cfg->mapped_sz); + } + + if (-1 != cfg->fd) { + close(cfg->fd); + } + return 0; +} + +/* + * create file at rank 0, open elsewhere + */ +static inline +int test_create_file(test_cfg* cfg, const char* filepath, int access) +{ + FILE* fp = NULL; + const char* fmode; + int fd = -1; + int create_flags = O_CREAT | O_TRUNC; + int create_mode = 0600; // S_IRUSR | S_IWUSR + + assert(NULL != cfg); + + if (cfg->use_stdio) { + fmode = test_access_to_stdio_mode(access); + } + + // rank 0 creates or all ranks create if using file-per-process + if (cfg->rank == 0 || cfg->io_pattern == IO_PATTERN_NN) { + if (cfg->use_stdio) { + fp = fopen(filepath, fmode); + if (NULL == fp) { + test_print(cfg, "ERROR: fopen(%s) failed", filepath); + return -1; + } + cfg->fp = fp; + } else { + fd = open(filepath, access | create_flags, create_mode); + if (-1 == fd) { + test_print(cfg, "ERROR: open(%s, CREAT) failed", filepath); + return -1; + } + cfg->fd = fd; + cfg->fd_access = access; + } + } + + if (cfg->io_pattern == IO_PATTERN_N1) { + // barrier enforces create before open + test_barrier(cfg); + + // other ranks just do normal open + if (cfg->rank != 0) { + return test_open_file(cfg, filepath, access); + } + } + return 0; +} + + +/* + * map a segment of an already open file + */ +static inline +int test_map_file(test_cfg* cfg, off_t off, size_t len) +{ + void* addr; + int prot; + + assert(NULL != cfg); + + if (!cfg->use_mapio) { + test_print(cfg, "ERROR: cfg.use_mapio is not enabled"); + return -1; + } else if (-1 == cfg->fd) { + test_print(cfg, "ERROR: cfg.fd is invalid, must create/open first"); + return -1; + } + + prot = test_access_to_mmap_prot(cfg->fd_access); + addr = mmap(NULL, len, prot, MAP_SHARED, cfg->fd, off); + if (MAP_FAILED == addr) { + test_print(cfg, "ERROR: mmap() failed"); + return -1; + } + cfg->mapped = addr; + cfg->mapped_sz = len; + cfg->mapped_off = off; + return 0; +} + +/* + * wait for input at rank 0, barrier everywhere + */ +static inline +void test_pause(test_cfg* cfg, const char* fmt, ...) +{ + assert(NULL != cfg); + + if (cfg->rank == 0) { + va_list args; + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); + + fprintf(stderr, "press ENTER to continue ... "); + (void) getchar(); + } + test_barrier(cfg); +} + +/* + * abort the test everywhere + */ +static inline +void test_abort(test_cfg* cfg, int rc) +{ + assert(NULL != cfg); + + if (cfg->use_mpi) { + MPI_Abort(MPI_COMM_WORLD, rc); + } else { + exit(rc); + } +} + +/* + * initialize test using argv to set config + */ +static inline +int test_init(int argc, char** argv, + test_cfg* cfg) +{ + int rc; + + if (NULL == cfg) { + fprintf(stderr, "INTERNAL ERROR: %s() : cfg is NULL\n", __func__); + return -1; + } + + test_config_init(cfg); + + rc = test_process_argv(cfg, argc, argv); + if (rc) { + fprintf(stderr, "ERROR: failed to process command-line args"); + return rc; + } + + if (cfg->use_mpi) { + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &(cfg->n_ranks)); + MPI_Comm_rank(MPI_COMM_WORLD, &(cfg->rank)); + } else { + cfg->n_ranks = 1; + } + + if (cfg->verbose) { + // must come after test_mpi_init() to pick up MPI info + test_config_print(cfg); + } + + if (cfg->use_unifyfs) { +#ifndef DISABLE_UNIFYFS + if (cfg->debug) { + test_pause(cfg, "Before unifyfs_mount()"); + } + rc = unifyfs_mount(cfg->mountpt, cfg->rank, cfg->n_ranks, cfg->app_id); + if (rc) { + test_print(cfg, "ERROR: unifyfs_mount() failed (rc=%d)", rc); + test_abort(cfg, rc); + } +#endif + test_barrier(cfg); + } else { + if (cfg->debug) { + test_pause(cfg, "Finished test initialization"); + } + } + + return 0; +} + +/* + * finalize test and free allocated config state + */ +static inline +void test_fini(test_cfg* cfg) +{ + if (NULL == cfg) { + fprintf(stderr, "INTERNAL ERROR: %s() : cfg is NULL\n", __func__); + return; + } + + test_close_file(cfg); + + if (cfg->use_unifyfs) { +#ifndef DISABLE_UNIFYFS + int rc = unifyfs_unmount(); + if (rc) { + test_print(cfg, "ERROR: unifyfs_unmount() failed (rc=%d)", rc); + } +#endif + } + + if (cfg->use_mpi) { + MPI_Finalize(); + } + + if (NULL != cfg->filename) { + free(cfg->filename); + } + + if (NULL != cfg->mountpt) { + free(cfg->mountpt); + } + + memset(cfg, 0, sizeof(test_cfg)); +} + +/* + * Various C equivalents of bash commands (dd, test, mktemp, du, stat, ...) + */ +int dd_cmd(test_cfg* cfg, char* infile, char* outfile, unsigned long bs, + unsigned long count, unsigned long seek); +int test_cmd(char* expression, char* path); +char* mktemp_cmd(test_cfg* cfg, char* tmpdir); +long du_cmd(test_cfg* cfg, char* filename, int apparent_size); +int sync_cmd(test_cfg* cfg, char* filename); +int stat_cmd(test_cfg* cfg, char* filename); + +#endif /* UNIFYFS_TEST_UTIL_H */ diff --git a/examples/src/testutil_rdwr.h b/examples/src/testutil_rdwr.h new file mode 100644 index 000000000..1ed21813f --- /dev/null +++ b/examples/src/testutil_rdwr.h @@ -0,0 +1,419 @@ +/* + * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2019, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#ifndef UNIFYFS_TESTUTIL_RDWR_H +#define UNIFYFS_TESTUTIL_RDWR_H + +/* -------- Write Helper Methods -------- */ + +static inline +int issue_write_req(test_cfg* cfg, struct aiocb* req) +{ + int rc, err; + ssize_t ss; + size_t written, remaining; + off_t off; + void* src; + + assert(NULL != cfg); + + errno = 0; + if (cfg->use_aio) { // aio_write(2) + rc = aio_write(req); + if (-1 == rc) { + test_print(cfg, "aio_write() failed"); + } + return rc; + } else if (cfg->use_mapio) { // mmap(2) + return ENOTSUP; + } else if (cfg->use_prdwr) { // pwrite(2) + written = 0; + remaining = req->aio_nbytes; + do { + src = (void*)((char*)req->aio_buf + written); + ss = pwrite(req->aio_fildes, src, remaining, + (req->aio_offset + written)); + if (-1 == ss) { + err = errno; + if ((EINTR == err) || (EAGAIN == err)) { + continue; + } + test_print(cfg, "pwrite() failed"); + return -1; + } + written += (size_t)ss; + remaining -= (size_t)ss; + } while (remaining); + } else if (cfg->use_stdio) { // fwrite(3) + return ENOTSUP; + } else if (cfg->use_vecio) { // writev(2) + return EINVAL; + } else { // write(2) + written = 0; + remaining = req->aio_nbytes; + off = lseek(req->aio_fildes, req->aio_offset, SEEK_SET); + if (-1 == off) { + test_print(cfg, "lseek() failed"); + return -1; + } + do { + src = (void*)((char*)req->aio_buf + written); + ss = write(req->aio_fildes, src, remaining); + if (-1 == ss) { + err = errno; + if ((EINTR == err) || (EAGAIN == err)) { + continue; + } + test_print(cfg, "write() failed"); + return -1; + } + written += (size_t)ss; + remaining -= (size_t)ss; + } while (remaining); + } + return 0; +} + +static inline +int issue_write_req_batch(test_cfg* cfg, size_t n_reqs, struct aiocb* reqs) +{ + int rc, ret, lio_mode; + size_t i; + + assert(NULL != cfg); + + if (cfg->use_lio) { // lio_listio(2) + struct aiocb* lio_vec[n_reqs]; + for (i = 0; i < n_reqs; i++) { + lio_vec[i] = reqs + i; + } + lio_mode = LIO_WAIT; + if (cfg->use_aio) { + lio_mode = LIO_NOWAIT; + } + errno = 0; + rc = lio_listio(lio_mode, lio_vec, (int)n_reqs, NULL); + if (-1 == rc) { + test_print(cfg, "lio_listio() failed"); + } + return rc; + } else if (cfg->use_mapio) { // mmap(2) + return ENOTSUP; + } else if (cfg->use_vecio) { // writev(2) + return ENOTSUP; + } else { + ret = 0; + for (i = 0; i < n_reqs; i++) { + rc = issue_write_req(cfg, reqs + i); + if (rc) { + test_print(cfg, "write req %zu failed", i); + ret = -1; + } + } + return ret; + } +} + +static inline +int wait_write_req(test_cfg* cfg, struct aiocb* req) +{ + assert(NULL != cfg); + + if (cfg->use_aio) { + ssize_t ss; + const struct aiocb* creq = (const struct aiocb*)req; + int rc = aio_suspend(&creq, 1, NULL); + if (-1 == rc) { + test_print(cfg, "aio_suspend() failed"); + return rc; + } + do { + rc = aio_error(creq); + if (EINPROGRESS == rc) { + continue; + } + if (rc) { + errno = rc; + test_print(cfg, "aio_error() reported async write failure"); + } + break; + } while (1); + ss = aio_return(req); + if (ss != req->aio_nbytes) { + test_print(cfg, "partial async write (%zd of %zu)", + ss, req->aio_nbytes); + return -1; + } + } + return 0; +} + +static inline +int wait_write_req_batch(test_cfg* cfg, size_t n_reqs, struct aiocb* reqs) +{ + int rc, ret = 0; + size_t i; + + assert(NULL != cfg); + + for (i = 0; i < n_reqs; i++) { + rc = wait_write_req(cfg, reqs + i); + if (rc) { + test_print(cfg, "write req %zu failed", i); + ret = -1; + } + } + return ret; +} + +static inline +int write_sync(test_cfg* cfg) +{ + int rc; + + assert(NULL != cfg); + + if (NULL != cfg->fp) { // fflush(3) + rc = fflush(cfg->fp); + if (-1 == rc) { + test_print(cfg, "fflush() failed"); + return -1; + } + } else if (NULL != cfg->mapped) { // msync(2) + rc = msync(cfg->mapped, cfg->mapped_sz, MS_SYNC); + if (-1 == rc) { + test_print(cfg, "msync() failed"); + return -1; + } + } else if (-1 != cfg->fd) { // fsync(2) + rc = fsync(cfg->fd); + if (-1 == rc) { + test_print(cfg, "fsync() failed"); + return -1; + } + } + return 0; +} + +/* -------- Read Helper Methods -------- */ + +static inline +int issue_read_req(test_cfg* cfg, struct aiocb* req) +{ + int rc, err; + ssize_t ss; + size_t nread, remaining; + off_t off; + void* dst; + + assert(NULL != cfg); + + errno = 0; + if (cfg->use_aio) { // aio_read(2) + rc = aio_read(req); + if (-1 == rc) { + test_print(cfg, "aio_read() failed"); + } + return rc; + } else if (cfg->use_mapio) { // mmap(2) + return ENOTSUP; + } else if (cfg->use_prdwr) { // pread(2) + nread = 0; + remaining = req->aio_nbytes; + do { + dst = (void*)((char*)req->aio_buf + nread); + ss = pread(req->aio_fildes, dst, remaining, + (req->aio_offset + nread)); + if (-1 == ss) { + err = errno; + if ((EINTR == err) || (EAGAIN == err)) { + continue; + } + test_print(cfg, "pread() failed"); + return -1; + } else if (0 == ss) { + test_print(cfg, "pread() EOF"); + return -1; + } + nread += (size_t)ss; + remaining -= (size_t)ss; + } while (remaining); + } else if (cfg->use_stdio) { // fread(3) + return ENOTSUP; + } else if (cfg->use_vecio) { // readv(2) + return EINVAL; + } else { // read(2) + nread = 0; + remaining = req->aio_nbytes; + off = lseek(req->aio_fildes, req->aio_offset, SEEK_SET); + if (-1 == off) { + test_print(cfg, "lseek() failed"); + return -1; + } + do { + dst = (void*)((char*)req->aio_buf + nread); + ss = read(req->aio_fildes, dst, remaining); + if (-1 == ss) { + err = errno; + if ((EINTR == err) || (EAGAIN == err)) { + continue; + } + test_print(cfg, "read() failed"); + return -1; + } else if (0 == ss) { + test_print(cfg, "read() EOF"); + return -1; + } + nread += (size_t)ss; + remaining -= (size_t)ss; + } while (remaining); + } + return 0; +} + +static inline +int issue_read_req_batch(test_cfg* cfg, size_t n_reqs, struct aiocb* reqs) +{ + int rc, ret, lio_mode; + size_t i; + + assert(NULL != cfg); + + if (cfg->use_lio) { // lio_listio(2) + struct aiocb* lio_vec[n_reqs]; + for (i = 0; i < n_reqs; i++) { + lio_vec[i] = reqs + i; + } + lio_mode = LIO_WAIT; + if (cfg->use_aio) { + lio_mode = LIO_NOWAIT; + } + errno = 0; + rc = lio_listio(lio_mode, lio_vec, (int)n_reqs, NULL); + if (-1 == rc) { + test_print(cfg, "lio_listio() failed"); + } + return rc; + } else if (cfg->use_mapio) { // mmap(2) + return ENOTSUP; + } else if (cfg->use_vecio) { // readv(2) + return ENOTSUP; + } else { + ret = 0; + for (i = 0; i < n_reqs; i++) { + rc = issue_read_req(cfg, reqs + i); + if (rc) { + test_print(cfg, "read req %zu failed", i); + ret = -1; + } + } + return ret; + } +} + +static inline +int wait_read_req(test_cfg* cfg, struct aiocb* req) +{ + assert(NULL != cfg); + + if (cfg->use_aio) { + ssize_t ss; + const struct aiocb* creq = (const struct aiocb*)req; + int rc = aio_suspend(&creq, 1, NULL); + if (-1 == rc) { + test_print(cfg, "aio_suspend() failed"); + return rc; + } + do { + rc = aio_error(creq); + if (EINPROGRESS == rc) { + continue; + } + if (rc) { + errno = rc; + test_print(cfg, "aio_error() reported async read failure"); + } + break; + } while (1); + ss = aio_return(req); + if (ss != req->aio_nbytes) { + test_print(cfg, "partial async read (%zd of %zu)", + ss, req->aio_nbytes); + return -1; + } + } + return 0; +} + +static inline +int wait_read_req_batch(test_cfg* cfg, size_t n_reqs, struct aiocb* reqs) +{ + int rc, ret = 0; + size_t i; + + assert(NULL != cfg); + + for (i = 0; i < n_reqs; i++) { + rc = wait_read_req(cfg, reqs + i); + if (rc) { + test_print(cfg, "read req %zu failed", i); + ret = -1; + } + } + return ret; +} + +static inline +int check_read_req(test_cfg* cfg, struct aiocb* req) +{ + int ret = 0; + + assert(NULL != cfg); + + if (cfg->io_check) { + uint64_t error_offset = 0; + uint64_t len = (uint64_t) req->aio_nbytes; + uint64_t off = (uint64_t) req->aio_offset; + int rc = lipsum_check((const char*)req->aio_buf, len, off, + &error_offset); + if (-1 == rc) { + test_print(cfg, "data check failed at offset %" PRIu64, + error_offset); + ret = -1; + } + } + return ret; +} + +static inline +int check_read_req_batch(test_cfg* cfg, size_t n_reqs, struct aiocb* reqs) +{ + int ret = 0; + + assert(NULL != cfg); + + if (cfg->io_check) { + size_t i; + for (i = 0; i < n_reqs; i++) { + int rc = check_read_req(cfg, reqs + i); + if (rc) { + test_print(cfg, "read req %zu data check failed", i); + ret = -1; + } + } + } + return ret; +} + +#endif /* UNIFYFS_TESTUTIL_RDWR_H */ diff --git a/examples/src/transfer.c b/examples/src/transfer.c new file mode 100644 index 000000000..a01e36702 --- /dev/null +++ b/examples/src/transfer.c @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2017, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "testlib.h" + +static int rank; +static int total_ranks; +static int rank_worker; +static int parallel; +static int debug; + +static char* mountpoint = "/unifyfs"; /* unifyfs mountpoint */ +static int unmount; /* unmount unifyfs after running the test */ + +static char* srcpath; +static char* dstpath; + +static unsigned long bufsize = 64 * (1 << 10); + +static struct option long_opts[] = { + { "debug", 0, 0, 'd' }, + { "help", 0, 0, 'h' }, + { "mount", 1, 0, 'm' }, + { "parallel", 0, 0, 'p' }, + { "rank", 1, 0, 'r' }, + { "unmount", 0, 0, 'u' }, + { 0, 0, 0, 0}, +}; + +static char* short_opts = "dhm:pr:u"; + +static const char* usage_str = + "\n" + "Usage: %s [options...] \n" + "\n" + "Available options:\n" + " -d, --debug pause before running test\n" + " (handy for attaching in debugger)\n" + " -h, --help help message\n" + " -m, --mount= use for unifyfs\n" + " (default: /unifyfs)\n" + " -p, --parallel parallel transfer\n" + " -r, --rank= use for transfer (default: 0)\n" + " -u, --unmount unmount the filesystem after test\n" + "\n"; + +static char* program; + +static void print_usage(void) +{ + test_print_once(rank, usage_str, program); + exit(0); +} + +int main(int argc, char** argv) +{ + int ret = 0; + int ch = 0; + int optidx = 0; + struct stat sb = { 0, }; + + program = basename(strdup(argv[0])); + + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &total_ranks); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + while ((ch = getopt_long(argc, argv, + short_opts, long_opts, &optidx)) >= 0) { + switch (ch) { + case 'b': + bufsize = strtoul(optarg, NULL, 0); + break; + + case 'd': + debug = 1; + break; + + case 'm': + mountpoint = strdup(optarg); + break; + + case 'p': + parallel = 1; + break; + + case 'r': + rank_worker = atoi(optarg); + break; + + case 'u': + unmount = 1; + break; + + case 'h': + default: + print_usage(); + break; + } + } + + if (argc - optind != 2) { + print_usage(); + } + + srcpath = strdup(argv[optind++]); + dstpath = strdup(argv[optind++]); + + if (srcpath[strlen(srcpath) - 1] == '/') { + srcpath[strlen(srcpath) - 1] = '\0'; + } + + if (debug) { + test_pause(rank, "Attempting to mount"); + } + + ret = unifyfs_mount(mountpoint, rank, total_ranks, 0); + if (ret) { + test_print(rank, "unifyfs_mount failed (return = %d)", ret); + goto out; + } + + if (parallel) { + ret = unifyfs_transfer_file_parallel(srcpath, dstpath); + if (ret) { + test_print(rank, "copy failed (%d: %s)", ret, strerror(ret)); + } + } else { + if (rank_worker >= total_ranks) { + test_print(rank, "%d is not a valid rank"); + goto out; + } + + MPI_Barrier(MPI_COMM_WORLD); + + if (rank == rank_worker) { + ret = unifyfs_transfer_file_serial(srcpath, dstpath); + if (ret) { + test_print(rank, "copy failed (%d: %s)", ret, strerror(ret)); + } + } + } + + free(dstpath); + free(srcpath); + + MPI_Barrier(MPI_COMM_WORLD); + + if (unmount) { + unifyfs_unmount(); + } + +out: + MPI_Finalize(); + + return ret; +} + diff --git a/examples/src/write.c b/examples/src/write.c new file mode 100644 index 000000000..5e8e794f2 --- /dev/null +++ b/examples/src/write.c @@ -0,0 +1,276 @@ +/* + * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2019, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#include "testutil.h" +#include "testutil_rdwr.h" + +// generate N-to-1 or N-to-N writes according to test config +size_t generate_write_reqs(test_cfg* cfg, char* srcbuf, + struct aiocb** reqs_out) +{ + off_t blk_off, chk_off; + size_t i, j, ndx = 0; + size_t blk_sz = cfg->block_sz; + size_t tran_sz = cfg->chunk_sz; + size_t n_tran_per_blk = blk_sz / tran_sz; + + size_t num_reqs = cfg->n_blocks * n_tran_per_blk; + struct aiocb* req; + struct aiocb* reqs = calloc(num_reqs, sizeof(struct aiocb)); + if (NULL == reqs) { + *reqs_out = NULL; + return 0; + } + + if (!cfg->io_check) { + // fill srcbuf with unique character per rank + int byte = (int)'0' + cfg->rank; + memset(srcbuf, byte, tran_sz); + } + + req = reqs; + for (i = 0; i < cfg->n_blocks; i++) { + if (IO_PATTERN_N1 == cfg->io_pattern) { + // interleaved block writes + blk_off = (i * blk_sz * cfg->n_ranks) + + (cfg->rank * blk_sz); + } else { // IO_PATTERN_NN + // blocked writes + blk_off = (i * blk_sz); + } + + if (cfg->io_check) { + // generate lipsum in source block + lipsum_generate((srcbuf + ndx), blk_sz, blk_off); + } + + for (j = 0; j < n_tran_per_blk; j++) { + chk_off = blk_off + (j * tran_sz); + + req->aio_fildes = cfg->fd; + req->aio_buf = (void*)(srcbuf + ndx); + req->aio_nbytes = tran_sz; + req->aio_offset = chk_off; + req->aio_lio_opcode = LIO_WRITE; + + req++; + ndx += tran_sz; + } + } + + *reqs_out = reqs; + return num_reqs; +} + +/* -------- Main Program -------- */ + +/* Description: + * + * [ Mode 1: N-to-1 shared-file ] + * Each rank writes cfg.n_blocks blocks of size cfg.block_sz to the + * shared file, using I/O operation sizes of cfg.chunk_sz. Blocks are + * rank-interleaved (i.e., the block at offset 0 is written by rank 0, + * the block at offset cfg.block_sz is written by rank 1, and so on). + * After writing all blocks, the written data is synced (laminated). + * Rank 0 then checks that the file size is as expected. + * + * [ Mode 2: N-to-N file-per-process ] + * Each rank writes cfg.n_blocks blocks of size cfg.block_sz using + * I/O operation sizes of cfg.chunk_sz. After writing all blocks, + * the written data is synced (laminated). Each rank then checks that + * the file size is as expected. + * + * [ Options for Both Modes ] + * cfg.use_aio - when enabled, aio(7) will be used for issuing and + * completion of writes. + * + * cfg.use_lio - when enabled, lio_listio(3) will be used for batching + * writes. When cfg.use_aio is also enabled, the mode will be + * LIO_NOWAIT. + * + * cfg.use_mapio - support is not yet implemented. When enabled, + * direct memory loads and stores will be used for writes. + * + * cfg.use_prdwr - when enabled, pwrite(2) will be used. + * + * cfg.use_stdio - when enabled, fwrite(2) will be used. + * + * cfg.use_vecio - support is not yet implemented. When enabled, + * writev(2) will be used for batching writes. + * + * cfg.io_check - when enabled, lipsum data is used when writing + * the file. + * + * cfg.io_shuffle - ignored. + */ + +int main(int argc, char* argv[]) +{ + char* wr_buf; + char* target_file; + struct aiocb* reqs; + size_t num_reqs = 0; + int rc; + + test_cfg test_config; + test_cfg* cfg = &test_config; + test_timer time_wr; + test_timer time_sync; + + timer_init(&time_wr, "write"); + timer_init(&time_sync, "sync"); + + rc = test_init(argc, argv, cfg); + if (rc) { + fprintf(stderr, "ERROR - Test %s initialization failed!", + argv[0]); + fflush(NULL); + return rc; + } + + if (!test_config.use_mpi) { + fprintf(stderr, "ERROR - Test %s requires MPI!", + argv[0]); + fflush(NULL); + return -1; + } + + target_file = test_target_filename(cfg); + test_print_verbose_once(cfg, "DEBUG: creating target file %s", + target_file); + rc = test_create_file(cfg, target_file, O_RDWR); + if (rc) { + test_abort(cfg, rc); + } + + // generate write requests + test_print_verbose_once(cfg, "DEBUG: generating write requests"); + wr_buf = calloc(test_config.n_blocks, test_config.block_sz); + if (NULL == wr_buf) { + test_abort(cfg, ENOMEM); + } + num_reqs = generate_write_reqs(cfg, wr_buf, &reqs); + if (0 == num_reqs) { + test_abort(cfg, ENOMEM); + } + + // do writes + test_print_verbose_once(cfg, "DEBUG: starting write requests"); + test_barrier(cfg); + timer_start(&time_wr); + rc = issue_write_req_batch(cfg, num_reqs, reqs); + if (rc) { + test_abort(cfg, rc); + } + rc = wait_write_req_batch(cfg, num_reqs, reqs); + if (rc) { + test_abort(cfg, rc); + } + timer_stop(&time_wr); + test_print_verbose_once(cfg, "DEBUG: finished write requests"); + + // sync/laminate + timer_start(&time_sync); + rc = write_sync(cfg); + if (rc) { + test_abort(cfg, rc); + } + timer_stop(&time_sync); + test_barrier(cfg); + test_print_verbose_once(cfg, "DEBUG: finished sync"); + + if ((test_config.rank == 0) || + (IO_PATTERN_NN == test_config.io_pattern)) { + /* laminate by removing write bits */ + chmod(target_file, 0400); + } + test_print_verbose_once(cfg, "DEBUG: finished lamination"); + + // post-write cleanup + free(wr_buf); + free(reqs); + reqs = NULL; + + // close file + rc = test_close_file(cfg); + if (rc) { + test_abort(cfg, rc); + } + + // file size check + size_t rank_bytes = test_config.n_blocks * test_config.block_sz; + size_t total_bytes = rank_bytes * test_config.n_ranks; + size_t expected = total_bytes; + if (IO_PATTERN_NN == test_config.io_pattern) { + expected = rank_bytes; + } + if ((test_config.rank == 0) || + (IO_PATTERN_NN == test_config.io_pattern)) { + struct stat s; + rc = stat(target_file, &s); + if (-1 == rc) { + test_print(cfg, "ERROR - stat(%s) failed", target_file); + } else { + if (s.st_size != expected) { + test_print(cfg, "ERROR - file size check failed - " + "actual size is %zu B, expected %zu B", + s.st_size, expected); + } + } + } + + // calculate achieved bandwidth rates + double max_write_time, max_sync_time; + double write_bw, aggr_write_bw, eff_write_bw; + + max_write_time = test_reduce_double_max(cfg, time_wr.elapsed_sec); + max_sync_time = test_reduce_double_max(cfg, time_sync.elapsed_sec); + + write_bw = bandwidth_mib(rank_bytes, time_wr.elapsed_sec); + aggr_write_bw = test_reduce_double_sum(cfg, write_bw); + eff_write_bw = bandwidth_mib(total_bytes, max_write_time); + + test_print_once(cfg, + "\n" + "I/O pattern: %s\n" + "I/O block size: %.2lf KiB\n" + "I/O request size: %.2lf KiB\n" + "Number of processes: %d\n" + "Each process wrote: %.2lf MiB\n" + "Total data written: %.2lf MiB\n" + "Maximum write time: %.6lf sec\n" + "Maximum sync time: %.6lf sec\n" + "Aggregate write bandwidth: %.3lf MiB/s\n" + "Effective write bandwidth: %.3lf MiB/s\n", + io_pattern_str(test_config.io_pattern), + bytes_to_kib(test_config.block_sz), + bytes_to_kib(test_config.chunk_sz), + test_config.n_ranks, + bytes_to_mib(rank_bytes), + bytes_to_mib(total_bytes), + max_write_time, + max_sync_time, + aggr_write_bw, + eff_write_bw); + + // cleanup + free(target_file); + + timer_fini(&time_wr); + timer_fini(&time_sync); + + test_fini(cfg); + + return 0; +} diff --git a/examples/src/writeread.c b/examples/src/writeread.c new file mode 100644 index 000000000..c104a5ee9 --- /dev/null +++ b/examples/src/writeread.c @@ -0,0 +1,335 @@ +/* + * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2019, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#include "testutil.h" +#include "testutil_rdwr.h" + +// generate N-to-1 or N-to-N writes according to test config +size_t generate_write_reqs(test_cfg* cfg, char* srcbuf, + struct aiocb** reqs_out) +{ + off_t blk_off, chk_off; + size_t i, j, ndx = 0; + size_t blk_sz = cfg->block_sz; + size_t tran_sz = cfg->chunk_sz; + size_t n_tran_per_blk = blk_sz / tran_sz; + int rankbyte = (int)'0' + cfg->rank; + + size_t num_reqs = cfg->n_blocks * n_tran_per_blk; + struct aiocb* req; + struct aiocb* reqs = calloc(num_reqs, sizeof(struct aiocb)); + if (NULL == reqs) { + *reqs_out = NULL; + return 0; + } + + req = reqs; + for (i = 0; i < cfg->n_blocks; i++) { + if (IO_PATTERN_N1 == cfg->io_pattern) { + // interleaved block writes + blk_off = (i * blk_sz * cfg->n_ranks) + + (cfg->rank * blk_sz); + } else { // IO_PATTERN_NN + // blocked writes + blk_off = (i * blk_sz); + } + + if (cfg->io_check) { + // generate lipsum in source block + lipsum_generate((srcbuf + ndx), blk_sz, blk_off); + } else { + // fill srcbuf with unique rank character + memset((srcbuf + ndx), rankbyte, blk_sz); + } + + for (j = 0; j < n_tran_per_blk; j++) { + chk_off = blk_off + (j * tran_sz); + + req->aio_fildes = cfg->fd; + req->aio_buf = (void*)(srcbuf + ndx); + req->aio_nbytes = tran_sz; + req->aio_offset = chk_off; + req->aio_lio_opcode = LIO_WRITE; + + req++; + ndx += tran_sz; + } + } + + *reqs_out = reqs; + return num_reqs; +} + +// generate N-to-1 or N-to-N reads according to test config +size_t generate_read_reqs(test_cfg* cfg, char* dstbuf, + struct aiocb** reqs_out) +{ + int read_rank = cfg->rank; + off_t blk_off, chk_off; + size_t i, j, ndx = 0; + size_t blk_sz = cfg->block_sz; + size_t tran_sz = cfg->chunk_sz; + size_t n_tran_per_blk = blk_sz / tran_sz; + + size_t num_reqs = cfg->n_blocks * n_tran_per_blk; + struct aiocb* req; + struct aiocb* reqs = calloc(num_reqs, sizeof(struct aiocb)); + if (NULL == reqs) { + *reqs_out = NULL; + return 0; + } + + if (cfg->io_shuffle) { + // 0 reads data written by N-1, N-1 reads from 0, etc. + read_rank = (cfg->n_ranks - 1) - cfg->rank; + } + + req = reqs; + for (i = 0; i < cfg->n_blocks; i++) { + if (IO_PATTERN_N1 == cfg->io_pattern) { + // interleaved block writes + blk_off = (i * blk_sz * cfg->n_ranks) + + (read_rank * blk_sz); + } else { // IO_PATTERN_NN + // blocked writes + blk_off = (i * blk_sz); + } + + for (j = 0; j < n_tran_per_blk; j++) { + chk_off = blk_off + (j * tran_sz); + + req->aio_fildes = cfg->fd; + req->aio_buf = (void*)(dstbuf + ndx); + req->aio_nbytes = tran_sz; + req->aio_offset = chk_off; + req->aio_lio_opcode = LIO_READ; + + req++; + ndx += tran_sz; + } + } + + *reqs_out = reqs; + return num_reqs; +} + + +/* -------- Main Program -------- */ + +/* Description: + * + * [ Mode 1: N-to-1 shared-file ] + * Each rank writes cfg.n_blocks blocks of size cfg.block_sz to the + * shared file, using I/O operation sizes of cfg.chunk_sz. Blocks are + * rank-interleaved (i.e., the block at offset 0 is written by rank 0, + * the block at offset cfg.block_sz is written by rank 1, and so on). + * After writing all blocks, the written data is synced (laminated). + * Each rank then reads back the written blocks using I/O operation + * sizes of cfg.chunk_sz. If cfg.io_shuffle is enabled, each rank will + * read different blocks than it wrote. + * + * [ Mode 2: N-to-N file-per-process ] + * Each rank writes cfg.n_blocks blocks of size cfg.block_sz using + * I/O operation sizes of cfg.chunk_sz. The cfg.io_shuffle option is + * ignored. + * + * [ Options for Both Modes ] + * cfg.use_aio - when enabled, aio(7) will be used for issuing and + * completion of reads and writes. + * + * cfg.use_lio - when enabled, lio_listio(3) will be used for batching + * reads and writes. When cfg.use_aio is also enabled, the mode will + * be LIO_NOWAIT. + * + * cfg.use_mapio - support is not yet implemented. When enabled, + * direct memory loads and stores will be used for reads and writes. + * + * cfg.use_prdwr - when enabled, pread(2) and pwrite(2) will be used. + * + * cfg.use_stdio - when enabled, fread(2) and fwrite(2) will be used. + * + * cfg.use_vecio - support is not yet implemented. When enabled, + * readv(2) and writev(2) will be used for batching reads and writes. + * + * cfg.io_check - when enabled, lipsum data is used when writing + * the file and verified when reading. + */ + +int main(int argc, char* argv[]) +{ + char* wr_buf; + char* rd_buf; + char* target_file; + struct aiocb* reqs; + size_t num_reqs = 0; + int rc; + + test_cfg test_config; + test_cfg* cfg = &test_config; + test_timer time_wr; + test_timer time_rd; + test_timer time_sync; + + timer_init(&time_wr, "write"); + timer_init(&time_rd, "read"); + timer_init(&time_sync, "sync"); + + rc = test_init(argc, argv, cfg); + if (rc) { + fprintf(stderr, "ERROR - Test %s initialization failed!", + argv[0]); + fflush(NULL); + return rc; + } + + if (!test_config.use_mpi) { + fprintf(stderr, "ERROR - Test %s requires MPI!", + argv[0]); + fflush(NULL); + return -1; + } + + target_file = test_target_filename(cfg); + test_print_verbose_once(cfg, "DEBUG: creating target file %s", + target_file); + rc = test_create_file(cfg, target_file, O_RDWR); + if (rc) { + test_abort(cfg, rc); + } + + // generate write requests + test_print_verbose_once(cfg, "DEBUG: generating write requests"); + wr_buf = calloc(test_config.n_blocks, test_config.block_sz); + if (NULL == wr_buf) { + test_abort(cfg, ENOMEM); + } + num_reqs = generate_write_reqs(cfg, wr_buf, &reqs); + if (0 == num_reqs) { + test_abort(cfg, ENOMEM); + } + + // do writes + test_print_verbose_once(cfg, "DEBUG: starting write requests"); + test_barrier(cfg); + timer_start(&time_wr); + rc = issue_write_req_batch(cfg, num_reqs, reqs); + if (rc) { + test_abort(cfg, rc); + } + rc = wait_write_req_batch(cfg, num_reqs, reqs); + if (rc) { + test_abort(cfg, rc); + } + timer_stop(&time_wr); + test_print_verbose_once(cfg, "DEBUG: finished write requests"); + + // sync/laminate + timer_start(&time_sync); + rc = write_sync(cfg); + if (rc) { + test_abort(cfg, rc); + } + timer_stop(&time_sync); + test_barrier(cfg); + test_print_verbose_once(cfg, "DEBUG: finished sync"); + + // post-write cleanup + free(wr_buf); + free(reqs); + reqs = NULL; + + // generate read requests + test_print_verbose_once(cfg, "DEBUG: generating read requests"); + rd_buf = calloc(test_config.n_blocks, test_config.block_sz); + if (NULL == rd_buf) { + test_abort(cfg, ENOMEM); + } + num_reqs = generate_read_reqs(cfg, rd_buf, &reqs); + if (0 == num_reqs) { + test_abort(cfg, ENOMEM); + } + + // do reads + test_print_verbose_once(cfg, "DEBUG: starting read requests"); + test_barrier(cfg); + timer_start(&time_rd); + rc = issue_read_req_batch(cfg, num_reqs, reqs); + if (rc) { + test_abort(cfg, rc); + } + rc = wait_read_req_batch(cfg, num_reqs, reqs); + if (rc) { + test_abort(cfg, rc); + } + timer_stop(&time_rd); + test_barrier(cfg); + test_print_verbose_once(cfg, "DEBUG: finished read requests"); + + if (test_config.io_check) { + test_print_verbose_once(cfg, "DEBUG: verifying data"); + rc = check_read_req_batch(cfg, num_reqs, reqs); + } + + // post-read cleanup + free(rd_buf); + free(reqs); + reqs = NULL; + + // calculate achieved bandwidth rates + size_t rank_bytes = test_config.n_blocks * test_config.block_sz; + double write_bw, read_bw; + double aggr_write_bw, aggr_read_bw; + double max_write_time, max_read_time; + double min_sync_time, max_sync_time; + + write_bw = bandwidth_mib(rank_bytes, time_wr.elapsed_sec); + aggr_write_bw = test_reduce_double_sum(cfg, write_bw); + max_write_time = test_reduce_double_max(cfg, time_wr.elapsed_sec); + + min_sync_time = test_reduce_double_min(cfg, time_sync.elapsed_sec); + max_sync_time = test_reduce_double_max(cfg, time_sync.elapsed_sec); + + read_bw = bandwidth_mib(rank_bytes, time_rd.elapsed_sec); + aggr_read_bw = test_reduce_double_sum(cfg, read_bw); + max_read_time = test_reduce_double_max(cfg, time_rd.elapsed_sec); + + if (test_config.rank == 0) { + size_t total_bytes = rank_bytes * test_config.n_ranks; + double eff_write_bw, eff_read_bw; + eff_write_bw = bandwidth_mib(total_bytes, max_write_time); + eff_read_bw = bandwidth_mib(total_bytes, max_read_time); + + printf("Aggregate Write BW is %.3lf MiB/s\n" + "Effective Write BW is %.3lf MiB/s\n\n", + aggr_write_bw, eff_write_bw); + printf("Minimum Sync Time is %.6lf s\n" + "Maximum Sync Time is %.6lf s\n\n", + min_sync_time, max_sync_time); + printf("Aggregate Read BW is %.3lf MiB/s\n" + "Effective Read BW is %.3lf MiB/s\n\n", + aggr_read_bw, eff_read_bw); + fflush(stdout); + } + + // cleanup + free(target_file); + + timer_fini(&time_wr); + timer_fini(&time_rd); + timer_fini(&time_sync); + + test_fini(cfg); + + return 0; +} diff --git a/examples/src/writeread.f90 b/examples/src/writeread.f90 new file mode 100644 index 000000000..3c98fa6aa --- /dev/null +++ b/examples/src/writeread.f90 @@ -0,0 +1,58 @@ + program write_read_F + + implicit none + + include 'mpif.h' + include 'unifyfsf.h' + + character*1024 :: basefname = "file" + character*1024 :: fname, file_suffix + character*1024 :: prefix = "/unifyfs" + integer(kind=4) :: flag; + integer(kind=4) :: outflags; + integer(kind=4) :: valid; + + integer, parameter :: ni=20, nj=30, nk=45 + integer :: loop_count=5 + + integer(kind=8), dimension(ni,nj,nk) :: W1, R1 + + integer :: ierr, errors, all_errors, nprocs, mynod, ios + integer :: i,j,k,loop + + integer :: writeunit, readunit + integer(kind=8) :: nodeoff + +! integer (kind = 8) :: total_bytes_transferred + real (kind = 8) :: total_bytes_transferred + real (kind = 4) overall_transfer_rate + real (kind = 8) time0, time1, iter_time0, iter_time1 + + call MPI_INIT(ierr) + call MPI_COMM_SIZE(MPI_COMM_WORLD, nprocs, ierr) + call MPI_COMM_RANK(MPI_COMM_WORLD, mynod, ierr) + + call UNIFYFS_MOUNT(prefix, mynod, nprocs, 0, ierr) + + nodeoff=2**21 + + fname = trim(prefix) // "/" // trim(basefname) // ".ckpt" + + forall(i=1:ni,j=1:nj,k=1:nk) & + W1(i,j,k) = nodeoff*mynod+i+ni*(j-1+nj*(k-1)) + + writeunit = mynod + open(unit=writeunit,file=fname,form='unformatted',action='write') + + write(writeunit,iostat=ios) W1 + close(writeunit) + +! R1 = 0 +! open(unit=readunit,file=fname,form='unformatted',action='read') +! read(readunit,iostat=ios) R1 +! close(readunit) + + call UNIFYFS_UNMOUNT(ierr) + call MPI_FINALIZE(ierr) + + end program write_read_F diff --git a/extras/Makefile.am b/extras/Makefile.am index ff7613c82..007720c9d 100644 --- a/extras/Makefile.am +++ b/extras/Makefile.am @@ -1,8 +1,8 @@ -confdir = $(sysconfdir)/unifycr +confdir = $(sysconfdir)/unifyfs -conf_DATA = unifycr.conf +conf_DATA = unifyfs.conf -EXTRA_DIST = unifycr.conf.in +EXTRA_DIST = unifyfs.conf.in -CLEANFILES = unifycr.conf +CLEANFILES = unifyfs.conf diff --git a/extras/unifycr.conf.in b/extras/unifyfs.conf.in similarity index 81% rename from extras/unifycr.conf.in rename to extras/unifyfs.conf.in index dd4db5fee..365c7194e 100644 --- a/extras/unifycr.conf.in +++ b/extras/unifyfs.conf.in @@ -1,4 +1,4 @@ -# unifycr.conf +# unifyfs.conf # NOTE: settings with default values are commented out # @@ -7,11 +7,10 @@ # ';' end of line comment character, must be preceded by a space # SECTION: top-level configuration -[unifycr] +[unifyfs] # consistency = "LAMINATED" ; NONE | LAMINATED | POSIX # daemonize = on ; servers will become daemons -debug = on ; enable debug output (default: off) -# mountpoint = "/unifycr" ; mountpoint (i.e., prefix path) +# mountpoint = "/unifyfs" ; mountpoint (i.e., prefix path) # SECTION: client settings [client] @@ -20,12 +19,12 @@ max_files = 64 ; max open files per client (default: 128) # SECTION: log settings [log] # dir = /tmp ; log file directory path -# file = unifycrd.log ; log file name (server rank will be appended) +# file = unifyfsd.log ; log file name (server rank will be appended) verbosity = 5 ; logging verbosity level [0-5] (default: 0) # SECTION: metadata settings [meta] -# db_name = "unifycr_metadb" ; metadata datbase name +# db_name = "unifyfs_metadb" ; metadata datbase name db_path = "/var/tmp" ; metadata database directory path (default: /tmp) # SECTION: shared memory segment settings @@ -39,4 +38,3 @@ chunk_mem = 67108864 ; segment size for data chunks (default: 256 MiB) # data_dir = "/mnt/ssd" ; directory path for data spillover # meta_dir = "/mnt/ssd" ; directory path for metadata spillover size = 268435456 ; data spillover max size (default: 1 GiB) - diff --git a/m4/flatcc.m4 b/m4/flatcc.m4 index 6b28581ed..f4d7f6658 100644 --- a/m4/flatcc.m4 +++ b/m4/flatcc.m4 @@ -1,4 +1,4 @@ -AC_DEFUN([UNIFYCR_AC_FLATCC], [ +AC_DEFUN([UNIFYFS_AC_FLATCC], [ # preserve state of flags FLATCC_OLD_CFLAGS=$CFLAGS FLATCC_OLD_LDFLAGS=$LDFLAGS diff --git a/m4/gotcha.m4 b/m4/gotcha.m4 index 5cc432ea0..6d32ac5f3 100644 --- a/m4/gotcha.m4 +++ b/m4/gotcha.m4 @@ -1,4 +1,4 @@ -AC_DEFUN([UNIFYCR_AC_GOTCHA], [ +AC_DEFUN([UNIFYFS_AC_GOTCHA], [ # preserve state of flags GOTCHA_OLD_CFLAGS=$CFLAGS GOTCHA_OLD_CXXFLAGS=$CXXFLAGS diff --git a/m4/leveldb.m4 b/m4/leveldb.m4 index bb65dd1bc..2eed55189 100644 --- a/m4/leveldb.m4 +++ b/m4/leveldb.m4 @@ -1,4 +1,4 @@ -AC_DEFUN([UNIFYCR_AC_LEVELDB], [ +AC_DEFUN([UNIFYFS_AC_LEVELDB], [ # preserve state of flags LEVELDB_OLD_CFLAGS=$CFLAGS LEVELDB_OLD_LDFLAGS=$LDFLAGS @@ -6,7 +6,7 @@ AC_DEFUN([UNIFYCR_AC_LEVELDB], [ AC_ARG_WITH([leveldb], [AC_HELP_STRING([--with-leveldb=PATH], [path to installed libleveldb [default=/usr/local]])], [ LEVELDB_CFLAGS="-I${withval}/include" - LEVELDB_LDFLAGS="-L${withval}/lib" + LEVELDB_LDFLAGS="-L${withval}/lib -L${withval}/lib64" CFLAGS="$CFLAGS ${LEVELDB_CFLAGS}" LDFLAGS="$LDFLAGS ${LEVELDB_LDFLAGS}" ], []) diff --git a/m4/lx_find_mpi.m4 b/m4/lx_find_mpi.m4 index 755beaaed..1e1c84b20 100644 --- a/m4/lx_find_mpi.m4 +++ b/m4/lx_find_mpi.m4 @@ -155,7 +155,7 @@ AC_DEFUN([LX_QUERY_MPI_COMPILER], # set a shell variable that the caller can test outside this macro have_$3_mpi='yes' else - Echo Unable to find suitable MPI Compiler. Try setting $1. + echo Unable to find suitable MPI Compiler. Try setting $1. have_$3_mpi='no' fi ]) diff --git a/m4/margo.m4 b/m4/margo.m4 index 90b5f0264..0be0592d5 100644 --- a/m4/margo.m4 +++ b/m4/margo.m4 @@ -1,4 +1,4 @@ -AC_DEFUN([UNIFYCR_AC_MARGO], [ +AC_DEFUN([UNIFYFS_AC_MARGO], [ # preserve state of flags MARGO_OLD_CFLAGS=$CFLAGS MARGO_OLD_CXXFLAGS=$CXXFLAGS diff --git a/meta/README.md b/meta/README.md index b1d194639..e3bdddebf 100644 --- a/meta/README.md +++ b/meta/README.md @@ -2,7 +2,7 @@ MDHIM - Multi-Dimensional Hashing Indexing Middleware Description --------------- -This version of MDHIM package is customized to support UnifyCR. +This version of MDHIM package is customized to support UnifyFS. In order to test these extended interfaces and implementations, two test files are provided in the directory tests/singletests (range_bget.c and range_test.c). The test scripts for these two diff --git a/meta/src/Makefile.am b/meta/src/Makefile.am index 9798b38b2..4f4539ff1 100644 --- a/meta/src/Makefile.am +++ b/meta/src/Makefile.am @@ -35,7 +35,7 @@ AM_CPPFLAGS = -I$(top_srcdir)/meta/src/Mlog2 \ -I$(top_srcdir)/common/src \ -I$(top_srcdir)/server/src -AM_CFLAGS = -DLEVELDB_SUPPORT $(LEVELDB_CFLAGS) $(MPI_CFLAGS) +AM_CFLAGS = -DLEVELDB_SUPPORT $(LEVELDB_CFLAGS) $(MPI_CFLAGS) $(MARGO_CFLAGS) AM_CFLAGS += -Wall CLEANFILES = diff --git a/meta/src/client.c b/meta/src/client.c index c34b8943b..20a084f33 100644 --- a/meta/src/client.c +++ b/meta/src/client.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* diff --git a/meta/src/client.h b/meta/src/client.h index fa0082935..a744302eb 100644 --- a/meta/src/client.h +++ b/meta/src/client.h @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* diff --git a/meta/src/data_store.c b/meta/src/data_store.c index 4a09e374f..71fb760d5 100644 --- a/meta/src/data_store.c +++ b/meta/src/data_store.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* diff --git a/meta/src/data_store.h b/meta/src/data_store.h index 4736f18fd..284b143c7 100644 --- a/meta/src/data_store.h +++ b/meta/src/data_store.h @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* diff --git a/meta/src/ds_leveldb.c b/meta/src/ds_leveldb.c index 8828e0abf..b28453dc0 100644 --- a/meta/src/ds_leveldb.c +++ b/meta/src/ds_leveldb.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -218,7 +218,7 @@ static int cmp_byte_compare(void* arg, const char* a, size_t alen, return ret; } -static int cmp_unifycr_compare(void* arg, const char* a, size_t alen, +static int cmp_unifyfs_compare(void* arg, const char* a, size_t alen, const char* b, size_t blen) { int ret; @@ -329,9 +329,9 @@ int mdhim_leveldb_open(void **dbh, void **dbs, char *path, int flags, int key_ty mdhimdb->cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_string_compare, cmp_name); mdhimdb->compare = cmp_string_compare; break; - case MDHIM_UNIFYCR_KEY: - mdhimdb->cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_unifycr_compare, cmp_name); - mdhimdb->compare = cmp_unifycr_compare; + case MDHIM_UNIFYFS_KEY: + mdhimdb->cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_unifyfs_compare, cmp_name); + mdhimdb->compare = cmp_unifyfs_compare; default: mdhimdb->cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_byte_compare, cmp_name); mdhimdb->compare = cmp_byte_compare; @@ -963,9 +963,9 @@ int leveldb_batch_ranges(void *dbh, char **key, int32_t *key_len, start_ndx = 2 * i; end_ndx = start_ndx + 1; /* printf("range %d: fid is %d, start_offset=%zu end_offset=%zu\n", - * i, UNIFYCR_KEY_FID(key[start_ndx]), - * UNIFYCR_KEY_OFF(key[start_ndx]), - * UNIFYCR_KEY_OFF(key[end_ndx])); + * i, UNIFYFS_KEY_FID(key[start_ndx]), + * UNIFYFS_KEY_OFF(key[start_ndx]), + * UNIFYFS_KEY_OFF(key[end_ndx])); */ leveldb_process_range(iter, key[start_ndx], key[end_ndx], key_len[start_ndx], @@ -979,9 +979,9 @@ int leveldb_batch_ranges(void *dbh, char **key, int32_t *key_len, /* printf("out_records_cnt is %d\n", *out_records_cnt); * for (i = 0; i < *out_records_cnt; i++) { * printf("out %d: fid is %d, offset=%zu addr=%zu\n", - * i, UNIFYCR_KEY_FID((*out_keys)[i]), - * UNIFYCR_KEY_OFF((*out_keys)[i]), - * UNIFYCR_VAL_ADDR((*out_vals)[i])); + * i, UNIFYFS_KEY_FID((*out_keys)[i]), + * UNIFYFS_KEY_OFF((*out_keys)[i]), + * UNIFYFS_VAL_ADDR((*out_vals)[i])); * } * fflush(stdout); */ @@ -1023,7 +1023,7 @@ int leveldb_process_range(leveldb_iterator_t *iter, ret_key = leveldb_iter_key(iter, &tmp_key_len); if (!ret_key) return MDHIM_DB_ERROR; - else if (UNIFYCR_KEY_FID(ret_key) != UNIFYCR_KEY_FID(start_key)) + else if (UNIFYFS_KEY_FID(ret_key) != UNIFYFS_KEY_FID(start_key)) return 0; // last key matched fid, but not offset @@ -1033,7 +1033,7 @@ int leveldb_process_range(leveldb_iterator_t *iter, if (!ret_key) return MDHIM_DB_ERROR; - if (UNIFYCR_KEY_FID(start_key) != UNIFYCR_KEY_FID(ret_key)) { + if (UNIFYFS_KEY_FID(start_key) != UNIFYFS_KEY_FID(ret_key)) { // mismatch on fid, check previous K-V leveldb_iter_prev(iter); if (!leveldb_iter_valid(iter)) { @@ -1043,7 +1043,7 @@ int leveldb_process_range(leveldb_iterator_t *iter, ret_key = leveldb_iter_key(iter, &tmp_key_len); if (!ret_key) return MDHIM_DB_ERROR; - else if (UNIFYCR_KEY_FID(start_key) != UNIFYCR_KEY_FID(ret_key)) + else if (UNIFYFS_KEY_FID(start_key) != UNIFYFS_KEY_FID(ret_key)) return 0; prev_flag = 1; @@ -1054,13 +1054,13 @@ int leveldb_process_range(leveldb_iterator_t *iter, if (!ret_val) return MDHIM_DB_ERROR; - unsigned long start_off = UNIFYCR_KEY_OFF(start_key); - unsigned long end_off = UNIFYCR_KEY_OFF(end_key); + unsigned long start_off = UNIFYFS_KEY_OFF(start_key); + unsigned long end_off = UNIFYFS_KEY_OFF(end_key); if (prev_flag) { // ret_key is previous K-V with matching fid - unsigned long prev_st = UNIFYCR_KEY_OFF(ret_key); - unsigned long prev_end = prev_st + UNIFYCR_VAL_LEN(ret_val) - 1; + unsigned long prev_st = UNIFYFS_KEY_OFF(ret_key); + unsigned long prev_end = prev_st + UNIFYFS_VAL_LEN(ret_val) - 1; if (start_off > prev_end) { /* prev_s......prev_e; ...... start..end */ return 0; @@ -1077,18 +1077,18 @@ int leveldb_process_range(leveldb_iterator_t *iter, tmp_end = end_off; } - assert((UNIFYCR_KEY_SZ == tmp_key_len) && - (UNIFYCR_VAL_SZ == tmp_val_len)); - char *ret_out_key = calloc(1, UNIFYCR_KEY_SZ); - char *ret_out_val = calloc(1, UNIFYCR_VAL_SZ); + assert((UNIFYFS_KEY_SZ == tmp_key_len) && + (UNIFYFS_VAL_SZ == tmp_val_len)); + char *ret_out_key = calloc(1, UNIFYFS_KEY_SZ); + char *ret_out_val = calloc(1, UNIFYFS_VAL_SZ); - memcpy(ret_out_key, ret_key, UNIFYCR_KEY_SZ); - UNIFYCR_KEY_OFF(ret_out_key) = start_off; + memcpy(ret_out_key, ret_key, UNIFYFS_KEY_SZ); + UNIFYFS_KEY_OFF(ret_out_key) = start_off; - memcpy(ret_out_val, ret_val, UNIFYCR_VAL_SZ); - UNIFYCR_VAL_ADDR(ret_out_val) = UNIFYCR_VAL_ADDR(ret_val) + memcpy(ret_out_val, ret_val, UNIFYFS_VAL_SZ); + UNIFYFS_VAL_ADDR(ret_out_val) = UNIFYFS_VAL_ADDR(ret_val) + (start_off - prev_st); - UNIFYCR_VAL_LEN(ret_out_val) = tmp_end - start_off + 1; + UNIFYFS_VAL_LEN(ret_out_val) = tmp_end - start_off + 1; add_kv(out_keys, out_keys_len, out_vals, out_vals_len, @@ -1098,7 +1098,7 @@ int leveldb_process_range(leveldb_iterator_t *iter, return 0; - } else if (UNIFYCR_KEY_OFF(ret_key) == start_off) { + } else if (UNIFYFS_KEY_OFF(ret_key) == start_off) { // exact match on start offset return handle_next_half(iter, start_key, end_key, out_keys, out_keys_len, @@ -1120,7 +1120,7 @@ int leveldb_process_range(leveldb_iterator_t *iter, ret_key = leveldb_iter_key(iter, &tmp_key_len); if (!ret_key) return MDHIM_DB_ERROR; - else if (UNIFYCR_KEY_FID(ret_key) != UNIFYCR_KEY_FID(start_key)) { + else if (UNIFYFS_KEY_FID(ret_key) != UNIFYFS_KEY_FID(start_key)) { leveldb_iter_next(iter); return handle_next_half(iter, start_key, end_key, out_keys, out_keys_len, @@ -1132,8 +1132,8 @@ int leveldb_process_range(leveldb_iterator_t *iter, if (!ret_val) return MDHIM_DB_ERROR; - unsigned long prev_st = UNIFYCR_KEY_OFF(ret_key); - unsigned long prev_end = prev_st + UNIFYCR_VAL_LEN(ret_val) - 1; + unsigned long prev_st = UNIFYFS_KEY_OFF(ret_key); + unsigned long prev_end = prev_st + UNIFYFS_VAL_LEN(ret_val) - 1; if (start_off <= prev_end) { int found_end = 0; @@ -1150,17 +1150,17 @@ int leveldb_process_range(leveldb_iterator_t *iter, * start..................end */ - assert((UNIFYCR_KEY_SZ == tmp_key_len) && - (UNIFYCR_VAL_SZ == tmp_val_len)); - char *ret_out_key = (char *) calloc(1, UNIFYCR_KEY_SZ); - char *ret_out_val = (char *) calloc(1, UNIFYCR_VAL_SZ); + assert((UNIFYFS_KEY_SZ == tmp_key_len) && + (UNIFYFS_VAL_SZ == tmp_val_len)); + char *ret_out_key = (char *) calloc(1, UNIFYFS_KEY_SZ); + char *ret_out_val = (char *) calloc(1, UNIFYFS_VAL_SZ); - memcpy(ret_out_key, ret_key, UNIFYCR_KEY_SZ); - UNIFYCR_KEY_OFF(ret_out_key) = start_off; + memcpy(ret_out_key, ret_key, UNIFYFS_KEY_SZ); + UNIFYFS_KEY_OFF(ret_out_key) = start_off; - memcpy(ret_out_val, ret_val, UNIFYCR_VAL_SZ); - UNIFYCR_VAL_LEN(ret_out_val) = tmp_end - start_off + 1; - UNIFYCR_VAL_ADDR(ret_out_val) = UNIFYCR_VAL_ADDR(ret_val) + + memcpy(ret_out_val, ret_val, UNIFYFS_VAL_SZ); + UNIFYFS_VAL_LEN(ret_out_val) = tmp_end - start_off + 1; + UNIFYFS_VAL_ADDR(ret_out_val) = UNIFYFS_VAL_ADDR(ret_val) + (start_off - prev_st); add_kv(out_keys, out_keys_len, @@ -1174,7 +1174,7 @@ int leveldb_process_range(leveldb_iterator_t *iter, } // start at next to find rest of range - UNIFYCR_KEY_OFF(start_key) = UNIFYCR_KEY_OFF(next_ret_key); + UNIFYFS_KEY_OFF(start_key) = UNIFYFS_KEY_OFF(next_ret_key); leveldb_iter_next(iter); } else { /* start between prev and next, one of two cases: @@ -1210,13 +1210,13 @@ int handle_next_half(leveldb_iterator_t *iter, if (!ret_val) return MDHIM_DB_ERROR; - assert((UNIFYCR_KEY_SZ == tmp_key_len) && - (UNIFYCR_VAL_SZ == tmp_val_len)); + assert((UNIFYFS_KEY_SZ == tmp_key_len) && + (UNIFYFS_VAL_SZ == tmp_val_len)); - unsigned long curr_off = UNIFYCR_KEY_OFF(ret_key); - unsigned long curr_end = curr_off + UNIFYCR_VAL_LEN(ret_val) - 1; + unsigned long curr_off = UNIFYFS_KEY_OFF(ret_key); + unsigned long curr_end = curr_off + UNIFYFS_VAL_LEN(ret_val) - 1; - unsigned long end_off = UNIFYCR_KEY_OFF(end_key); + unsigned long end_off = UNIFYFS_KEY_OFF(end_key); if (curr_off > end_off) { // start..end precedes current K-V offset @@ -1226,17 +1226,17 @@ int handle_next_half(leveldb_iterator_t *iter, char *ret_out_key; char *ret_out_val; - ret_out_key = (char *) calloc(1, UNIFYCR_KEY_SZ); - ret_out_val = (char *) calloc(1, UNIFYCR_VAL_SZ); - memcpy(ret_out_key, ret_key, UNIFYCR_KEY_SZ); - memcpy(ret_out_val, ret_val, UNIFYCR_VAL_SZ); + ret_out_key = (char *) calloc(1, UNIFYFS_KEY_SZ); + ret_out_val = (char *) calloc(1, UNIFYFS_VAL_SZ); + memcpy(ret_out_key, ret_key, UNIFYFS_KEY_SZ); + memcpy(ret_out_val, ret_val, UNIFYFS_VAL_SZ); if (end_off <= curr_end) { // found end in current K-V, add slice /* curr_s.........curr_e [start]....end */ - UNIFYCR_VAL_LEN(ret_out_val) = end_off - curr_off + 1; + UNIFYFS_VAL_LEN(ret_out_val) = end_off - curr_off + 1; add_kv(out_keys, out_keys_len, out_vals, out_vals_len, @@ -1265,7 +1265,7 @@ int handle_next_half(leveldb_iterator_t *iter, ret_key = leveldb_iter_key(iter, (size_t *)&tmp_key_len); if (!ret_key) return MDHIM_DB_ERROR; - else if (UNIFYCR_KEY_FID(ret_key) != UNIFYCR_KEY_FID(start_key)) { + else if (UNIFYFS_KEY_FID(ret_key) != UNIFYFS_KEY_FID(start_key)) { // fid mismatch break; } @@ -1274,25 +1274,25 @@ int handle_next_half(leveldb_iterator_t *iter, if (!ret_val) return MDHIM_DB_ERROR; - curr_off = UNIFYCR_KEY_OFF(ret_key); - curr_end = curr_off + UNIFYCR_VAL_LEN(ret_val) - 1; + curr_off = UNIFYFS_KEY_OFF(ret_key); + curr_end = curr_off + UNIFYFS_VAL_LEN(ret_val) - 1; if (curr_off > end_off) { // current K-V starts after end break; } - assert((UNIFYCR_KEY_SZ == tmp_key_len) && - (UNIFYCR_VAL_SZ == tmp_val_len)); - ret_out_key = (char *) calloc(1, UNIFYCR_KEY_SZ); - ret_out_val = (char *) calloc(1, UNIFYCR_VAL_SZ); - memcpy(ret_out_key, ret_key, UNIFYCR_KEY_SZ); - memcpy(ret_out_val, ret_val, UNIFYCR_VAL_SZ); + assert((UNIFYFS_KEY_SZ == tmp_key_len) && + (UNIFYFS_VAL_SZ == tmp_val_len)); + ret_out_key = (char *) calloc(1, UNIFYFS_KEY_SZ); + ret_out_val = (char *) calloc(1, UNIFYFS_VAL_SZ); + memcpy(ret_out_key, ret_key, UNIFYFS_KEY_SZ); + memcpy(ret_out_val, ret_val, UNIFYFS_VAL_SZ); if (curr_end >= end_off) { // found end in current K-V, add slice found_end = 1; - UNIFYCR_VAL_LEN(ret_out_val) = end_off - curr_off + 1; + UNIFYFS_VAL_LEN(ret_out_val) = end_off - curr_off + 1; } // else, range fully covers current K-V, add it add_kv(out_keys, out_keys_len, diff --git a/meta/src/ds_leveldb.h b/meta/src/ds_leveldb.h index 36980bcf3..124aeaca1 100644 --- a/meta/src/ds_leveldb.h +++ b/meta/src/ds_leveldb.h @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -47,7 +47,7 @@ #include "partitioner.h" #include "data_store.h" -#include "unifycr_metadata.h" +#include "unifyfs_metadata.h" /* Function pointer for comparator in C */ typedef int (*mdhim_store_cmp_fn_t)(void* arg, const char* a, size_t alen, diff --git a/meta/src/ds_mysql.c b/meta/src/ds_mysql.c index e63ce7c9e..de9cde98a 100644 --- a/meta/src/ds_mysql.c +++ b/meta/src/ds_mysql.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* diff --git a/meta/src/ds_mysql.h b/meta/src/ds_mysql.h index e3911feef..15d87c441 100644 --- a/meta/src/ds_mysql.h +++ b/meta/src/ds_mysql.h @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* diff --git a/meta/src/indexes.c b/meta/src/indexes.c index cdabae8a6..359179d9a 100644 --- a/meta/src/indexes.c +++ b/meta/src/indexes.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -249,10 +249,10 @@ int update_stat(struct mdhim_t *md, struct index_t *index, void *key, uint32_t k /* printf("is float key\n"); fflush(stdout); */ - } else if (index->key_type != MDHIM_UNIFYCR_KEY){ + } else if (index->key_type != MDHIM_UNIFYFS_KEY){ val1 = (void *) malloc(sizeof(uint64_t)); val2 = (void *) malloc(sizeof(uint64_t)); - /* printf("is not unifycr key\n"); + /* printf("is not unifyfs key\n"); fflush(stdout); */ } @@ -260,7 +260,7 @@ int update_stat(struct mdhim_t *md, struct index_t *index, void *key, uint32_t k val1 = NULL; val2 = NULL; /* - printf("is unifycr key\n"); + printf("is unifyfs key\n"); fflush(stdout); */ } @@ -286,7 +286,7 @@ int update_stat(struct mdhim_t *md, struct index_t *index, void *key, uint32_t k } else if (index->key_type == MDHIM_BYTE_KEY) { *(unsigned long *)val1 = get_byte_num(key, key_len); *(unsigned long *)val2 = *(unsigned long *)val1; - } else if (index->key_type == MDHIM_UNIFYCR_KEY) { + } else if (index->key_type == MDHIM_UNIFYFS_KEY) { val1 = get_meta_pair(key, key_len); val2 = get_meta_pair(key, key_len); } @@ -311,8 +311,8 @@ int update_stat(struct mdhim_t *md, struct index_t *index, void *key, uint32_t k // fflush(stdout); gettimeofday(&cmpstart, NULL); - if (index->key_type == MDHIM_UNIFYCR_KEY && os ) { - if (unifycr_compare(os->min, val1) > 0) { + if (index->key_type == MDHIM_UNIFYFS_KEY && os ) { + if (unifyfs_compare(os->min, val1) > 0) { /* printf("freeing %x, va1 addr is %x\n", os->min, val1); fflush(stdout); @@ -324,7 +324,7 @@ int update_stat(struct mdhim_t *md, struct index_t *index, void *key, uint32_t k free(val1); } - if (unifycr_compare(os->max, val2) < 0) { + if (unifyfs_compare(os->max, val2) < 0) { /* printf("freeing %xb,bbb\n", os->max); fflush(stdout); @@ -375,7 +375,7 @@ int update_stat(struct mdhim_t *md, struct index_t *index, void *key, uint32_t k } } - if (!float_type && os && index->key_type != MDHIM_UNIFYCR_KEY) { + if (!float_type && os && index->key_type != MDHIM_UNIFYFS_KEY) { if (*(uint64_t *)os->min > *(uint64_t *)val1) { /* printf("freeing %x3\n", os->min); @@ -694,7 +694,7 @@ struct index_t *create_local_index(struct mdhim_t *md, int db_type, int key_type MPI_Barrier(md->mdhim_client_comm); //Check that the key type makes sense - if (key_type < MDHIM_INT_KEY || key_type > MDHIM_UNIFYCR_KEY) { + if (key_type < MDHIM_INT_KEY || key_type > MDHIM_UNIFYFS_KEY) { mlog(MDHIM_CLIENT_CRIT, "MDHIM - Invalid key type specified"); return NULL; } @@ -842,7 +842,7 @@ struct index_t *create_global_index(struct mdhim_t *md, int server_factor, MPI_Barrier(md->mdhim_client_comm); //Check that the key type makes sense - if (key_type < MDHIM_INT_KEY || key_type > MDHIM_UNIFYCR_KEY) { + if (key_type < MDHIM_INT_KEY || key_type > MDHIM_UNIFYFS_KEY) { mlog(MDHIM_CLIENT_CRIT, "MDHIM - Invalid key type specified"); return NULL; } @@ -1299,7 +1299,7 @@ int pack_stats(struct index_t *index, void *buf, int size, fstat->slice = stat->key; fstat->num = stat->num; - if (index->key_type == MDHIM_UNIFYCR_KEY) { + if (index->key_type == MDHIM_UNIFYFS_KEY) { /* printf("bbbefore:min fid is %ld, min offset is %ld, \ max fid is %ld, max offset is %ld\n", \ *((ulong *)stat->min), *((ulong *)stat->min+1), \ @@ -1368,7 +1368,7 @@ int get_stat_flush_global(struct mdhim_t *md, struct index_t *index) { //Determine the size of the buffers to send based on the number and type of stats if ((ret = is_float_key(index->key_type)) == 1 || \ - index->key_type == MDHIM_UNIFYCR_KEY) { + index->key_type == MDHIM_UNIFYFS_KEY) { float_type = 1; stat_size = sizeof(struct mdhim_db_fstat); } else { @@ -1386,7 +1386,7 @@ int get_stat_flush_global(struct mdhim_t *md, struct index_t *index) { num_items = 0; } ret = is_float_key(index->key_type); - if (ret == 1 || index->key_type == MDHIM_UNIFYCR_KEY) + if (ret == 1 || index->key_type == MDHIM_UNIFYFS_KEY) sendsize = num_items * sizeof(struct mdhim_db_fstat); else sendsize = num_items * sizeof(struct mdhim_db_istat); @@ -1511,7 +1511,7 @@ int get_stat_flush_global(struct mdhim_t *md, struct index_t *index) { if (float_type) { stat->min = (void *) malloc(sizeof(long double)); stat->max = (void *) malloc(sizeof(long double)); - if (index->key_type == MDHIM_UNIFYCR_KEY) { + if (index->key_type == MDHIM_UNIFYFS_KEY) { struct mdhim_db_fstat * tmp_stat; tmp_stat = (struct mdhim_db_fstat *)tstat; memcpy((char *)stat->min, (char *)&(tmp_stat->dmin), sizeof(ulong)); diff --git a/meta/src/indexes.h b/meta/src/indexes.h index 98ca9c78f..d40c7daa5 100644 --- a/meta/src/indexes.h +++ b/meta/src/indexes.h @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* diff --git a/meta/src/local_client.c b/meta/src/local_client.c index ea66cf1c4..0a5d64a20 100644 --- a/meta/src/local_client.c +++ b/meta/src/local_client.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* diff --git a/meta/src/local_client.h b/meta/src/local_client.h index 0fb777f97..42e6c4a12 100644 --- a/meta/src/local_client.h +++ b/meta/src/local_client.h @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* diff --git a/meta/src/mdhim.c b/meta/src/mdhim.c index ab7588f78..d89cd9544 100644 --- a/meta/src/mdhim.c +++ b/meta/src/mdhim.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -895,33 +895,6 @@ struct mdhim_brm_t *mdhimBDelete(struct mdhim_t *md, struct index_t *index, return brm_head; } -int unlink_cb(const char *fpath,\ - const struct stat *sb, int typeflag, struct FTW *ftwbuf) -{ - int rv = remove(fpath); - - if (rv) - perror(fpath); - - return rv; -} - -int rmrf(char *path) -{ - return nftw(path, unlink_cb, 64, FTW_DEPTH | FTW_PHYS); -} - -int mdhimSanitize(char *dbfilename, char *statfilename, char *manifestfilename) { - int rc = 0; - - rc = rmrf(dbfilename); - rc = rmrf(statfilename); - rc = unlink(manifestfilename); - - return rc; - -} - /** * Retrieves statistics from all the range servers - collective call diff --git a/meta/src/mdhim.h b/meta/src/mdhim.h index d67034720..4a5366dd0 100644 --- a/meta/src/mdhim.h +++ b/meta/src/mdhim.h @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -172,8 +172,6 @@ struct secondary_bulk_info *mdhimCreateSecondaryBulkInfo(struct index_t *seconda int **secondary_key_lens, int *num_keys, int info_type); void mdhimReleaseSecondaryBulkInfo(struct secondary_bulk_info *si); -int mdhimSanitize(char *dbfilename, char *statfilename, char *manifestfilename); -int rmrf(char *path); #ifdef __cplusplus } diff --git a/meta/src/mdhim_options.c b/meta/src/mdhim_options.c index c2c2e5399..dd19e33cb 100644 --- a/meta/src/mdhim_options.c +++ b/meta/src/mdhim_options.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* diff --git a/meta/src/mdhim_options.h b/meta/src/mdhim_options.h index 65d049632..31e7370f1 100644 --- a/meta/src/mdhim_options.h +++ b/meta/src/mdhim_options.h @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* diff --git a/meta/src/mdhim_private.c b/meta/src/mdhim_private.c index 15d3097f7..dc499c5ef 100644 --- a/meta/src/mdhim_private.c +++ b/meta/src/mdhim_private.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* diff --git a/meta/src/mdhim_private.h b/meta/src/mdhim_private.h index 09f512715..f6177b2a7 100644 --- a/meta/src/mdhim_private.h +++ b/meta/src/mdhim_private.h @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* diff --git a/meta/src/messages.c b/meta/src/messages.c index a7d34a8ae..ce452f50d 100644 --- a/meta/src/messages.c +++ b/meta/src/messages.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* diff --git a/meta/src/messages.h b/meta/src/messages.h index 12ee13f0a..9419b9eb0 100644 --- a/meta/src/messages.h +++ b/meta/src/messages.h @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* diff --git a/meta/src/partitioner.c b/meta/src/partitioner.c index f8f89e276..37c171bcb 100644 --- a/meta/src/partitioner.c +++ b/meta/src/partitioner.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -395,7 +395,7 @@ int get_slice_num(struct mdhim_t *md, struct index_t *index, void *key, int key_ /* Convert the key to a slice number */ slice_num = key_num/index->mdhim_max_recs_per_slice; - if (key_type == MDHIM_UNIFYCR_KEY) { + if (key_type == MDHIM_UNIFYFS_KEY) { unsigned long *meta_pair = get_meta_pair(key, key_len); unsigned long surplus = meta_pair[1]; unsigned long highval = (meta_pair[0] << 1); @@ -1026,7 +1026,7 @@ rangesrv_list *get_range_servers_from_range(struct mdhim_t *md, struct index_t * ret_rp->first_key = cur_stat->min; } else { - if (unifycr_compare(ret_rp->first_key, cur_stat->min) > 0 ) { + if (unifyfs_compare(ret_rp->first_key, cur_stat->min) > 0 ) { ret_rp->first_key = cur_stat->min; } } diff --git a/meta/src/partitioner.h b/meta/src/partitioner.h index a25b53d2f..9707af159 100644 --- a/meta/src/partitioner.h +++ b/meta/src/partitioner.h @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -64,7 +64,7 @@ extern "C" #define MDHIM_STRING_KEY 5 //An arbitrary sized key #define MDHIM_BYTE_KEY 6 -#define MDHIM_UNIFYCR_KEY 7 +#define MDHIM_UNIFYFS_KEY 7 //Maximum length of a key #define MAX_KEY_LEN 1048576 diff --git a/meta/src/range_server.c b/meta/src/range_server.c index e566cc26e..75739dd09 100644 --- a/meta/src/range_server.c +++ b/meta/src/range_server.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -50,7 +50,7 @@ #include "mdhim_options.h" #include "partitioner.h" #include "range_server.h" -#include "unifycr_metadata.h" +#include "unifyfs_metadata.h" #include "uthash.h" int recv_counter = 0; @@ -85,11 +85,11 @@ double starttime=0; int putflag = 1; -int unifycr_compare(const char* a, const char* b) { +int unifyfs_compare(const char* a, const char* b) { int rc; - unifycr_key_t *keya = (unifycr_key_t *)a; - unifycr_key_t *keyb = (unifycr_key_t *)b; - rc = unifycr_key_compare(keya, keyb); + unifyfs_key_t *keya = (unifyfs_key_t *)a; + unifyfs_key_t *keyb = (unifyfs_key_t *)b; + rc = unifyfs_key_compare(keya, keyb); return rc; } diff --git a/meta/src/range_server.h b/meta/src/range_server.h index 8f1b477c0..bfcbfeb8e 100644 --- a/meta/src/range_server.h +++ b/meta/src/range_server.h @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -92,6 +92,6 @@ int range_server_init_comm(struct mdhim_t *md); int range_server_stop(struct mdhim_t *md); int range_server_add_oreq(struct mdhim_t *md, MPI_Request *req, void *msg); //Add an outstanding request int range_server_clean_oreqs(struct mdhim_t *md); //Clean outstanding reqs -int unifycr_compare(const char *a, const char *b); +int unifyfs_compare(const char *a, const char *b); #endif diff --git a/meta/tests/single_tests/bput-bget.c b/meta/tests/single_tests/bput-bget.c index e7deeca8a..2348306cf 100644 --- a/meta/tests/single_tests/bput-bget.c +++ b/meta/tests/single_tests/bput-bget.c @@ -139,7 +139,7 @@ int main(int argc, char **argv) { db_opts->manifest_path = manifest_path; db_opts->db_name = db_name; db_opts->db_type = LEVELDB; - db_opts->db_key_type = MDHIM_UNIFYCR_KEY; + db_opts->db_key_type = MDHIM_UNIFYFS_KEY; db_opts->debug_level = MLOG_CRIT; db_opts->max_recs_per_slice = rangesz; db_opts->rserver_factor = serratio; diff --git a/meta/tests/single_tests/range_bget.c b/meta/tests/single_tests/range_bget.c index d1f358536..73dd8c836 100644 --- a/meta/tests/single_tests/range_bget.c +++ b/meta/tests/single_tests/range_bget.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -152,7 +152,7 @@ int main(int argc, char **argv) { db_opts->db_name = db_name; db_opts->db_type = LEVELDB; - db_opts->db_key_type = MDHIM_UNIFYCR_KEY; + db_opts->db_key_type = MDHIM_UNIFYFS_KEY; db_opts->debug_level = MLOG_CRIT; db_opts->max_recs_per_slice = rangesz; db_opts->rserver_factor = serratio; diff --git a/meta/tests/single_tests/range_test.c b/meta/tests/single_tests/range_test.c index 5cfacb446..bea723082 100644 --- a/meta/tests/single_tests/range_test.c +++ b/meta/tests/single_tests/range_test.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -144,7 +144,7 @@ int main(int argc, char **argv) { db_opts->db_name = db_name; db_opts->db_type = LEVELDB; - db_opts->db_key_type = MDHIM_UNIFYCR_KEY; + db_opts->db_key_type = MDHIM_UNIFYFS_KEY; db_opts->debug_level = MLOG_CRIT; db_opts->max_recs_per_slice = rangesz; db_opts->rserver_factor = serratio; diff --git a/scripts/README.md b/scripts/README.md index 1bc1cb48e..de41649e3 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -4,5 +4,5 @@ The asytle tool can be used to apply much of the required code styling used in the project and `checkpatch.sh` can be used to check that uncommitted changes meet the coding style -Our [Style Guides](https://unifycr.readthedocs.io/en/dev/style-guides.html) +Our [Style Guides](https://unifyfs.readthedocs.io/en/dev/style-guides.html) have our complete documentation coding conventions. diff --git a/scripts/checkpatch.sh b/scripts/checkpatch.sh index 0104c0adb..ded03b541 100755 --- a/scripts/checkpatch.sh +++ b/scripts/checkpatch.sh @@ -2,7 +2,7 @@ # # This is a wrapper around checkpatch.pl which is a code style-checking # script borrowed from the Linux kernel. The wrapper calls checkpatch.pl -# with arguments to make it UnifyCR style-friendly. +# with arguments to make it UnifyFS style-friendly. # basedir=$(dirname "$0") @@ -10,7 +10,7 @@ checkpatch_cmd=$basedir/linux_kernel_checkpatch/checkpatch.pl # # These are checkpatch.pl message types to ignore for cases where -# UnifyCR deviates from the Linux kernel coding standards. +# UnifyFS deviates from the Linux kernel coding standards. # # See 'scripts/linux_kernel_checkpatch/checkpatch.pl --list-types' # for all message types diff --git a/scripts/linux_kernel_checkpatch/checkpatch.pl b/scripts/linux_kernel_checkpatch/checkpatch.pl index 577b1f316..49af5338b 100755 --- a/scripts/linux_kernel_checkpatch/checkpatch.pl +++ b/scripts/linux_kernel_checkpatch/checkpatch.pl @@ -3787,7 +3787,7 @@ sub process { # Should start with a space. #$to =~ s/^(\S)/ $1/; - # UNIFYCR: Should not start with a space. + # UNIFYFS: Should not start with a space. $to =~ s/^\s+(\S)/$1/; # Should not end with a space. $to =~ s/\s+$//; @@ -4316,7 +4316,7 @@ sub process { } } } elsif ($ctx =~ /Wx[^WCE]|[^WCE]xW/) { - # UNIFYCR: trips on things like "extern FILE* dbg_stream;" + # UNIFYFS: trips on things like "extern FILE* dbg_stream;" # #if (ERROR("SPACING", # "need consistent spacing around '$op' $at\n" . $hereptr)) { @@ -5211,6 +5211,12 @@ sub process { $allowed = 0; } + # Allow while conditional that comes at end of a do-while loop + if ($cond =~ /^(\}\swhile)/) { + #print "APW: ALLOWED: cond<$cond>\n"; + $allowed = 1; + } + # Allow any conditional directives if ($line =~ /^(.*#)/) { #print "APW: ALLOWED: pre<$1>\n"; diff --git a/scripts/unifycr.astyle b/scripts/unifyfs.astyle similarity index 100% rename from scripts/unifycr.astyle rename to scripts/unifyfs.astyle diff --git a/server/src/Makefile.am b/server/src/Makefile.am index e674b7eb6..89338dbf6 100644 --- a/server/src/Makefile.am +++ b/server/src/Makefile.am @@ -1,41 +1,53 @@ -bin_PROGRAMS = unifycrd - -unifycrd_SOURCES = arraylist.c \ - arraylist.h \ - unifycr_cmd_handler.c \ - unifycr_cmd_handler.h \ - unifycr_init.c \ - unifycr_init.h \ - unifycr_metadata.c \ - unifycr_metadata.h \ - unifycr_request_manager.c \ - unifycr_request_manager.h \ - unifycr_service_manager.c \ - unifycr_service_manager.h \ - unifycr_sock.c \ - unifycr_sock.h \ - unifycr_global.h \ - unifycr_setup.h \ - unifycr_server.h - -unifycrd_LDFLAGS = -static - -unifycrd_LDADD = $(top_builddir)/common/src/libunifycr_common.la \ - $(top_builddir)/meta/src/libmdhim.a \ - $(LEVELDB_LDFLAGS) $(LEVELDB_LIBS) \ - $(MPI_CLDFLAGS) \ - $(MERCURY_LDFLAGS) $(MERCURY_LIBS) \ - $(ARGOBOTS_LDFLAGS) $(ARGOBOTS_LIBS) \ - $(MARGO_LDFLAGS) $(MARGO_LIBS) \ - $(FLATCC_LDFLAGS) $(FLATCC_LIBS) \ - -lpthread -lm -lstdc++ -lrt - -AM_CPPFLAGS = -I$(top_srcdir)/common/src \ - -I$(top_srcdir)/client/src \ - -I$(top_srcdir)/meta/src \ - -I$(top_srcdir)/meta/src/uthash \ - -I$(top_srcdir)/meta/src/Mlog2 - -AM_CFLAGS = -Wall $(MPI_CFLAGS) $(MERCURY_CFLAGS) $(ARGOBOTS_CFLAGS) $(MARGO_CFLAGS) $(FLATCC_CFLAGS) +noinst_LIBRARIES = libunifyfsd.a + +libunifyfsd_a_SOURCES = \ + arraylist.c \ + arraylist.h \ + margo_server.c \ + margo_server.h \ + unifyfs_cmd_handler.c \ + unifyfs_global.h \ + unifyfs_metadata.c \ + unifyfs_metadata.h \ + unifyfs_request_manager.c \ + unifyfs_request_manager.h \ + unifyfs_service_manager.c \ + unifyfs_service_manager.h \ + unifyfs_sock.c \ + unifyfs_sock.h + +bin_PROGRAMS = unifyfsd + +unifyfsd_SOURCES = unifyfs_init.c + +unifyfsd_LDFLAGS = -static \ + $(LEVELDB_LDFLAGS) \ + $(MARGO_LDFLAGS) \ + $(FLATCC_LDFLAGS) + +unifyfsd_LDADD = \ + libunifyfsd.a \ + $(top_builddir)/common/src/libunifyfs_common.la \ + $(top_builddir)/meta/src/libmdhim.a \ + $(LEVELDB_LIBS) \ + $(MPI_CLDFLAGS) \ + $(MARGO_LIBS) \ + $(FLATCC_LIBS) \ + -lpthread -lm -lstdc++ -lrt + +AM_CPPFLAGS = \ + -I$(top_srcdir)/common/src \ + -I$(top_srcdir)/client/src \ + -I$(top_srcdir)/meta/src \ + -I$(top_srcdir)/meta/src/uthash \ + -I$(top_srcdir)/meta/src/Mlog2 + +AM_CFLAGS = -Wall \ + $(LEVELDB_CFLAGS) \ + $(MPI_CFLAGS) \ + $(MERCURY_CFLAGS) \ + $(ARGOBOTS_CFLAGS) \ + $(MARGO_CFLAGS) \ + $(FLATCC_CFLAGS) CLEANFILES = $(bin_PROGRAMS) diff --git a/server/src/arraylist.c b/server/src/arraylist.c index ff4aa1066..68b057676 100644 --- a/server/src/arraylist.c +++ b/server/src/arraylist.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* diff --git a/server/src/arraylist.h b/server/src/arraylist.h index 5fe72a3f1..b33737614 100644 --- a/server/src/arraylist.h +++ b/server/src/arraylist.h @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* diff --git a/server/src/margo_server.c b/server/src/margo_server.c new file mode 100644 index 000000000..96d9ebdcd --- /dev/null +++ b/server/src/margo_server.c @@ -0,0 +1,300 @@ +/* + * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2017-2019, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +// common headers +#include "unifyfs_keyval.h" +#include "unifyfs_client_rpcs.h" +#include "unifyfs_server_rpcs.h" +#include "unifyfs_rpc_util.h" + +// server headers +#include "unifyfs_global.h" +#include "margo_server.h" + +// global variables +ServerRpcContext_t* unifyfsd_rpc_context; +bool margo_use_tcp = true; +bool margo_lazy_connect; // = false + +static const char* PROTOCOL_MARGO_SHM = "na+sm://"; +static const char* PROTOCOL_MARGO_VERBS = "ofi+verbs://"; +static const char* PROTOCOL_MARGO_TCP = "bmi+tcp://"; + +/* setup_remote_target - Initializes the server-server margo target */ +static margo_instance_id setup_remote_target(void) +{ + /* initialize margo */ + hg_return_t hret; + hg_addr_t addr_self; + char self_string[128]; + hg_size_t self_string_sz = sizeof(self_string); + margo_instance_id mid; + const char* margo_protocol; + + if (margo_use_tcp) { + margo_protocol = PROTOCOL_MARGO_TCP; + } else { + margo_protocol = PROTOCOL_MARGO_VERBS; + } + + mid = margo_init(margo_protocol, MARGO_SERVER_MODE, 1, 4); + if (mid == MARGO_INSTANCE_NULL) { + LOGERR("margo_init(%s)", margo_protocol); + return mid; + } + + /* figure out what address this server is listening on */ + hret = margo_addr_self(mid, &addr_self); + if (hret != HG_SUCCESS) { + LOGERR("margo_addr_self()"); + margo_finalize(mid); + return MARGO_INSTANCE_NULL; + } + hret = margo_addr_to_string(mid, + self_string, &self_string_sz, + addr_self); + if (hret != HG_SUCCESS) { + LOGERR("margo_addr_to_string()"); + margo_addr_free(mid, addr_self); + margo_finalize(mid); + return MARGO_INSTANCE_NULL; + } + LOGDBG("margo RPC server: %s", self_string); + margo_addr_free(mid, addr_self); + + /* publish rpc address of server for remote servers */ + rpc_publish_remote_server_addr(self_string); + + return mid; +} + +/* register server-server RPCs */ +static void register_server_server_rpcs(margo_instance_id mid) +{ + unifyfsd_rpc_context->rpcs.hello_id = + MARGO_REGISTER(mid, "server_hello_rpc", + server_hello_in_t, server_hello_out_t, + server_hello_rpc); + + unifyfsd_rpc_context->rpcs.request_id = + MARGO_REGISTER(mid, "server_request_rpc", + server_request_in_t, server_request_out_t, + server_request_rpc); + + unifyfsd_rpc_context->rpcs.chunk_read_request_id = + MARGO_REGISTER(mid, "chunk_read_request_rpc", + chunk_read_request_in_t, chunk_read_request_out_t, + chunk_read_request_rpc); + + unifyfsd_rpc_context->rpcs.chunk_read_response_id = + MARGO_REGISTER(mid, "chunk_read_response_rpc", + chunk_read_response_in_t, chunk_read_response_out_t, + chunk_read_response_rpc); +} + +/* setup_local_target - Initializes the client-server margo target */ +static margo_instance_id setup_local_target(void) +{ + /* initialize margo */ + hg_return_t hret; + hg_addr_t addr_self; + char self_string[128]; + hg_size_t self_string_sz = sizeof(self_string); + margo_instance_id mid; + + mid = margo_init(PROTOCOL_MARGO_SHM, MARGO_SERVER_MODE, 1, -1); + if (mid == MARGO_INSTANCE_NULL) { + LOGERR("margo_init(%s)", PROTOCOL_MARGO_SHM); + return mid; + } + + /* figure out what address this server is listening on */ + hret = margo_addr_self(mid, &addr_self); + if (hret != HG_SUCCESS) { + LOGERR("margo_addr_self()"); + margo_finalize(mid); + return MARGO_INSTANCE_NULL; + } + hret = margo_addr_to_string(mid, + self_string, &self_string_sz, + addr_self); + if (hret != HG_SUCCESS) { + LOGERR("margo_addr_to_string()"); + margo_addr_free(mid, addr_self); + margo_finalize(mid); + return MARGO_INSTANCE_NULL; + } + LOGDBG("shared-memory margo RPC server: %s", self_string); + margo_addr_free(mid, addr_self); + + /* publish rpc address of server for local clients */ + rpc_publish_local_server_addr(self_string); + + return mid; +} + +/* register client-server RPCs */ +static void register_client_server_rpcs(margo_instance_id mid) +{ + MARGO_REGISTER(mid, "unifyfs_mount_rpc", + unifyfs_mount_in_t, unifyfs_mount_out_t, + unifyfs_mount_rpc); + + MARGO_REGISTER(mid, "unifyfs_unmount_rpc", + unifyfs_unmount_in_t, unifyfs_unmount_out_t, + unifyfs_unmount_rpc); + + MARGO_REGISTER(mid, "unifyfs_metaget_rpc", + unifyfs_metaget_in_t, unifyfs_metaget_out_t, + unifyfs_metaget_rpc); + + MARGO_REGISTER(mid, "unifyfs_metaset_rpc", + unifyfs_metaset_in_t, unifyfs_metaset_out_t, + unifyfs_metaset_rpc); + + MARGO_REGISTER(mid, "unifyfs_fsync_rpc", + unifyfs_fsync_in_t, unifyfs_fsync_out_t, + unifyfs_fsync_rpc); + + MARGO_REGISTER(mid, "unifyfs_filesize_rpc", + unifyfs_filesize_in_t, unifyfs_filesize_out_t, + unifyfs_filesize_rpc); + + MARGO_REGISTER(mid, "unifyfs_read_rpc", + unifyfs_read_in_t, unifyfs_read_out_t, + unifyfs_read_rpc) + + MARGO_REGISTER(mid, "unifyfs_mread_rpc", + unifyfs_mread_in_t, unifyfs_mread_out_t, + unifyfs_mread_rpc); +} + +/* margo_server_rpc_init + * + * Initialize the server's Margo RPC functionality, for + * both intra-node (client-server shared memory) and + * inter-node (server-server). + */ +int margo_server_rpc_init(void) +{ + int rc = UNIFYFS_SUCCESS; + + if (NULL == unifyfsd_rpc_context) { + /* create rpc server context */ + unifyfsd_rpc_context = calloc(1, sizeof(ServerRpcContext_t)); + assert(unifyfsd_rpc_context); + } + + margo_instance_id mid; + mid = setup_local_target(); + if (mid == MARGO_INSTANCE_NULL) { + rc = UNIFYFS_FAILURE; + } else { + unifyfsd_rpc_context->shm_mid = mid; + register_client_server_rpcs(mid); + } + + mid = setup_remote_target(); + if (mid == MARGO_INSTANCE_NULL) { + rc = UNIFYFS_FAILURE; + } else { + unifyfsd_rpc_context->svr_mid = mid; + register_server_server_rpcs(mid); + } + + return rc; +} + +/* margo_server_rpc_finalize + * + * Finalize the server's Margo RPC functionality, for + * both intra-node (client-server shared memory) and + * inter-node (server-server). + */ +int margo_server_rpc_finalize(void) +{ + int rc = UNIFYFS_SUCCESS; + + if (NULL != unifyfsd_rpc_context) { + /* define a temporary to refer to context */ + ServerRpcContext_t* ctx = unifyfsd_rpc_context; + unifyfsd_rpc_context = NULL; + + rpc_clean_local_server_addr(); + + /* shut down margo */ + margo_finalize(ctx->svr_mid); + /* NOTE: 2nd call to margo_finalize() sometimes crashes - Margo bug? */ + margo_finalize(ctx->shm_mid); + + /* free memory allocated for context structure */ + free(ctx); + } + + return rc; +} + +/* margo_connect_servers + * + * Using address strings found in glb_servers, resolve + * each peer server's margo address. + */ +int margo_connect_servers(void) +{ + int rc; + int ret = (int)UNIFYFS_SUCCESS; + size_t i; + hg_return_t hret; + + for (i = 0; i < glb_num_servers; i++) { + int remote_mpi_rank = -1; + char* mpi_rank_str = NULL; + char* margo_addr_str = NULL; + + // NOTE: this really doesn't belong here, and will eventually go away + rc = unifyfs_keyval_lookup_remote(i, key_unifyfsd_mpi_rank, + &mpi_rank_str); + if ((int)UNIFYFS_SUCCESS == rc) { + remote_mpi_rank = atoi(mpi_rank_str); + free(mpi_rank_str); + } else { + LOGERR("server index=%zu - MPI rank lookup failed", i); + ret = (int)UNIFYFS_FAILURE; + } + glb_servers[i].mpi_rank = remote_mpi_rank; + + margo_addr_str = rpc_lookup_remote_server_addr(i); + glb_servers[i].margo_svr_addr_str = margo_addr_str; + if (NULL != margo_addr_str) { + LOGDBG("server index=%zu, mpi_rank=%d, margo_addr=%s", + i, remote_mpi_rank, margo_addr_str); + if (!margo_lazy_connect) { + glb_servers[i].margo_svr_addr = HG_ADDR_NULL; + hret = margo_addr_lookup(unifyfsd_rpc_context->svr_mid, + glb_servers[i].margo_svr_addr_str, + &(glb_servers[i].margo_svr_addr)); + if (hret != HG_SUCCESS) { + LOGERR("server index=%zu - margo_addr_lookup(%s) failed", + i, margo_addr_str); + ret = (int)UNIFYFS_FAILURE; + } + } + } else { + LOGERR("server index=%zu - margo addr string lookup failed", i); + ret = (int)UNIFYFS_FAILURE; + } + } + + return ret; +} diff --git a/server/src/margo_server.h b/server/src/margo_server.h new file mode 100644 index 000000000..2a9f1a298 --- /dev/null +++ b/server/src/margo_server.h @@ -0,0 +1,41 @@ +#ifndef _MARGO_SERVER_H +#define _MARGO_SERVER_H + +/******************************************** + * + * margo_server.h + * + * Declarations for the server's use of Margo + * + *********************************************/ + +#include +#include +#include + +#include + +typedef struct ServerRpcIds { + hg_id_t hello_id; + hg_id_t request_id; + hg_id_t chunk_read_request_id; + hg_id_t chunk_read_response_id; +} server_rpcs_t; + +typedef struct ServerRpcContext { + margo_instance_id shm_mid; + margo_instance_id svr_mid; + server_rpcs_t rpcs; +} ServerRpcContext_t; + +extern ServerRpcContext_t* unifyfsd_rpc_context; + +extern bool margo_use_tcp; +extern bool margo_lazy_connect; + +int margo_server_rpc_init(void); +int margo_server_rpc_finalize(void); + +int margo_connect_servers(void); + +#endif // MARGO_SERVER_H diff --git a/server/src/unifycr_cmd_handler.h b/server/src/unifycr_cmd_handler.h deleted file mode 100644 index 074921d42..000000000 --- a/server/src/unifycr_cmd_handler.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * - * Copyright 2017, UT-Battelle, LLC. - * - * LLNL-CODE-741539 - * All rights reserved. - * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. - */ - -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * Copyright (c) 2017, Florida State University. Contributions from - * the Computer Architecture and Systems Research Laboratory (CASTL) - * at the Department of Computer Science. - * - * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror - * LLNL-CODE-728877. All rights reserved. - * - * This file is part of burstfs. - * For details, see https://github.com/llnl/burstfs - * Please read https://github.com/llnl/burstfs/LICENSE for full license text. - */ - -#ifndef UNIFYCR_CMD_HANDLER_H -#define UNIFYCR_CMD_HANDLER_H - -#endif diff --git a/server/src/unifycr_init.c b/server/src/unifycr_init.c deleted file mode 100644 index 5410d345d..000000000 --- a/server/src/unifycr_init.c +++ /dev/null @@ -1,859 +0,0 @@ -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * - * Copyright 2017, UT-Battelle, LLC. - * - * LLNL-CODE-741539 - * All rights reserved. - * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. - */ - -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * Copyright (c) 2017, Florida State University. Contributions from - * the Computer Architecture and Systems Research Laboratory (CASTL) - * at the Department of Computer Science. - * - * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror - * LLNL-CODE-728877. All rights reserved. - * - * This file is part of burstfs. - * For details, see https://github.com/llnl/burstfs - * Please read https://github.com/llnl/burstfs/LICENSE for full license text. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "unifycr_metadata.h" -#include "unifycr_log.h" -#include "unifycr_sock.h" -#include "unifycr_init.h" -#include "unifycr_const.h" -#include "unifycr_shm.h" -#include "arraylist.h" -#include "unifycr_global.h" -#include "unifycr_cmd_handler.h" -#include "unifycr_service_manager.h" -#include "unifycr_request_manager.h" -#include "unifycr_runstate.h" - -#include "unifycr_clientcalls_rpc.h" -#include "unifycr_server.h" - -int* local_rank_lst; -int local_rank_cnt; -int glb_rank, glb_size; - -arraylist_t* app_config_list; -pthread_t data_thrd; -arraylist_t* thrd_list; - -int invert_sock_ids[MAX_NUM_CLIENTS]; /*records app_id for each sock_id*/ - -unifycr_cfg_t server_cfg; - -static const int IO_POOL_SIZE = 2; - -/* publishes server RPC address to a place where clients can find it */ -static void addr_publish_server_rpc(const char* addr) -{ - /* TODO: support other publish modes like PMIX */ - - /* write server address to /dev/shm/ for client on node to - * read from */ - FILE* fp = fopen("/dev/shm/svr_id", "w+"); - if (fp != NULL) { - fprintf(fp, "%s", addr); - fclose(fp); - } else { - LOGERR("Error writing server rpc addr to file `/dev/shm/svr_id'"); - } -} - -/* find_address - Resolves a name string to an address structure. - * - */ -static int find_address( - hg_class_t* hg_class, - hg_context_t* hg_context, - const char* arg, - char* addr_string) -{ - /* figure out what address this server is listening on */ - hg_addr_t addr_self; - int ret = HG_Addr_self(hg_class, &addr_self); - if (ret != HG_SUCCESS) { - LOGERR("HG_Addr_self()"); - HG_Context_destroy(hg_context); - HG_Finalize(hg_class); - return (-1); - } - char addr_self_string[128]; - hg_size_t addr_self_string_sz = 128; - ret = HG_Addr_to_string(hg_class, addr_self_string, - &addr_self_string_sz, addr_self); - if (ret != HG_SUCCESS) { - LOGERR("HG_Addr_self()"); - HG_Context_destroy(hg_context); - HG_Finalize(hg_class); - return (-1); - } - HG_Addr_free(hg_class, addr_self); - - printf("%s\n", addr_self_string); - - //add to address string here - addr_string = strcpy(addr_string, addr_self_string); - - printf("addr string:%s\n", addr_string); - - /* publish rpc address of server for local clients */ - addr_publish_server_rpc(addr_self_string); - - return 0; -} - -/* setup_sm_target - Initializes the shared-memory target. - * - * Not used for FUSE deployment. - * - */ -static margo_instance_id setup_sm_target() -{ - /* initialize margo */ - hg_return_t hret; - margo_instance_id mid; - hg_addr_t addr_self; - char addr_self_string[128]; - hg_size_t addr_self_string_sz = 128; - mid = margo_init(SMSVR_ADDR_STR, MARGO_SERVER_MODE, - 1, 1); - assert(mid); - - /* figure out what address this server is listening on */ - hret = margo_addr_self(mid, &addr_self); - if (hret != HG_SUCCESS) { - LOGERR("margo_addr_self()"); - margo_finalize(mid); - return NULL; - } - hret = margo_addr_to_string(mid, addr_self_string, - &addr_self_string_sz, addr_self); - if (hret != HG_SUCCESS) { - LOGERR("margo_addr_to_string()"); - margo_addr_free(mid, addr_self); - margo_finalize(mid); - return NULL; - } - margo_addr_free(mid, addr_self); - - printf("# accepting RPCs on address \"%s\"\n", addr_self_string); - - /* publish rpc address of server for local clients */ - addr_publish_server_rpc(addr_self_string); - - //margo_instance_id mid = margo_init_pool(*progress_pool, handler_pool, - // hg_context); - //assert(mid); - - MARGO_REGISTER(mid, "unifycr_mount_rpc", - unifycr_mount_in_t, unifycr_mount_out_t, - unifycr_mount_rpc); - - MARGO_REGISTER(mid, "unifycr_unmount_rpc", - unifycr_unmount_in_t, unifycr_unmount_out_t, - unifycr_unmount_rpc); - - MARGO_REGISTER(mid, "unifycr_metaget_rpc", - unifycr_metaget_in_t, unifycr_metaget_out_t, - unifycr_metaget_rpc); - - MARGO_REGISTER(mid, "unifycr_metaset_rpc", - unifycr_metaset_in_t, unifycr_metaset_out_t, - unifycr_metaset_rpc); - - MARGO_REGISTER(mid, "unifycr_fsync_rpc", - unifycr_fsync_in_t, unifycr_fsync_out_t, - unifycr_fsync_rpc); - - MARGO_REGISTER(mid, "unifycr_filesize_rpc", - unifycr_filesize_in_t, unifycr_filesize_out_t, - unifycr_filesize_rpc); - - MARGO_REGISTER(mid, "unifycr_read_rpc", - unifycr_read_in_t, unifycr_read_out_t, - unifycr_read_rpc); - - MARGO_REGISTER(mid, "unifycr_mread_rpc", - unifycr_mread_in_t, unifycr_mread_out_t, - unifycr_mread_rpc); - - return mid; -} - -/* setup_verbs_target - Initializes the inter-server target. - * - */ -#if 0 -static margo_instance_id setup_verbs_target(ABT_pool& progress_pool) -{ - hg_class_t* hg_class; - if (usetcp) { - hg_class = HG_Init(TCPSVR_ADDR_STR, HG_TRUE); - } else { - hg_class = HG_Init(VERBSVR_ADDR_STR, HG_TRUE); - } - - if (!hg_class) { - LOGERR("HG_Init() for verbs_target"); - assert(false); - } - hg_context_t* hg_context = HG_Context_create(hg_class); - if (!hg_context) { - LOGERR("HG_Context_create() for verbs_target"); - HG_Finalize(hg_class); - assert(false); - } - - /* figure out what address this server is listening on */ - char* addr_string = "cci+"; - if (usetcp) { - if (find_address(hg_class, hg_context, TCPSVR_ADDR_STR, - addr_string) == -1) { - assert(false); - } - } else { - if (find_address(hg_class, hg_context, VERBSVR_ADDR_STR, - addr_string) == -1) { - assert(false); - } - } - printf("finished find_address\n"); - - ABT_xstream handler_xstream; - ABT_pool handler_pool; - int ret = ABT_snoozer_xstream_create(1, &handler_pool, &handler_xstream); - if (ret != 0) { - LOGERR("ABT_snoozer_xstream_create()"); - assert(false); - } - printf("finished handler create\n"); - - margo_instance_id mid = margo_init_pool(progress_pool, handler_pool, - hg_context); - printf("finished pool init\n"); - - printf("Starting register is verbs\n"); - //MARGO_REGISTER(mid, "unifycr_mdread_rpc", - // unifycr_mdread_in_t, unifycr_mdread_out_t, - // unifycr_mdread_rpc); - - /*MARGO_REGISTER(mid, "unifycr_mdwrite_rpc", - unifycr_mdwrite_in_t, unifycr_mdwrite_out_t, - unifycr_mdwrite_rpc); - - MARGO_REGISTER(mid, "unifycr_mdchkdir_rpc", - unifycr_mdchkdir_in_t, unifycr_mdchkdir_out_t, - unifycr_mdchkdir_rpc); - - MARGO_REGISTER(mid, "unifycr_mdaddfile_rpc", - unifycr_mdaddfile_in_t, unifycr_mdaddfile_out_t, - unifycr_mdaddfile_rpc); - - MARGO_REGISTER(mid, "unifycr_mdopen_rpc", - unifycr_mdopen_in_t, unifycr_mdopen_out_t, - unifycr_mdopen_rpc); - - MARGO_REGISTER(mid, "unifycr_mdclose_rpc", - unifycr_mdclose_in_t, unifycr_mdclose_out_t, - unifycr_mdclose_rpc); - - MARGO_REGISTER(mid, "unifycr_mdgetfilestat_rpc", - unifycr_mdgetfilestat_in_t, unifycr_mdgetfilestat_out_t, - unifycr_mdgetfilestat_rpc); - - MARGO_REGISTER(mid, "unifycr_mdgetdircontents_rpc", - unifycr_mdgetdircontents_in_t, unifycr_mdgetdircontents_out_t, - unifycr_mdgetdircontents_rpc);*/ - - printf("Completed register in verbs\n"); - - server_addresses->at(my_rank).string_address = addr_string; - char fname[16]; - sprintf(fname, "%d.addr", my_rank); - - //TODO: Add file support for C - //std::ofstream myfile; - //myfile.open (fname); - //myfile << addr_string << std::endl; - //myfile.close(); - - return mid; -} -#endif -/* unifycr_server_rpc_init - Initializes the server-side RPC functionality - * for inter-server communication. - * - */ -margo_instance_id unifycr_server_rpc_init() -{ - //assert(unifycr_rpc_context); - /* set up argobots */ - /*int ret = ABT_init(0, NULL); - if ( ret != 0 ) { - LOGERR("ABT_init()"); - assert(false); - }*/ - - /* set primary ES to idle without polling */ - /* - ret = ABT_snoozer_xstream_self_set(); - if ( ret != 0 ) { - LOGERR("ABT_snoozer_xstream_self_set()"); - assert(false); - } - - - ABT_xstream* io_xstreams = (ABT_xstream*)malloc(sizeof(ABT_xstream) * IO_POOL_SIZE); - assert(io_xstreams); - - ABT_pool io_pool; - ret = ABT_snoozer_xstream_create(IO_POOL_SIZE, &io_pool, io_xstreams); - assert(ret == 0); - - aid = abt_io_init_pool(io_pool); - assert(aid); - - - ABT_xstream progress_xstream; - ret = ABT_xstream_self(&progress_xstream); - if ( ret != 0 ) { - LOGERR("ABT_xstream_self()"); - assert(false); - } - ABT_pool progress_pool; - - ret = ABT_xstream_get_main_pools(progress_xstream, 1, &progress_pool); - if ( ret != 0 ) { - LOGERR("ABT_xstream_get_main_pools()"); - assert(false); - } - - ABT_xstream progress_xstream; - ABT_pool progress_pool; - ret = ABT_snoozer_xstream_create(1, &progress_pool, &progress_xstream); - if ( ret != 0 ) { - LOGERR("ABT_snoozer_xstream_create()"); - assert(false); - }*/ - return setup_sm_target(); -#ifdef VERBS_TARGET - ABT_xstream progress_xstream2; - ABT_pool progress_pool2; - ret = ABT_snoozer_xstream_create(1, &progress_pool2, &progress_xstream2); - if (ret != 0) { - LOGERR("ABT_snoozer_xstream_create()"); - assert(false); - } - return setup_verbs_target(progress_pool2); -#endif -} - -/* - * Perform steps to create a daemon process: - * - * 1. Fork and exit from parent so child runs in the background - * 2. Set the daemon umask to 0 so file modes passed to open() and - * mkdir() fully control access modes - * 3. Call setsid() to create a new session and detach from controlling tty - * 4. Change current working directory to / so daemon doesn't block - * filesystem unmounts - * 5. close STDIN, STDOUT, and STDERR - * 6. Fork again to abdicate session leader position to guarantee - * daemon cannot reacquire a controlling TTY - * - */ -static void daemonize(void) -{ - pid_t pid; - pid_t sid; - int rc; - - pid = fork(); - - if (pid < 0) { - LOGERR("fork failed: %s", strerror(errno)); - exit(1); - } - - if (pid > 0) { - exit(0); - } - - umask(0); - - sid = setsid(); - if (sid < 0) { - LOGERR("setsid failed: %s", strerror(errno)); - exit(1); - } - - rc = chdir("/"); - if (rc < 0) { - LOGERR("chdir failed: %s", strerror(errno)); - exit(1); - } - - close(STDIN_FILENO); - close(STDOUT_FILENO); - close(STDERR_FILENO); - - pid = fork(); - if (pid < 0) { - LOGERR("fork failed: %s", strerror(errno)); - exit(1); - } else if (pid > 0) { - exit(0); - } -} - -int main(int argc, char* argv[]) -{ - int provided; - int rc; - int srvr_rank_idx = 0; - bool daemon = true; - char dbg_fname[UNIFYCR_MAX_FILENAME] = {0}; - - rc = unifycr_config_init(&server_cfg, argc, argv); - if (rc != 0) { - exit(1); - } - - rc = configurator_bool_val(server_cfg.unifycr_daemonize, &daemon); - if (rc != 0) { - exit(1); - } - if (daemon) { - daemonize(); - } - - rc = unifycr_write_runstate(&server_cfg); - if (rc != (int)UNIFYCR_SUCCESS) { - exit(1); - } - - rc = MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); - - if (rc != MPI_SUCCESS) { - exit(1); - } - - rc = MPI_Comm_rank(MPI_COMM_WORLD, &glb_rank); - if (rc != MPI_SUCCESS) { - exit(1); - } - - rc = MPI_Comm_size(MPI_COMM_WORLD, &glb_size); - if (rc != MPI_SUCCESS) { - exit(1); - } - - rc = CountTasksPerNode(glb_rank, glb_size); - if (rc < 0) { - exit(1); - } - - snprintf(dbg_fname, sizeof(dbg_fname), "%s/%s.%d", - server_cfg.log_dir, server_cfg.log_file, glb_rank); - - rc = unifycr_log_open(dbg_fname); - if (rc != ULFS_SUCCESS) - LOG(LOG_ERR, "%s", - unifycr_error_enum_description((unifycr_error_e)rc)); - - app_config_list = arraylist_create(); - if (app_config_list == NULL) { - LOG(LOG_ERR, "%s", - unifycr_error_enum_description(UNIFYCR_ERROR_NOMEM)); - exit(1); - } - - thrd_list = arraylist_create(); - if (thrd_list == NULL) { - LOG(LOG_ERR, "%s", - unifycr_error_enum_description(UNIFYCR_ERROR_NOMEM)); - exit(1); - } - - //TODO: replace with unifycr_server_rpc_init?? - margo_instance_id mid = unifycr_server_rpc_init(); - -#if defined(UNIFYCR_MULTIPLE_DELEGATORS) - srvr_rank_idx = find_rank_idx(glb_rank, local_rank_lst, - local_rank_cnt); -#endif - rc = sock_init_server(srvr_rank_idx); - if (rc != 0) { - LOG(LOG_ERR, "%s", - unifycr_error_enum_description(UNIFYCR_ERROR_SOCKET)); - exit(1); - } - - /* launch the service manager */ - rc = pthread_create(&data_thrd, NULL, sm_service_reads, NULL); - if (rc != 0) { - LOG(LOG_ERR, "%s", - unifycr_error_enum_description(UNIFYCR_ERROR_THRDINIT)); - exit(1); - } - - rc = meta_init_store(&server_cfg); - if (rc != 0) { - LOG(LOG_ERR, "%s", - unifycr_error_enum_description(UNIFYCR_ERROR_MDINIT)); - exit(1); - } - - LOGDBG("finished service initialization"); - - MPI_Barrier(MPI_COMM_WORLD); - while (1) { - rc = sock_wait_cli_cmd(); -#if 0 - if (rc != ULFS_SUCCESS) { - int ret = sock_handle_error(rc); - if (ret != 0) { - LOG(LOG_ERR, "%s", - unifycr_error_enum_description((unifycr_error_e)ret)); - exit(1); - } - - } else { - int sock_id = sock_get_id(); - if (sock_id != 0) { - char* cmd = sock_get_cmd_buf(sock_id); - //TODO: remove this loop and replace with rpc calls? - int cmd_rc = delegator_handle_command(cmd, sock_id); - if (cmd_rc != ULFS_SUCCESS) { - LOG(LOG_ERR, "%s", - unifycr_error_enum_description((unifycr_error_e)ret)); - return ret; - } - } - } -#endif - } - - margo_wait_for_finalize(mid); - MPI_Barrier(MPI_COMM_WORLD); - MPI_Finalize(); - - LOGDBG("terminating service"); - rc = unifycr_clean_runstate(&server_cfg); - - return 0; -} - -/** -* count the number of delegators per node, and -* the rank of each delegator, the results are stored -* in local_rank_cnt and local_rank_lst. -* @param numTasks: number of processes in the communicator -* @return success/error code, local_rank_cnt and local_rank_lst. -*/ -static int CountTasksPerNode(int rank, int numTasks) -{ - char localhost[HOST_NAME_MAX]; - char hostname[HOST_NAME_MAX]; - int resultsLen = HOST_NAME_MAX; - - MPI_Status status; - int rc; - - rc = MPI_Get_processor_name(localhost, &resultsLen); - if (rc != 0) { - return -1; - } - - int i; - if (numTasks > 0) { - if (rank == 0) { - /* a container of (rank, host) mappings*/ - name_rank_pair_t* host_set = - (name_rank_pair_t*)malloc(numTasks - * sizeof(name_rank_pair_t)); - /* MPI_receive all hostnames, and compare to local hostname */ - for (i = 1; i < numTasks; i++) { - rc = MPI_Recv(hostname, HOST_NAME_MAX, - MPI_CHAR, MPI_ANY_SOURCE, - MPI_ANY_TAG, - MPI_COMM_WORLD, &status); - - if (rc != 0) { - return -1; - } - strcpy(host_set[i].hostname, hostname); - host_set[i].rank = status.MPI_SOURCE; - } - strcpy(host_set[0].hostname, localhost); - host_set[0].rank = 0; - - /*sort according to the hostname*/ - qsort(host_set, numTasks, sizeof(name_rank_pair_t), - compare_name_rank_pair); - - /* rank_cnt: records the number of processes on each node - * rank_set: the list of ranks for each node - * */ - int** rank_set = (int**)malloc(numTasks * sizeof(int*)); - int* rank_cnt = (int*)malloc(numTasks * sizeof(int)); - - int cursor = 0, set_counter = 0; - for (i = 1; i < numTasks; i++) { - if (strcmp(host_set[i].hostname, - host_set[i - 1].hostname) == 0) - ; /*do nothing*/ - else { - // find a different rank, so switch to a new set - int j, k = 0; - rank_set[set_counter] = - (int*)malloc((i - cursor) * sizeof(int)); - rank_cnt[set_counter] = i - cursor; - for (j = cursor; j <= i - 1; j++) { - rank_set[set_counter][k] = host_set[j].rank; - k++; - } - - set_counter++; - cursor = i; - } - } - - - /*fill rank_cnt and rank_set entry for the last node*/ - int j = 0; - - rank_set[set_counter] = - (int*)malloc((i - cursor) * sizeof(int)); - rank_cnt[set_counter] = numTasks - cursor; - for (i = cursor; i <= numTasks - 1; i++) { - rank_set[set_counter][j] = host_set[i].rank; - j++; - } - set_counter++; - - /*broadcast the rank_cnt and rank_set information to each - * rank*/ - int root_set_no = -1; - for (i = 0; i < set_counter; i++) { - for (j = 0; j < rank_cnt[i]; j++) { - if (rank_set[i][j] != 0) { - rc = MPI_Send(&rank_cnt[i], 1, MPI_INT, - rank_set[i][j], 0, MPI_COMM_WORLD); - if (rc != 0) { - return -1; - } - - /*send the local rank set to the corresponding rank*/ - rc = MPI_Send(rank_set[i], rank_cnt[i], MPI_INT, - rank_set[i][j], 0, MPI_COMM_WORLD); - if (rc != 0) { - return -1; - } - } else { - root_set_no = i; - } - } - } - - - /* root process set its own local rank set and rank_cnt*/ - if (root_set_no >= 0) { - local_rank_lst = malloc(rank_cnt[root_set_no] * sizeof(int)); - for (i = 0; i < rank_cnt[root_set_no]; i++) { - local_rank_lst[i] = rank_set[root_set_no][i]; - } - local_rank_cnt = rank_cnt[root_set_no]; - } - - for (i = 0; i < set_counter; i++) { - free(rank_set[i]); - } - free(rank_cnt); - free(host_set); - free(rank_set); - - } else { - /* non-root process performs MPI_send to send - * hostname to root node */ - rc = MPI_Send(localhost, HOST_NAME_MAX, MPI_CHAR, - 0, 0, MPI_COMM_WORLD); - if (rc != 0) { - return -1; - } - /*receive the local rank count */ - rc = MPI_Recv(&local_rank_cnt, 1, MPI_INT, 0, - 0, MPI_COMM_WORLD, &status); - if (rc != 0) { - return -1; - } - - /* receive the the local rank list */ - local_rank_lst = (int*)malloc(local_rank_cnt * sizeof(int)); - rc = MPI_Recv(local_rank_lst, local_rank_cnt, MPI_INT, 0, - 0, MPI_COMM_WORLD, &status); - if (rc != 0) { - free(local_rank_lst); - return -1; - } - - } - - qsort(local_rank_lst, local_rank_cnt, sizeof(int), - compare_int); - - // scatter ranks out - } else { - return -1; - } - - return 0; -} - -#if defined(UNIFYCR_MULTIPLE_DELEGATORS) -static int find_rank_idx(int my_rank, - int* local_rank_lst, int local_rank_cnt) -{ - int i; - for (i = 0; i < local_rank_cnt; i++) { - if (local_rank_lst[i] == my_rank) { - return i; - } - } - - return -1; - -} -#endif - -static int compare_int(const void* a, const void* b) -{ - const int* ptr_a = a; - const int* ptr_b = b; - - if (*ptr_a - *ptr_b > 0) { - return 1; - } - - if (*ptr_a - *ptr_b < 0) { - return -1; - } - - return 0; -} - -static int compare_name_rank_pair(const void* a, const void* b) -{ - const name_rank_pair_t* pair_a = a; - const name_rank_pair_t* pair_b = b; - - if (strcmp(pair_a->hostname, pair_b->hostname) > 0) { - return 1; - } - - if (strcmp(pair_a->hostname, pair_b->hostname) < 0) { - return -1; - } - - return 0; -} - -static int unifycr_exit() -{ - int rc = ULFS_SUCCESS; - - /* notify the threads of request manager to exit*/ - int i, j; - for (i = 0; i < arraylist_size(thrd_list); i++) { - /* wait for resource manager thread to exit */ - thrd_ctrl_t* thrd_ctrl = (thrd_ctrl_t*)arraylist_get(thrd_list, i); - rm_cmd_exit(thrd_ctrl); - } - - /* all threads have joined back, so release our control strucutres */ - arraylist_free(thrd_list); - - /* sanitize the shared memory and delete the log files - * */ - int app_sz = arraylist_size(app_config_list); - - /* iterate over each active application and free resources */ - for (i = 0; i < app_sz; i++) { - /* get pointer to app config for this app_id */ - app_config_t* app = - (app_config_t*)arraylist_get(app_config_list, i); - - /* skip to next app_id if this is empty */ - if (app == NULL) { - continue; - } - - /* free resources allocate for each client */ - for (j = 0; j < MAX_NUM_CLIENTS; j++) { - /* release request buffer shared memory region */ - if (app->shm_req_bufs[j] != NULL) { - unifycr_shm_free(app->req_buf_name[j], - app->req_buf_sz, (void**)&(app->shm_req_bufs[j])); - } - - /* release receive buffer shared memory region */ - if (app->shm_recv_bufs[j] != NULL) { - unifycr_shm_free(app->recv_buf_name[j], - app->recv_buf_sz, (void**)&(app->shm_recv_bufs[j])); - } - - /* release super block shared memory region */ - if (app->shm_superblocks[j] != NULL) { - unifycr_shm_free(app->super_buf_name[j], - app->superblock_sz, (void**)&(app->shm_superblocks[j])); - } - - /* close spill log file and delete it */ - if (app->spill_log_fds[j] > 0) { - close(app->spill_log_fds[j]); - unlink(app->spill_log_name[j]); - } - - /* close spill log index file and delete it */ - if (app->spill_index_log_fds[j] > 0) { - close(app->spill_index_log_fds[j]); - unlink(app->spill_index_log_name[j]); - } - } - } - - /* shutdown the metadata service*/ - meta_sanitize(); - - /* TODO: notify the service threads to exit */ - - /* destroy the sockets except for the ones - * for acks*/ - sock_sanitize(); - - return rc; -} diff --git a/server/src/unifycr_init.h b/server/src/unifycr_init.h deleted file mode 100644 index c07ad0727..000000000 --- a/server/src/unifycr_init.h +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * - * Copyright 2017, UT-Battelle, LLC. - * - * LLNL-CODE-741539 - * All rights reserved. - * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. - */ - -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * Copyright (c) 2017, Florida State University. Contributions from - * the Computer Architecture and Systems Research Laboratory (CASTL) - * at the Department of Computer Science. - * - * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror - * LLNL-CODE-728877. All rights reserved. - * - * This file is part of burstfs. - * For details, see https://github.com/llnl/burstfs - * Please read https://github.com/llnl/burstfs/LICENSE for full license text. - */ - -#ifndef UNIFYCR_INIT_H -#define UNIFYCR_INIT_H -#include -#include "unifycr_const.h" - -/* - * structure that records the information of - * each application launched by srun. - * */ -typedef struct { - char hostname[HOST_NAME_MAX]; - int rank; -} name_rank_pair_t; - -typedef struct { - pthread_t thrd; - pthread_cond_t thrd_cond; -} thread_ctrl_t; - -static int CountTasksPerNode(int rank, int numTasks); -static int compare_name_rank_pair(const void* a, const void* b); -static int compare_int(const void* a, const void* b); - -#if defined(UNIFYCR_MULTIPLE_DELEGATORS) -static int find_rank_idx(int my_rank, - int* local_rank_lst, int local_rank_cnt); -#endif - -static int unifycr_exit(); -#endif diff --git a/server/src/unifycr_metadata.c b/server/src/unifycr_metadata.c deleted file mode 100644 index cc9c59d1e..000000000 --- a/server/src/unifycr_metadata.c +++ /dev/null @@ -1,736 +0,0 @@ -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * - * Copyright 2017, UT-Battelle, LLC. - * - * LLNL-CODE-741539 - * All rights reserved. - * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. - */ - -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * Copyright (c) 2017, Florida State University. Contributions from - * the Computer Architecture and Systems Research Laboratory (CASTL) - * at the Department of Computer Science. - * - * Written by: Teng Wang, Adam Moody, Wekuan Yu, Kento Sato, Kathryn Mohror - * LLNL-CODE-728877. All rights reserved. - * - * This file is part of burstfs. - * For details, see https://github.com/llnl/burstfs - * Please read https://github.com/llnl/burstfs/LICNSE for full license text. - */ - -#include -#include -#include -#include -#include - -#include "arraylist.h" -#include "indexes.h" -#include "mdhim.h" -#include "unifycr_log.h" -#include "unifycr_metadata.h" -#include "unifycr_clientcalls_rpc.h" -#include "ucr_read_builder.h" - -unifycr_key_t** unifycr_keys; -unifycr_val_t** unifycr_vals; - -fattr_key_t** fattr_keys; -fattr_val_t** fattr_vals; - -char* manifest_path; - -struct mdhim_brm_t* brm, *brmp; -struct mdhim_bgetrm_t* bgrm, *bgrmp; - -mdhim_options_t* db_opts; -struct mdhim_t* md; - -int md_size; -int unifycr_key_lens[MAX_META_PER_SEND] = {0}; -int unifycr_val_lens[MAX_META_PER_SEND] = {0}; - -int fattr_key_lens[MAX_FILE_CNT_PER_NODE] = {0}; -int fattr_val_lens[MAX_FILE_CNT_PER_NODE] = {0}; - -struct index_t* unifycr_indexes[2]; -size_t max_recs_per_slice; - -void debug_log_key_val(const char* ctx, - unifycr_key_t* key, - unifycr_val_t* val) -{ - if ((key != NULL) && (val != NULL)) { - LOGDBG("@%s - key(fid=%lu, offset=%lu), " - "val(del=%d, len=%lu, addr=%lu, app=%d, rank=%d)", - ctx, key->fid, key->offset, - val->delegator_id, val->len, val->addr, val->app_id, val->rank); - } else if (key != NULL) { - LOGDBG("@%s - key(fid=%lu, offset=%lu)", - ctx, key->fid, key->offset); - } -} - -int unifycr_key_compare(unifycr_key_t* a, unifycr_key_t* b) -{ - assert((NULL != a) && (NULL != b)); - if (a->fid == b->fid) { - if (a->offset == b->offset) { - return 0; - } else if (a->offset < b->offset) { - return -1; - } else { - return 1; - } - } else if (a->fid < b->fid) { - return -1; - } else { - return 1; - } -} - -/** -* initialize the key-value store -*/ -int meta_init_store(unifycr_cfg_t* cfg) -{ - int rc, ser_ratio; - size_t path_len; - long svr_ratio, range_sz; - MPI_Comm comm = MPI_COMM_WORLD; - - if (cfg == NULL) { - return -1; - } - - db_opts = calloc(1, sizeof(struct mdhim_options_t)); - if (db_opts == NULL) { - return -1; - } - - /* UNIFYCR_META_DB_PATH: file that stores the key value pair*/ - db_opts->db_path = strdup(cfg->meta_db_path); - if (db_opts->db_path == NULL) { - return -1; - } - - db_opts->manifest_path = NULL; - db_opts->db_type = LEVELDB; - db_opts->db_create_new = 1; - - /* META_SERVER_RATIO: number of metadata servers = - number of processes/META_SERVER_RATIO */ - svr_ratio = 0; - rc = configurator_int_val(cfg->meta_server_ratio, &svr_ratio); - if (rc != 0) { - return -1; - } - ser_ratio = (int) svr_ratio; - db_opts->rserver_factor = ser_ratio; - - db_opts->db_paths = NULL; - db_opts->num_paths = 0; - db_opts->num_wthreads = 1; - - path_len = strlen(db_opts->db_path) + strlen(MANIFEST_FILE_NAME) + 1; - manifest_path = malloc(path_len); - if (manifest_path == NULL) { - return -1; - } - sprintf(manifest_path, "%s/%s", db_opts->db_path, MANIFEST_FILE_NAME); - db_opts->manifest_path = manifest_path; - - db_opts->db_name = strdup(cfg->meta_db_name); - if (db_opts->db_name == NULL) { - return -1; - } - - db_opts->db_key_type = MDHIM_UNIFYCR_KEY; - db_opts->debug_level = MLOG_CRIT; - - /* indices/attributes are striped to servers according - * to UnifyCR_META_RANGE_SIZE. - */ - range_sz = 0; - rc = configurator_int_val(cfg->meta_range_size, &range_sz); - if (rc != 0) { - return -1; - } - max_recs_per_slice = (size_t) range_sz; - db_opts->max_recs_per_slice = (uint64_t) range_sz; - - md = mdhimInit(&comm, db_opts); - - /*this index is created for storing index metadata*/ - unifycr_indexes[0] = md->primary_index; - - /*this index is created for storing file attribute metadata*/ - unifycr_indexes[1] = create_global_index(md, ser_ratio, 1, - LEVELDB, MDHIM_INT_KEY, "file_attr"); - - MPI_Comm_size(md->mdhim_comm, &md_size); - - rc = meta_init_indices(); - if (rc != 0) { - return -1; - } - - return 0; - -} - -/** -* initialize the key and value list used to -* put/get key-value pairs -* ToDo: split once the number of metadata exceeds MAX_META_PER_SEND -*/ -int meta_init_indices() -{ - - int i; - - /*init index metadata*/ - unifycr_keys = (unifycr_key_t**)malloc(MAX_META_PER_SEND - * sizeof(unifycr_key_t*)); - - unifycr_vals = (unifycr_val_t**)malloc(MAX_META_PER_SEND - * sizeof(unifycr_val_t*)); - - for (i = 0; i < MAX_META_PER_SEND; i++) { - unifycr_keys[i] = (unifycr_key_t*)malloc(sizeof(unifycr_key_t)); - if (unifycr_keys[i] == NULL) { - return (int)UNIFYCR_ERROR_NOMEM; - } - memset(unifycr_keys[i], 0, sizeof(unifycr_key_t)); - } - - for (i = 0; i < MAX_META_PER_SEND; i++) { - unifycr_vals[i] = (unifycr_val_t*)malloc(sizeof(unifycr_val_t)); - if (unifycr_vals[i] == NULL) { - return (int)UNIFYCR_ERROR_NOMEM; - } - memset(unifycr_vals[i], 0, sizeof(unifycr_val_t)); - } - - /*init attribute metadata*/ - fattr_keys = (fattr_key_t**)malloc(MAX_FILE_CNT_PER_NODE - * sizeof(fattr_key_t*)); - - fattr_vals = (fattr_val_t**)malloc(MAX_FILE_CNT_PER_NODE - * sizeof(fattr_val_t*)); - - for (i = 0; i < MAX_FILE_CNT_PER_NODE; i++) { - fattr_keys[i] = (fattr_key_t*)malloc(sizeof(fattr_key_t)); - if (fattr_keys[i] == NULL) { - return (int)UNIFYCR_ERROR_NOMEM; - } - memset(fattr_keys[i], 0, sizeof(fattr_key_t)); - } - - for (i = 0; i < MAX_FILE_CNT_PER_NODE; i++) { - fattr_vals[i] = (fattr_val_t*)malloc(sizeof(fattr_val_t)); - if (fattr_vals[i] == NULL) { - return (int)UNIFYCR_ERROR_NOMEM; - } - memset(fattr_vals[i], 0, sizeof(fattr_val_t)); - } - - return 0; -} - -/** -* store the file attribute to the key-value store -* @param ptr_fattr: file attribute received from the client -* @return success/error code -*/ -int meta_process_attr_set(unifycr_file_attr_t* ptr_fattr) -{ - int rc; - fattr_key_t local_key; - fattr_val_t local_val; - - assert(NULL != ptr_fattr); - - local_key = (fattr_key_t) ptr_fattr->gfid; - memset(&local_val, 0, sizeof(local_val)); - strncpy(local_val.fname, ptr_fattr->filename, sizeof(local_val.fname)); - local_val.file_attr = ptr_fattr->file_attr; - - /* LOGDBG("rank:%d, setting fattr key:%d, value:%s", - * glb_rank, local_key, local_val.fname); */ - md->primary_index = unifycr_indexes[1]; - brm = mdhimPut(md, &local_key, sizeof(fattr_key_t), - &local_val, sizeof(fattr_val_t), - NULL, NULL); - if (!brm || brm->error) { - rc = (int)UNIFYCR_ERROR_MDHIM; - } else { - rc = UNIFYCR_SUCCESS; - } - - mdhim_full_release_msg(brm); - - return rc; -} - - -/* get the file attribute from the key-value store -* @param ptr_fattr: file attribute to return to the client -* @return success/error code -*/ - -int meta_process_attr_get(unifycr_file_attr_t* ptr_fattr) -{ - int rc; - - assert(NULL != ptr_fattr); - - fattr_key_t local_key = (fattr_key_t) ptr_fattr->gfid; - - md->primary_index = unifycr_indexes[1]; - bgrm = mdhimGet(md, md->primary_index, - &local_key, sizeof(fattr_key_t), MDHIM_GET_EQ); - if (!bgrm || bgrm->error) { - rc = (int)UNIFYCR_ERROR_MDHIM; - } else { - fattr_val_t* ptr_val = (fattr_val_t*) bgrm->values[0]; - - /* LOGDBG("rank:%d, getting fattr key:%d", - * glb_rank, ptr_fattr->gfid); */ - ptr_fattr->file_attr = ptr_val->file_attr; - strncpy(ptr_fattr->filename, ptr_val->fname, - sizeof(ptr_fattr->filename)); - - rc = UNIFYCR_SUCCESS; - } - - mdhim_full_release_msg(bgrm); - return rc; -} - -/*synchronize all the indices and file attributes -* to the key-value store -* @param sock_id: the connection id in poll_set of the delegator -* @return success/error code -*/ - -int meta_process_fsync(int app_id, int client_side_id, int gfid) -{ - int ret = 0; - - app_config_t* app_config = (app_config_t*)arraylist_get(app_config_list, - app_id); - - size_t num_entries = - *((size_t*)(app_config->shm_superblocks[client_side_id] - + app_config->meta_offset)); - - /* indices are stored in the superblock shared memory - * created by the client*/ - int page_sz = getpagesize(); - unifycr_index_t* meta_payload = - (unifycr_index_t*)(app_config->shm_superblocks[client_side_id] - + app_config->meta_offset + page_sz); - - md->primary_index = unifycr_indexes[0]; - - size_t i; - size_t used_entries = 0; - for (i = 0; i < num_entries; i++) { - if (gfid != meta_payload[i].fid) { - continue; - } - - unifycr_keys[used_entries]->fid = meta_payload[i].fid; - unifycr_keys[used_entries]->offset = meta_payload[i].file_pos; - - unifycr_vals[used_entries]->addr = meta_payload[i].mem_pos; - unifycr_vals[used_entries]->len = meta_payload[i].length; - unifycr_vals[used_entries]->delegator_id = glb_rank; - unifycr_vals[used_entries]->app_id = app_id; - unifycr_vals[used_entries]->rank = client_side_id; - - // debug_log_key_val("before put", - // unifycr_keys[used_entries], - // unifycr_vals[used_entries]); - - unifycr_key_lens[used_entries] = sizeof(unifycr_key_t); - unifycr_val_lens[used_entries] = sizeof(unifycr_val_t); - used_entries++; - } - - // print_fsync_indices(unifycr_keys, unifycr_vals, used_entries); - - brm = mdhimBPut(md, (void**)(&unifycr_keys[0]), unifycr_key_lens, - (void**)(&unifycr_vals[0]), unifycr_val_lens, used_entries, - NULL, NULL); - brmp = brm; - while (brmp) { - if (brmp->error < 0) { - LOGDBG("mdhimBPut returned error %d", brmp->error); - ret = (int)UNIFYCR_ERROR_MDHIM; - } - brm = brmp; - brmp = brmp->next; - mdhim_full_release_msg(brm); - } - - md->primary_index = unifycr_indexes[1]; - - num_entries = - *((size_t*)(app_config->shm_superblocks[client_side_id] - + app_config->fmeta_offset)); - - /* file attributes are stored in the superblock shared memory - * created by the client*/ - unifycr_file_attr_t* attr_payload = - (unifycr_file_attr_t*)(app_config->shm_superblocks[client_side_id] - + app_config->fmeta_offset + page_sz); - - used_entries = 0; - for (i = 0; i < num_entries; i++) { - if (gfid != attr_payload[i].gfid) { - continue; - } - - *fattr_keys[used_entries] = attr_payload[i].gfid; - fattr_vals[used_entries]->file_attr = attr_payload[i].file_attr; - strcpy(fattr_vals[used_entries]->fname, attr_payload[i].filename); - - fattr_key_lens[used_entries] = sizeof(fattr_key_t); - fattr_val_lens[used_entries] = sizeof(fattr_val_t); - used_entries++; - } - - brm = mdhimBPut(md, (void**)(&fattr_keys[0]), fattr_key_lens, - (void**)(&fattr_vals[0]), fattr_val_lens, used_entries, - NULL, NULL); - brmp = brm; - while (brmp) { - if (brmp->error < 0) { - LOGDBG("mdhimBPut returned error %d", brmp->error); - ret = (int)UNIFYCR_ERROR_MDHIM; - } - brm = brmp; - brmp = brmp->next; - mdhim_full_release_msg(brm); - } - - if (ret == (int)UNIFYCR_ERROR_MDHIM) { - LOGDBG("Rank - %d: Error inserting keys/values into MDHIM", - md->mdhim_rank); - } - - return ret; -} - -/* get the locations of all the requested file segment from - * the key-value store. - * @param app_id: client's application id - * @param client_id: client-side process id - * @param thrd_id: the thread created for processing - * its client's read requests. - * @param cli_rank: the client process's rank in its - * own application, used for debug purpose - * @param gfid: global file id - * @param offset: start offset for segment - * @param length: segment length - * @del_req_set: contains metadata information for all - * the read requests, such as the locations of the - * requested segments - * @return success/error code - */ -int meta_read_get(int app_id, int client_id, int thrd_id, int cli_rank, - int gfid, size_t offset, size_t length, - msg_meta_t* del_req_set) -{ - /* assume we'll succeed, set to error otherwise */ - int rc = (int)UNIFYCR_SUCCESS; - - LOGDBG("app_id:%d, client_id:%d, thrd_id:%d, " - "fid:%d, offset:%zu, length:%zu", - app_id, client_id, thrd_id, gfid, offset, length); - - /* create key to describe first byte we'll read */ - unifycr_keys[0]->fid = gfid; - unifycr_keys[0]->offset = offset; - - /* create key to describe last byte we'll read */ - unifycr_keys[1]->fid = gfid; - unifycr_keys[1]->offset = offset + length - 1; - - unifycr_key_lens[0] = sizeof(unifycr_key_t); - unifycr_key_lens[1] = sizeof(unifycr_key_t); - - /* get list of values for these keys */ - md->primary_index = unifycr_indexes[0]; - bgrm = mdhimBGet(md, md->primary_index, (void**)unifycr_keys, - unifycr_key_lens, 2, MDHIM_RANGE_BGET); - - /* track number of items we got */ - int tot_num = 0; - - /* iterate over each item returned by get */ - while (bgrm) { - bgrmp = bgrm; - /* check that we didn't hit an error with this item */ - if (bgrmp->error < 0) { - rc = (int)UNIFYCR_ERROR_MDHIM; - } - - /* iterate over each key and value in this item */ - unifycr_key_t* key; - unifycr_val_t* val; - size_t i; - for (i = 0; i < bgrmp->num_keys; i++) { - /* get pointer to key and value */ - key = (unifycr_key_t*) bgrmp->keys[i]; - val = (unifycr_val_t*) bgrmp->values[i]; - - /* get pointer to next send_msg structure */ - send_msg_t* msg = &(del_req_set->msg_meta[tot_num]); - memset(msg, 0, sizeof(send_msg_t)); - tot_num++; - - /* rank of the remote delegator */ - msg->dest_delegator_rank = val->delegator_id; - - /* dest_client_id and dest_app_id uniquely identify the remote - * physical log file that contains the requested segments */ - msg->dest_app_id = val->app_id; - msg->dest_client_id = val->rank; - - /* physical offset of the requested file segment on the log file */ - msg->dest_offset = (size_t) val->addr; - msg->length = (size_t) val->len; - - /* src_app_id and src_cli_id identifies the requested client */ - msg->src_cli_id = client_id; - msg->src_app_id = app_id; - - /* src_offset is the logical offset of the shared file */ - msg->src_offset = (size_t) key->offset; - - msg->src_dbg_rank = cli_rank; - msg->src_delegator_rank = glb_rank; - msg->src_fid = (int) key->fid; - msg->src_thrd = thrd_id; - } - - /* advance to next item in list */ - bgrm = bgrmp->next; - mdhim_full_release_msg(bgrmp); - } - - /* record total number of entries */ - del_req_set->num = tot_num; - - return rc; -} - -/* get the locations of all the requested file segments from - * the key-value store. - * @param app_id: client's application id - * @param client_id: client-side process id - * @param thrd_id: the thread created for processing - * its client's read requests. - * @param dbg_rank: the client process's rank in its - * own application, used for debug purpose - * @param reqbuf: memory buffer that contains - * the client's read requests - * @param req_cnt: number of read requests - * @del_req_set: contains metadata information for all - * the read requests, such as the locations of the - * requested segments - * @return success/error code - */ -int meta_batch_get(int app_id, int client_id, int thrd_id, int dbg_rank, - void* reqbuf, size_t req_cnt, - msg_meta_t* del_req_set) -{ - LOGDBG("app_id:%d, client_id:%d, thrd_id:%d", - app_id, client_id, thrd_id); - - unifycr_ReadRequest_table_t readRequest = unifycr_ReadRequest_as_root(reqbuf); - unifycr_Extent_vec_t extents = unifycr_ReadRequest_extents(readRequest); - size_t extents_len = unifycr_Extent_vec_len(extents); - assert(extents_len == req_cnt); - - int fid; - size_t i, ndx, eoff, elen; - for (i = 0; i < extents_len; i++) { - ndx = 2 * i; - fid = unifycr_Extent_fid(unifycr_Extent_vec_at(extents, i)); - eoff = unifycr_Extent_offset(unifycr_Extent_vec_at(extents, i)); - elen = unifycr_Extent_length(unifycr_Extent_vec_at(extents, i)); - LOGDBG("fid:%d, offset:%zu, length:%zu", fid, eoff, elen); - - unifycr_keys[ndx]->fid = fid; - unifycr_keys[ndx]->offset = eoff; - unifycr_keys[ndx + 1]->fid = fid; - unifycr_keys[ndx + 1]->offset = eoff + elen - 1; - unifycr_key_lens[ndx] = sizeof(unifycr_key_t); - unifycr_key_lens[ndx + 1] = sizeof(unifycr_key_t); - } - - md->primary_index = unifycr_indexes[0]; - bgrm = mdhimBGet(md, md->primary_index, (void**)unifycr_keys, - unifycr_key_lens, 2 * extents_len, MDHIM_RANGE_BGET); - - int rc = 0; - int tot_num = 0; - unifycr_key_t* key; - unifycr_val_t* val; - send_msg_t* msg; - - while (bgrm) { - bgrmp = bgrm; - if (bgrmp->error < 0) { - rc = (int)UNIFYCR_ERROR_MDHIM; - } - - for (i = 0; i < bgrmp->num_keys; i++) { - key = (unifycr_key_t*) bgrmp->keys[i]; - val = (unifycr_val_t*) bgrmp->values[i]; - // debug_log_key_val("after get", key, val); - - msg = &(del_req_set->msg_meta[tot_num]); - memset(msg, 0, sizeof(send_msg_t)); - tot_num++; - - /* rank of the remote delegator */ - msg->dest_delegator_rank = val->delegator_id; - - /* dest_client_id and dest_app_id uniquely identify the remote - * physical log file that contains the requested segments */ - msg->dest_client_id = val->rank; - msg->dest_app_id = val->app_id; - - /* physical offset of the requested file segment on the log file */ - msg->dest_offset = (size_t) val->addr; - msg->length = (size_t) val->len; - - /* src_app_id and src_cli_id identifies the requested client */ - msg->src_app_id = app_id; - msg->src_cli_id = client_id; - - /* src_offset is the logical offset of the shared file */ - msg->src_offset = (size_t) key->offset; - - msg->src_dbg_rank = dbg_rank; - msg->src_delegator_rank = glb_rank; - msg->src_fid = (int) key->fid; - msg->src_thrd = thrd_id; - } - bgrm = bgrmp->next; - mdhim_full_release_msg(bgrmp); - } - - del_req_set->num = tot_num; -// print_bget_indices(app_id, client_id, del_req_set->msg_meta, tot_num); - - return rc; -} - -void print_bget_indices(int app_id, int cli_id, - send_msg_t* msgs, int tot_num) -{ - int i; - for (i = 0; i < tot_num; i++) { - LOGDBG("index:dbg_rank:%d, dest_offset:%zu, " - "dest_del_rank:%d, dest_cli_id:%d, dest_app_id:%d, " - "length:%zu, src_app_id:%d, src_cli_id:%d, src_offset:%zu, " - "src_del_rank:%d, src_fid:%d, num:%d", - msgs[i].src_dbg_rank, msgs[i].dest_offset, - msgs[i].dest_delegator_rank, msgs[i].dest_client_id, - msgs[i].dest_app_id, msgs[i].length, - msgs[i].src_app_id, msgs[i].src_cli_id, - msgs[i].src_offset, msgs[i].src_delegator_rank, - msgs[i].src_fid, tot_num); - } -} - -void print_fsync_indices(unifycr_key_t** unifycr_keys, - unifycr_val_t** unifycr_vals, - size_t num_entries) -{ - size_t i; - for (i = 0; i < num_entries; i++) { - LOGDBG("fid:%lu, offset:%lu, addr:%lu, len:%lu, del_id:%d", - unifycr_keys[i]->fid, unifycr_keys[i]->offset, - unifycr_vals[i]->addr, unifycr_vals[i]->len, - unifycr_vals[i]->delegator_id); - } -} - -int meta_free_indices() -{ - int i; - for (i = 0; i < MAX_META_PER_SEND; i++) { - free(unifycr_keys[i]); - } - free(unifycr_keys); - - for (i = 0; i < MAX_META_PER_SEND; i++) { - free(unifycr_vals[i]); - } - free(unifycr_vals); - - for (i = 0; i < MAX_FILE_CNT_PER_NODE; i++) { - free(fattr_keys[i]); - } - free(fattr_keys); - - for (i = 0; i < MAX_FILE_CNT_PER_NODE; i++) { - free(fattr_vals[i]); - } - free(fattr_vals); - - return 0; -} - -int meta_sanitize() -{ - int rc = ULFS_SUCCESS; - - meta_free_indices(); - - char dbfilename[UNIFYCR_MAX_FILENAME] = {0}; - char statfilename[UNIFYCR_MAX_FILENAME] = {0}; - char manifestname[UNIFYCR_MAX_FILENAME] = {0}; - - char dbfilename1[UNIFYCR_MAX_FILENAME] = {0}; - char statfilename1[UNIFYCR_MAX_FILENAME] = {0}; - char manifestname1[UNIFYCR_MAX_FILENAME] = {0}; - sprintf(dbfilename, "%s/%s-%d-%d", md->db_opts->db_path, - md->db_opts->db_name, unifycr_indexes[0]->id, md->mdhim_rank); - - sprintf(statfilename, "%s_stats", dbfilename); - sprintf(manifestname, "%s%d_%d_%d", md->db_opts->manifest_path, - unifycr_indexes[0]->type, - unifycr_indexes[0]->id, md->mdhim_rank); - - sprintf(dbfilename1, "%s/%s-%d-%d", md->db_opts->db_path, - md->db_opts->db_name, unifycr_indexes[1]->id, md->mdhim_rank); - - sprintf(statfilename1, "%s_stats", dbfilename1); - sprintf(manifestname1, "%s%d_%d_%d", md->db_opts->manifest_path, - unifycr_indexes[1]->type, - unifycr_indexes[1]->id, md->mdhim_rank); - - mdhimClose(md); - rc = mdhimSanitize(dbfilename, statfilename, manifestname); - rc = mdhimSanitize(dbfilename1, statfilename1, manifestname1); - - mdhim_options_destroy(db_opts); - return rc; -} diff --git a/server/src/unifycr_metadata.h b/server/src/unifycr_metadata.h deleted file mode 100644 index 203cc0080..000000000 --- a/server/src/unifycr_metadata.h +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * - * Copyright 2017, UT-Battelle, LLC. - * - * LLNL-CODE-741539 - * All rights reserved. - * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. - */ - -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * Copyright (c) 2017, Florida State University. Contributions from - * the Computer Architecture and Systems Research Laboratory (CASTL) - * at the Department of Computer Science. - * - * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror - * LLNL-CODE-728877. All rights reserved. - * - * This file is part of burstfs. - * For details, see https://github.com/llnl/burstfs - * Please read https://github.com/llnl/burstfs/LICENSE for full license text. - */ - -#ifndef UNIFYCR_METADATA_H -#define UNIFYCR_METADATA_H - -#include "unifycr_const.h" -#include "unifycr_global.h" -#include "unifycr_meta.h" - -#define MANIFEST_FILE_NAME "mdhim_manifest_" - -typedef struct { - unsigned long fid; - unsigned long offset; -} unifycr_key_t; - -#define UNIFYCR_KEY_SZ (sizeof(unifycr_key_t)) - -#define UNIFYCR_KEY_FID(keyp) (((unifycr_key_t*)keyp)->fid) -#define UNIFYCR_KEY_OFF(keyp) (((unifycr_key_t*)keyp)->offset) - -int unifycr_key_compare(unifycr_key_t* a, unifycr_key_t* b); - -typedef struct { - unsigned long addr; - unsigned long len; - int delegator_id; - int app_id; - int rank; -} unifycr_val_t; - -#define UNIFYCR_VAL_SZ (sizeof(unifycr_val_t)) - -#define UNIFYCR_VAL_ADDR(valp) (((unifycr_val_t*)valp)->addr) -#define UNIFYCR_VAL_LEN(valp) (((unifycr_val_t*)valp)->len) - -void debug_log_key_val(const char* ctx, - unifycr_key_t* key, - unifycr_val_t* val); - -int meta_sanitize(); -int meta_init_store(unifycr_cfg_t* cfg); -void print_bget_indices(int app_id, int client_id, - send_msg_t* index_set, int tot_num); -int meta_process_fsync(int app_id, int client_id, int gfid); -int meta_read_get(int app_id, int client_id, int thrd_id, int dbg_rank, - int gfid, size_t offset, size_t length, - msg_meta_t* del_req_set); -int meta_batch_get(int app_id, int client_id, int thrd_id, int dbg_rank, - void* reqbuf, size_t req_cnt, - msg_meta_t* del_req_set); -int meta_init_indices(); -int meta_free_indices(); -void print_fsync_indices(unifycr_key_t** unifycr_keys, - unifycr_val_t** unifycr_vals, size_t num_entries); - -int meta_process_attr_get(unifycr_file_attr_t* ptr_attr_val); -int meta_process_attr_set(unifycr_file_attr_t* ptr_attr_val); - -#endif diff --git a/server/src/unifycr_request_manager.c b/server/src/unifycr_request_manager.c deleted file mode 100644 index d34266c28..000000000 --- a/server/src/unifycr_request_manager.c +++ /dev/null @@ -1,898 +0,0 @@ -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * - * Copyright 2017, UT-Battelle, LLC. - * - * LLNL-CODE-741539 - * All rights reserved. - * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. - */ - -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * Copyright (c) 2017, Florida State University. Contribuions from - * the Computer Architecture and Systems Research Laboratory (CASTL) - * at the Department of Computer Science. - * - * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror - * LLNL-CODE-728877. All rights reserved. - * - * This file is part of burstfs. - * For details, see https://github.com/llnl/burstfs - * Please read https://github.com/llnl/burstfs/LICENSE for full license text. - */ - -#include -#include -#include -#include -#include -#include "unifycr_log.h" -#include "unifycr_request_manager.h" -#include "unifycr_const.h" -#include "unifycr_global.h" -#include "unifycr_metadata.h" -#include "unifycr_sock.h" - -/* One request manager thread is created for each client that a - * delegator serves. The main thread of the delegator assigns - * work to the request manager thread to retrieve data and send - * it back to the client. - * - * To start, given a read request from the client (via rpc) - * the handler function on the main delegator first queries the - * key/value store using the given file id and byte range to obtain - * the meta data on the physical location of the file data. This - * meta data provides the host delegator rank, the app/client - * ids that specify the log file on the remote delegator, the - * offset within the log file and the length of data. The rpc - * handler function sorts the meta data by host delegator rank, - * generates read requests, and inserts those into a list on a - * data structure shared with the request manager (del_req_set). - * - * The request manager thread coordinates with the main thread - * using a lock and a condition variable to protect the shared data - * structure and impose flow control. When assigned work, the - * request manager thread packs and sends request messages to - * service manager threads on remote delegators via MPI send. - * It waits for data to be sent back, and unpacks the read replies - * in each message into a shared memory buffer for the client. - * When the shared memory is full or all data has been received, - * it signals the client process to process the read replies. - * It iterates with the client until all incoming read replies - * have been transferred. */ - -static void print_send_msgs(send_msg_t* send_metas, - int msg_cnt) -{ - int i; - for (i = 0; i < msg_cnt; i++) { - LOGDBG("src_offset:%zu, byte_cnt:%zu", - send_metas[i].src_offset, - send_metas[i].length); - } -} - -static void print_remote_del_reqs(int app_id, int cli_id, - del_req_stat_t* del_req_stat) -{ - int i; - for (i = 0; i < del_req_stat->del_cnt; i++) { - LOGDBG("remote_delegator:%d, req_cnt:%d", - del_req_stat->req_stat[i].del_id, - del_req_stat->req_stat[i].req_cnt); - } -} - -#if 0 // NOT CURRENTLY USED -static void print_recv_msg(int app_id, int cli_id, - int thrd_id, - shm_meta_t* msg) -{ - LOGDBG("recv_msg: app_id:%d, cli_id:%d, thrd_id:%d, " - "fid:%d, offset:%ld, len:%ld", - app_id, cli_id, thrd_id, msg->src_fid, - msg->offset, msg->length); -} -#endif - -/* order read requests by destination delegator rank */ -static int compare_delegators(const void* a, const void* b) -{ - const send_msg_t* msg_a = a; - const send_msg_t* msg_b = b; - int rank_a = msg_a->dest_delegator_rank; - int rank_b = msg_b->dest_delegator_rank; - - if (rank_a == rank_b) { - return 0; - } else if (rank_a < rank_b) { - return -1; - } else { - return 1; - } -} - -/************************ - * These functions are called by the rpc handler to assign work - * to the request manager thread - ***********************/ - -/* given an app_id, client_id and global file id, - * compute and return file size for specified file - */ -int rm_cmd_filesize( - int app_id, /* app_id for requesting client */ - int client_id, /* client_id for requesting client */ - int gfid, /* global file id of read request */ - size_t* outsize) /* output file size */ -{ - /* get pointer to app structure for this app id */ - app_config_t* app_config = - (app_config_t*)arraylist_get(app_config_list, app_id); - - /* get thread id for this client */ - int thrd_id = app_config->thrd_idxs[client_id]; - - /* look up thread control structure */ - thrd_ctrl_t* thrd_ctrl = - (thrd_ctrl_t*)arraylist_get(thrd_list, thrd_id); - - /* wait for lock for shared data structures holding requests - * and condition variable */ - pthread_mutex_lock(&thrd_ctrl->thrd_lock); - - /* set offset and length to request *all* key/value pairs - * for this file */ - size_t offset = 0; - - /* want to pick the highest integer offset value a file - * could have here */ - // TODO: would like to unsed max for unsigned long, but - // that fails to return any keys for some reason - size_t length = (SIZE_MAX >> 1) - 1; - - /* get the locations of all the read requests from the - * key-value store*/ - int rc = meta_read_get(app_id, client_id, thrd_id, 0, - gfid, offset, length, thrd_ctrl->del_req_set); - - /* compute our file size by iterating over each file - * segment and taking the max logical offset */ - int i; - size_t filesize = 0; - for (i = 0; i < thrd_ctrl->del_req_set->num; i++) { - /* get pointer to next send_msg structure */ - send_msg_t* msg = &(thrd_ctrl->del_req_set->msg_meta[i]); - - /* get last byte offset for this segment of the file */ - size_t last_offset = msg->src_offset + msg->length; - - /* update our filesize if this offset is bigger than the current max */ - if (last_offset > filesize) { - filesize = last_offset; - } - } - - /* done updating shared variables, release the lock */ - pthread_mutex_unlock(&thrd_ctrl->thrd_lock); - - *outsize = filesize; - return rc; -} - -/* read function for one requested extent, - * called from rpc handler to fill shared data structures - * with read requests to be handled by the delegator thread - * returns before requests are handled - */ -int rm_cmd_read( - int app_id, /* app_id for requesting client */ - int client_id, /* client_id for requesting client */ - int gfid, /* global file id of read request */ - size_t offset, /* logical file offset of read request */ - size_t length) /* number of bytes to read */ -{ - /* get pointer to app structure for this app id */ - app_config_t* app_config = - (app_config_t*)arraylist_get(app_config_list, app_id); - - /* get thread id for this client */ - int thrd_id = app_config->thrd_idxs[client_id]; - - /* client app rank for debugging */ - int cli_rank = app_config->dbg_ranks[client_id]; - - /* look up thread control structure */ - thrd_ctrl_t* thrd_ctrl = - (thrd_ctrl_t*)arraylist_get(thrd_list, thrd_id); - - /* wait for lock for shared data structures holding requests - * and condition variable */ - pthread_mutex_lock(&thrd_ctrl->thrd_lock); - - /* get the locations of all the read requests from the - * key-value store*/ - int rc = meta_read_get(app_id, client_id, thrd_id, cli_rank, - gfid, offset, length, thrd_ctrl->del_req_set); - - /* sort read requests to be sent to the same delegators. */ - qsort(thrd_ctrl->del_req_set->msg_meta, - thrd_ctrl->del_req_set->num, - sizeof(send_msg_t), compare_delegators); - - /* debug print */ - print_send_msgs(thrd_ctrl->del_req_set->msg_meta, - thrd_ctrl->del_req_set->num); - - /* get pointer to list of delegator stat objects to record - * delegator rank and count of requests for each delegator */ - per_del_stat_t* req_stat = thrd_ctrl->del_req_stat->req_stat; - - /* get pointer to send message structures, one for each request */ - send_msg_t* msg_meta = thrd_ctrl->del_req_set->msg_meta; - - /* record rank of first delegator we'll send to */ - req_stat[0].del_id = msg_meta[0].dest_delegator_rank; - - /* initialize request count for first delegator to 1 */ - req_stat[0].req_cnt = 1; - - /* iterate over read requests and count number of requests - * to be sent to each delegator */ - int del_cnt = 0; - int i; - for (i = 1; i < thrd_ctrl->del_req_set->num; i++) { - int cur_rank = msg_meta[i].dest_delegator_rank; - int prev_rank = msg_meta[i-1].dest_delegator_rank; - if (cur_rank == prev_rank) { - /* another message for the current delegator */ - req_stat[del_cnt].req_cnt++; - } else { - /* got a new delegator, set the rank */ - req_stat[del_cnt].del_id = msg_meta[i].dest_delegator_rank; - - /* initialize the request count */ - req_stat[del_cnt].req_cnt = 1; - - /* increment the delegator count */ - del_cnt++; - } - } - del_cnt++; - - /* record total number of delegators we'll send requests to */ - thrd_ctrl->del_req_stat->del_cnt = del_cnt; - - /* debug print */ - print_remote_del_reqs(app_id, thrd_id, thrd_ctrl->del_req_stat); - - /* wake up the request manager thread for the requesting client */ - if (!thrd_ctrl->has_waiting_delegator) { - /* delegator thread is not waiting, but we are in critical - * section, we just added requests so we must wait for delegator - * to signal us that it's reached the critical section before - * we escaple so we don't overwrite these requests before it - * has had a chance to process them */ - thrd_ctrl->has_waiting_dispatcher = 1; - pthread_cond_wait(&thrd_ctrl->thrd_cond, &thrd_ctrl->thrd_lock); - - /* delegator thread has signaled us that it's now waiting, - * so signal it to go ahead and then release the lock, - * so it can start */ - thrd_ctrl->has_waiting_dispatcher = 0; - pthread_cond_signal(&thrd_ctrl->thrd_cond); - } else { - /* have a delegator thread waiting on condition variable, - * signal it to begin processing the requests we just added */ - pthread_cond_signal(&thrd_ctrl->thrd_cond); - } - - /* done updating shared variables, release the lock */ - pthread_mutex_unlock(&thrd_ctrl->thrd_lock); - - return rc; -} - -/** -* send the read requests to the remote delegators -* -* @param app_id: application id -* @param client_id: client id for requesting process -* @param gfid: global file id -* @param req_num: number of read requests -* @param reqbuf: read requests buffer -* @return success/error code -*/ -int rm_cmd_mread(int app_id, int client_id, int gfid, - size_t req_num, void* reqbuf) -{ - int rc; - int dbg_rank = -1; - - app_config_t* app_config = - (app_config_t*)arraylist_get(app_config_list, app_id); - - int thrd_id = app_config->thrd_idxs[client_id]; - thrd_ctrl_t* thrd_ctrl = (thrd_ctrl_t*)arraylist_get(thrd_list, thrd_id); - - pthread_mutex_lock(&thrd_ctrl->thrd_lock); - - /* get the locations of all the read requests from the key-value store*/ - LOGDBG("calling meta_batch_get with req_num:%zu, thrd_id:%d", - req_num, thrd_id); - rc = meta_batch_get(app_id, client_id, thrd_id, 0, - reqbuf, req_num, - thrd_ctrl->del_req_set); - LOGDBG("completed meta_batch_get, rc=%d", rc); - - /* - * group together the read requests - * to be sent to the same delegators. - * */ - qsort(thrd_ctrl->del_req_set->msg_meta, - thrd_ctrl->del_req_set->num, - sizeof(send_msg_t), compare_delegators); - print_send_msgs(thrd_ctrl->del_req_set->msg_meta, - thrd_ctrl->del_req_set->num); - thrd_ctrl->del_req_stat->req_stat[0].req_cnt = 1; - - int i, del_cnt = 0; - thrd_ctrl->del_req_stat->req_stat[0].del_id = - thrd_ctrl->del_req_set->msg_meta[0].dest_delegator_rank; - - /* calculate the number of read requests - * to be sent to each delegator*/ - for (i = 1; i < thrd_ctrl->del_req_set->num; i++) { - if (thrd_ctrl->del_req_set->msg_meta[i].dest_delegator_rank == - thrd_ctrl->del_req_set->msg_meta[i - 1].dest_delegator_rank) { - thrd_ctrl->del_req_stat->req_stat[del_cnt].req_cnt++; - } else { - del_cnt++; - thrd_ctrl->del_req_stat->req_stat[del_cnt].req_cnt = 1; - thrd_ctrl->del_req_stat->req_stat[del_cnt].del_id = - thrd_ctrl->del_req_set->msg_meta[i].dest_delegator_rank; - } - } - del_cnt++; - - thrd_ctrl->del_req_stat->del_cnt = del_cnt; - - print_remote_del_reqs(app_id, thrd_id, thrd_ctrl->del_req_stat); - - LOGDBG("wake up the service thread for the requesting client"); - /*wake up the service thread for the requesting client*/ - if (!thrd_ctrl->has_waiting_delegator) { - LOGDBG("has waiting delegator"); - thrd_ctrl->has_waiting_dispatcher = 1; - pthread_cond_wait(&thrd_ctrl->thrd_cond, &thrd_ctrl->thrd_lock); - LOGDBG("has waiting dispatcherr"); - thrd_ctrl->has_waiting_dispatcher = 0; - pthread_cond_signal(&thrd_ctrl->thrd_cond); - LOGDBG("signaled"); - } else { - LOGDBG("does not have waiting delegator"); - pthread_cond_signal(&thrd_ctrl->thrd_cond); - } - LOGDBG("woked"); - pthread_mutex_unlock(&thrd_ctrl->thrd_lock); - LOGDBG("unlocked"); - - return rc; -} - -/* function called by main thread to instruct - * resource manager thread to exit, - * returns UNIFYCR_SUCCESS on success */ -int rm_cmd_exit(thrd_ctrl_t* thrd_ctrl) -{ - /* grab the lock */ - pthread_mutex_lock(&thrd_ctrl->thrd_lock); - - /* if delegator thread is not waiting in critical - * section, let's wait on it to come back */ - if (!thrd_ctrl->has_waiting_delegator) { - /* delegator thread is not in critical section, - * tell it we've got something and signal it */ - thrd_ctrl->has_waiting_dispatcher = 1; - pthread_cond_wait(&thrd_ctrl->thrd_cond, &thrd_ctrl->thrd_lock); - - /* we're no longer waiting */ - thrd_ctrl->has_waiting_dispatcher = 0; - } - - /* inform delegator thread that it's time to exit */ - thrd_ctrl->exit_flag = 1; - - /* free storage holding shared data structures */ - free(thrd_ctrl->del_req_set); - free(thrd_ctrl->del_req_stat->req_stat); - free(thrd_ctrl->del_req_stat); - - /* signal delegator thread */ - pthread_cond_signal(&thrd_ctrl->thrd_cond); - - /* release the lock */ - pthread_mutex_unlock(&thrd_ctrl->thrd_lock); - - /* wait for delegator thread to exit */ - void* status; - pthread_join(thrd_ctrl->thrd, &status); - - return UNIFYCR_SUCCESS; -} - -/************************ - * These functions define the logic of the request manager thread - ***********************/ - -/** -* pack the the requests to be sent to the same -* delegator. -* ToDo: pack and send multiple rounds if the -* total request sizes is larger than REQ_BUF_LEN -* @param rank: source rank that sends the requests -* @param req_msg_buf: request buffer -* @param req_num: number of read requests -* @param *tot_sz: the total data size to read in these -* packed read requests -* @return success/error code -*/ -static int rm_pack_send_requests( - char* req_msg_buf, /* pointer to buffer to pack requests into */ - send_msg_t* send_metas, /* request objects to be packed */ - int req_cnt, /* number of requests */ - size_t* tot_sz) /* total data payload size we're requesting */ -{ - /* tot_sz records the aggregate data size - * requested in this transfer */ - - /* send format: - * (int) cmd - specifies type of message (XFER_COMM_DATA) - * (int) req_num - number of requests in message - * {sequence of send_meta_t requests} */ - - /* get pointer to start of send buffer */ - char* ptr = req_msg_buf; - memset(ptr, 0, REQ_BUF_LEN); - - /* pack command */ - int cmd = XFER_COMM_DATA; - memcpy(ptr, &cmd, sizeof(int)); - ptr += sizeof(int); - - /* pack request count */ - memcpy(ptr, &req_cnt, sizeof(int)); - ptr += sizeof(int); - - /* pack each request into the send buffer, - * total up incoming bytes as we go */ - int i; - size_t bytes = 0; - for (i = 0; i < req_cnt; i++) { - /* accumulate data size of this request */ - bytes += send_metas[i].length; - } - - /* copy requests into buffer */ - memcpy(ptr, send_metas, (req_cnt * sizeof(send_msg_t))); - ptr += (req_cnt * sizeof(send_msg_t)); - - /* increment running total size of data bytes */ - (*tot_sz) += bytes; - - /* return number of bytes used to pack requests */ - size_t packed_size = ptr - req_msg_buf; - return (int)packed_size; -} - -/** -* send the read requests to the remote delegator service managers -* @return success/error code -*/ -static int rm_send_remote_requests( - thrd_ctrl_t* thrd_ctrl, /* lists delegators and read requests */ - size_t* tot_sz) /* returns total data payload to be read */ -{ - int i = 0; - - /* ToDo: Transfer the message in multiple - * rounds when total size > the size of - * send_msg_buf - * */ - - /* use this variable to total up number of incoming data bytes */ - *tot_sz = 0; - - /* get pointer to send buffer */ - char* sendbuf = thrd_ctrl->del_req_msg_buf; - - /* get pointer to start of read request array, - * and initialize index to point to first element */ - send_msg_t* msgs = thrd_ctrl->del_req_set->msg_meta; - int msg_cursor = 0; - - /* iterate over each delegator we need to send requests to */ - for (i = 0; i < thrd_ctrl->del_req_stat->del_cnt; i++) { - /* pointer to start of requests for this delegator */ - send_msg_t* reqs = msgs + msg_cursor; - - /* number of requests for this delegator */ - int req_num = thrd_ctrl->del_req_stat->req_stat[i].req_cnt; - - /* pack requests into send buffer, get size of packed data, - * increase total number of data payload we will get back */ - int packed_size = rm_pack_send_requests(sendbuf, reqs, - req_num, tot_sz); - - /* get rank of target delegator */ - int del_rank = thrd_ctrl->del_req_stat->req_stat[i].del_id; - - /* send requests */ - MPI_Send(sendbuf, packed_size, MPI_BYTE, - del_rank, CLI_DATA_TAG, MPI_COMM_WORLD); - - /* advance to requests for next delegator */ - msg_cursor += req_num; - } - - return ULFS_SUCCESS; -} - -/* signal the client process for it to start processing read - * data in shared memory */ -static int client_signal(int app_id, int client_id, int flag) -{ - /* signal client on socket */ -// int rc = sock_notify_cli(client_id, COMM_DIGEST); -// return rc; - - /* lookup our data structure for this app id */ - app_config_t* app_config = - (app_config_t*)arraylist_get(app_config_list, app_id); - - /* get pointer to flag in shared memory */ - volatile int* ptr_flag = - (volatile int*)app_config->shm_recv_bufs[client_id]; - - /* set flag to 1 to signal client */ - *ptr_flag = flag; - - /* TODO: MEM_FLUSH */ - - return UNIFYCR_SUCCESS; -} - -/* wait until client has processed all read data in shared memory */ -static int client_wait(int app_id, int client_id) -{ - /* specify time to sleep between checking flag in shared - * memory indicating client has processed data */ - struct timespec shm_wait_tm; - shm_wait_tm.tv_sec = 0; - shm_wait_tm.tv_nsec = SHM_WAIT_INTERVAL; - - /* lookup our data structure for this app id */ - app_config_t* app_config = - (app_config_t*)arraylist_get(app_config_list, app_id); - - /* get pointer to flag in shared memory */ - volatile int* ptr_flag = - (volatile int*)app_config->shm_recv_bufs[client_id]; - - /* TODO: MEM_FETCH */ - - /* wait for client to set flag to 0 */ - while (*ptr_flag != 0) { - /* not there yet, sleep for a while */ - nanosleep(&shm_wait_tm, NULL); - - /* TODO: MEM_FETCH */ - } - - return UNIFYCR_SUCCESS; -} - -/** -* parse the read replies from message received from service manager, -* deliver replies back to client -* -* @param app_id: client's application id -* @param client_id: socket index in the poll_set -* for that client -* @param recv_msg_buf: buffer for received message -* packed read requests -* @param ptr_tot_sz: total data size to receive -* @return success/error code -*/ -static int rm_process_received_msg( - int app_id, /* client app id to get shared memory */ - int client_id, /* client_id to client */ - char* recv_msg_buf, /* pointer to receive buffer */ - size_t* ptr_tot_sz) /* decrements total data received */ -{ - /* assume we'll succeed in processing the message */ - int rc = UNIFYCR_SUCCESS; - - /* look up client app config based on client id */ - app_config_t* app_config = - (app_config_t*)arraylist_get(app_config_list, app_id); - - /* format of read replies in shared memory - * (int) flag - used for signal between delegator and client - * (int) size - bytes consumed for shared memory read replies - * (int) num - number of read replies - * {sequence of shm_meta_t} - read replies */ - - /* number of bytes in header (3 ints right now) */ - size_t header_size = 3 * sizeof(int); - - /* get pointer to shared memory buffer for this client */ - char* shmbuf = (char*) app_config->shm_recv_bufs[client_id]; - - /* get pointer to flag in shared memory that we'll set - * to signal to client that data is ready */ - //int* ptr_flag = (int*)shmbuf; - shmbuf += sizeof(int); - - /* get pointer to slot in shared memory to write bytes - * consumed by read replies */ - int* ptr_size = (int*)shmbuf; - shmbuf += sizeof(int); - - /* get pointer to slot in shared memory to write number - * of read replies */ - int* ptr_num = (int*)shmbuf; - shmbuf += sizeof(int); - - /* read current size and count from shared memory - * because they may not be zero? */ - int shm_offset = *ptr_size; - int shm_count = *ptr_num; - - /* format of recv_msg_buf: - * (int) num - number of read replies packed in message - * {sequence of recv_msg_t containing read replies} */ - - /* get pointer to start of receive buffer */ - char* msgptr = recv_msg_buf; - - /* extract number of read requests in this message */ - int num = *(int*)msgptr; - msgptr += sizeof(int); - - /* unpack each read reply */ - int j; - for (j = 0; j < num; j++) { - /* point to first read reply in message */ - recv_msg_t* msg = (recv_msg_t*)msgptr; - msgptr += sizeof(recv_msg_t); - - /* compute max byte that will be consumed by copying - * data for this message into shared memory buffer */ - size_t msg_size = sizeof(shm_meta_t) + msg->length; - size_t need_size = header_size + shm_offset + msg_size; - - /* check that there is space for this message */ - if (need_size > app_config->recv_buf_sz) { - /* client-side receive buffer is full, - * inform client to start reading */ - client_signal(app_id, client_id, 1); - - /* wait for client to read data */ - client_wait(app_id, client_id); - - /* TODO: MEM_FETCH */ - - /* refresh our packing values now that client - * has processed entries */ - shm_offset = *ptr_size; - shm_count = *ptr_num; - } - - /* fill the next message in the shared buffer */ - shm_count++; - - /* TODO: we should probably add a field to track errors */ - - /* get pointer in shared memory for next read reply */ - shm_meta_t* shmmsg = (shm_meta_t*)(shmbuf + shm_offset); - - /* copy in header for this read request */ - shmmsg->src_fid = msg->src_fid; - shmmsg->offset = msg->src_offset; - shmmsg->length = msg->length; - shmmsg->errcode = msg->errcode; - shm_offset += sizeof(shm_meta_t); - - /* copy data for this read request */ - memcpy(shmbuf + shm_offset, msgptr, msg->length); - shm_offset += msg->length; - - /* advance to next read reply in message buffer */ - msgptr += msg->length; - - /* decrement number of bytes processed from total */ - *ptr_tot_sz -= msg->length; - } - - /* record total bytes and number of read requests in buffer */ - *ptr_size = shm_offset; - *ptr_num = shm_count; - - return rc; -} - -/** -* receive the requested data returned from service managers -* as a result of the read requests we sent to them -* -* @param app_id: app id for incoming data -* @param client_id: client id for incoming data -* @param tot_sz: total data size to receive (excludes header bytes) -* @return success/error code -*/ -static int rm_receive_remote_message( - thrd_ctrl_t* thrd_ctrl, /* contains pointer to receive buffer */ - size_t tot_sz) /* number of incoming data payload bytes */ -{ - /* assume we'll succeed */ - int rc = ULFS_SUCCESS; - - /* get app id and client id that we'll be serving, - * app id associates thread with a namespace (mountpoint) - * the client id associates the thread with a particular - * client process id */ - int app_id = thrd_ctrl->app_id; - int client_id = thrd_ctrl->client_id; - - /* lookup our data structure for this app id */ - app_config_t* app_config = - (app_config_t*)arraylist_get(app_config_list, app_id); - - /* get thread id for this client (used for MPI tags) */ - int thrd_id = app_config->thrd_idxs[client_id]; - - /* service manager will incorporate our thread id in tag, - * to distinguish between target request manager threads */ - int tag = SER_DATA_TAG + thrd_id; - - /* array of MPI_Request objects for window of posted receives */ - MPI_Request recv_req[RECV_BUF_CNT] = {MPI_REQUEST_NULL}; - - /* get number of receives to post and size of each buffer */ - int recv_buf_cnt = RECV_BUF_CNT; - int recv_buf_len = (int) RECV_BUF_LEN; - - /* post a window of receive buffers for incoming data */ - int i; - for (i = 0; i < recv_buf_cnt; i++) { - /* post buffer for incoming receive */ - MPI_Irecv(thrd_ctrl->del_recv_msg_buf[i], recv_buf_len, MPI_BYTE, - MPI_ANY_SOURCE, tag, MPI_COMM_WORLD, &recv_req[i]); - } - - /* spin until we have received all incoming data */ - while (tot_sz > 0) { - /* wait for any receive to come in */ - int index; - MPI_Status status; - MPI_Waitany(recv_buf_cnt, recv_req, &index, &status); - - /* got a new message, get pointer to message buffer */ - char* buf = thrd_ctrl->del_recv_msg_buf[index]; - - /* unpack the data into client shared memory, - * this will internally signal client and wait - * for data to be processed if shared memory - * buffer is filled */ - int tmp_rc = rm_process_received_msg( - app_id, client_id, buf, &tot_sz); - if (tmp_rc != ULFS_SUCCESS) { - rc = tmp_rc; - } - - /* done processing, repost this receive buffer */ - MPI_Irecv(thrd_ctrl->del_recv_msg_buf[index], recv_buf_len, MPI_BYTE, - MPI_ANY_SOURCE, tag, MPI_COMM_WORLD, &recv_req[index]); - } - - /* cancel posted MPI receives */ - for (i = 0; i < recv_buf_cnt; i++) { - MPI_Status status; - MPI_Cancel(&recv_req[i]); - MPI_Wait(&recv_req[i], &status); - } - - /* signal client that we're now done writing data (flag=2) */ - client_signal(app_id, client_id, 2); - - /* wait for client to read data */ - client_wait(app_id, client_id); - - return rc; -} - -/** -* entry point for request manager thread, one thread is created -* for each client process, client informs thread of a set of read -* requests, thread retrieves data for client and notifies client -* when data is ready -* -* delegate the read requests for the delegator thread's client. Each -* delegator thread handles one connection to one client-side rank. -* -* @param arg: pointer to control structure for the delegator thread -* -* @return NULL -*/ -void* rm_delegate_request_thread(void* arg) -{ - /* get pointer to our thread control structure */ - thrd_ctrl_t* thrd_ctrl = (thrd_ctrl_t*) arg; - - /* loop forever to handle read requests from the client, - * new requests are added to a list on a shared data structure - * with main thread, new items inserted by the rpc handler */ - while (1) { - /* grab lock */ - pthread_mutex_lock(&thrd_ctrl->thrd_lock); - - /* inform dispatcher that we're waiting for work - * inside the critical section */ - thrd_ctrl->has_waiting_delegator = 1; - - /* if dispatcher is waiting on us, signal it to go ahead, - * this coordination ensures that we'll be the next thread - * to grab the lock after the dispatcher has assigned us - * some work (rather than the dispatcher grabbing the lock - * and assigning yet more work) */ - if (thrd_ctrl->has_waiting_dispatcher == 1) { - pthread_cond_signal(&thrd_ctrl->thrd_cond); - } - - /* release lock and wait to be signaled by dispatcher */ - pthread_cond_wait(&thrd_ctrl->thrd_cond, &thrd_ctrl->thrd_lock); - - /* set flag to indicate we're no longer waiting */ - thrd_ctrl->has_waiting_delegator = 0; - - /* go do work ... */ - - /* release lock and bail out if we've been told to exit */ - if (thrd_ctrl->exit_flag == 1) { - pthread_mutex_unlock(&thrd_ctrl->thrd_lock); - break; - } - - /* this will hold the total number of bytes we expect - * to come in, compute in send, subtracted in receive */ - size_t tot_sz = 0; - - /* send read requests to remote servers */ - int rc = rm_send_remote_requests(thrd_ctrl, &tot_sz); - if (rc != UNIFYCR_SUCCESS) { - /* release lock and exit if we hit an error */ - pthread_mutex_unlock(&thrd_ctrl->thrd_lock); - return NULL; - } - - /* wait for data to come back from servers */ - rc = rm_receive_remote_message(thrd_ctrl, tot_sz); - if (rc != UNIFYCR_SUCCESS) { - /* release lock and exit if we hit an error */ - pthread_mutex_unlock(&thrd_ctrl->thrd_lock); - return NULL; - } - - /* release lock */ - pthread_mutex_unlock(&thrd_ctrl->thrd_lock); - } - - return NULL; -} diff --git a/server/src/unifycr_request_manager.h b/server/src/unifycr_request_manager.h deleted file mode 100644 index c11b3e978..000000000 --- a/server/src/unifycr_request_manager.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * - * Copyright 2017, UT-Battelle, LLC. - * - * LLNL-CODE-741539 - * All rights reserved. - * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. - */ - -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * Copyright (c) 2017, Florida State University. Contributions from - * the Computer Architecture and Systems Research Laboratory (CASTL) - * at the Department of Computer Science. - * - * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror - * LLNL-CODE-728877. All rights reserved. - * - * This file is part of burstfs. - * For details, see https://github.com/llnl/burstfs - * Please read https://github.com/llnl/burstfs/LICENSE for full license text. - */ - -#ifndef UNIFYCR_REQUEST_MANAGER_H -#define UNIFYCR_REQUEST_MANAGER_H - -#include "unifycr_const.h" -#include "unifycr_global.h" - -/* entry point for starting a request manager thread */ -void* rm_delegate_request_thread(void* arg); - -/* functions called by rpc handlers to assign work - * to request managre threads */ -int rm_cmd_mread(int app_id, int client_id, int gfid, - size_t req_num, void* reqbuf); - -int rm_cmd_read(int app_id, int client_id, int gfid, - size_t offset, size_t length); - -int rm_cmd_filesize(int app_id, int client_id, int gfid, size_t* outsize); - -/* function called by main thread to instruct - * resource manager thread to exit, - * returns UNIFYCR_SUCCESS on success */ -int rm_cmd_exit(thrd_ctrl_t* thrd_ctrl); - -#endif diff --git a/server/src/unifycr_server.h b/server/src/unifycr_server.h deleted file mode 100644 index 3b61cd95e..000000000 --- a/server/src/unifycr_server.h +++ /dev/null @@ -1,64 +0,0 @@ -#ifndef __UNIFYCR_SERVER_H -#define __UNIFYCR_SERVER_H - -/******************************************************************************* - * - * unifycr_server.h - * - * Declarations for the UCR global server. - * - *******************************************************************************/ - -#include -#include -#include - -#include -#include -#include - -typedef struct ServerRpcContext { - margo_instance_id mid; - hg_context_t* hg_context; - hg_class_t* hg_class ; - hg_id_t unifycr_read_rpc_id; - hg_id_t unifycr_fsync_rpc_id; - hg_id_t unifycr_mount_rpc_id; - hg_id_t unifycr_unmount_rpc_id; - hg_id_t unifycr_metaget_rpc_id; - hg_id_t unifycr_metaset_rpc_id; - //hg_id_t write_rpc_id; - //hg_id_t chkdir_rpc_id; - //hg_id_t addfile_rpc_id; - //hg_id_t open_rpc_id; - //hg_id_t close_rpc_id; - //hg_id_t getfilestat_rpc_id; - //hg_id_t getdircontents_rpc_id; - //hg_id_t readtransfer_rpc_id; -} ServerRpcContext_t; - -static const char* SMSVR_ADDR_STR = "na+sm://"; -static const char* VERBSVR_ADDR_STR = "cci+verbs"; -static const char* TCPSVR_ADDR_STR = "cci+tcp"; - -extern bool usetcp; - -extern uint16_t total_rank; -extern uint16_t my_rank; - -typedef struct ServerAddress { - char* string_address; - hg_addr_t svr_addr; -} ServerAddress_t; - -extern char** server_addresses; - -extern ServerRpcContext_t* unifycr_rpc_context; - -margo_instance_id unifycr_server_rpc_init(); - -void unifycr_server_addresses_init(); - -int unifycr_inter_server_client_init(); - -#endif diff --git a/server/src/unifycr_service_manager.c b/server/src/unifycr_service_manager.c deleted file mode 100644 index 7b639ec5b..000000000 --- a/server/src/unifycr_service_manager.c +++ /dev/null @@ -1,1493 +0,0 @@ -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * - * Copyright 2017, UT-Battelle, LLC. - * - * LLNL-CODE-741539 - * All rights reserved. - * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. - */ - -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * Copyright (c) 2017, Florida State University. Contributions from - * the Computer Architecture and Systems Research Laboratory (CASTL) - * at the Department of Computer Science. - * - * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror - * LLNL-CODE-728877. All rights reserved. - * - * This file is part of burstfs. - * For details, see https://github.com/llnl/burstfs - * Please read https://github.com/llnl/burstfs/LICENSE for full license text. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "mpi.h" - -#include "unifycr_log.h" -#include "unifycr_sock.h" -#include "unifycr_service_manager.h" -#include "unifycr_const.h" -#include "unifycr_global.h" -#include "arraylist.h" - -/* The service manager thread runs in a loop waiting for incoming - * requests. When it receives a message, it unpacks all read - * requests and appends them to the service_msgs list. It delays - * for some time in acting on requests in the hopes of buffering - * read bursts to make I/O more efficient. If no read request - * has come in, and the delay time out has expired, and there are - * pening read requests to service, then it services all requests. - * - * It first creates a set of read tasks based on the set of read - * requests. The read requests are sorted by source file and then - * by offset, and read requests that refer to contiguous data - * regions are merged into a large read task. Read tasks are - * added to a read_task list. - * - * The read tasks are executed to copy data into a read buffer. - * Data that is copied from shared memory simply uses memcpy(). - * Data that spans shared memory and the spillover file uses - * memcpy with pread. Data that is fully in the spillover file - * are read with async I/O (aio) operations. The aio operations - * are added to a pending_reads queue that is later waited on. - * - * After all data has been read, results are packed into send - * buffers. This is done by matching read tasks with corresponding - * read requests. When send buffers are filled with read replies - * (acks), they are sent back to requesting delegators with - * MPI_Isend calls. The MPI_Request objects for those isends are - * added to a pending_sends list, which is later waited on. - * - * After replying to all service_msgs, the service manager - * thread again starts listening for more incoming requests */ - -/* records info needed to build read reply to send back to - * requesting delegator, these are appened in an ack list - * which records a set of read replies before being sent */ -typedef struct { - recv_msg_t msg; /* header information for read reply */ - char* addr; /* address of data in read buffer */ -} ack_meta_t; - -/* this records info about outstanding sends to delegators, - * including the MPI request that must be waited on to - * determine when send has completed, these are added to - * an pending sends list when sent and then iterated over - * while waiting for all pending sends to complete */ -typedef struct { - MPI_Request req; /* MPI request for outstanding isend */ - MPI_Status stat; /* status for test call */ - int src_rank; /* target delegator rank */ - int src_thrd; /* target delegator thread */ -} ack_stat_t; - -/* records info about read tasks, which are generated by - * unpacking incoming read requests from remote delegators, - * it encodes information about a range of read requests - * that generated the read task, read requests that refer - * to contiguous data may be merged into a single read task, - * and the app_id and client_id are used to determine - * the memory/files holding the source data */ -typedef struct { - int start_idx; /* index of starting read request */ - int end_idx; /* index of ending read request */ - int size; /* size of read operation */ - int app_id; /* app id holding requested data */ - int cli_id; /* client id holding requested data */ - int arrival_time; /* time stamp when read request arrived */ -} read_task_t; - -/* defines an array of read tasks */ -typedef struct { - read_task_t* read_tasks; /* list of read tasks */ - int num; /* number of active read tasks */ - int cap; /* total capacity of read task list */ -} task_set_t; - -/* defines an array of read requests, generated by unpacking - * read requests from requesting delegators */ -typedef struct { - send_msg_t* msg; /* buffer of read requests */ - int num; /* number of active read requests in buffer */ - int cap; /* total capacity of read request list */ -} service_msgs_t; - -/* tracks an outstanding read operation, these are generated - * from read tasks and added to a list of pending read operations, - * which are later waited on before sending data back to requesting - * delegators */ -typedef struct { - int err_submit; /* indicates whether read was submitted */ - struct aiocb read_cb; /* structure for outstanding aio read */ - int index; /* index in read task list for this read */ - int start_pos; /* starting byte offset into read request */ - int end_pos; /* ending byte offset into read request */ - char* mem_pos; /* buffer holding read data */ -} pended_read_t; - -/* defines a list of read replies to be sent to a delegator */ -typedef struct { - arraylist_t* ack_list; /* list of read replies for delegator */ - int rank_id; /* rank of remote delegator to send to */ - int thrd_id; /* thread id of remote delegator */ - int src_cli_rank; /* rank of client that initiated read */ - size_t src_sz; /* total data size in read replies */ - int start_cursor; /* offset within ack_list */ -} rank_ack_meta_t; - -/* defines an list of reply data for different delegators, - * list is ordered by (rank,thread) of delegator for fast lookup */ -typedef struct { - rank_ack_meta_t* ack_metas; /* read reply data for a delegator */ - int num; /* number of items in list */ -} rank_ack_task_t; - -/* records list of read requests from requesting delegators */ -static service_msgs_t service_msgs; - -/* list of read tasks that must be executed, - * generated from read requests */ -static task_set_t read_task_set; - -/* list of read reply data for each delegator */ -static rank_ack_task_t rank_ack_task; - -/* list of outstanding read operations */ -static arraylist_t* pended_reads; - -/* list of outstanding send operations */ -static arraylist_t* pended_sends; - -/* list of buffers to be used in send operations */ -static arraylist_t* send_buf_list; - -/* counter for time waited before creating read tasks, - * used to buffer multiple read requests before responding - * with the idea that reads come in bursts */ -static int wait_time = 0; - -/* tracks running total of bytes in current read burst */ -static long burst_data_sz = 0; - -/* this will point to a buffer to hold read data while gathering it - * from source memory/files and before being packed into send buffers - * for sending read replies */ -static char* read_buf; - -/* sort read requests into log files by - * app id, then client id, then log offset */ -static int compare_send_msg(const void* a, const void* b) -{ - const send_msg_t* ptr_a = a; - const send_msg_t* ptr_b = b; - - if (ptr_a->dest_app_id > ptr_b->dest_app_id) { - return 1; - } - - if (ptr_a->dest_app_id < ptr_b->dest_app_id) { - return -1; - } - - if (ptr_a->dest_client_id > ptr_b->dest_client_id) { - return 1; - } - - if (ptr_a->dest_client_id < ptr_b->dest_client_id) { - return -1; - } - - if (ptr_a->dest_offset > ptr_b->dest_offset) { - return 1; - } - - if (ptr_a->dest_offset < ptr_b->dest_offset) { - return -1; - } - - return 0; -} - -/* starts a new read task based on read request in service_msgs - * at given index */ -static void reset_read_tasks(task_set_t* read_task_set, - service_msgs_t* service_msgs, int index) -{ - /* get pointer to service message at given index */ - send_msg_t* msg = &service_msgs->msg[index]; - - /* get pointer to current read task */ - int idx = read_task_set->num; - read_task_t* read_task = &read_task_set->read_tasks[idx]; - - /* copy fields from message to read task */ - read_task->start_idx = index; - read_task->end_idx = index; - read_task->size = msg->length; - read_task->app_id = msg->dest_app_id; - read_task->cli_id = msg->dest_client_id; - read_task->arrival_time = msg->arrival_time; -} - -/** -* Cluster read requests based on file -* offset and ages; Each log file is uniquely -* identified by client-side app_id and client_id, so -* from these two, we can locate the target -* log file (generated by the client-side program). -* @param: service_msgs: a list of received messages -* @param: read_task_set: a list of read tasks containing -* the clustered read requests -* return success/error -*/ -static int sm_cluster_reads( - task_set_t* read_task_set, - service_msgs_t* service_msgs) -{ - /* sort service messages by log file (app_id, client_id) - * and then by offset within each log file */ - qsort(service_msgs->msg, service_msgs->num, - sizeof(send_msg_t), compare_send_msg); - - /* initialize count of read tasks */ - read_task_set->num = 0; - - /* create read task given first service message */ - reset_read_tasks(read_task_set, service_msgs, 0); - read_task_set->num++; - - /* iterate over each service message and create read tasks, - * will merge multiple read requests into read tasks - * when two requests refer to contiguous data in a log file */ - int i; - for (i = 1; i < service_msgs->num; i++) { - /* get pointer to current service message */ - send_msg_t* msg = &service_msgs->msg[i]; - - /* get pointer to preivous read task */ - read_task_t* last_read = - &read_task_set->read_tasks[read_task_set->num - 1]; - - /* check whether current message reads from the same log file, - * as our last read task */ - if (last_read->app_id != msg->dest_app_id || - last_read->cli_id != msg->dest_client_id) { - /* reading on a different local log file, - * so create a new read task for this message */ - reset_read_tasks(read_task_set, service_msgs, i); - read_task_set->num++; - } else { - /* this message reads from the same log file as our last - * read request */ - - /* get pointer to previous read request */ - send_msg_t* last_msg = &service_msgs->msg[i - 1]; - - /* see if we can tack current message on to - * previous read request */ - long last_offset = last_msg->dest_offset + last_msg->length; - if (last_offset == msg->dest_offset) { - /* current message starts where last read request - * ends, so append it to last read request if no larger - * than read_block_size */ - - /* the size of individual read should be smaller - * than read_block_size, if read size is larger it - * needs to be split into the unit of READ_BLOCK_SIZE */ - if (last_read->size + msg->length <= READ_BLOCK_SIZE) { - /* tack current message on previous read request */ - last_read->end_idx = i; - last_read->size += msg->length; - - /* update minimum arrival time */ - if (msg->arrival_time < last_read->arrival_time) { - last_read->arrival_time = msg->arrival_time; - } - } else { - /* if larger than read block size, start a new - * read task, here the data size requested by - * individual read request should be smaller - * than read_block_size. The larger one - * has already been split by the initiator */ - reset_read_tasks(read_task_set, service_msgs, i); - read_task_set->num++; - } - } else { - /* not contiguous from the last offset, - * start a new read request */ - reset_read_tasks(read_task_set, service_msgs, i); - read_task_set->num++; - } - } - } - - return UNIFYCR_SUCCESS; -} - -/* compare by rank and then thread in increasing order */ -static int compare_rank_thrd(int src_rank, int src_thrd, - int cmp_rank, int cmp_thrd) -{ - if (src_rank > cmp_rank) { - return 1; - } - if (src_rank < cmp_rank) { - return -1; - } - if (src_thrd > cmp_thrd) { - return 1; - } - if (src_thrd < cmp_thrd) { - return -1; - } - return 0; -} - -/* returns index where delegator (rank,thread), - * should be in list, caller must check whether - * delegator at that position matches */ -static int find_ack_meta( - rank_ack_task_t* rank_ack_task, - int src_rank, - int src_thrd, - int* found) -{ - /* assume we won't find item */ - *found = 0; - - /* if nothing in list, place this as first item */ - if (rank_ack_task->num == 0) { - return 0; - } - - rank_ack_meta_t* metas = rank_ack_task->ack_metas; - - /* if list has one item, compare to that item */ - if (rank_ack_task->num == 1) { - /* compare to first item */ - int cmp = compare_rank_thrd(src_rank, src_thrd, - metas[0].rank_id, metas[0].thrd_id); - if (cmp < 0) { - /* item is smaller than first element */ - return 0; - } else if (cmp > 0) { - /* item is smaller than first element */ - return 1; - } else { - /* item matches first element */ - *found = 1; - return 0; - } - } - - /* execute binary search for item location in list */ - int left = 0; - int right = rank_ack_task->num - 1; - int mid = (left + right) / 2; - while (right > left + 1) { - /* compare item to middle element */ - int cmp = compare_rank_thrd(src_rank, src_thrd, - metas[mid].rank_id, metas[mid].thrd_id); - if (cmp > 0) { - /* item is larger than middle item, - * so bump left range up */ - left = mid; - } else if (cmp < 0) { - /* item is smaller than middle item, - * so move right down to middle */ - right = mid; - } else { - /* found an exact match with middle element */ - *found = 1; - return mid; - } - - /* update middle */ - mid = (left + right) / 2; - } - - /* two elements left in the list, compare to left element */ - int cmp_left = compare_rank_thrd(src_rank, src_thrd, - metas[left].rank_id, metas[left].thrd_id); - if (cmp_left < 0) { - /* item should come before left element */ - return left; - } else if (cmp_left == 0) { - /* item matches left item */ - *found = 1; - return left; - } - - /* item is larger than left element, so compare to right */ - int cmp_right = compare_rank_thrd(src_rank, src_thrd, - metas[right].rank_id, metas[right].thrd_id); - if (cmp_right < 0) { - /* item should come before right element */ - return right; - } else if (cmp_right == 0) { - /* item matches right element */ - *found = 1; - return right; - } - - /* otherwise, item must be larger than right element */ - return (right + 1); -} - -/* insert read reply list for specified (rank_id, thrd_id) delegator - * into ack list, keep ordered by rank,thread for fast lookup */ -static int insert_ack_meta( - rank_ack_task_t* rank_ack_task, - ack_meta_t* ack, - int pos, - int rank_id, - int thrd_id, - int src_cli_rank) -{ - /* get pointer to array of read reply data for delegators */ - rank_ack_meta_t* metas = rank_ack_task->ack_metas; - - /* check whether insert location is in middle of the list */ - if (pos < rank_ack_task->num) { - /* need to insert in the middle, bump all entries - * past pos up a slot */ - int i; - for (i = rank_ack_task->num - 1; i >= pos; i--) { - metas[i + 1] = metas[i]; - } - } - - /* get pointer to ack meta data structure */ - rank_ack_meta_t* ack_meta = &metas[pos]; - - /* initialize with values */ - ack_meta->ack_list = arraylist_create(); - ack_meta->rank_id = rank_id; - ack_meta->thrd_id = thrd_id; - ack_meta->src_cli_rank = src_cli_rank; - ack_meta->src_sz = ack->msg.length; - ack_meta->start_cursor = 0; - - /* check that we were able to create a new ack_list */ - if (ack_meta->ack_list == NULL) { - return (int)UNIFYCR_ERROR_NOMEM; - } - - /* insert ack_meta into our list */ - arraylist_add(ack_meta->ack_list, ack); - - /* increment the number of entries in our list */ - rank_ack_task->num++; - - return UNIFYCR_SUCCESS; -} - -/* send back ack to the remote delegator - * packs read replies into a send buffer, sends data with isend, - * adds record of send to pending sends list */ -static int sm_ack_remote_delegator(rank_ack_meta_t* ack_meta) -{ - int i; - - /* allocate more send buffers if we're at capacity */ - if (send_buf_list->size == send_buf_list->cap) { - /* at capacity in our list, allocate more space, - * double capacity in array */ - size_t new_cap = 2 * send_buf_list->cap; - send_buf_list->elems = (void**)realloc(send_buf_list->elems, - new_cap * sizeof(void*)); - - /* intialize pointers in new portion of array */ - for (i = send_buf_list->cap; i < new_cap; i++) { - send_buf_list->elems[i] = NULL; - } - - /* record new capacity */ - send_buf_list->cap = new_cap; - } - - /* attempt to reuse allocated buffer if we can */ - if (send_buf_list->elems[send_buf_list->size] == NULL) { - /* need to allocate a new buffer */ - send_buf_list->elems[send_buf_list->size] = - malloc(SEND_BLOCK_SIZE); - } - - /* get pointer to send buffer */ - char* send_msg_buf = send_buf_list->elems[send_buf_list->size]; - send_buf_list->size++; - - /* running total number of bytes we'll send */ - int send_sz = 0; - - /* compute number of read replies we'll send */ - size_t ack_count = arraylist_size(ack_meta->ack_list); - size_t ack_start = ack_meta->start_cursor; - int len = ack_count - ack_start; - - /* copy in number of read replies to message */ - memcpy(send_msg_buf + send_sz, &len, sizeof(int)); - send_sz += sizeof(int); - - /* pack read replies into send buffer */ - for (i = ack_start; i < ack_count; i++) { - /* get pointer to read reply header */ - ack_meta_t* meta = - (ack_meta_t*)arraylist_get(ack_meta->ack_list, i); - - /* copy read reply header to send buffer */ - memcpy(send_msg_buf + send_sz, &(meta->msg), sizeof(recv_msg_t)); - send_sz += sizeof(recv_msg_t); - - /* copy file data to send buffer */ - size_t length = (size_t) meta->msg.length; - memcpy(send_msg_buf + send_sz, meta->addr, length); - send_sz += (int) length; - } - - /* get rank and thread id of remote delegator */ - int del_rank = ack_meta->rank_id; - int del_thread = ack_meta->thrd_id; - - /* allocate a new ack stat structure to track details - * of pending send */ - ack_stat_t* ack_stat = (ack_stat_t*)malloc(sizeof(ack_stat_t)); - ack_stat->src_rank = del_rank; - ack_stat->src_thrd = del_thread; - - /* send read replies to delegator (rank and thread), - * record MPI request in ack stat structure to wait later */ - MPI_Isend(send_msg_buf, send_sz, MPI_BYTE, - del_rank, SER_DATA_TAG + del_thread, - MPI_COMM_WORLD, &(ack_stat->req)); - - /* add item to our list of pending sends */ - arraylist_add(pended_sends, ack_stat); - - return UNIFYCR_SUCCESS; -} - -/* - * insert a message to an entry of ack (read reply) list corresponding - * to its destination delegator. - * - * @param: rank_ack_task: a list of ack for each (rank, thrd) pair - * @param: mem_addr: address of data to be acked in mem_pool - * @param service_msgs and index: identifies msg to be inserted - * to ack_lst - * @param src_offset: offset of the requested segment on the logical - * file (not the physical log file on SSD. - * E.g. for N-1 pattern, logical - * offset is the offset on the shared file) - * @param len: length of the message - * */ -static int insert_to_ack_list( - rank_ack_task_t* rank_ack_task, - char* mem_addr, - service_msgs_t* service_msgs, - int index, - size_t src_offset, - size_t len, - int errcode) -{ - int rc = 0; - - /* get pointer to read request we are replying to */ - send_msg_t* msg = &service_msgs->msg[index]; - - /* allocate a new structure to record ack meta data */ - ack_meta_t* ack = (ack_meta_t*)malloc(sizeof(ack_meta_t)); - - /* the src_offset might start from any position in the message, so - * make it a separate parameter */ - ack->msg.src_fid = msg->src_fid; /* global file id */ - ack->msg.src_offset = src_offset; /* offset in file */ - ack->msg.length = len; /* length of data */ - ack->msg.errcode = errcode; /* error code for read (pass/fail) */ - ack->addr = mem_addr; /* pointer to data in read buffer */ - - /* after setting the ack for this message, link it - * to a ack list based on its destination. */ - - int src_rank = msg->src_delegator_rank; - int src_thrd = msg->src_thrd; - int src_cli_rank = msg->src_dbg_rank; - - /* find the position in the list for target delegator */ - int found = 0; - int pos = find_ack_meta(rank_ack_task, src_rank, src_thrd, &found); - - /* check whether delegator at this position is the target */ - if (! found) { - /* it's not, so insert new entry for the target into the - * list at this position */ - rc = insert_ack_meta(rank_ack_task, ack, pos, - src_rank, src_thrd, src_cli_rank); - } else { - /* found the corresponding entry for target delegator */ - rank_ack_meta_t* ack_meta = &rank_ack_task->ack_metas[pos]; - - /* compute number of read replies waiting to be sent */ - int num_entries = arraylist_size(ack_meta->ack_list); - int num_to_ack = num_entries - ack_meta->start_cursor; - - /* number of bytes needed to pack existing read replies */ - size_t curr_bytes = (num_to_ack * sizeof(ack_meta_t)) + - ack_meta->src_sz; - - /* number of bytes to pack this read reply */ - size_t bytes = sizeof(ack_meta_t) + ack->msg.length; - - /* check whether we can fit this data into the - * existing send block */ - if (curr_bytes + bytes > SEND_BLOCK_SIZE) { - /* not enough room, send the current list of read replies */ - rc = sm_ack_remote_delegator(ack_meta); - - /* start a new list */ - ack_meta->src_sz = ack->msg.length; - arraylist_add(ack_meta->ack_list, ack); - - /* start_cursor records the starting ack - * for the next send */ - ack_meta->start_cursor += num_to_ack; - - /* check that our reads and sends completed ok */ - if (rc < 0) { - return (int)UNIFYCR_ERROR_SEND; - } - } else { - /* current read reply fits in send buffer, - * add it to the list */ - ack_meta->src_sz += ack->msg.length; - arraylist_add(ack_meta->ack_list, ack); - } - } - - return UNIFYCR_SUCCESS; -} - -/* insert the data read for each element in read task list to read - * reply list, data will be sent later */ -static int batch_insert_to_ack_list( - rank_ack_task_t* rank_ack_task, /* read reply list for delegator */ - read_task_t* read_task, /* data returned from read task */ - service_msgs_t* service_msgs, /* read requests */ - char* mem_addr, /* memory loc to copy read data */ - int start_offset, /* first offset in read task */ - int end_offset, /* last offset in read task */ - int errcode) /* error code on read (pass/fail) */ -{ - /* search for the starting read request in service msgs for - * a given region of read_task_t identified by start_offset - * and size of read task */ - - /* read_task - * start_offset end_offset - * read_task_t: -----------******************------------ - * service_msgs:[ ],[*********],[***], [******],[ ] - * - * */ - - /* find index in read requests such that it contains starting - * offset for read task */ - int idx = read_task->start_idx; - int cur_offset = 0; - while (1) { - /* get pointer to current read request */ - send_msg_t* msg = &service_msgs->msg[idx]; - - /* check whether end offset of this read request comes - * at or after starting offset of read task */ - if (cur_offset + msg->length >= start_offset) { - /* starting offset of read task falls between start - * and end offset of this read request */ - break; - } - - /* move on to next read request */ - cur_offset += msg->length; - idx++; - } - - /* compute leading bytes in read request that this read task - * does not overlap with */ - int skip_offset = start_offset - cur_offset; - - /* insert read replies for leading read request and all middle - * read requests */ - int first = 1; - int mem_pos = 0; - while (1) { - /* get pointer to read request */ - send_msg_t* msg = &service_msgs->msg[idx]; - - /* stop if we reached the read request that contains the last - * byte of our read task */ - if (cur_offset + msg->length >= end_offset + 1) { - /* ending byte in read task comes before ending byte - * in read request */ - break; - } - - /* compute length and offset of read request that we cover with - * this read task, assume it's the whole read request */ - long length = msg->length; - long offset = msg->src_offset; - if (first == 1) { - /* in the first read request, the read task may start - * partway through, so skip any leading bytes */ - length = msg->length - skip_offset; - offset = msg->src_offset + skip_offset; - first = 0; - } - - /* add entry to read reply list */ - insert_to_ack_list(rank_ack_task, mem_addr + mem_pos, - service_msgs, idx, offset, length, errcode); - - /* update our offset into read reply buffer */ - mem_pos += length; - - /* update offset into read task data */ - cur_offset += length; - - /* move on to next read request */ - idx++; - } - - /* the read task may not fully fill the ending read request */ - if (mem_pos < end_offset - start_offset + 1) { - /* compute remaining length of read task */ - long length = (end_offset - start_offset + 1) - mem_pos; - - /* starting offset for read request */ - long offset = service_msgs->msg[idx].src_offset; - - /* add entry to read reply list */ - insert_to_ack_list(rank_ack_task, mem_addr + mem_pos, - service_msgs, idx, offset, length, errcode); - } - - return UNIFYCR_SUCCESS; -} - -/** -* Wait until all data are read and sent -* @param: read_task_set: a list of reading task -* @param: service_msgs: a list of received read requests -* @param: a list of send tasks for acknowledging the -* requesting delegators -* return success/error -*/ -static int sm_wait_until_digested(task_set_t* read_task_set, - service_msgs_t* service_msgs, - rank_ack_task_t* read_ack_task) -{ - int i; - - /* pointer to array of boolean flags for whether we need to test - * outstanding operations */ - int* flags = NULL; - - /* get number of pending read operations */ - int num_pended_reads = arraylist_size(pended_reads); - if (num_pended_reads > 0) { - /* allocate space for pending flags */ - size_t flags_size = num_pended_reads * sizeof(int); - flags = (int*)malloc(flags_size); - - /* set all flags to 0 */ - memset(flags, 0, flags_size); - } - - /* wait for all pending read operations to complete */ - long counter = 0; - while (1) { - /* check whether we have processed all reads */ - if (counter == num_pended_reads) { - break; - } - - /* iterate over pending reads */ - for (i = 0; i < num_pended_reads; i++) { - /* get meta data for this pending read */ - pended_read_t* pendread = - (pended_read_t*)arraylist_get(pended_reads, i); - - /* don't need to check if we failed to even submit - * the read operation */ - if (flags[i] != 1 && pendread->err_submit) { - /* mark that this read operation has completed */ - flags[i] = 1; - counter++; - - /* failed to submit, so we failed to read */ - int errcode = (int)UNIFYCR_ERROR_READ; - - /* add read reply to ack_list as failed */ - batch_insert_to_ack_list(read_ack_task, - &read_task_set->read_tasks[pendread->index], - service_msgs, - pendread->mem_pos, - pendread->start_pos, - pendread->end_pos, - errcode); - - continue; - } - - /* check whether this read has completed */ - if (flags[i] != 1 && - aio_error(&pendread->read_cb) != EINPROGRESS) { - /* mark that this read operation has completed */ - flags[i] = 1; - counter++; - - /* assume that we succeeded */ - int errcode = UNIFYCR_SUCCESS; - - /* get return code for read operation */ - ssize_t ret_code = aio_return(&pendread->read_cb); - - /* check that read completed without error */ - if (ret_code != (ssize_t)pendread->read_cb.aio_nbytes) { - /* read error or short read, - * consider either case as a read error */ - errcode = (int)UNIFYCR_ERROR_READ; - } - - /* add read reply to ack_list */ - batch_insert_to_ack_list(read_ack_task, - &read_task_set->read_tasks[pendread->index], - service_msgs, - pendread->mem_pos, - pendread->start_pos, - pendread->end_pos, - errcode); - } - } - } - - /* free flags array */ - if (flags != NULL) { - free(flags); - flags = NULL; - } - - /* reset our list of pending read operations */ - arraylist_reset(pended_reads); - - /* read operations have completed, - * send data to delegators */ - - /* iterate over list of delegators and send remaining acks */ - for (i = 0; i < read_ack_task->num; i++) { - /* get data structure for this delegator */ - rank_ack_meta_t* ack_meta = &read_ack_task->ack_metas[i]; - - /* get total number of read replies in this list */ - long tmp_sz = arraylist_size(ack_meta->ack_list); - - /* if we have read replies we have yet to send, - * send them now */ - if (ack_meta->start_cursor < tmp_sz) { - /* got some read replies, send them */ - int ret_code = sm_ack_remote_delegator(ack_meta); - if (ret_code != UNIFYCR_SUCCESS) { - return ret_code; - } - } - } - - /* sends issued, now wait for them to complete */ - - /* get number of outstanding sends - * initiated in sm_ack_remote_delegator */ - int num_pended_sends = arraylist_size(pended_sends); - if (num_pended_sends > 0) { - /* got some outstandings sends, allocate space for flags */ - size_t flags_size = num_pended_sends * sizeof(int); - flags = (int*)malloc(flags_size); - - /* set all flags to 0 */ - memset(flags, 0, flags_size); - } - - /* wait until all acks are sent */ - counter = 0; - while (1) { - /* check whether we have waited on all sends */ - if (counter == num_pended_sends) { - break; - } - - /* iterate over each send we issued */ - for (i = 0; i < num_pended_sends; i++) { - /* if send is still outstanding, check whether it's done */ - if (flags[i] == 0) { - /* still outstanding, get data for this send */ - ack_stat_t* ack_stat = arraylist_get(pended_sends, i); - - /* test whether send is complete */ - int ret_code = MPI_Test(&ack_stat->req, &flags[i], - &ack_stat->stat); - if (ret_code != MPI_SUCCESS) { - return (int)UNIFYCR_ERROR_RECV; - } - - /* if send completed, bump our counter */ - if (flags[i] != 0) { - counter++; - } - } - } - } - - /* free flags array */ - if (flags != NULL) { - free(flags); - flags = NULL; - } - - /* clear our list of sends */ - arraylist_reset(pended_sends); - - /* reset our list of read replies */ - for (i = 0; i < read_ack_task->num; i++) { - rank_ack_meta_t* ack_meta = &(read_ack_task->ack_metas[i]); - - arraylist_reset(ack_meta->ack_list); - ack_meta->rank_id = -1; - ack_meta->thrd_id = -1; - ack_meta->src_sz = 0; - ack_meta->start_cursor = 0; - } - read_ack_task->num = 0; - - /* TODO: might be nice to free some send buffers here */ - - /* set active send buffers back to 0, - * we do not free send buffers so that we do not have to - * allocate them again */ - send_buf_list->size = 0; - - return UNIFYCR_SUCCESS; -} - -static int compare_read_task(const void* a, const void* b) -{ - const read_task_t* ptr_a = a; - const read_task_t* ptr_b = b; - - if (ptr_a->size > ptr_b->size) { - return 1; - } - - if (ptr_a->size < ptr_b->size) { - return -1; - } - - if (ptr_a->arrival_time > ptr_b->arrival_time) { - return 1; - } - - if (ptr_a->arrival_time < ptr_b->arrival_time) { - return -1; - } - - return 0; -} - -/** -* Read and send the data via pipelined read, copy and send -* @param: read_task_set: a list of reading task -* @param: service_msgs: a list of received read requests -* @param: a list of send tasks for acknowledging the -* requesting delegators -* return success/error -*/ -static int sm_read_send_pipe(task_set_t* read_task_set, - service_msgs_t* service_msgs, - rank_ack_task_t* rank_ack_task) -{ - /* sort read tasks by size and then by arrival time */ - qsort(read_task_set->read_tasks, read_task_set->num, - sizeof(read_task_t), compare_read_task); - - /* points to offset in read reply buffer */ - long buf_cursor = 0; - - /* iterate over read tasks and pack read replies into send buffer */ - int i; - for (i = 0; i < read_task_set->num; i++) { - /* get pointer to current read task */ - read_task_t* read_task = &read_task_set->read_tasks[i]; - - /* get size of data we are to read */ - size_t size = read_task->size; - - /* check whether we have room in the read buffer to hold data */ - if (buf_cursor + size > READ_BUF_SZ) { - /* no room, wait until reads complete and send - * out replies */ - sm_wait_until_digested(read_task_set, - service_msgs, rank_ack_task); - - /* reset to start of read buffer */ - buf_cursor = 0; - } - - /* get app id and client id for this read task, - * defines log files holding data */ - int app_id = read_task->app_id; - int cli_id = read_task->cli_id; - - /* look up app config for given app id */ - app_config_t* app_config = - arraylist_get(app_config_list, app_id); - assert(app_config); - - /* get index of starting service message */ - int start_idx = read_task->start_idx; - - /* get offset in log file */ - send_msg_t* msg = &service_msgs->msg[start_idx]; - long offset = msg->dest_offset; - - /* prepare read opertions based on data location */ - if (offset + read_task->size <= app_config->data_size) { - /* requested data in read_task is totally in shared memory, - * get pointer to position in shared memory */ - char* log_ptr = app_config->shm_superblocks[cli_id] + - app_config->data_offset + offset; - - /* copy data into read reply buffer */ - char* buf_ptr = read_buf + buf_cursor; - memcpy(buf_ptr, log_ptr, size); - buf_cursor += size; - - /* we assume memcpy worked */ - int errcode = UNIFYCR_SUCCESS; - - /* prepare read reply meta data */ - batch_insert_to_ack_list(rank_ack_task, read_task, - service_msgs, buf_ptr, 0, size - 1, errcode); - } else if (offset < app_config->data_size) { - /* part of the requested data is in shared memory, - * compute size in shared memory */ - long sz_from_mem = app_config->data_size - offset; - - /* get pointer to position in shared memory */ - char* log_ptr = app_config->shm_superblocks[cli_id] + - app_config->data_offset + offset; - - /* copy data into read reply buffer */ - char* buf_ptr = read_buf + buf_cursor; - memcpy(buf_ptr, log_ptr, sz_from_mem); - buf_cursor += sz_from_mem; - - /* we assume memcpy worked */ - int errcode = UNIFYCR_SUCCESS; - - /* compute size in spillover file */ - long sz_from_ssd = size - sz_from_mem; - - /* read data from spillover file */ - int fd = app_config->spill_log_fds[cli_id]; - ssize_t nread = pread(fd, read_buf + buf_cursor, - sz_from_ssd, 0); - if (nread != (ssize_t)sz_from_ssd) { - /* read error or short read, - * consider either case to be an error */ - errcode = (int)UNIFYCR_ERROR_READ; - } - buf_cursor += sz_from_ssd; - - /* prepare read reply meta data */ - batch_insert_to_ack_list(rank_ack_task, read_task, - service_msgs, buf_ptr, 0, size - 1, errcode); - } else { - /* all requested data in the current read task - * are in spillover files */ - int fd = app_config->spill_log_fds[cli_id]; - - /* allocate empty pending read structure */ - pended_read_t* ptr = - (pended_read_t*)malloc(sizeof(pended_read_t)); - - /* fill in aio fields */ - memset(&ptr->read_cb, 0, sizeof(struct aiocb)); - ptr->read_cb.aio_fildes = fd; - ptr->read_cb.aio_buf = read_buf + buf_cursor; - ptr->read_cb.aio_offset = offset - app_config->data_size; - ptr->read_cb.aio_nbytes = size; - - /* index of read task for this pending read */ - ptr->index = i; - - /* offset locations in generating read request */ - ptr->start_pos = 0; - ptr->end_pos = size - 1; - - /* send buffer location to copy data when complete */ - ptr->mem_pos = read_buf + buf_cursor; - - /* submit read as aio operation */ - ptr->err_submit = 0; - int rc = aio_read(&ptr->read_cb); - if (rc < 0) { - /* remember that we failed to submit this read */ - ptr->err_submit = 1; - } - buf_cursor += size; - - /* enqueue entry in our list of pending reads */ - arraylist_add(pended_reads, ptr); - } - - /* update accounting for burst size */ - burst_data_sz += size; - } - - /* have initiated all read tasks, wait for them to finish - * and send results to delegators */ - sm_wait_until_digested(read_task_set, service_msgs, rank_ack_task); - - return UNIFYCR_SUCCESS; -} - -/** -* decode the message received from request_manager -* @param receive buffer that contains the requests -* @return success/error code -*/ -static int sm_decode_msg(char* recv_msg_buf) -{ - /* get pointer to start of receive buffer */ - char* ptr = recv_msg_buf; - - /* advance past command */ - ptr += sizeof(int); - - /* extract number of read requests in message */ - int num = *((int*)ptr); - ptr += sizeof(int); - - /* get pointer to read request */ - send_msg_t* msg = (send_msg_t*)ptr; - - /* allocate a larger array of service requests if needed */ - if (num + service_msgs.num >= service_msgs.cap) { - /* get a larger buffer (2x what is currently needed) */ - size_t count = 2 * (num + service_msgs.num); - - /* allocate larger buffer (2x what is currently needed) */ - size_t bytes = count * sizeof(send_msg_t); - service_msgs.msg = - (send_msg_t*)realloc(service_msgs.msg, bytes); - if (service_msgs.msg == NULL) { - /* failed to allocate memory */ - return (int)UNIFYCR_ERROR_NOMEM; - } - - /* got the memory, increase the capacity */ - service_msgs.cap = count; - - /* allocate corresponding space for read tasks */ - bytes = count * sizeof(read_task_t); - read_task_set.read_tasks = - (read_task_t*)realloc(read_task_set.read_tasks, bytes); - if (read_task_set.read_tasks == NULL) { - /* failed to allocate memory */ - return (int)UNIFYCR_ERROR_NOMEM; - } - - /* got the memory, increase the capacity */ - read_task_set.cap = count; - } - - /* get current timestamp as integer */ - int now = (int)time(NULL); - - /* unpack read requests to fill in service messages */ - int iter; - for (iter = 0; iter < num; iter++) { - /* copy values from read request */ - service_msgs.msg[service_msgs.num] = msg[iter]; - - /* set time stamp on when we received this request */ - service_msgs.msg[service_msgs.num].arrival_time = now; - - /* increment the number of service requests - * and go to next read request */ - service_msgs.num++; - } - - return UNIFYCR_SUCCESS; -} - -/* clean up state and resources allocated by service manager thread - * before shutting down */ -static int sm_exit() -{ - int rc = UNIFYCR_SUCCESS; - - arraylist_free(pended_reads); - arraylist_free(pended_sends); - arraylist_free(send_buf_list); - - free(service_msgs.msg); - free(read_task_set.read_tasks); - - int i; - for (i = 0; i < rank_ack_task.num; i++) { - arraylist_free(rank_ack_task.ack_metas[i].ack_list); - } - - return rc; -} - -/* return code for service manager thread, - * this is a static, since we return its address - * from the pthread entry point function */ -static int sm_rc = 0; - -/* entry point for the service thread, service the read requests - * received from the requesting delegators, executes a loop constantly - * waiting on incoming message, for each message that comes in, - * appends read requests to service_msgs list, if no message has - * arrived, and after some wait time (to catch bursty reads), - * then convert read requests into read tasks, execute read tasks - * to build read replies, and send read replies back to delegators */ -void* sm_service_reads(void* ctx) -{ - /* allocate a buffer to hold read data before packing - * into send buffers */ - read_buf = calloc(1, READ_BUF_SZ); - if (read_buf == NULL) { - // TODO: we need a better way to handle this case - fprintf(stderr, "Error allocating buffer!!!\n"); - exit(-1); - } - - /* initialize value on how long to wait before processing - * incoming read requests */ - long bursty_interval = MAX_BURSTY_INTERVAL / 10; - - /* tracks how much data we process in each burst */ - burst_data_sz = 0; - - /* intialize our data structure for holding read requests */ - size_t bytes = MAX_META_PER_SEND * sizeof(send_msg_t); - service_msgs.msg = (send_msg_t*)malloc(bytes); - service_msgs.num = 0; - service_msgs.cap = MAX_META_PER_SEND; - - /* intialize our data structure for holding read tasks */ - bytes = MAX_META_PER_SEND * sizeof(read_task_t); - read_task_set.read_tasks = (read_task_t*)malloc(bytes); - read_task_set.num = 0; - read_task_set.cap = MAX_META_PER_SEND; - - /* allocate a list to track pending data read operations */ - pended_reads = arraylist_create(); - if (pended_reads == NULL) { - sm_rc = (int)UNIFYCR_ERROR_NOMEM; - return (void*)&sm_rc; - } - - /* allocate a list to track outstanding sends back to - * requesting delegators */ - pended_sends = arraylist_create(); - if (pended_sends == NULL) { - sm_rc = (int)UNIFYCR_ERROR_NOMEM; - return (void*)&sm_rc; - } - - /* allocate a list to track pending send operations - * that need to be initiated back to requesting delegators */ - send_buf_list = arraylist_create(); - if (send_buf_list == NULL) { - sm_rc = (int)UNIFYCR_ERROR_NOMEM; - return (void*)&sm_rc; - } - - /* allocate memory to hold meta data for read replies */ - bytes = glb_size * MAX_NUM_CLIENTS * sizeof(rank_ack_meta_t); - rank_ack_task.num = 0; - rank_ack_task.ack_metas = (rank_ack_meta_t*)malloc(bytes); - - /* initialize each ack_meta structure, keep one for each potential - * application client - * (num delegaotrs * max num clients per delegator) */ - int i; - for (i = 0; i < glb_size * MAX_NUM_CLIENTS; i++) { - /* get pointer to current structure */ - rank_ack_meta_t* ack_meta = &rank_ack_task.ack_metas[i]; - - /* initialize fields */ - ack_meta->ack_list = NULL; - ack_meta->rank_id = -1; - ack_meta->thrd_id = -1; - ack_meta->src_sz = 0; - ack_meta->start_cursor = 0; - } - - /* received message format: - * cmd, req_num, a list of read requests */ - - /* buffer to hold received msg, - * since this is large, malloc it instead of declare it on stack - * (mitigate problems thread stack size limits) */ - char* req_msg_buf = (char*)malloc(REQ_BUF_LEN); - - /* listen and server incoming requests until signaled to exit */ - MPI_Status status; - int flag = 0; - while (! flag) { - /* post a receive for incoming request */ - MPI_Request request; - MPI_Irecv(req_msg_buf, REQ_BUF_LEN, MPI_BYTE, - MPI_ANY_SOURCE, CLI_DATA_TAG, - MPI_COMM_WORLD, &request); - - /* test whether we received anything */ - int irecv_flag = 0; - int return_code = MPI_Test(&request, &irecv_flag, &status); - if (return_code != MPI_SUCCESS) { - sm_rc = (int)UNIFYCR_ERROR_RECV; - return (void*)&sm_rc; - } - - /* as long as we keep receiving requests, we'll keep skipping - * the while loop below (and its sleep) and keep appending - * items to our read request queue */ - - /* - * keep receiving the read request - * until the end of a anticipated - * bursty behavior - * */ - while (! irecv_flag) { - /* if we have not received anything, sleep */ - if (bursty_interval > MIN_SLEEP_INTERVAL) { - usleep(SLEEP_INTERVAL); /* wait an interval */ - wait_time += SLEEP_INTERVAL; - } - - /* a bursty behavior is considered end when - * wait time is larger than BURSTY_INTERVAL - * */ - if (wait_time >= bursty_interval || - bursty_interval <= MIN_SLEEP_INTERVAL) { - /* if time to wait has expired, and if we have some - * queued read requests, do some work */ - if (service_msgs.num > 0) { - /* run through list of read requests and generate - * read tasks, merge requests for contiguous data - * into a single read task */ - int rc = sm_cluster_reads(&read_task_set, - &service_msgs); - if (rc != 0) { - sm_rc = rc; - return (void*)&sm_rc; - } - - /* execute read tasks, wait for them to complete, - * then pack read replies, send to delegators, and - * finally wait for sends to complete */ - rc = sm_read_send_pipe(&read_task_set, - &service_msgs, &rank_ack_task); - if (rc != 0) { - sm_rc = rc; - return (void*)&sm_rc; - } - - /* have processed all read requests and read tasks, - * prep them for next message */ - service_msgs.num = 0; - read_task_set.num = 0; - - /* determine how long to wait next time based on - * how much data we just processed in this burst */ - if (burst_data_sz >= LARGE_BURSTY_DATA) { - /* for large bursts above a threshold, - * wait for a fixed amount of time */ - bursty_interval = MAX_BURSTY_INTERVAL; - } else { - /* for smaller bursts, set delay proportionally - * to burst size we just processed */ - bursty_interval = - (long)((double)burst_data_sz / 1048576) * - SLEEP_SLICE_PER_UNIT; - } - - /* reset our burst size counter */ - burst_data_sz = 0; - } - - /* reset our timer */ - wait_time = 0; - } - - /* test for receive again, will break while loop - * if we find something */ - return_code = MPI_Test(&request, &irecv_flag, &status); - if (return_code != MPI_SUCCESS) { - sm_rc = (int)UNIFYCR_ERROR_RECV; - return (void*)&sm_rc; - } - } - - /* got a message, reset wait time */ - wait_time = 0; - - /* first value of receive buffer is integer holding command */ - int cmd = *((int*)req_msg_buf); - - /* check whether this is a request for data */ - if (cmd == XFER_COMM_DATA) { - /* got a request for data, append read requests in message - * to our service_msgs list */ - sm_decode_msg(req_msg_buf); - } - - /* check whether we were told to exit */ - if (cmd == XFER_COMM_EXIT) { - /* time to exit, free off data structures */ - sm_exit(); - - /* set flag to exit */ - flag = 1; - } - } - - /* free receive buffer */ - free(req_msg_buf); - req_msg_buf = NULL; - - return (void*)&sm_rc; -} diff --git a/server/src/unifycr_service_manager.h b/server/src/unifycr_service_manager.h deleted file mode 100644 index 74c1bcad2..000000000 --- a/server/src/unifycr_service_manager.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * - * Copyright 2017, UT-Battelle, LLC. - * - * LLNL-CODE-741539 - * All rights reserved. - * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. - */ - -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * Copyright (c) 2017, Florida State University. Contributions from - * the Computer Architecture and Systems Research Laboratory (CASTL) - * at the Department of Computer Science. - * - * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror - * LLNL-CODE-728877. All rights reserved. - * - * This file is part of burstfs. - * For details, see https://github.com/llnl/burstfs - * Please read https://github.com/llnl/burstfs/LICENSE for full license text. - */ - -#ifndef UNIFYCR_SERVICE_MANAGER_H -#define UNIFYCR_SERVICE_MANAGER_H - -void* sm_service_reads(void* ctx); - -#endif diff --git a/server/src/unifycr_setup.h b/server/src/unifycr_setup.h deleted file mode 100644 index cb45d639e..000000000 --- a/server/src/unifycr_setup.h +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * - * Copyright 2017, UT-Battelle, LLC. - * - * LLNL-CODE-741539 - * All rights reserved. - * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. - */ - -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * Copyright (c) 2017, Florida State University. Contributions from - * the Computer Architecture and Systems Research Laboratory (CASTL) - * at the Department of Computer Science. - * - * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror - * LLNL-CODE-728877. All rights reserved. - * - * This file is part of burstfs. - * For details, see https://github.com/llnl/burstfs - * Please read https://github.com/llnl/burstfs/LICENSE for full license text. - */ - -#ifndef UNIFYCR_SETUP_H -#define UNIFYCR_SETUP_H -#include "unifycr_const.h" -extern int* local_rank_lst; -extern int rank, size; -extern arraylist_t* app_config_list; -extern int invert_sock_ids[MAX_NUM_CLIENTS]; -#endif diff --git a/server/src/unifycr_sock.c b/server/src/unifycr_sock.c deleted file mode 100644 index ef6112401..000000000 --- a/server/src/unifycr_sock.c +++ /dev/null @@ -1,293 +0,0 @@ -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * - * Copyright 2017, UT-Battelle, LLC. - * - * LLNL-CODE-741539 - * All rights reserved. - * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. - */ - -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * Copyright (c) 2017, Florida State University. Contributions from - * the Computer Architecture and Systems Research Laboratory (CASTL) - * at the Department of Computer Science. - * - * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror - * LLNL-CODE-728877. All rights reserved. - * - * This file is part of burstfs. - * For details, see https://github.com/llnl/burstfs - * Please read https://github.com/llnl/burstfs/LICENSE for full license text. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "arraylist.h" -#include "unifycr_log.h" -#include "unifycr_const.h" -#include "unifycr_global.h" -#include "unifycr_pmix.h" -#include "unifycr_setup.h" -#include "unifycr_sock.h" - -int server_rank_idx; - -int server_sockfd; -int client_sockfd; -int num_fds = 0; - -//int thrd_pipe_fd[2] = {0}; - -struct pollfd poll_set[MAX_NUM_CLIENTS]; -struct sockaddr_un server_address; -char cmd_buf[MAX_NUM_CLIENTS][CMD_BUF_SIZE]; -char ack_buf[MAX_NUM_CLIENTS][CMD_BUF_SIZE]; -int ack_msg[3] = {0}; - -int detached_sock_id = -1; -int cur_sock_id = 1; - -/** -* initialize the listening socket on this delegator -* @return success/error code -*/ -int sock_init_server(int srvr_id) -{ - int rc; - char sock_path[UNIFYCR_MAX_FILENAME]; - - snprintf(sock_path, sizeof(sock_path), "%s.%d.%d", - SOCKET_PATH, getuid(), srvr_id); - - server_sockfd = socket(AF_UNIX, SOCK_STREAM, 0); - - memset(&server_address, 0, sizeof(server_address)); - server_address.sun_family = AF_UNIX; - strcpy(server_address.sun_path, sock_path); - int server_len = sizeof(server_address); - unlink(sock_path); - - rc = bind(server_sockfd, (struct sockaddr*)&server_address, - (socklen_t)server_len); - if (rc != 0) { - close(server_sockfd); - return -1; - } - - rc = listen(server_sockfd, MAX_NUM_CLIENTS); - if (rc != 0) { - close(server_sockfd); - return -1; - } - LOGDBG("domain socket path is %s", sock_path); - - int flag = fcntl(server_sockfd, F_GETFL); - fcntl(server_sockfd, F_SETFL, flag | O_NONBLOCK); - poll_set[0].fd = server_sockfd; //add - poll_set[0].events = POLLIN | POLLHUP; - poll_set[0].revents = 0; - num_fds++; - LOGDBG("completed sock init server"); - -#ifdef HAVE_PMIX_H - // publish domain socket path - unifycr_pmix_publish(pmix_key_unifycrd_socket, sock_path); -#endif - - return 0; -} - -int sock_add(int fd) -{ - LOGDBG("sock_adding fd: %d", fd); - if (num_fds == MAX_NUM_CLIENTS) { - return -1; - } - int flag = fcntl(fd, F_GETFL); - fcntl(fd, F_SETFL, flag | O_NONBLOCK); - poll_set[num_fds].fd = fd; - poll_set[num_fds].events = POLLIN | POLLHUP; - poll_set[num_fds].revents = 0; - num_fds++; - return 0; -} - -void sock_reset() -{ - int i; - - for (i = 0; i < num_fds; i++) { - poll_set[i].events = POLLIN | POLLHUP; - poll_set[i].revents = 0; - } -} - -int sock_remove(int idx) -{ - /* in this case, we simply disable the disconnected - * file descriptor. */ - poll_set[idx].fd = -1; - return 0; -} - -/* - * send command to the client to let the client digest the - * data in the shared receive buffer - * @param: sock_id: socket index in poll_set - * @param: cmd: command type - * - * */ -int sock_notify_cli(int sock_id, int cmd) -{ - memset(ack_buf[sock_id], 0, sizeof(ack_buf[sock_id])); - - LOGDBG("sock notifying fd: %d", client_sockfd); - - memcpy(ack_buf[sock_id], &cmd, sizeof(int)); - int rc = write(client_sockfd, - ack_buf[sock_id], sizeof(ack_buf[sock_id])); - - if (rc < 0) { - return (int)UNIFYCR_ERROR_WRITE; - } - return ULFS_SUCCESS; -} - - -/* - * wait for the client-side command - * */ - -int sock_wait_cli_cmd() -{ - int rc, i; - - sock_reset(); - rc = poll(poll_set, num_fds, -1); - if (rc <= 0) { - return (int)UNIFYCR_ERROR_POLL; - } else { - LOGDBG("in wait_cli_cmd"); - for (i = 0; i < num_fds; i++) { - if (poll_set[i].fd != -1 && poll_set[i].revents != 0) { - if (i == 0 && poll_set[i].revents == POLLIN) { - int client_len = sizeof(struct sockaddr_un); - - struct sockaddr_un client_address; - client_sockfd = accept(server_sockfd, - (struct sockaddr*)&client_address, - (socklen_t*)&client_len); - LOGDBG("accepted client on socket %d", client_sockfd); - rc = sock_add(client_sockfd); - if (rc < 0) { - return (int)UNIFYCR_ERROR_SOCKET_FD_EXCEED; - } else { - cur_sock_id = i; - return ULFS_SUCCESS; - } - } else if (i != 0 && poll_set[i].revents == POLLIN) { - int bytes_read = read(poll_set[i].fd, - cmd_buf[i], CMD_BUF_SIZE); - if (bytes_read == 0) { - sock_remove(i); - detached_sock_id = i; - return (int)UNIFYCR_ERROR_SOCK_DISCONNECT; - } - cur_sock_id = i; - return ULFS_SUCCESS; - } else { - if (i == 0) { - return (int)UNIFYCR_ERROR_SOCK_LISTEN; - } else { - detached_sock_id = i; - if (i != 0 && poll_set[i].revents == POLLHUP) { - sock_remove(i); - return (int)UNIFYCR_ERROR_SOCK_DISCONNECT; - } else { - sock_remove(i); - return (int)UNIFYCR_ERROR_SOCK_OTHER; - - } - } - } - } - } - } - - return ULFS_SUCCESS; - -} - -int sock_ack_cli(int sock_id, int ret_sz) -{ - int rc = write(poll_set[sock_id].fd, - ack_buf[sock_id], ret_sz); - if (rc < 0) { - return (int)UNIFYCR_ERROR_SOCK_OTHER; - } - return ULFS_SUCCESS; -} - -int sock_handle_error(int sock_error_no) -{ - return ULFS_SUCCESS; -} - -int sock_get_error_id() -{ - return detached_sock_id; -} - -char* sock_get_cmd_buf(int sock_id) -{ - return cmd_buf[sock_id]; -} - -char* sock_get_ack_buf(int sock_id) -{ - return (char*)ack_buf[sock_id]; -} - -int sock_get_id() -{ - return 0; -} - -void sock_sanitize_cli(int client_id) -{ - /* close socket for this client id - * and set fd back to -1 */ - close(poll_set[client_id].fd); - poll_set[client_id].fd = -1; -} - -int sock_sanitize() -{ - int i; - char tmp_str[UNIFYCR_MAX_FILENAME] = {0}; - - for (i = 0; i < num_fds; i++) { - if (poll_set[i].fd > 0) { - close(poll_set[i].fd); - } - } - - snprintf(tmp_str, sizeof(tmp_str), "%s%d", - SOCKET_PATH, server_rank_idx); - unlink(tmp_str); - return 0; -} diff --git a/server/src/unifycr_cmd_handler.c b/server/src/unifyfs_cmd_handler.c similarity index 54% rename from server/src/unifycr_cmd_handler.c rename to server/src/unifyfs_cmd_handler.c index 27bbba1d7..5df7f5a78 100644 --- a/server/src/unifycr_cmd_handler.c +++ b/server/src/unifyfs_cmd_handler.c @@ -2,14 +2,14 @@ * Copyright (c) 2017, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2017-2019, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -27,40 +27,31 @@ * Please read https://github.com/llnl/burstfs/LICENSE for full license text. */ -#include -#include -#include -#include +// system headers +#include #include -#include -#include -#include -#include -#include -#include -#include "unifycr_log.h" -#include "unifycr_global.h" -#include "unifycr_meta.h" -#include "unifycr_cmd_handler.h" -#include "unifycr_request_manager.h" -#include "unifycr_setup.h" -#include "unifycr_const.h" -#include "unifycr_sock.h" -#include "unifycr_metadata.h" -#include "unifycr_shm.h" -#include "../../client/src/unifycr_clientcalls_rpc.h" +#include + +// server components +#include "unifyfs_global.h" +#include "unifyfs_metadata.h" +#include "unifyfs_request_manager.h" + +// margo rpcs +#include "margo_server.h" +#include "unifyfs_client_rpcs.h" +#include "unifyfs_rpc_util.h" /** * attach to the client-side shared memory * @param app_config: application information * @param app_id: the server-side - * @param sock_id: position in poll_set in unifycr_sock.h + * @param sock_id: position in poll_set in unifyfs_sock.h * @return success/error code */ -static int attach_to_shm( - app_config_t* app_config, - int app_id, - int client_side_id) +static int attach_to_shm(app_config_t* app_config, + int app_id, + int client_side_id) { char shm_name[GEN_STR_LEN] = {0}; @@ -71,34 +62,34 @@ static int attach_to_shm( /* define name of superblock region for this client */ sprintf(shm_name, "%d-super-%d", app_id, client_side_id); - /* copy name of superblock region */ - strcpy(app_config->super_buf_name[client_side_id], shm_name); - /* attach to superblock */ - void* addr = unifycr_shm_alloc(shm_name, app_config->superblock_sz); + void* addr = unifyfs_shm_alloc(shm_name, app_config->superblock_sz); if (addr == NULL) { LOGERR("Failed to attach to superblock %s", shm_name); - return (int)UNIFYCR_ERROR_SHMEM; + return (int)UNIFYFS_ERROR_SHMEM; } app_config->shm_superblocks[client_side_id] = addr; + /* copy name of superblock region */ + strcpy(app_config->super_buf_name[client_side_id], shm_name); + /* attach shared request buffer, a request buffer is created by each * client to convey the client-side read request to the delegator */ /* define name of request buffer region for this client */ sprintf(shm_name, "%d-req-%d", app_id, client_side_id); - /* copy name of request buffer region */ - strcpy(app_config->req_buf_name[client_side_id], shm_name); - /* attach to request buffer region */ - addr = unifycr_shm_alloc(shm_name, app_config->req_buf_sz); + addr = unifyfs_shm_alloc(shm_name, app_config->req_buf_sz); if (addr == NULL) { LOGERR("Failed to attach to request buffer %s", shm_name); - return (int)UNIFYCR_ERROR_SHMEM; + return (int)UNIFYFS_ERROR_SHMEM; } app_config->shm_req_bufs[client_side_id] = addr; + /* copy name of request buffer region */ + strcpy(app_config->req_buf_name[client_side_id], shm_name); + /* initialize shared receive buffer, a request buffer is created * by each client for the delegator to temporarily buffer the * received data for this client */ @@ -106,18 +97,23 @@ static int attach_to_shm( /* define name of receive buffer region for this client */ sprintf(shm_name, "%d-recv-%d", app_id, client_side_id); - /* copy name of request buffer region */ - strcpy(app_config->recv_buf_name[client_side_id], shm_name); - /* attach to request buffer region */ - addr = unifycr_shm_alloc(shm_name, app_config->recv_buf_sz); + addr = unifyfs_shm_alloc(shm_name, app_config->recv_buf_sz); if (addr == NULL) { LOGERR("Failed to attach to receive buffer %s", shm_name); - return (int)UNIFYCR_ERROR_SHMEM; + return (int)UNIFYFS_ERROR_SHMEM; } app_config->shm_recv_bufs[client_side_id] = addr; + shm_header_t* shm_hdr = (shm_header_t*)addr; + pthread_mutex_init(&(shm_hdr->sync), NULL); + shm_hdr->meta_cnt = 0; + shm_hdr->bytes = 0; + shm_hdr->state = SHMEM_REGION_EMPTY; - return ULFS_SUCCESS; + /* copy name of request buffer region */ + strcpy(app_config->recv_buf_name[client_side_id], shm_name); + + return UNIFYFS_SUCCESS; } /** @@ -126,7 +122,7 @@ static int attach_to_shm( * overflows. * @param app_config: application information * @param app_id: the server-side application id - * @param sock_id: position in poll_set in unifycr_sock.h + * @param sock_id: position in poll_set in unifyfs_sock.h * @return success/error code */ static int open_log_file(app_config_t* app_config, @@ -136,7 +132,7 @@ static int open_log_file(app_config_t* app_config, * have one of these per app_id and client_id, * client writes data to spill over file when it fills * memory storage */ - char path[UNIFYCR_MAX_FILENAME] = {0}; + char path[UNIFYFS_MAX_FILENAME] = {0}; snprintf(path, sizeof(path), "%s/spill_%d_%d.log", app_config->external_spill_dir, app_id, client_side_id); @@ -146,9 +142,9 @@ static int open_log_file(app_config_t* app_config, /* open spill over file for reading */ app_config->spill_log_fds[client_side_id] = open(path, O_RDONLY, 0666); if (app_config->spill_log_fds[client_side_id] < 0) { - printf("rank:%d, openning file %s failure\n", glb_rank, path); + printf("rank:%d, opening file %s failure\n", glb_mpi_rank, path); fflush(stdout); - return (int)UNIFYCR_ERROR_FILE; + return (int)UNIFYFS_ERROR_FILE; } /* build name of spill over index file, @@ -164,142 +160,17 @@ static int open_log_file(app_config_t* app_config, app_config->spill_index_log_fds[client_side_id] = open(path, O_RDONLY, 0666); if (app_config->spill_index_log_fds[client_side_id] < 0) { - printf("rank:%d, openning index file %s failure\n", glb_rank, path); + printf("rank:%d, opening index file %s failure\n", glb_mpi_rank, path); fflush(stdout); - return (int)UNIFYCR_ERROR_FILE; + return (int)UNIFYFS_ERROR_FILE; } - return ULFS_SUCCESS; + return UNIFYFS_SUCCESS; } -/* create a request manager thread for the given app_id - * and client_id, returns pointer to thread control structure - * on success and NULL on failure */ -static thrd_ctrl_t* unifycr_rm_thrd_create(int app_id, int client_id) -{ - /* allocate a new thread control structure */ - size_t bytes = sizeof(thrd_ctrl_t); - thrd_ctrl_t* thrd_ctrl = (thrd_ctrl_t*)malloc(bytes); - if (thrd_ctrl == NULL) { - LOGERR("Failed to allocate structure for request " - "manager thread for app_id=%d client_id=%d", - app_id, client_id); - return NULL; - } - memset(thrd_ctrl, 0, bytes); - - /* allocate an array for listing read requests from client */ - bytes = sizeof(msg_meta_t); - thrd_ctrl->del_req_set = (msg_meta_t*)malloc(bytes); - if (thrd_ctrl->del_req_set == NULL) { - LOGERR("Failed to allocate read request structure for request " - "manager thread for app_id=%d client_id=%d", - app_id, client_id); - free(thrd_ctrl); - return NULL; - } - memset(thrd_ctrl->del_req_set, 0, bytes); - - /* allocate structure for tracking outstanding read requests - * this delegator has with service managers on other nodes */ - bytes = sizeof(del_req_stat_t); - thrd_ctrl->del_req_stat = (del_req_stat_t*)malloc(bytes); - if (thrd_ctrl->del_req_stat == NULL) { - LOGERR("Failed to allocate delegator structure for request " - "manager thread for app_id=%d client_id=%d", - app_id, client_id); - free(thrd_ctrl->del_req_set); - free(thrd_ctrl); - return NULL; - } - memset(thrd_ctrl->del_req_set, 0, bytes); - - /* allocate a structure to track requests we have on each - * remote service manager */ - bytes = sizeof(per_del_stat_t) * glb_size; - thrd_ctrl->del_req_stat->req_stat = (per_del_stat_t*)malloc(bytes); - if (thrd_ctrl->del_req_stat->req_stat == NULL) { - LOGERR("Failed to allocate per-delegator structure for request " - "manager thread for app_id=%d client_id=%d", - app_id, client_id); - free(thrd_ctrl->del_req_stat); - free(thrd_ctrl->del_req_set); - free(thrd_ctrl); - return NULL; - } - memset(thrd_ctrl->del_req_stat->req_stat, 0, bytes); - - /* initialize lock for shared data structures between - * main thread and request delegator thread */ - int rc = pthread_mutex_init(&(thrd_ctrl->thrd_lock), NULL); - if (rc != 0) { - LOGERR("pthread_mutex_init failed for request " - "manager thread app_id=%d client_id=%d rc=%d (%s)", - app_id, client_id, rc, strerror(rc)); - free(thrd_ctrl->del_req_stat->req_stat); - free(thrd_ctrl->del_req_stat); - free(thrd_ctrl->del_req_set); - free(thrd_ctrl); - return NULL; - } - - /* initailize condition variable to flow control - * work between main thread and request delegator thread */ - rc = pthread_cond_init(&(thrd_ctrl->thrd_cond), NULL); - if (rc != 0) { - LOGERR("pthread_cond_init failed for request " - "manager thread app_id=%d client_id=%d rc=%d (%s)", - app_id, client_id, rc, strerror(rc)); - pthread_mutex_destroy(&(thrd_ctrl->thrd_lock)); - free(thrd_ctrl->del_req_stat->req_stat); - free(thrd_ctrl->del_req_stat); - free(thrd_ctrl->del_req_set); - free(thrd_ctrl); - return NULL; - } - - /* record app and client id this thread will be serving */ - thrd_ctrl->app_id = app_id; - thrd_ctrl->client_id = client_id; - - /* initialize flow control flags */ - thrd_ctrl->exit_flag = 0; - thrd_ctrl->has_waiting_delegator = 0; - thrd_ctrl->has_waiting_dispatcher = 0; - - /* TODO: can't we just tack this address onto tmp_config structure? */ - /* insert our thread control structure into our list of - * active request manager threads, important to do this before - * launching thread since it uses list to lookup its structure */ - rc = arraylist_add(thrd_list, thrd_ctrl); - if (rc != 0) { - pthread_cond_destroy(&(thrd_ctrl->thrd_cond)); - pthread_mutex_destroy(&(thrd_ctrl->thrd_lock)); - free(thrd_ctrl->del_req_stat->req_stat); - free(thrd_ctrl->del_req_stat); - free(thrd_ctrl->del_req_set); - free(thrd_ctrl); - return NULL; - } - /* launch request manager thread */ - rc = pthread_create(&(thrd_ctrl->thrd), NULL, - rm_delegate_request_thread, (void*)thrd_ctrl); - if (rc != 0) { - LOGERR("pthread_create failed for request " - "manager thread app_id=%d client_id=%d rc=%d (%s)", - app_id, client_id, rc, strerror(rc)); - pthread_cond_destroy(&(thrd_ctrl->thrd_cond)); - pthread_mutex_destroy(&(thrd_ctrl->thrd_lock)); - free(thrd_ctrl->del_req_stat->req_stat); - free(thrd_ctrl->del_req_stat); - free(thrd_ctrl->del_req_set); - free(thrd_ctrl); - return NULL; - } - return thrd_ctrl; -} +/* BEGIN MARGO CLIENT-SERVER RPC HANDLER FUNCTIONS */ /* called by client to register with the server, client provides a * structure of values on input, some of which specify global @@ -312,14 +183,15 @@ static thrd_ctrl_t* unifycr_rm_thrd_create(int app_id, int client_id) * server attaches to client shared memory regions, opens files * holding spill over data, and launchers request manager for * client */ -static void unifycr_mount_rpc(hg_handle_t handle) +static void unifyfs_mount_rpc(hg_handle_t handle) { int rc; + int ret = (int)UNIFYFS_SUCCESS; /* get input params */ - unifycr_mount_in_t in; - int ret = margo_get_input(handle, &in); - assert(ret == HG_SUCCESS); + unifyfs_mount_in_t in; + hg_return_t hret = margo_get_input(handle, &in); + assert(hret == HG_SUCCESS); /* read app_id and client_id from input */ int app_id = in.app_id; @@ -368,6 +240,7 @@ static void unifycr_mount_rpc(hg_handle_t handle) tmp_config->shm_superblocks[i] = NULL; tmp_config->spill_log_fds[i] = -1; tmp_config->spill_index_log_fds[i] = -1; + tmp_config->client_addr[i] = HG_ADDR_NULL; } /* insert new app_config into our list, indexed by app_id */ @@ -375,8 +248,16 @@ static void unifycr_mount_rpc(hg_handle_t handle) if (rc != 0) { ret = rc; } + } else { + LOGDBG("using existing app_config for app_id=%d", app_id); } + /* convert client_addr_str sent in input struct to margo hg_addr_t, + * which is the address type needed to call rpc functions, etc */ + hret = margo_addr_lookup(unifyfsd_rpc_context->shm_mid, + in.client_addr_str, + &(tmp_config->client_addr[client_id])); + /* record client id of process on this node */ tmp_config->client_ranks[client_id] = client_id; @@ -385,260 +266,283 @@ static void unifycr_mount_rpc(hg_handle_t handle) /* attach to shared memory regions of this client */ rc = attach_to_shm(tmp_config, app_id, client_id); - if (rc != ULFS_SUCCESS) { + if (rc != UNIFYFS_SUCCESS) { + LOGERR("attach_to_shm() failed for app_id=%d client_id=%d rc=%d", + app_id, client_id, rc); ret = rc; } /* open spill over files for this client */ rc = open_log_file(tmp_config, app_id, client_id); if (rc < 0) { + LOGERR("open_log_file() failed for app_id=%d client_id=%d rc=%d", + app_id, client_id, rc); ret = rc; } /* create request manager thread */ - thrd_ctrl_t* thrd_ctrl = unifycr_rm_thrd_create(app_id, client_id); - if (thrd_ctrl != NULL) { + reqmgr_thrd_t* rm_thrd = unifyfs_rm_thrd_create(app_id, client_id); + if (rm_thrd != NULL) { /* TODO: seems like it would be cleaner to avoid thread_list * and instead just record address to struct */ /* remember id for thread control for this client */ - tmp_config->thrd_idxs[client_id] = arraylist_size(thrd_list) - 1; + tmp_config->thrd_idxs[client_id] = rm_thrd->thrd_ndx; } else { /* failed to create request manager thread */ - ret = UNIFYCR_FAILURE; + LOGERR("unifyfs_rm_thrd_create() failed for app_id=%d client_id=%d", + app_id, client_id); + ret = UNIFYFS_FAILURE; } /* build output structure to return to caller */ - unifycr_mount_out_t out; + unifyfs_mount_out_t out; out.ret = ret; out.max_recs_per_slice = max_recs_per_slice; /* send output back to caller */ - hg_return_t hret = margo_respond(handle, &out); + hret = margo_respond(handle, &out); assert(hret == HG_SUCCESS); /* free margo resources */ margo_free_input(handle, &in); margo_destroy(handle); } -DEFINE_MARGO_RPC_HANDLER(unifycr_mount_rpc) +DEFINE_MARGO_RPC_HANDLER(unifyfs_mount_rpc) -static void unifycr_unmount_rpc(hg_handle_t handle) +static void unifyfs_unmount_rpc(hg_handle_t handle) { /* get input params */ - unifycr_unmount_in_t in; - int margo_ret = margo_get_input(handle, &in); - assert(margo_ret == HG_SUCCESS); + unifyfs_unmount_in_t in; + hg_return_t hret = margo_get_input(handle, &in); + assert(hret == HG_SUCCESS); /* read app_id and client_id from input */ int app_id = in.app_id; int client_id = in.local_rank_idx; + /* build output structure to return to caller */ + unifyfs_unmount_out_t out; + out.ret = UNIFYFS_SUCCESS; + + /* send output back to caller */ + hret = margo_respond(handle, &out); + assert(hret == HG_SUCCESS); + + /* free margo resources */ + margo_free_input(handle, &in); + margo_destroy(handle); + /* lookup app_config for given app_id */ app_config_t* app_config = - (app_config_t *) arraylist_get(app_config_list, app_id); - - /* TODO: do cleanup here */ + (app_config_t*) arraylist_get(app_config_list, app_id); /* get thread id for this client */ int thrd_id = app_config->thrd_idxs[client_id]; /* look up thread control structure */ - thrd_ctrl_t* thrd_ctrl = (thrd_ctrl_t *)arraylist_get(thrd_list, thrd_id); + reqmgr_thrd_t* thrd_ctrl = rm_get_thread(thrd_id); /* shutdown the delegator thread */ rm_cmd_exit(thrd_ctrl); /* detach from the request shared memory */ - unifycr_shm_free(app_config->req_buf_name[client_id], - app_config->req_buf_sz, - (void**)&(app_config->shm_req_bufs[client_id])); - app_config->shm_req_bufs[client_id] = NULL; + if (NULL != app_config->shm_req_bufs[client_id]) { + unifyfs_shm_free(app_config->req_buf_name[client_id], + app_config->req_buf_sz, + (void**)&(app_config->shm_req_bufs[client_id])); + } /* detach from the read shared memory buffer */ - unifycr_shm_free(app_config->recv_buf_name[client_id], - app_config->recv_buf_sz, - (void**)&(app_config->shm_recv_bufs[client_id])); - app_config->shm_recv_bufs[client_id] = NULL; - - /* destroy the sockets except for the ones for acks */ - sock_sanitize_cli(client_id); - - /* build output structure to return to caller */ - unifycr_unmount_out_t out; - out.ret = UNIFYCR_SUCCESS; - - /* send output back to caller */ - hg_return_t hret = margo_respond(handle, &out); - assert(hret == HG_SUCCESS); + if (NULL != app_config->shm_recv_bufs[client_id]) { + unifyfs_shm_free(app_config->recv_buf_name[client_id], + app_config->recv_buf_sz, + (void**)&(app_config->shm_recv_bufs[client_id])); + } - /* free margo resources */ - margo_free_input(handle, &in); - margo_destroy(handle); + /* free margo hg_addr_t client addresses in app_config struct */ + margo_addr_free(unifyfsd_rpc_context->shm_mid, + app_config->client_addr[client_id]); } -DEFINE_MARGO_RPC_HANDLER(unifycr_unmount_rpc) +DEFINE_MARGO_RPC_HANDLER(unifyfs_unmount_rpc) /* returns file meta data including file size and file name * given a global file id */ -static void unifycr_metaget_rpc(hg_handle_t handle) +static void unifyfs_metaget_rpc(hg_handle_t handle) { /* get input params */ - unifycr_metaget_in_t in; - int ret = margo_get_input(handle, &in); - assert(ret == HG_SUCCESS); + unifyfs_metaget_in_t in; + hg_return_t hret = margo_get_input(handle, &in); + assert(hret == HG_SUCCESS); /* given the global file id, look up file attributes * from key/value store */ - unifycr_file_attr_t attr_val; - attr_val.gfid = in.gfid; - ret = meta_process_attr_get(&attr_val); + unifyfs_file_attr_t attr_val; + + int ret = unifyfs_get_file_attribute(in.gfid, &attr_val); /* build our output values */ - unifycr_metaget_out_t out; - out.st_size = attr_val.file_attr.st_size; + unifyfs_metaget_out_t out; + out.gfid = attr_val.gfid; + out.mode = attr_val.mode; + out.uid = attr_val.uid; + out.gid = attr_val.gid; + out.size = attr_val.size; + out.atime = attr_val.atime; + out.mtime = attr_val.mtime; + out.ctime = attr_val.ctime; out.filename = attr_val.filename; - out.ret = ret; + out.is_laminated = attr_val.is_laminated; + out.ret = ret; /* send output back to caller */ - hg_return_t hret = margo_respond(handle, &out); + hret = margo_respond(handle, &out); assert(hret == HG_SUCCESS); /* free margo resources */ margo_free_input(handle, &in); margo_destroy(handle); } -DEFINE_MARGO_RPC_HANDLER(unifycr_metaget_rpc) +DEFINE_MARGO_RPC_HANDLER(unifyfs_metaget_rpc) /* given a global file id and a file name, * record key/value entry for this file */ -static void unifycr_metaset_rpc(hg_handle_t handle) +static void unifyfs_metaset_rpc(hg_handle_t handle) { /* get input params */ - unifycr_metaset_in_t in; - int ret = margo_get_input(handle, &in); - assert(ret == HG_SUCCESS); + unifyfs_metaset_in_t in; + hg_return_t hret = margo_get_input(handle, &in); + assert(hret == HG_SUCCESS); /* store file name for given global file id */ - unifycr_file_attr_t fattr; + unifyfs_file_attr_t fattr; memset(&fattr, 0, sizeof(fattr)); fattr.gfid = in.gfid; strncpy(fattr.filename, in.filename, sizeof(fattr.filename)); - /* TODO: unifycr_metaset_in_t is missing struct stat info */ - ret = meta_process_attr_set(&fattr); + fattr.mode = in.mode; + fattr.uid = in.uid; + fattr.gid = in.gid; + fattr.size = in.size; + fattr.atime = in.atime; + fattr.mtime = in.mtime; + fattr.ctime = in.ctime; + fattr.is_laminated = in.is_laminated; + + int ret = unifyfs_set_file_attribute(&fattr); /* build our output values */ - unifycr_metaset_out_t out; + unifyfs_metaset_out_t out; out.ret = ret; /* return to caller */ - hg_return_t hret = margo_respond(handle, &out); + hret = margo_respond(handle, &out); assert(hret == HG_SUCCESS); /* free margo resources */ margo_free_input(handle, &in); margo_destroy(handle); } -DEFINE_MARGO_RPC_HANDLER(unifycr_metaset_rpc) +DEFINE_MARGO_RPC_HANDLER(unifyfs_metaset_rpc) /* given app_id, client_id, and a global file id as input, * read extent location metadata from client shared memory * and insert corresponding key/value pairs into global index */ -static void unifycr_fsync_rpc(hg_handle_t handle) +static void unifyfs_fsync_rpc(hg_handle_t handle) { /* get input params */ - unifycr_fsync_in_t in; - int ret = margo_get_input(handle, &in); - assert(ret == HG_SUCCESS); + unifyfs_fsync_in_t in; + hg_return_t hret = margo_get_input(handle, &in); + assert(hret == HG_SUCCESS); /* given global file id, read index metadata from client and * insert into global index key/value store */ - ret = meta_process_fsync(in.app_id, in.local_rank_idx, in.gfid); + int ret = rm_cmd_fsync(in.app_id, in.local_rank_idx, in.gfid); /* build our output values */ - unifycr_metaset_out_t out; + unifyfs_metaset_out_t out; out.ret = ret; /* return to caller */ - hg_return_t hret = margo_respond(handle, &out); + hret = margo_respond(handle, &out); assert(hret == HG_SUCCESS); /* free margo resources */ margo_free_input(handle, &in); margo_destroy(handle); } -DEFINE_MARGO_RPC_HANDLER(unifycr_fsync_rpc) +DEFINE_MARGO_RPC_HANDLER(unifyfs_fsync_rpc) /* given an app_id, client_id, global file id, * return current file size */ -static void unifycr_filesize_rpc(hg_handle_t handle) +static void unifyfs_filesize_rpc(hg_handle_t handle) { /* get input params */ - unifycr_read_in_t in; - int ret = HG_Get_input(handle, &in); - assert(ret == HG_SUCCESS); + unifyfs_filesize_in_t in; + hg_return_t hret = margo_get_input(handle, &in); + assert(hret == HG_SUCCESS); /* read data for a single read request from client, * returns data to client through shared memory */ size_t filesize; - ret = rm_cmd_filesize(in.app_id, in.local_rank_idx, - in.gfid, &filesize); + int ret = rm_cmd_filesize(in.app_id, in.local_rank_idx, + in.gfid, &filesize); /* build our output values */ - unifycr_filesize_out_t out; + unifyfs_filesize_out_t out; out.ret = (int32_t) ret; out.filesize = (hg_size_t) filesize; /* return to caller */ - hg_return_t hret = margo_respond(handle, &out); + hret = margo_respond(handle, &out); assert(hret == HG_SUCCESS); /* free margo resources */ margo_free_input(handle, &in); margo_destroy(handle); } -DEFINE_MARGO_RPC_HANDLER(unifycr_filesize_rpc) +DEFINE_MARGO_RPC_HANDLER(unifyfs_filesize_rpc) /* given an app_id, client_id, global file id, an offset, and a length, * initiate read operation to lookup and return data, * client synchronizes with server again later when data is available * to be copied into user buffers */ -static void unifycr_read_rpc(hg_handle_t handle) +static void unifyfs_read_rpc(hg_handle_t handle) { /* get input params */ - unifycr_read_in_t in; - int ret = margo_get_input(handle, &in); - assert(ret == HG_SUCCESS); + unifyfs_read_in_t in; + hg_return_t hret = margo_get_input(handle, &in); + assert(hret == HG_SUCCESS); /* read data for a single read request from client, * returns data to client through shared memory */ - ret = rm_cmd_read(in.app_id, in.local_rank_idx, - in.gfid, in.offset, in.length); + int ret = rm_cmd_read(in.app_id, in.local_rank_idx, + in.gfid, in.offset, in.length); /* build our output values */ - unifycr_read_out_t out; + unifyfs_read_out_t out; out.ret = ret; /* return to caller */ - hg_return_t hret = margo_respond(handle, &out); + hret = margo_respond(handle, &out); assert(hret == HG_SUCCESS); /* free margo resources */ margo_free_input(handle, &in); margo_destroy(handle); } -DEFINE_MARGO_RPC_HANDLER(unifycr_read_rpc) +DEFINE_MARGO_RPC_HANDLER(unifyfs_read_rpc) /* given an app_id, client_id, global file id, and a count * of read requests, follow by list of offset/length tuples * initiate read requests for data, * client synchronizes with server again later when data is available * to be copied into user buffers */ -static void unifycr_mread_rpc(hg_handle_t handle) +static void unifyfs_mread_rpc(hg_handle_t handle) { /* get input params */ - unifycr_mread_in_t in; - int ret = margo_get_input(handle, &in); - assert(ret == HG_SUCCESS); + unifyfs_mread_in_t in; + hg_return_t hret = margo_get_input(handle, &in); + assert(hret == HG_SUCCESS); /* allocate buffer to hold array of read requests */ hg_size_t size = in.bulk_size; @@ -653,20 +557,21 @@ static void unifycr_mread_rpc(hg_handle_t handle) /* register local target buffer for bulk access */ hg_bulk_t bulk_handle; - hg_return_t hret = margo_bulk_create(mid, 1, &buffer, - &size, HG_BULK_WRITE_ONLY, &bulk_handle); + hret = margo_bulk_create(mid, 1, &buffer, &size, + HG_BULK_WRITE_ONLY, &bulk_handle); + assert(hret == HG_SUCCESS); /* get list of read requests */ - hret = margo_bulk_transfer(mid, HG_BULK_PULL, - hgi->addr, in.bulk_handle, 0, bulk_handle, 0, size); + hret = margo_bulk_transfer(mid, HG_BULK_PULL, hgi->addr, + in.bulk_handle, 0, bulk_handle, 0, size); assert(hret == HG_SUCCESS); /* initiate read operations to fetch data for read requests */ - ret = rm_cmd_mread(in.app_id, in.local_rank_idx, - in.gfid, in.read_count, buffer); + int ret = rm_cmd_mread(in.app_id, in.local_rank_idx, + in.read_count, buffer); /* build our output values */ - unifycr_mread_out_t out; + unifyfs_mread_out_t out; out.ret = ret; /* return to caller */ @@ -679,4 +584,4 @@ static void unifycr_mread_rpc(hg_handle_t handle) free(buffer); margo_destroy(handle); } -DEFINE_MARGO_RPC_HANDLER(unifycr_mread_rpc) +DEFINE_MARGO_RPC_HANDLER(unifyfs_mread_rpc) diff --git a/server/src/unifycr_global.h b/server/src/unifyfs_global.h similarity index 58% rename from server/src/unifycr_global.h rename to server/src/unifyfs_global.h index e8c71b0ca..9e5f9c01e 100644 --- a/server/src/unifycr_global.h +++ b/server/src/unifyfs_global.h @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -27,19 +27,53 @@ * Please read https://github.com/llnl/burstfs/LICENSE for full license text. */ -#ifndef UNIFYCR_GLOBAL_H -#define UNIFYCR_GLOBAL_H +#ifndef UNIFYFS_GLOBAL_H +#define UNIFYFS_GLOBAL_H -#include +// system headers +#include +#include +#include +#include #include -#include "unifycr_configurator.h" +#include +#include +#include +#include + +// common headers #include "arraylist.h" +#include "unifyfs_const.h" +#include "unifyfs_log.h" +#include "unifyfs_meta.h" +#include "unifyfs_shm.h" +#include "unifyfs_sock.h" + +#include +#include + +extern arraylist_t* app_config_list; +extern arraylist_t* rm_thrd_list; + +extern char glb_host[UNIFYFS_MAX_HOSTNAME]; +extern int glb_mpi_rank, glb_mpi_size; + +extern size_t max_recs_per_slice; /* defines commands for messages sent to service manager threads */ typedef enum { - XFER_COMM_DATA, /* message contains read requests */ - XFER_COMM_EXIT, /* indicates service manager thread should exit */ -} service_cmd_lst_t; + SVC_CMD_INVALID = 0, + SVC_CMD_RDREQ_MSG = 1, /* read requests (send_msg_t) */ + SVC_CMD_RDREQ_CHK, /* read requests (chunk_read_req_t) */ + SVC_CMD_EXIT, /* service manager thread should exit */ +} service_cmd_e; + +typedef enum { + READ_REQUEST_TAG = 5001, + READ_RESPONSE_TAG = 6001, + CHUNK_REQUEST_TAG = 7001, + CHUNK_RESPONSE_TAG = 8001 +} service_tag_e; /* this defines a read request as sent from the request manager to the * service manager, it contains info about the physical location of @@ -52,7 +86,7 @@ typedef enum { * * it also contains a return address to use in the read reply that * the service manager sends back to the request manager: - * + * * src_delegator_rank - rank of requesting delegator process * src_thrd - thread id of request manager (used to compute MPI tag) * src_app_id, src_cli_id @@ -84,10 +118,10 @@ typedef struct { * back to request manager, data payload of length bytes immediately * follows the header */ typedef struct { - size_t src_offset; /* logical offset in file */ + size_t src_offset; /* file offset */ size_t length; /* number of bytes */ - int src_fid; /* global file id */ - int errcode; /* indicates whether read was successful */ + int src_fid; /* global file id */ + int errcode; /* indicates whether read was successful */ } recv_msg_t; /* defines a fixed-length list of read requests */ @@ -96,68 +130,48 @@ typedef struct { send_msg_t msg_meta[MAX_META_PER_SEND]; /* list of requests */ } msg_meta_t; -/* one entry per delegator for which we have active read requests, - * records rank of delegator and request count */ -typedef struct { - int req_cnt; /* number of requests to this delegator */ - int del_id; /* rank of delegator */ -} per_del_stat_t; +// NEW READ REQUEST STRUCTURES +typedef enum { + READREQ_INIT = 0, + READREQ_STARTED, /* chunk requests issued */ + READREQ_PARTIAL_COMPLETE, /* some reads completed */ + READREQ_COMPLETE /* all reads completed */ +} readreq_status_e; -/* records list of delegator information (rank, req count) for - * set of delegators we have active read requests for */ -typedef struct { - per_del_stat_t *req_stat; /* delegator rank and request count */ - int del_cnt; /* number of delegators we have read requests for */ -} del_req_stat_t; - -/* this structure is created by the main thread for each request - * manager thread, contains shared data structures where main thread - * issues read requests and request manager processes them, contains - * condition variable and lock for coordination between threads */ typedef struct { - /* request manager thread */ - pthread_t thrd; - - /* condition variable to synchronize request manager thread - * and main thread delivering work */ - pthread_cond_t thrd_cond; - - /* lock for shared data structures (variables below) */ - pthread_mutex_t thrd_lock; + size_t nbytes; /* size of data chunk */ + size_t offset; /* file offset */ + size_t log_offset; /* remote log offset */ + int log_app_id; /* remote log application id */ + int log_client_id; /* remote log client id */ +} chunk_read_req_t; - /* flag indicating that request manager thread is waiting - * for work inside of critical region */ - int has_waiting_delegator; - - /* flag indicating main thread is in critical section waiting - * for request manager thread */ - int has_waiting_dispatcher; - - /* a list of read requests to be sent to each delegator, - * main thread adds items to this list, request manager - * processes them */ - msg_meta_t *del_req_set; - - /* statistics of read requests to be sent to each delegator */ - del_req_stat_t *del_req_stat; - - /* buffer to build read request messages */ - char del_req_msg_buf[REQ_BUF_LEN]; - - /* memory for posting receives for incoming read reply messages - * from the service threads */ - char del_recv_msg_buf[RECV_BUF_CNT][RECV_BUF_LEN]; - - /* flag set by main thread indicating whether request - * manager thread should exit */ - int exit_flag; +typedef struct { + size_t offset; /* file offset */ + size_t nbytes; /* requested read size */ + ssize_t read_rc; /* bytes read (or negative error code) */ +} chunk_read_resp_t; - /* app_id this thread is serving */ - int app_id; +typedef struct { + int rank; /* remote delegator rank */ + int rdreq_id; /* read-request id */ + int app_id; /* app id of requesting client process */ + int client_id; /* client id of requesting client process */ + int num_chunks; /* number of chunk requests/responses */ + readreq_status_e status; /* summary status for chunk reads */ + size_t total_sz; /* total size of data requested */ + chunk_read_req_t* reqs; /* @RM: subarray of server_read_req_t.chunks + * @SM: received requests buffer */ + chunk_read_resp_t* resp; /* @RM: received responses buffer + * @SM: allocated responses buffer */ +} remote_chunk_reads_t; - /* client_id this thread is serving */ - int client_id; -} thrd_ctrl_t; +typedef struct { + size_t length; /* length of data to read */ + size_t offset; /* file offset */ + int gfid; /* global file id */ + int errcode; /* request completion status */ +} client_read_req_t; /* one of these structures is created for each app id, * it contains info for each client like names, file descriptors, @@ -197,37 +211,43 @@ typedef struct { int spill_index_log_fds[MAX_NUM_CLIENTS]; /* spillover index */ /* shared memory pointers */ - char *shm_superblocks[MAX_NUM_CLIENTS]; /* superblock data */ - char *shm_req_bufs[MAX_NUM_CLIENTS]; /* read request shm */ - char *shm_recv_bufs[MAX_NUM_CLIENTS]; /* read reply shm */ + char* shm_superblocks[MAX_NUM_CLIENTS]; /* superblock data */ + char* shm_req_bufs[MAX_NUM_CLIENTS]; /* read request shm */ + char* shm_recv_bufs[MAX_NUM_CLIENTS]; /* read reply shm */ + + /* client address for rpc invocation */ + hg_addr_t client_addr[MAX_NUM_CLIENTS]; /* file names */ - char super_buf_name[MAX_NUM_CLIENTS][UNIFYCR_MAX_FILENAME]; - char req_buf_name[MAX_NUM_CLIENTS][UNIFYCR_MAX_FILENAME]; - char recv_buf_name[MAX_NUM_CLIENTS][UNIFYCR_MAX_FILENAME]; - char spill_log_name[MAX_NUM_CLIENTS][UNIFYCR_MAX_FILENAME]; - char spill_index_log_name[MAX_NUM_CLIENTS][UNIFYCR_MAX_FILENAME]; + char super_buf_name[MAX_NUM_CLIENTS][UNIFYFS_MAX_FILENAME]; + char req_buf_name[MAX_NUM_CLIENTS][UNIFYFS_MAX_FILENAME]; + char recv_buf_name[MAX_NUM_CLIENTS][UNIFYFS_MAX_FILENAME]; + char spill_log_name[MAX_NUM_CLIENTS][UNIFYFS_MAX_FILENAME]; + char spill_index_log_name[MAX_NUM_CLIENTS][UNIFYFS_MAX_FILENAME]; /* directory holding spill over files */ - char external_spill_dir[UNIFYCR_MAX_FILENAME]; + char external_spill_dir[UNIFYFS_MAX_FILENAME]; } app_config_t; typedef int fattr_key_t; typedef struct { - char fname[UNIFYCR_MAX_FILENAME]; + char fname[UNIFYFS_MAX_FILENAME]; struct stat file_attr; } fattr_val_t; -extern arraylist_t *app_config_list; -extern arraylist_t *thrd_list; - int invert_sock_ids[MAX_NUM_CLIENTS]; -extern pthread_t data_thrd; -extern int glb_rank, glb_size; -extern int *local_rank_lst; -extern int local_rank_cnt; -extern size_t max_recs_per_slice; +typedef struct { + //char* hostname; + char* margo_svr_addr_str; + hg_addr_t margo_svr_addr; + int mpi_rank; +} server_info_t; + +extern char glb_host[UNIFYFS_MAX_HOSTNAME]; +extern size_t glb_num_servers; +extern server_info_t* glb_servers; + -#endif +#endif // UNIFYFS_GLOBAL_H diff --git a/server/src/unifyfs_init.c b/server/src/unifyfs_init.c new file mode 100644 index 000000000..4781d341f --- /dev/null +++ b/server/src/unifyfs_init.c @@ -0,0 +1,676 @@ +/* + * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2017, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +/* + * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * Copyright (c) 2017, Florida State University. Contributions from + * the Computer Architecture and Systems Research Laboratory (CASTL) + * at the Department of Computer Science. + * + * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror + * LLNL-CODE-728877. All rights reserved. + * + * This file is part of burstfs. + * For details, see https://github.com/llnl/burstfs + * Please read https://github.com/llnl/burstfs/LICENSE for full license text. + */ + +// system headers +#include +#include +#include + +// common headers +#include "unifyfs_configurator.h" +#include "unifyfs_keyval.h" +#include "unifyfs_runstate.h" + +// server components +#include "unifyfs_global.h" +#include "unifyfs_metadata.h" +#include "unifyfs_request_manager.h" +#include "unifyfs_service_manager.h" + +// margo rpcs +#include "margo_server.h" + +int glb_mpi_rank, glb_mpi_size; +char glb_host[UNIFYFS_MAX_HOSTNAME]; + +size_t glb_num_servers; // size of glb_servers array +server_info_t* glb_servers; // array of server_info_t + +arraylist_t* app_config_list; + +int invert_sock_ids[MAX_NUM_CLIENTS]; /*records app_id for each sock_id*/ + +unifyfs_cfg_t server_cfg; + +static int unifyfs_exit(void); + +#if defined(UNIFYFS_MULTIPLE_DELEGATORS) +int* local_rank_lst; +int local_rank_cnt; + +/* + * structure that records the information of + * each application launched by srun. + * */ +typedef struct { + char hostname[UNIFYFS_MAX_HOSTNAME]; + int rank; +} name_rank_pair_t; + +static int compare_name_rank_pair(const void* a, const void* b); +static int compare_int(const void* a, const void* b); +static int CountTasksPerNode(int rank, int numTasks); +static int find_rank_idx(int my_rank); +#endif + +/* + * Perform steps to create a daemon process: + * + * 1. Fork and exit from parent so child runs in the background + * 2. Set the daemon umask to 0 so file modes passed to open() and + * mkdir() fully control access modes + * 3. Call setsid() to create a new session and detach from controlling tty + * 4. Change current working directory to / so daemon doesn't block + * filesystem unmounts + * 5. close STDIN, STDOUT, and STDERR + * 6. Fork again to abdicate session leader position to guarantee + * daemon cannot reacquire a controlling TTY + * + */ +static void daemonize(void) +{ + pid_t pid; + pid_t sid; + int rc; + + pid = fork(); + + if (pid < 0) { + LOGERR("fork failed: %s", strerror(errno)); + exit(1); + } + + if (pid > 0) { + exit(0); + } + + umask(0); + + sid = setsid(); + if (sid < 0) { + LOGERR("setsid failed: %s", strerror(errno)); + exit(1); + } + + rc = chdir("/"); + if (rc < 0) { + LOGERR("chdir failed: %s", strerror(errno)); + exit(1); + } + + close(STDIN_FILENO); + close(STDOUT_FILENO); + close(STDERR_FILENO); + + pid = fork(); + if (pid < 0) { + LOGERR("fork failed: %s", strerror(errno)); + exit(1); + } else if (pid > 0) { + exit(0); + } +} + +static int time_to_exit; +void exit_request(int sig) +{ +#ifdef HAVE_STRSIGNAL + const char* sigstr = strsignal(sig); + LOGDBG("got signal %s", sigstr); +#endif + + switch (sig) { + case SIGINT: + case SIGQUIT: + case SIGTERM: + time_to_exit = 1; + LOGDBG("exit requested"); + break; + default: + LOGERR("unhandled signal %d", sig); + break; + } +} + +static int allocate_servers(size_t n_servers) +{ + glb_num_servers = n_servers; + glb_servers = (server_info_t*) calloc(n_servers, sizeof(server_info_t)); + if (NULL == glb_servers) { + LOGERR("failed to allocate server_info array"); + return (int)UNIFYFS_ERROR_NOMEM; + } + return (int)UNIFYFS_SUCCESS; +} + +static int process_servers_hostfile(const char* hostfile) +{ + int rc; + size_t i, cnt; + FILE* fp = NULL; + char hostbuf[UNIFYFS_MAX_HOSTNAME+1]; + + if (NULL == hostfile) { + return (int)UNIFYFS_ERROR_INVAL; + } + fp = fopen(hostfile, "r"); + if (!fp) { + LOGERR("failed to open hostfile %s", hostfile); + return (int)UNIFYFS_FAILURE; + } + + // scan first line: number of hosts + rc = fscanf(fp, "%zu\n", &cnt); + if (1 != rc) { + LOGERR("failed to scan hostfile host count"); + fclose(fp); + return (int)UNIFYFS_FAILURE; + } + rc = allocate_servers(cnt); + if ((int)UNIFYFS_SUCCESS != rc) { + fclose(fp); + return (int)UNIFYFS_FAILURE; + } + + // scan host lines + for (i = 0; i < cnt; i++) { + memset(hostbuf, 0, sizeof(hostbuf)); + rc = fscanf(fp, "%s\n", hostbuf); + if (1 != rc) { + LOGERR("failed to scan hostfile host line %zu", i); + fclose(fp); + return (int)UNIFYFS_FAILURE; + } + //glb_servers[i].hostname = strdup(hostbuf); + // NOTE: following assumes one server per host + if (0 == strcmp(glb_host, hostbuf)) { + //glb_svr_rank = (int)i; + LOGDBG("found myself at hostfile index=%zu, mpi_rank=%d", + i, glb_mpi_rank); + } + } + fclose(fp); + + return (int)UNIFYFS_SUCCESS; +} + +int main(int argc, char* argv[]) +{ + int provided; + int rc; + int kv_rank, kv_nranks; + bool daemon = true; + struct sigaction sa; + char rank_str[16] = {0}; + char dbg_fname[UNIFYFS_MAX_FILENAME] = {0}; + + rc = unifyfs_config_init(&server_cfg, argc, argv); + if (rc != 0) { + exit(1); + } + server_cfg.ptype = UNIFYFS_SERVER; + + rc = configurator_bool_val(server_cfg.unifyfs_daemonize, &daemon); + if (rc != 0) { + exit(1); + } + if (daemon) { + daemonize(); + } + + // setup clean termination by signal + memset(&sa, 0, sizeof(struct sigaction)); + sa.sa_handler = exit_request; + rc = sigemptyset(&sa.sa_mask); + rc = sigaction(SIGINT, &sa, NULL); + rc = sigaction(SIGQUIT, &sa, NULL); + rc = sigaction(SIGTERM, &sa, NULL); + + app_config_list = arraylist_create(); + if (app_config_list == NULL) { + LOGERR("%s", unifyfs_error_enum_description(UNIFYFS_ERROR_NOMEM)); + exit(1); + } + + rm_thrd_list = arraylist_create(); + if (rm_thrd_list == NULL) { + LOGERR("%s", unifyfs_error_enum_description(UNIFYFS_ERROR_NOMEM)); + exit(1); + } + + rc = MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); + if (rc != MPI_SUCCESS) { + exit(1); + } + + rc = MPI_Comm_rank(MPI_COMM_WORLD, &glb_mpi_rank); + if (rc != MPI_SUCCESS) { + exit(1); + } + + rc = MPI_Comm_size(MPI_COMM_WORLD, &glb_mpi_size); + if (rc != MPI_SUCCESS) { + exit(1); + } + + // start logging + gethostname(glb_host, sizeof(glb_host)); + snprintf(dbg_fname, sizeof(dbg_fname), "%s/%s.%s.%d", + server_cfg.log_dir, server_cfg.log_file, glb_host, glb_mpi_rank); + rc = unifyfs_log_open(dbg_fname); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("%s", unifyfs_error_enum_description((unifyfs_error_e)rc)); + } + + if (NULL != server_cfg.server_hostfile) { + rc = process_servers_hostfile(server_cfg.server_hostfile); + if (rc != (int)UNIFYFS_SUCCESS) { + LOGERR("failed to gather server information"); + exit(1); + } + } + + kv_rank = glb_mpi_rank; + kv_nranks = glb_mpi_size; + rc = unifyfs_keyval_init(&server_cfg, &kv_rank, &kv_nranks); + if (rc != (int)UNIFYFS_SUCCESS) { + exit(1); + } + if (glb_mpi_rank != kv_rank) { + LOGDBG("mismatch on MPI (%d) vs kvstore (%d) rank", + glb_mpi_rank, kv_rank); + } + if (glb_mpi_size != kv_nranks) { + LOGDBG("mismatch on MPI (%d) vs kvstore (%d) num ranks", + glb_mpi_size, kv_nranks); + } + + // TEMPORARY: remove once we fully eliminate use of MPI in sever + snprintf(rank_str, sizeof(rank_str), "%d", glb_mpi_rank); + rc = unifyfs_keyval_publish_remote(key_unifyfsd_mpi_rank, rank_str); + if (rc != (int)UNIFYFS_SUCCESS) { + exit(1); + } + + if (NULL == server_cfg.server_hostfile) { + //glb_svr_rank = kv_rank; + rc = allocate_servers((size_t)kv_nranks); + } + + rc = unifyfs_write_runstate(&server_cfg); + if (rc != (int)UNIFYFS_SUCCESS) { + exit(1); + } + + LOGDBG("initializing rpc service"); + rc = configurator_bool_val(server_cfg.margo_tcp, &margo_use_tcp); + rc = margo_server_rpc_init(); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("%s", unifyfs_error_enum_description(UNIFYFS_ERROR_MARGO)); + exit(1); + } + + MPI_Barrier(MPI_COMM_WORLD); + + LOGDBG("connecting rpc servers"); + rc = margo_connect_servers(); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("%s", unifyfs_error_enum_description(UNIFYFS_ERROR_MARGO)); + exit(1); + } + +#if defined(UNIFYFS_USE_DOMAIN_SOCKET) + int srvr_rank_idx = 0; +#if defined(UNIFYFS_MULTIPLE_DELEGATORS) + rc = CountTasksPerNode(glb_mpi_rank, glb_mpi_size); + if (rc < 0) { + exit(1); + } + srvr_rank_idx = find_rank_idx(glb_mpi_rank); +#endif // UNIFYFS_MULTIPLE_DELEGATORS + LOGDBG("creating server domain socket"); + rc = sock_init_server(srvr_rank_idx); + if (rc != 0) { + LOGERR("%s", unifyfs_error_enum_description(UNIFYFS_ERROR_SOCKET)); + exit(1); + } +#endif // UNIFYFS_USE_DOMAIN_SOCKET + + /* launch the service manager */ + LOGDBG("launching service manager thread"); + rc = svcmgr_init(); + if (rc != (int)UNIFYFS_SUCCESS) { + LOGERR("launch failed - %s", unifyfs_error_enum_description(rc)); + exit(1); + } + + LOGDBG("initializing metadata store"); + rc = meta_init_store(&server_cfg); + if (rc != 0) { + LOGERR("%s", unifyfs_error_enum_description(UNIFYFS_ERROR_MDINIT)); + exit(1); + } + + MPI_Barrier(MPI_COMM_WORLD); + LOGDBG("finished service initialization"); + + while (1) { +#if defined(UNIFYFS_USE_DOMAIN_SOCKET) + int timeout_ms = 2000; /* in milliseconds */ + rc = sock_wait_cmd(timeout_ms); + if (rc != UNIFYFS_SUCCESS) { + // we ignore disconnects, they are expected + if (rc != UNIFYFS_ERROR_SOCK_DISCONNECT) { + LOGDBG("domain socket error %s", + unifyfs_error_enum_description((unifyfs_error_e)rc)); + time_to_exit = 1; + } + } +#else + sleep(1); +#endif // UNIFYFS_USE_DOMAIN_SOCKET + if (time_to_exit) { + LOGDBG("starting service shutdown"); + break; + } + } + + LOGDBG("stopping service manager thread"); + rc = svcmgr_fini(); + + LOGDBG("cleaning run state"); + rc = unifyfs_clean_runstate(&server_cfg); + + return unifyfs_exit(); +} + +#if defined(UNIFYFS_MULTIPLE_DELEGATORS) +/* count the number of delegators per node, and + * the rank of each delegator, the results are stored + * in local_rank_cnt and local_rank_lst. + * @param numTasks: number of processes in the communicator + * @return success/error code */ +static int CountTasksPerNode(int rank, int numTasks) +{ + char localhost[UNIFYFS_MAX_HOSTNAME]; + char hostname[UNIFYFS_MAX_HOSTNAME]; + int resultsLen = UNIFYFS_MAX_HOSTNAME; + + MPI_Status status; + int i, j, rc; + + if (numTasks < 0) { + return -1; + } + + rc = MPI_Get_processor_name(localhost, &resultsLen); + if (rc != 0) { + return -1; + } + + if (rank == 0) { + /* a container of (rank, host) mappings */ + name_rank_pair_t* host_set = + (name_rank_pair_t*)calloc(numTasks, sizeof(name_rank_pair_t)); + /* MPI_Recv all hostnames, and compare to local hostname */ + for (i = 1; i < numTasks; i++) { + rc = MPI_Recv(hostname, UNIFYFS_MAX_HOSTNAME, + MPI_CHAR, MPI_ANY_SOURCE, + MPI_ANY_TAG, + MPI_COMM_WORLD, &status); + if (rc != 0) { + return -1; + } + strcpy(host_set[i].hostname, hostname); + host_set[i].rank = status.MPI_SOURCE; + } + strcpy(host_set[0].hostname, localhost); + host_set[0].rank = 0; + + /* sort by hostname */ + qsort(host_set, numTasks, sizeof(name_rank_pair_t), + compare_name_rank_pair); + + /* rank_cnt: records the number of processes on each host + * rank_set: the list of ranks for each host */ + int** rank_set = (int**)calloc(numTasks, sizeof(int*)); + int* rank_cnt = (int*)calloc(numTasks, sizeof(int)); + + int cursor = 0; + int set_counter = 0; + for (i = 1; i < numTasks; i++) { + if (strcmp(host_set[i].hostname, + host_set[i - 1].hostname) != 0) { + // found a different host, so switch to a new set + int hiter, riter = 0; + rank_set[set_counter] = + (int*)calloc((i - cursor), sizeof(int)); + rank_cnt[set_counter] = i - cursor; + for (hiter = cursor; hiter < i; hiter++, riter++) { + rank_set[set_counter][riter] = host_set[hiter].rank; + } + + set_counter++; + cursor = i; + } + } + + /* fill rank_cnt and rank_set entry for the last host */ + + rank_set[set_counter] = + (int*)calloc((i - cursor), sizeof(int)); + rank_cnt[set_counter] = numTasks - cursor; + j = 0; + for (i = cursor; i < numTasks; i++, j++) { + rank_set[set_counter][j] = host_set[i].rank; + } + set_counter++; + + /* broadcast rank_set information */ + int root_set_no = -1; + for (i = 0; i < set_counter; i++) { + /* send rank set to each of its ranks */ + for (j = 0; j < rank_cnt[i]; j++) { + if (rank_set[i][j] != 0) { + rc = MPI_Send(&rank_cnt[i], 1, MPI_INT, + rank_set[i][j], 0, MPI_COMM_WORLD); + if (rc != 0) { + return -1; + } + rc = MPI_Send(rank_set[i], rank_cnt[i], MPI_INT, + rank_set[i][j], 0, MPI_COMM_WORLD); + if (rc != 0) { + return -1; + } + } else { + root_set_no = i; + local_rank_cnt = rank_cnt[i]; + local_rank_lst = (int*)calloc(rank_cnt[i], sizeof(int)); + memcpy(local_rank_lst, rank_set[i], + (local_rank_cnt * sizeof(int))) + } + } + } + + for (i = 0; i < set_counter; i++) { + free(rank_set[i]); + } + free(rank_cnt); + free(host_set); + free(rank_set); + } else { /* non-root rank */ + /* MPI_Send hostname to root */ + rc = MPI_Send(localhost, UNIFYFS_MAX_HOSTNAME, MPI_CHAR, + 0, 0, MPI_COMM_WORLD); + if (rc != 0) { + return -1; + } + /* receive the local rank set count */ + rc = MPI_Recv(&local_rank_cnt, 1, MPI_INT, 0, + 0, MPI_COMM_WORLD, &status); + if (rc != 0) { + return -1; + } + /* receive the the local rank set */ + local_rank_lst = (int*)calloc(local_rank_cnt, sizeof(int)); + rc = MPI_Recv(local_rank_lst, local_rank_cnt, MPI_INT, 0, + 0, MPI_COMM_WORLD, &status); + if (rc != 0) { + free(local_rank_lst); + return -1; + } + } + + /* sort by rank */ + qsort(local_rank_lst, local_rank_cnt, sizeof(int), compare_int); + + return 0; +} + +static int find_rank_idx(int my_rank) +{ + int i; + assert(local_rank_lst != NULL); + for (i = 0; i < local_rank_cnt; i++) { + if (local_rank_lst[i] == my_rank) { + return i; + } + } + return -1; +} + +static int compare_name_rank_pair(const void* a, const void* b) +{ + const name_rank_pair_t* pair_a = a; + const name_rank_pair_t* pair_b = b; + return strcmp(pair_a->hostname, pair_b->hostname); +} + +static int compare_int(const void* a, const void* b) +{ + int aval = *(const int*)a; + int bval = *(const int*)b; + return aval - bval; +} + +#endif // UNIFYFS_MULTIPLE_DELEGATORS + + +static int unifyfs_exit(void) +{ + int rc = UNIFYFS_SUCCESS; + + /* shutdown rpc service */ + LOGDBG("stopping rpc service"); + margo_server_rpc_finalize(); + +#if defined(UNIFYFS_USE_DOMAIN_SOCKET) + /* close remaining sockets */ + LOGDBG("closing sockets"); + sock_sanitize(); +#endif + + /* finalize kvstore service*/ + LOGDBG("finalizing kvstore service"); + unifyfs_keyval_fini(); + + /* TODO: notify the service threads to exit */ + + /* notify the request manager threads to exit*/ + LOGDBG("stopping request manager threads"); + int i, j; + for (i = 0; i < arraylist_size(rm_thrd_list); i++) { + /* request and wait for request manager thread exit */ + reqmgr_thrd_t* thrd_ctrl = + (reqmgr_thrd_t*) arraylist_get(rm_thrd_list, i); + rm_cmd_exit(thrd_ctrl); + } + arraylist_free(rm_thrd_list); + + /* sanitize the shared memory and delete the log files + * */ + int app_sz = arraylist_size(app_config_list); + + /* iterate over each active application and free resources */ + for (i = 0; i < app_sz; i++) { + /* get pointer to app config for this app_id */ + app_config_t* app = + (app_config_t*)arraylist_get(app_config_list, i); + + /* skip to next app_id if this is empty */ + if (app == NULL) { + continue; + } + + /* free resources allocate for each client */ + for (j = 0; j < MAX_NUM_CLIENTS; j++) { + /* release request buffer shared memory region */ + if (app->shm_req_bufs[j] != NULL) { + unifyfs_shm_free(app->req_buf_name[j], app->req_buf_sz, + (void**)&(app->shm_req_bufs[j])); + } + + /* release receive buffer shared memory region */ + if (app->shm_recv_bufs[j] != NULL) { + unifyfs_shm_free(app->recv_buf_name[j], app->recv_buf_sz, + (void**)&(app->shm_recv_bufs[j])); + } + + /* release super block shared memory region */ + if (app->shm_superblocks[j] != NULL) { + unifyfs_shm_free(app->super_buf_name[j], app->superblock_sz, + (void**)&(app->shm_superblocks[j])); + } + + /* close spill log file and delete it */ + if (app->spill_log_fds[j] > 0) { + close(app->spill_log_fds[j]); + unlink(app->spill_log_name[j]); + } + + /* close spill log index file and delete it */ + if (app->spill_index_log_fds[j] > 0) { + close(app->spill_index_log_fds[j]); + unlink(app->spill_index_log_name[j]); + } + } + } + + /* shutdown the metadata service*/ + LOGDBG("stopping metadata service"); + meta_sanitize(); + + LOGDBG("finalizing MPI"); + MPI_Finalize(); + + LOGDBG("all done!"); + unifyfs_log_close(); + + return rc; +} diff --git a/server/src/unifyfs_metadata.c b/server/src/unifyfs_metadata.c new file mode 100644 index 000000000..0f4436a65 --- /dev/null +++ b/server/src/unifyfs_metadata.c @@ -0,0 +1,487 @@ +/* + * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2017-2019, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +/* + * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * Copyright (c) 2017, Florida State University. Contributions from + * the Computer Architecture and Systems Research Laboratory (CASTL) + * at the Department of Computer Science. + * + * Written by: Teng Wang, Adam Moody, Wekuan Yu, Kento Sato, Kathryn Mohror + * LLNL-CODE-728877. All rights reserved. + * + * This file is part of burstfs. + * For details, see https://github.com/llnl/burstfs + * Please read https://github.com/llnl/burstfs/LICNSE for full license text. + */ + +// NOTE: following two lines needed for nftw(), MUST COME FIRST IN FILE +#define _XOPEN_SOURCE 500 +#include + +// common headers +#include "unifyfs_client_rpcs.h" +#include "ucr_read_builder.h" + +// server headers +#include "unifyfs_global.h" +#include "unifyfs_metadata.h" + +// MDHIM headers +#include "indexes.h" +#include "mdhim.h" + + +unifyfs_key_t** unifyfs_keys; +unifyfs_val_t** unifyfs_vals; + +fattr_key_t** fattr_keys; +fattr_val_t** fattr_vals; + +char* manifest_path; + +struct mdhim_brm_t* brm, *brmp; +struct mdhim_bgetrm_t* bgrm, *bgrmp; + +struct mdhim_t* md; +int md_size; + +struct index_t* unifyfs_indexes[2]; +size_t max_recs_per_slice; + +void debug_log_key_val(const char* ctx, + unifyfs_key_t* key, + unifyfs_val_t* val) +{ + if ((key != NULL) && (val != NULL)) { + LOGDBG("@%s - key(fid=%d, offset=%lu), " + "val(del=%d, len=%lu, addr=%lu, app=%d, rank=%d)", + ctx, key->fid, key->offset, + val->delegator_rank, val->len, val->addr, + val->app_id, val->rank); + } else if (key != NULL) { + LOGDBG("@%s - key(fid=%d, offset=%lu)", + ctx, key->fid, key->offset); + } +} + +int unifyfs_key_compare(unifyfs_key_t* a, unifyfs_key_t* b) +{ + assert((NULL != a) && (NULL != b)); + if (a->fid == b->fid) { + if (a->offset == b->offset) { + return 0; + } else if (a->offset < b->offset) { + return -1; + } else { + return 1; + } + } else if (a->fid < b->fid) { + return -1; + } else { + return 1; + } +} + +/* initialize the key-value store */ +int meta_init_store(unifyfs_cfg_t* cfg) +{ + int rc, ratio; + size_t path_len; + long svr_ratio, range_sz; + MPI_Comm comm = MPI_COMM_WORLD; + + if (cfg == NULL) { + return -1; + } + + mdhim_options_t* db_opts = mdhim_options_init(); + if (db_opts == NULL) { + return -1; + } + mdhim_options_set_db_type(db_opts, LEVELDB); + mdhim_options_set_db_name(db_opts, cfg->meta_db_name); + mdhim_options_set_key_type(db_opts, MDHIM_UNIFYFS_KEY); + mdhim_options_set_debug_level(db_opts, MLOG_CRIT); + + /* UNIFYFS_META_DB_PATH: root directory for metadata */ + mdhim_options_set_db_path(db_opts, cfg->meta_db_path); + + /* number of metadata servers = + * number of unifyfs servers / UNIFYFS_META_SERVER_RATIO */ + svr_ratio = 0; + rc = configurator_int_val(cfg->meta_server_ratio, &svr_ratio); + if (rc != 0) { + return -1; + } + ratio = (int) svr_ratio; + mdhim_options_set_server_factor(db_opts, ratio); + + /* indices/attributes are striped to servers according + * to config setting for UNIFYFS_META_RANGE_SIZE. */ + range_sz = 0; + rc = configurator_int_val(cfg->meta_range_size, &range_sz); + if (rc != 0) { + return -1; + } + max_recs_per_slice = (size_t) range_sz; + mdhim_options_set_max_recs_per_slice(db_opts, (uint64_t)range_sz); + + md = mdhimInit(&comm, db_opts); + + /*this index is created for storing index metadata*/ + unifyfs_indexes[0] = md->primary_index; + + /*this index is created for storing file attribute metadata*/ + unifyfs_indexes[1] = create_global_index(md, ratio, 1, + LEVELDB, MDHIM_INT_KEY, + "file_attr"); + + MPI_Comm_size(md->mdhim_comm, &md_size); + + rc = meta_init_indices(); + if (rc != 0) { + return -1; + } + + return 0; +} + +/* initialize the key and value list used to put/get key-value pairs + * TODO: split once the number of metadata exceeds MAX_META_PER_SEND */ +int meta_init_indices(void) +{ + int i; + + /* init index metadata */ + unifyfs_keys = (unifyfs_key_t**) + calloc(MAX_META_PER_SEND, sizeof(unifyfs_key_t*)); + if (unifyfs_keys == NULL) { + return (int)UNIFYFS_ERROR_NOMEM; + } + + unifyfs_vals = (unifyfs_val_t**) + calloc(MAX_META_PER_SEND, sizeof(unifyfs_val_t*)); + if (unifyfs_vals == NULL) { + return (int)UNIFYFS_ERROR_NOMEM; + } + + for (i = 0; i < MAX_META_PER_SEND; i++) { + unifyfs_keys[i] = (unifyfs_key_t*) calloc(1, sizeof(unifyfs_key_t)); + if (unifyfs_keys[i] == NULL) { + return (int)UNIFYFS_ERROR_NOMEM; + } + + unifyfs_vals[i] = (unifyfs_val_t*) calloc(1, sizeof(unifyfs_val_t)); + if (unifyfs_vals[i] == NULL) { + return (int)UNIFYFS_ERROR_NOMEM; + } + } + + /* init attribute metadata */ + fattr_keys = (fattr_key_t**) + calloc(MAX_FILE_CNT_PER_NODE, sizeof(fattr_key_t*)); + if (fattr_keys == NULL) { + return (int)UNIFYFS_ERROR_NOMEM; + } + + fattr_vals = (fattr_val_t**) + calloc(MAX_FILE_CNT_PER_NODE, sizeof(fattr_val_t*)); + if (fattr_vals == NULL) { + return (int)UNIFYFS_ERROR_NOMEM; + } + + for (i = 0; i < MAX_FILE_CNT_PER_NODE; i++) { + fattr_keys[i] = (fattr_key_t*) calloc(1, sizeof(fattr_key_t)); + if (fattr_keys[i] == NULL) { + return (int)UNIFYFS_ERROR_NOMEM; + } + + fattr_vals[i] = (fattr_val_t*) calloc(1, sizeof(fattr_val_t)); + if (fattr_vals[i] == NULL) { + return (int)UNIFYFS_ERROR_NOMEM; + } + } + + return 0; +} + +void print_bget_indices(int app_id, int cli_id, + send_msg_t* msgs, int tot_num) +{ + int i; + for (i = 0; i < tot_num; i++) { + LOGDBG("index:dbg_rank:%d, dest_offset:%zu, " + "dest_del_rank:%d, dest_cli_id:%d, dest_app_id:%d, " + "length:%zu, src_app_id:%d, src_cli_id:%d, src_offset:%zu, " + "src_del_rank:%d, src_fid:%d, num:%d", + msgs[i].src_dbg_rank, msgs[i].dest_offset, + msgs[i].dest_delegator_rank, msgs[i].dest_client_id, + msgs[i].dest_app_id, msgs[i].length, + msgs[i].src_app_id, msgs[i].src_cli_id, + msgs[i].src_offset, msgs[i].src_delegator_rank, + msgs[i].src_fid, tot_num); + } +} + +void print_fsync_indices(unifyfs_key_t** keys, + unifyfs_val_t** vals, + size_t num_entries) +{ + size_t i; + for (i = 0; i < num_entries; i++) { + LOGDBG("fid:%d, offset:%lu, addr:%lu, len:%lu, del_id:%d", + keys[i]->fid, keys[i]->offset, + vals[i]->addr, vals[i]->len, + vals[i]->delegator_rank); + } +} + +void meta_free_indices(void) +{ + int i; + if (NULL != unifyfs_keys) { + for (i = 0; i < MAX_META_PER_SEND; i++) { + if (NULL != unifyfs_keys[i]) { + free(unifyfs_keys[i]); + } + if (NULL != unifyfs_vals[i]) { + free(unifyfs_vals[i]); + } + } + free(unifyfs_keys); + free(unifyfs_vals); + } + if (NULL != fattr_keys) { + for (i = 0; i < MAX_FILE_CNT_PER_NODE; i++) { + if (NULL != fattr_keys[i]) { + free(fattr_keys[i]); + } + if (NULL != fattr_vals[i]) { + free(fattr_vals[i]); + } + } + free(fattr_keys); + free(fattr_vals); + } +} + +static int remove_cb(const char* fpath, const struct stat* sb, + int typeflag, struct FTW* ftwbuf) +{ + int rc = remove(fpath); + if (rc) { + LOGERR("failed to remove(%s)", fpath); + } + return rc; +} + +static int remove_mdhim_db_filetree(char* db_root_path) +{ + LOGDBG("remove MDHIM DB filetree at %s", db_root_path); + return nftw(db_root_path, remove_cb, 64, FTW_DEPTH | FTW_PHYS); +} + + +int meta_sanitize(void) +{ + int rc; + char dbpath[UNIFYFS_MAX_FILENAME] = {0}; + + snprintf(dbpath, sizeof(dbpath), "%s", md->db_opts->db_path); + + mdhimClose(md); + md = NULL; + + rc = remove_mdhim_db_filetree(dbpath); + if (rc) { + LOGERR("failure during MDHIM file tree removal"); + } + + meta_free_indices(); + + return UNIFYFS_SUCCESS; +} + +// New API +/* + * + */ +int unifyfs_set_file_attribute(unifyfs_file_attr_t* fattr_ptr) +{ + int rc = UNIFYFS_SUCCESS; + + int gfid = fattr_ptr->gfid; + + md->primary_index = unifyfs_indexes[1]; + brm = mdhimPut(md, &gfid, sizeof(int), + fattr_ptr, sizeof(unifyfs_file_attr_t), + NULL, NULL); + if (!brm || brm->error) { + // return UNIFYFS_ERROR_MDHIM on error + rc = (int)UNIFYFS_ERROR_MDHIM; + } + + mdhim_full_release_msg(brm); + + return rc; +} + +/* + * + */ +int unifyfs_set_file_attributes(int num_entries, + fattr_key_t** keys, int* key_lens, + unifyfs_file_attr_t** fattr_ptr, int* val_lens) +{ + int rc = UNIFYFS_SUCCESS; + + md->primary_index = unifyfs_indexes[1]; + brm = mdhimBPut(md, (void**)keys, key_lens, (void**)fattr_ptr, + val_lens, num_entries, NULL, NULL); + brmp = brm; + if (!brmp || brmp->error) { + rc = (int)UNIFYFS_ERROR_MDHIM; + LOGERR("Error inserting keys/values into MDHIM"); + } + + while (brmp) { + if (brmp->error < 0) { + rc = (int)UNIFYFS_ERROR_MDHIM; + break; + } + + brm = brmp; + brmp = brmp->next; + mdhim_full_release_msg(brm); + } + + return rc; +} + +/* + * + */ +int unifyfs_get_file_attribute(int gfid, + unifyfs_file_attr_t* attr_val_ptr) +{ + int rc = UNIFYFS_SUCCESS; + unifyfs_file_attr_t* tmp_ptr_attr; + + md->primary_index = unifyfs_indexes[1]; + bgrm = mdhimGet(md, md->primary_index, &gfid, + sizeof(int), MDHIM_GET_EQ); + if (!bgrm || bgrm->error) { + rc = (int)UNIFYFS_ERROR_MDHIM; + } else { + tmp_ptr_attr = (unifyfs_file_attr_t*)bgrm->values[0]; + memcpy(attr_val_ptr, tmp_ptr_attr, sizeof(unifyfs_file_attr_t)); + mdhim_full_release_msg(bgrm); + } + + return rc; +} + +/* + * + */ +int unifyfs_get_file_extents(int num_keys, unifyfs_key_t** keys, + int* key_lens, int* num_values, + unifyfs_keyval_t** keyvals) +{ + /* + * This is using a modified version of mdhim. The function will return all + * key-value pairs within the range of the key tuple. + * We need to re-evaluate this function to use different key-value stores. + */ + + int i; + int rc = UNIFYFS_SUCCESS; + int tot_num = 0; + + unifyfs_key_t* tmp_key; + unifyfs_val_t* tmp_val; + unifyfs_keyval_t* kviter = *keyvals; + + md->primary_index = unifyfs_indexes[0]; + bgrm = mdhimBGet(md, md->primary_index, (void**)keys, + key_lens, num_keys, MDHIM_RANGE_BGET); + + while (bgrm) { + bgrmp = bgrm; + if (bgrmp->error < 0) { + // TODO: need better error handling + rc = (int)UNIFYFS_ERROR_MDHIM; + return rc; + } + + if (tot_num < MAX_META_PER_SEND) { + for (i = 0; i < bgrmp->num_keys; i++) { + tmp_key = (unifyfs_key_t*)bgrmp->keys[i]; + tmp_val = (unifyfs_val_t*)bgrmp->values[i]; + memcpy(&(kviter->key), tmp_key, sizeof(unifyfs_key_t)); + memcpy(&(kviter->val), tmp_val, sizeof(unifyfs_val_t)); + kviter++; + tot_num++; + if (MAX_META_PER_SEND == tot_num) { + LOGERR("Error: maximum number of values!"); + rc = UNIFYFS_FAILURE; + break; + } + } + } + bgrm = bgrmp->next; + mdhim_full_release_msg(bgrmp); + } + + *num_values = tot_num; + + return rc; +} + +/* + * + */ +int unifyfs_set_file_extents(int num_entries, + unifyfs_key_t** keys, int* key_lens, + unifyfs_val_t** vals, int* val_lens) +{ + int rc = UNIFYFS_SUCCESS; + + md->primary_index = unifyfs_indexes[0]; + + brm = mdhimBPut(md, (void**)(keys), key_lens, + (void**)(vals), val_lens, num_entries, + NULL, NULL); + brmp = brm; + if (!brmp || brmp->error) { + rc = (int)UNIFYFS_ERROR_MDHIM; + LOGERR("Error inserting keys/values into MDHIM"); + } + + while (brmp) { + if (brmp->error < 0) { + rc = (int)UNIFYFS_ERROR_MDHIM; + break; + } + + brm = brmp; + brmp = brmp->next; + mdhim_full_release_msg(brm); + } + + return rc; +} + diff --git a/server/src/unifyfs_metadata.h b/server/src/unifyfs_metadata.h new file mode 100644 index 000000000..200c78742 --- /dev/null +++ b/server/src/unifyfs_metadata.h @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2017-2019, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +/* + * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * Copyright (c) 2017, Florida State University. Contributions from + * the Computer Architecture and Systems Research Laboratory (CASTL) + * at the Department of Computer Science. + * + * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror + * LLNL-CODE-728877. All rights reserved. + * + * This file is part of burstfs. + * For details, see https://github.com/llnl/burstfs + * Please read https://github.com/llnl/burstfs/LICENSE for full license text. + */ + +#ifndef UNIFYFS_METADATA_H +#define UNIFYFS_METADATA_H + +#include "unifyfs_configurator.h" +#include "unifyfs_global.h" + +#define MANIFEST_FILE_NAME "mdhim_manifest_" + +/** + * Key for a file extent + */ +typedef struct { + /** global file id */ + int fid; + /** logical file offset */ + size_t offset; +} unifyfs_key_t; + +#define UNIFYFS_KEY_SZ (sizeof(unifyfs_key_t)) +#define UNIFYFS_KEY_FID(keyp) (((unifyfs_key_t*)keyp)->fid) +#define UNIFYFS_KEY_OFF(keyp) (((unifyfs_key_t*)keyp)->offset) + +typedef struct { + size_t addr; /* data offset in server */ + size_t len; /* length of data at addr */ + int app_id; /* application id in server */ + int rank; /* client id in server */ + int delegator_rank; /* delegator/server rank hosting data */ +} unifyfs_val_t; + +#define UNIFYFS_VAL_SZ (sizeof(unifyfs_val_t)) +#define UNIFYFS_VAL_ADDR(valp) (((unifyfs_val_t*)valp)->addr) +#define UNIFYFS_VAL_LEN(valp) (((unifyfs_val_t*)valp)->len) + +/** + * key-value tuple for a file extent + */ +typedef struct { + /** key */ + unifyfs_key_t key; + /** value */ + unifyfs_val_t val; +} unifyfs_keyval_t; + +int unifyfs_key_compare(unifyfs_key_t* a, unifyfs_key_t* b); + +void debug_log_key_val(const char* ctx, + unifyfs_key_t* key, + unifyfs_val_t* val); + +int meta_sanitize(void); +int meta_init_store(unifyfs_cfg_t* cfg); +void print_bget_indices(int app_id, int client_id, + send_msg_t* index_set, int tot_num); + +int meta_init_indices(void); +void meta_free_indices(void); +void print_fsync_indices(unifyfs_key_t** unifyfs_keys, + unifyfs_val_t** unifyfs_vals, size_t num_entries); + +/** + * Retrieve a File attribute from the KV-Store. + * + * @param [in] gfid + * @param[out] *ptr_attr_val + * @return UNIFYFS_SUCCESS on success + */ +int unifyfs_get_file_attribute(int gfid, + unifyfs_file_attr_t* ptr_attr_val); + +/** + * Store a File attribute to the KV-Store. + * + * @param[in] *ptr_attr_val + * @return UNIFYFS_SUCCESS on success + */ +int unifyfs_set_file_attribute(unifyfs_file_attr_t* ptr_attr_val); + +/** + * Store File attributes to the KV-Store. + * + * @param[in] num_entries number of key value pairs to store + * @param[in] keys array storing the keys + * @param[in] key_lens array with the length of the elements in \p keys + * @param[in] vals array with the values + * @param[in] val_lens array with the length of the elements in \p vals + */ +int unifyfs_set_file_attributes(int num_entries, + fattr_key_t** keys, int* key_lens, + unifyfs_file_attr_t** vals, int* val_lens); + +/** + * Retrieve File extents from the KV-Store. + * + * @param[in] num_keys number of keys + * @param[in] keys array of keys to retrieve the values for + * @param[in] key_lens array with the length of the key in \p keys + * @param[out] num_values number of values in the keyval array + * @param[out] keyval array containing the key-value tuples found + * @return UNIFYFS_SUCCESS on success + */ +int unifyfs_get_file_extents(int num_keys, + unifyfs_key_t** keys, int* key_lens, + int* num_values, unifyfs_keyval_t** keyval); + +/** + * Store File extents in the KV-Store. + * + * @param [in] num_entries number of key value pairs to store + * @param[in] keys array storing the keys + * @param[in] key_lens array with the length of the elements in \p keys + * @param[in] vals array with the values + * @param[in] val_lens array with the length of the elements in \p vals + * @return UNIFYFS_SUCCESS on success + */ +int unifyfs_set_file_extents(int num_entries, unifyfs_key_t** keys, + int* key_lens, unifyfs_val_t** vals, + int* val_lens); + +#endif diff --git a/server/src/unifyfs_request_manager.c b/server/src/unifyfs_request_manager.c new file mode 100644 index 000000000..c3baa1e7e --- /dev/null +++ b/server/src/unifyfs_request_manager.c @@ -0,0 +1,2204 @@ +/* + * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2017-2019, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +/* + * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * Copyright (c) 2017, Florida State University. Contribuions from + * the Computer Architecture and Systems Research Laboratory (CASTL) + * at the Department of Computer Science. + * + * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror + * LLNL-CODE-728877. All rights reserved. + * + * This file is part of burstfs. + * For details, see https://github.com/llnl/burstfs + * Please read https://github.com/llnl/burstfs/LICENSE for full license text. + */ + +// system headers +#include +#include +#include +#include +#include +#include + +// general support +#include "unifyfs_global.h" +#include "unifyfs_log.h" + +// server components +#include "unifyfs_request_manager.h" +#include "unifyfs_service_manager.h" +#include "unifyfs_metadata.h" + +// margo rpcs +#include "unifyfs_server_rpcs.h" +#include "margo_server.h" +#include "ucr_read_builder.h" + + +#define RM_LOCK(rm) \ +do { \ + LOGDBG("locking RM[%d] state", rm->thrd_ndx); \ + pthread_mutex_lock(&(rm->thrd_lock)); \ +} while (0) + +#define RM_UNLOCK(rm) \ +do { \ + LOGDBG("unlocking RM[%d] state", rm->thrd_ndx); \ + pthread_mutex_unlock(&(rm->thrd_lock)); \ +} while (0) + +arraylist_t* rm_thrd_list; + +/* One request manager thread is created for each client that a + * delegator serves. The main thread of the delegator assigns + * work to the request manager thread to retrieve data and send + * it back to the client. + * + * To start, given a read request from the client (via rpc) + * the handler function on the main delegator first queries the + * key/value store using the given file id and byte range to obtain + * the meta data on the physical location of the file data. This + * meta data provides the host delegator rank, the app/client + * ids that specify the log file on the remote delegator, the + * offset within the log file and the length of data. The rpc + * handler function sorts the meta data by host delegator rank, + * generates read requests, and inserts those into a list on a + * data structure shared with the request manager (del_req_set). + * + * The request manager thread coordinates with the main thread + * using a lock and a condition variable to protect the shared data + * structure and impose flow control. When assigned work, the + * request manager thread packs and sends request messages to + * service manager threads on remote delegators via MPI send. + * It waits for data to be sent back, and unpacks the read replies + * in each message into a shared memory buffer for the client. + * When the shared memory is full or all data has been received, + * it signals the client process to process the read replies. + * It iterates with the client until all incoming read replies + * have been transferred. */ +/* create a request manager thread for the given app_id + * and client_id, returns pointer to thread control structure + * on success and NULL on failure */ + +/* Create Request Manager thread for application client */ +reqmgr_thrd_t* unifyfs_rm_thrd_create(int app_id, int client_id) +{ + /* allocate a new thread control structure */ + reqmgr_thrd_t* thrd_ctrl = (reqmgr_thrd_t*) + calloc(1, sizeof(reqmgr_thrd_t)); + if (thrd_ctrl == NULL) { + LOGERR("Failed to allocate structure for request " + "manager thread for app_id=%d client_id=%d", + app_id, client_id); + return NULL; + } + + /* allocate an array for listing read requests from client */ + thrd_ctrl->del_req_set = (msg_meta_t*)calloc(1, sizeof(msg_meta_t)); + if (thrd_ctrl->del_req_set == NULL) { + LOGERR("Failed to allocate read request structure for request " + "manager thread for app_id=%d client_id=%d", + app_id, client_id); + free(thrd_ctrl); + return NULL; + } + + /* allocate structure for tracking outstanding read requests + * this delegator has with service managers on other nodes */ + thrd_ctrl->del_req_stat = (del_req_stat_t*) + calloc(1, sizeof(del_req_stat_t)); + if (thrd_ctrl->del_req_stat == NULL) { + LOGERR("Failed to allocate delegator structure for request " + "manager thread for app_id=%d client_id=%d", + app_id, client_id); + free(thrd_ctrl->del_req_set); + free(thrd_ctrl); + return NULL; + } + + /* allocate a structure to track requests we have on each + * remote service manager */ + thrd_ctrl->del_req_stat->req_stat = (per_del_stat_t*) + calloc(glb_mpi_size, sizeof(per_del_stat_t)); + if (thrd_ctrl->del_req_stat->req_stat == NULL) { + LOGERR("Failed to allocate per-delegator structure for request " + "manager thread for app_id=%d client_id=%d", + app_id, client_id); + free(thrd_ctrl->del_req_stat); + free(thrd_ctrl->del_req_set); + free(thrd_ctrl); + return NULL; + } + + /* initialize lock for shared data structures between + * main thread and request delegator thread */ + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); + int rc = pthread_mutex_init(&(thrd_ctrl->thrd_lock), &attr); + if (rc != 0) { + LOGERR("pthread_mutex_init failed for request " + "manager thread app_id=%d client_id=%d rc=%d (%s)", + app_id, client_id, rc, strerror(rc)); + free(thrd_ctrl->del_req_stat->req_stat); + free(thrd_ctrl->del_req_stat); + free(thrd_ctrl->del_req_set); + free(thrd_ctrl); + return NULL; + } + + /* initailize condition variable to flow control + * work between main thread and request delegator thread */ + rc = pthread_cond_init(&(thrd_ctrl->thrd_cond), NULL); + if (rc != 0) { + LOGERR("pthread_cond_init failed for request " + "manager thread app_id=%d client_id=%d rc=%d (%s)", + app_id, client_id, rc, strerror(rc)); + pthread_mutex_destroy(&(thrd_ctrl->thrd_lock)); + free(thrd_ctrl->del_req_stat->req_stat); + free(thrd_ctrl->del_req_stat); + free(thrd_ctrl->del_req_set); + free(thrd_ctrl); + return NULL; + } + + /* record app and client id this thread will be serving */ + thrd_ctrl->app_id = app_id; + thrd_ctrl->client_id = client_id; + + /* initialize flow control flags */ + thrd_ctrl->exit_flag = 0; + thrd_ctrl->exited = 0; + thrd_ctrl->has_waiting_delegator = 0; + thrd_ctrl->has_waiting_dispatcher = 0; + + /* insert our thread control structure into our list of + * active request manager threads, important to do this before + * launching thread since it uses list to lookup its structure */ + rc = arraylist_add(rm_thrd_list, thrd_ctrl); + if (rc != 0) { + pthread_cond_destroy(&(thrd_ctrl->thrd_cond)); + pthread_mutex_destroy(&(thrd_ctrl->thrd_lock)); + free(thrd_ctrl->del_req_stat->req_stat); + free(thrd_ctrl->del_req_stat); + free(thrd_ctrl->del_req_set); + free(thrd_ctrl); + return NULL; + } + thrd_ctrl->thrd_ndx = arraylist_size(rm_thrd_list) - 1; + + /* launch request manager thread */ + rc = pthread_create(&(thrd_ctrl->thrd), NULL, + rm_delegate_request_thread, (void*)thrd_ctrl); + if (rc != 0) { + LOGERR("failed to create request manager thread for " + "app_id=%d client_id=%d - rc=%d (%s)", + app_id, client_id, rc, strerror(rc)); + pthread_cond_destroy(&(thrd_ctrl->thrd_cond)); + pthread_mutex_destroy(&(thrd_ctrl->thrd_lock)); + free(thrd_ctrl->del_req_stat->req_stat); + free(thrd_ctrl->del_req_stat); + free(thrd_ctrl->del_req_set); + free(thrd_ctrl); + return NULL; + } + + return thrd_ctrl; +} + +/* Lookup RM thread control structure */ +reqmgr_thrd_t* rm_get_thread(int thrd_id) +{ + return (reqmgr_thrd_t*) arraylist_get(rm_thrd_list, thrd_id); +} + +static void print_send_msgs(send_msg_t* send_metas, + int msg_cnt) +{ + int i; + send_msg_t* msg; + for (i = 0; i < msg_cnt; i++) { + msg = send_metas + i; + LOGDBG("msg[%d] gfid:%d length:%zu file_offset:%zu " + "dest_offset:%zu dest_app:%d dest_clid:%d", + i, msg->src_fid, msg->length, msg->src_offset, + msg->dest_offset, msg->dest_app_id, msg->dest_client_id); + } +} + +static void print_remote_del_reqs(int app_id, int cli_id, + del_req_stat_t* del_req_stat) +{ + int i; + for (i = 0; i < del_req_stat->del_cnt; i++) { + LOGDBG("remote_delegator:%d, req_cnt:%d", + del_req_stat->req_stat[i].del_id, + del_req_stat->req_stat[i].req_cnt); + } +} + +#if 0 // NOT CURRENTLY USED +static void print_recv_msg(int app_id, int cli_id, + int thrd_id, + shm_meta_t* msg) +{ + LOGDBG("recv_msg: app_id:%d, cli_id:%d, thrd_id:%d, " + "fid:%d, offset:%ld, len:%ld", + app_id, cli_id, thrd_id, msg->src_fid, + msg->offset, msg->length); +} +#endif + + +/* order keyvals by gfid, then host delegator rank */ +static int compare_kv_gfid_rank(const void* a, const void* b) +{ + const unifyfs_keyval_t* kv_a = a; + const unifyfs_keyval_t* kv_b = b; + + int gfid_a = kv_a->key.fid; + int gfid_b = kv_b->key.fid; + if (gfid_a == gfid_b) { + int rank_a = kv_a->val.delegator_rank; + int rank_b = kv_b->val.delegator_rank; + if (rank_a == rank_b) { + return 0; + } else if (rank_a < rank_b) { + return -1; + } else { + return 1; + } + } else if (gfid_a < gfid_b) { + return -1; + } else { + return 1; + } +} + +/* order read requests by destination delegator rank */ +static int compare_msg_delegators(const void* a, const void* b) +{ + const send_msg_t* msg_a = a; + const send_msg_t* msg_b = b; + int rank_a = msg_a->dest_delegator_rank; + int rank_b = msg_b->dest_delegator_rank; + + if (rank_a == rank_b) { + return 0; + } else if (rank_a < rank_b) { + return -1; + } else { + return 1; + } +} + +unifyfs_key_t** alloc_key_array(int elems) +{ + int size = elems * (sizeof(unifyfs_key_t*) + sizeof(unifyfs_key_t)); + + void* mem_block = calloc(size, sizeof(char)); + + unifyfs_key_t** array_ptr = mem_block; + unifyfs_key_t* key_ptr = (unifyfs_key_t*)(array_ptr + elems); + + for (int i = 0; i < elems; i++) { + array_ptr[i] = &key_ptr[i]; + } + + return (unifyfs_key_t**)mem_block; +} + +fattr_key_t** alloc_attr_key_array(int elems) +{ + int size = elems * (sizeof(fattr_key_t*) + sizeof(fattr_key_t)); + + void* mem_block = calloc(size, sizeof(char)); + + fattr_key_t** array_ptr = mem_block; + fattr_key_t* key_ptr = (fattr_key_t*)(array_ptr + elems); + + for (int i = 0; i < elems; i++) { + array_ptr[i] = &key_ptr[i]; + } + + return (fattr_key_t**)mem_block; +} + +unifyfs_val_t** alloc_value_array(int elems) +{ + int size = elems * (sizeof(unifyfs_val_t*) + sizeof(unifyfs_val_t)); + + void* mem_block = calloc(size, sizeof(char)); + + unifyfs_val_t** array_ptr = mem_block; + unifyfs_val_t* key_ptr = (unifyfs_val_t*)(array_ptr + elems); + + for (int i = 0; i < elems; i++) { + array_ptr[i] = &key_ptr[i]; + } + + return (unifyfs_val_t**)mem_block; +} + +void free_key_array(unifyfs_key_t** array) +{ + free(array); +} + +void free_value_array(unifyfs_val_t** array) +{ + free(array); +} + +void free_attr_key_array(fattr_key_t** array) +{ + free(array); +} + +static void debug_print_read_req(server_read_req_t* req) +{ + if (NULL != req) { + LOGDBG("server_read_req[%d] status=%d, gfid=%d, num_remote=%d", + req->req_ndx, req->status, req->extent.gfid, + req->num_remote_reads); + } +} + +static server_read_req_t* reserve_read_req(reqmgr_thrd_t* thrd_ctrl) +{ + server_read_req_t* rdreq = NULL; + RM_LOCK(thrd_ctrl); + if (thrd_ctrl->num_read_reqs < RM_MAX_ACTIVE_REQUESTS) { + if (thrd_ctrl->next_rdreq_ndx < (RM_MAX_ACTIVE_REQUESTS - 1)) { + rdreq = thrd_ctrl->read_reqs + thrd_ctrl->next_rdreq_ndx; + assert((rdreq->req_ndx == 0) && (rdreq->extent.gfid == 0)); + rdreq->req_ndx = thrd_ctrl->next_rdreq_ndx++; + } else { // search for unused slot + for (int i = 0; i < RM_MAX_ACTIVE_REQUESTS; i++) { + rdreq = thrd_ctrl->read_reqs + i; + if ((rdreq->req_ndx == 0) && (rdreq->extent.gfid == 0)) { + rdreq->req_ndx = i; + break; + } + } + } + thrd_ctrl->num_read_reqs++; + LOGDBG("reserved read req %d (active=%d, next=%d)", rdreq->req_ndx, + thrd_ctrl->num_read_reqs, thrd_ctrl->next_rdreq_ndx); + debug_print_read_req(rdreq); + } else { + LOGERR("maxed-out request manager read_reqs array!!"); + } + RM_UNLOCK(thrd_ctrl); + return rdreq; +} + +static int release_read_req(reqmgr_thrd_t* thrd_ctrl, + server_read_req_t* rdreq) +{ + int rc = (int)UNIFYFS_SUCCESS; + RM_LOCK(thrd_ctrl); + if (rdreq != NULL) { + LOGDBG("releasing read req %d", rdreq->req_ndx); + if (rdreq->req_ndx == (thrd_ctrl->next_rdreq_ndx - 1)) { + thrd_ctrl->next_rdreq_ndx--; + } + if (NULL != rdreq->chunks) { + free(rdreq->chunks); + } + if (NULL != rdreq->remote_reads) { + free(rdreq->remote_reads); + } + memset((void*)rdreq, 0, sizeof(server_read_req_t)); + thrd_ctrl->num_read_reqs--; + LOGDBG("after release (active=%d, next=%d)", + thrd_ctrl->num_read_reqs, thrd_ctrl->next_rdreq_ndx); + debug_print_read_req(rdreq); + } else { + rc = UNIFYFS_ERROR_INVAL; + LOGERR("NULL read_req"); + } + RM_UNLOCK(thrd_ctrl); + return rc; +} + +static void signal_new_requests(reqmgr_thrd_t* thrd_ctrl) +{ + // NOTE: this fn assumes thrd_ctrl->thrd_lock is locked + + /* wake up the request manager thread for the requesting client */ + if (!thrd_ctrl->has_waiting_delegator) { + /* delegator thread is not waiting, but we are in critical + * section, we just added requests so we must wait for delegator + * to signal us that it's reached the critical section before + * we escape so we don't overwrite these requests before it + * has had a chance to process them */ + thrd_ctrl->has_waiting_dispatcher = 1; + pthread_cond_wait(&thrd_ctrl->thrd_cond, &thrd_ctrl->thrd_lock); + + /* delegator thread has signaled us that it's now waiting */ + thrd_ctrl->has_waiting_dispatcher = 0; + } + /* have a delegator thread waiting on condition variable, + * signal it to begin processing the requests we just added */ + pthread_cond_signal(&thrd_ctrl->thrd_cond); +} + +static void signal_new_responses(reqmgr_thrd_t* thrd_ctrl) +{ + // NOTE: this fn assumes thrd_ctrl->thrd_lock is locked + + /* wake up the request manager thread */ + if (thrd_ctrl->has_waiting_delegator) { + /* have a delegator thread waiting on condition variable, + * signal it to begin processing the responses we just added */ + pthread_cond_signal(&thrd_ctrl->thrd_cond); + } +} + +/* issue remote chunk read requests for extent chunks + * contained within keyvals + */ +int create_request_messages(reqmgr_thrd_t* thrd_ctrl, + int client_rank, + int num_vals, + unifyfs_keyval_t* keyvals) +{ + int thrd_id = thrd_ctrl->thrd_ndx; + int app_id = thrd_ctrl->app_id; + int client_id = thrd_ctrl->client_id; + + /* wait for lock for shared data structures holding requests + * and condition variable */ + RM_LOCK(thrd_ctrl); + + // set up the thread_control delegator request set + // TODO: make this a function + int i; + for (i = 0; i < num_vals; i++) { + send_msg_t* meta = &(thrd_ctrl->del_req_set->msg_meta[i]); + memset(meta, 0, sizeof(send_msg_t)); + + debug_log_key_val(__func__, &keyvals[i].key, &keyvals[i].val); + + /* physical offset of the requested file segment on the log file */ + meta->dest_offset = keyvals[i].val.addr; + + /* rank of the remote delegator */ + meta->dest_delegator_rank = keyvals[i].val.delegator_rank; + + /* dest_client_id and dest_app_id uniquely identify the remote + * physical log file that contains the requested segments */ + meta->dest_app_id = keyvals[i].val.app_id; + meta->dest_client_id = keyvals[i].val.rank; + meta->length = (size_t)keyvals[i].val.len; + + /* src_app_id and src_cli_id identifies the requested client */ + meta->src_app_id = app_id; + meta->src_cli_id = client_id; + + /* src_offset is the logical offset of the shared file */ + meta->src_offset = keyvals[i].key.offset; + meta->src_delegator_rank = glb_mpi_rank; + meta->src_fid = keyvals[i].key.fid; + meta->src_dbg_rank = client_rank; + meta->src_thrd = thrd_id; + } + + thrd_ctrl->del_req_set->num = num_vals; + + if (num_vals > 1) { + /* sort read requests to be sent to the same delegators. */ + qsort(thrd_ctrl->del_req_set->msg_meta, + thrd_ctrl->del_req_set->num, + sizeof(send_msg_t), compare_msg_delegators); + } + + /* debug print */ + print_send_msgs(thrd_ctrl->del_req_set->msg_meta, + thrd_ctrl->del_req_set->num); + + /* get pointer to list of delegator stat objects to record + * delegator rank and count of requests for each delegator */ + per_del_stat_t* req_stat = thrd_ctrl->del_req_stat->req_stat; + + /* get pointer to send message structures, one for each request */ + send_msg_t* msg_meta = thrd_ctrl->del_req_set->msg_meta; + + /* iterate over read requests and count number of requests + * to be sent to each delegator */ + int del_ndx = 0; + req_stat[del_ndx].del_id = msg_meta[0].dest_delegator_rank; + req_stat[del_ndx].req_cnt = 1; + + for (i = 1; i < thrd_ctrl->del_req_set->num; i++) { + int cur_rank = msg_meta[i].dest_delegator_rank; + int prev_rank = msg_meta[i-1].dest_delegator_rank; + if (cur_rank == prev_rank) { + /* another message for the current delegator */ + req_stat[del_ndx].req_cnt++; + } else { + /* new delegator */ + del_ndx++; + req_stat[del_ndx].del_id = msg_meta[i].dest_delegator_rank; + req_stat[del_ndx].req_cnt = 1; + } + } + + /* record total number of delegators we'll send requests to */ + thrd_ctrl->del_req_stat->del_cnt = del_ndx + 1; + + /* debug print */ + print_remote_del_reqs(app_id, thrd_id, thrd_ctrl->del_req_stat); + + /* wake up the request manager thread for the requesting client */ + signal_new_requests(thrd_ctrl); + + /* done updating shared variables, release the lock */ + RM_UNLOCK(thrd_ctrl); + + return UNIFYFS_SUCCESS; +} + +/* issue remote chunk read requests for extent chunks + * listed within keyvals */ +int create_chunk_requests(reqmgr_thrd_t* thrd_ctrl, + server_read_req_t* rdreq, + int num_vals, + unifyfs_keyval_t* keyvals) +{ + int thrd_id = thrd_ctrl->thrd_ndx; + int app_id = thrd_ctrl->app_id; + int client_id = thrd_ctrl->client_id; + + chunk_read_req_t* all_chunk_reads = (chunk_read_req_t*) + calloc((size_t)num_vals, sizeof(chunk_read_req_t)); + if (NULL == all_chunk_reads) { + LOGERR("failed to allocate chunk-reads array"); + return UNIFYFS_ERROR_NOMEM; + } + + RM_LOCK(thrd_ctrl); + + LOGDBG("creating chunk requests for rdreq %d", rdreq->req_ndx); + + rdreq->chunks = all_chunk_reads; + + int i, curr_del; + int prev_del = -1; + int del_ndx = 0; + chunk_read_req_t* chk_read; + for (i = 0; i < num_vals; i++) { + /* count the delegators */ + curr_del = keyvals[i].val.delegator_rank; + if ((prev_del != -1) && (curr_del != prev_del)) { + del_ndx++; + } + prev_del = curr_del; + + /* create chunk-reads */ + debug_log_key_val(__func__, &keyvals[i].key, &keyvals[i].val); + chk_read = all_chunk_reads + i; + chk_read->nbytes = keyvals[i].val.len; + chk_read->offset = keyvals[i].key.offset; + chk_read->log_offset = keyvals[i].val.addr; + chk_read->log_app_id = keyvals[i].val.app_id; + chk_read->log_client_id = keyvals[i].val.rank; + } + + /* allocate per-delgator chunk-reads */ + int num_dels = del_ndx + 1; + rdreq->num_remote_reads = num_dels; + rdreq->remote_reads = (remote_chunk_reads_t*) + calloc((size_t)num_dels, sizeof(remote_chunk_reads_t)); + if (NULL == rdreq->remote_reads) { + LOGERR("failed to allocate remote-reads array"); + RM_UNLOCK(thrd_ctrl); + return UNIFYFS_ERROR_NOMEM; + } + + /* populate per-delegator chunk-reads info */ + size_t del_data_sz = 0; + remote_chunk_reads_t* del_reads; + prev_del = -1; + del_ndx = 0; + for (i = 0; i < num_vals; i++) { + curr_del = keyvals[i].val.delegator_rank; + if ((prev_del != -1) && (curr_del != prev_del)) { + /* record total data for previous delegator */ + del_reads = rdreq->remote_reads + del_ndx; + del_reads->total_sz = del_data_sz; + /* advance to next delegator */ + del_ndx++; + del_data_sz = 0; + } + prev_del = curr_del; + + /* update total data size for current delegator */ + del_data_sz += keyvals[i].val.len; + + del_reads = rdreq->remote_reads + del_ndx; + if (0 == del_reads->num_chunks) { + /* initialize structure */ + del_reads->rank = curr_del; + del_reads->rdreq_id = rdreq->req_ndx; + del_reads->reqs = all_chunk_reads + i; + del_reads->resp = NULL; + } + del_reads->num_chunks++; + } + del_reads = rdreq->remote_reads + del_ndx; + del_reads->total_sz = del_data_sz; + + /* wake up the request manager thread for the requesting client */ + signal_new_requests(thrd_ctrl); + RM_UNLOCK(thrd_ctrl); + + return UNIFYFS_SUCCESS; +} + +/************************ + * These functions are called by the rpc handler to assign work + * to the request manager thread + ***********************/ + +/* given an app_id, client_id and global file id, + * compute and return file size for specified file + */ +int rm_cmd_filesize( + int app_id, /* app_id for requesting client */ + int client_id, /* client_id for requesting client */ + int gfid, /* global file id of read request */ + size_t* outsize) /* output file size */ +{ + /* set offset and length to request *all* key/value pairs + * for this file */ + size_t offset = 0; + + /* want to pick the highest integer offset value a file + * could have here */ + // TODO: would like to unsed max for unsigned long, but + // that fails to return any keys for some reason + size_t length = (SIZE_MAX >> 1) - 1; + + /* get the locations of all the read requests from the + * key-value store*/ + unifyfs_key_t key1, key2; + + /* create key to describe first byte we'll read */ + key1.fid = gfid; + key1.offset = offset; + + /* create key to describe last byte we'll read */ + key2.fid = gfid; + key2.offset = offset + length - 1; + + unifyfs_keyval_t* keyvals; + unifyfs_key_t* unifyfs_keys[2] = {&key1, &key2}; + int key_lens[2] = {sizeof(unifyfs_key_t), sizeof(unifyfs_key_t)}; + + /* look up all entries in this range */ + int num_vals = 0; + keyvals = (unifyfs_keyval_t*) calloc(UNIFYFS_MAX_SPLIT_CNT, + sizeof(unifyfs_keyval_t)); + if (NULL == keyvals) { + LOGERR("failed to allocate keyvals"); + return UNIFYFS_ERROR_NOMEM; + } + + int rc = unifyfs_get_file_extents(2, unifyfs_keys, key_lens, + &num_vals, &keyvals); + /* TODO: if there are file extents not accounted for we should + * either return 0 for that date (holes) or EOF if reading past + * the end of the file */ + if (UNIFYFS_SUCCESS != rc) { + // we need to let the client know that there was an error + free(keyvals); + return UNIFYFS_FAILURE; + } + + /* compute our file size by iterating over each file + * segment and taking the max logical offset */ + int i; + size_t filesize = 0; + for (i = 0; i < num_vals; i++) { + /* get pointer to next key value pair */ + unifyfs_keyval_t* kv = &keyvals[i]; + + /* get last byte offset for this segment of the file */ + size_t last_offset = kv->key.offset + kv->val.len; + + /* update our filesize if this offset is bigger than the current max */ + if (last_offset > filesize) { + filesize = last_offset; + } + } + + // cleanup + free(keyvals); + + *outsize = filesize; + return rc; +} + +int create_gfid_chunk_reads(reqmgr_thrd_t* thrd_ctrl, + int gfid, int app_id, int client_id, + int num_keys, unifyfs_key_t** keys, int* keylens) +{ + // TODO: might want to get this from a memory pool + unifyfs_keyval_t* keyvals = calloc(UNIFYFS_MAX_SPLIT_CNT, + sizeof(unifyfs_keyval_t)); + if (NULL == keyvals) { + LOGERR("failed to allocate keyvals"); + return UNIFYFS_ERROR_NOMEM; + } + + int num_vals = 0; + int rc = unifyfs_get_file_extents(num_keys, keys, keylens, + &num_vals, &keyvals); + /* TODO: if there are file extents not accounted for we should + * either return 0 for that data (holes) or EOF if reading past + * the end of the file */ + if (UNIFYFS_SUCCESS != rc || num_vals == 0) { + /* failed to find any key / value pairs */ + rc = UNIFYFS_FAILURE; + } else { + if (num_vals > 1) { + /* sort keyvals by delegator */ + qsort(keyvals, (size_t)num_vals, sizeof(unifyfs_keyval_t), + compare_kv_gfid_rank); + } + server_read_req_t* rdreq = reserve_read_req(thrd_ctrl); + if (NULL == rdreq) { + rc = UNIFYFS_FAILURE; + } else { + rdreq->app_id = app_id; + rdreq->client_id = client_id; + rdreq->extent.gfid = gfid; + rdreq->extent.errcode = EINPROGRESS; + rc = create_chunk_requests(thrd_ctrl, rdreq, + num_vals, keyvals); + if (rc != (int)UNIFYFS_SUCCESS) { + release_read_req(thrd_ctrl, rdreq); + } + } + } + + // cleanup + free(keyvals); + + return rc; +} + +/* read function for one requested extent, + * called from rpc handler to fill shared data structures + * with read requests to be handled by the delegator thread + * returns before requests are handled + */ +int rm_cmd_read( + int app_id, /* app_id for requesting client */ + int client_id, /* client_id for requesting client */ + int gfid, /* global file id of read request */ + size_t offset, /* logical file offset of read request */ + size_t length) /* number of bytes to read */ +{ + /* get pointer to app structure for this app id */ + app_config_t* app_config = + (app_config_t*)arraylist_get(app_config_list, app_id); + + /* get thread id for this client */ + int thrd_id = app_config->thrd_idxs[client_id]; + + /* look up thread control structure */ + reqmgr_thrd_t* thrd_ctrl = rm_get_thread(thrd_id); + + /* get chunks corresponding to requested client read extent + * + * Generate a pair of keys for the read request, representing the start + * and end offset. MDHIM returns all key-value pairs that fall within + * the offset range. + * + * TODO: this is specific to the MDHIM in the source tree and not portable + * to other KV-stores. This needs to be revisited to utilize some + * other mechanism to retrieve all relevant key-value pairs from the + * KV-store. + */ + unifyfs_key_t key1, key2; + + /* create key to describe first byte we'll read */ + key1.fid = gfid; + key1.offset = offset; + + /* create key to describe last byte we'll read */ + key2.fid = gfid; + key2.offset = offset + length - 1; + + unifyfs_key_t* unifyfs_keys[2] = {&key1, &key2}; + int key_lens[2] = {sizeof(unifyfs_key_t), sizeof(unifyfs_key_t)}; + + return create_gfid_chunk_reads(thrd_ctrl, gfid, app_id, client_id, + 2, unifyfs_keys, key_lens); +} + +/* send the read requests to the remote delegators + * + * @param app_id: application id + * @param client_id: client id for requesting process + * @param gfid: global file id + * @param req_num: number of read requests + * @param reqbuf: read requests buffer + * @return success/error code */ +int rm_cmd_mread(int app_id, int client_id, + size_t req_num, void* reqbuf) +{ + /* get pointer to app structure for this app id */ + app_config_t* app_config = + (app_config_t*)arraylist_get(app_config_list, app_id); + + /* get thread id for this client */ + int thrd_id = app_config->thrd_idxs[client_id]; + + /* look up thread control structure */ + reqmgr_thrd_t* thrd_ctrl = rm_get_thread(thrd_id); + + /* get debug rank for this client */ + int cli_rank = app_config->dbg_ranks[client_id]; + + /* get the locations of all the read requests from the key-value store */ + unifyfs_ReadRequest_table_t readRequest = + unifyfs_ReadRequest_as_root(reqbuf); + unifyfs_Extent_vec_t extents = unifyfs_ReadRequest_extents(readRequest); + size_t extents_len = unifyfs_Extent_vec_len(extents); + assert(extents_len == req_num); + + // allocate key storage + unifyfs_key_t** unifyfs_keys; + int* key_lens; + size_t key_cnt = req_num * 2; + unifyfs_keys = alloc_key_array(key_cnt); + key_lens = (int*) calloc(key_cnt, sizeof(int)); + if ((NULL == unifyfs_keys) || + (NULL == key_lens)) { + // this is a fatal error + // TODO: we need better error handling + LOGERR("Error allocating buffers"); + return (int)UNIFYFS_ERROR_NOMEM; + } + + /* get chunks corresponding to requested client read extents */ + int rc, num_keys; + int fid = -1; + int last_fid = -1; + int ndx = 0; + size_t j, eoff, elen; + for (j = 0; j < req_num; j++) { + fid = unifyfs_Extent_fid(unifyfs_Extent_vec_at(extents, j)); + if (j && (fid != last_fid)) { + // create requests for all extents of last_fid + num_keys = ndx; + rc = create_gfid_chunk_reads(thrd_ctrl, last_fid, app_id, + client_id, num_keys, + unifyfs_keys, key_lens); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("Error creating chunk reads for gfid=%d", last_fid); + } + // reset ndx for current fid + ndx = 0; + } + + eoff = unifyfs_Extent_offset(unifyfs_Extent_vec_at(extents, j)); + elen = unifyfs_Extent_length(unifyfs_Extent_vec_at(extents, j)); + LOGDBG("gfid:%d, offset:%zu, length:%zu", fid, eoff, elen); + + /* Generate a pair of keys for each read request, representing + * the start and end offsets. MDHIM returns all key-value pairs that + * fall within the offset range. + * + * TODO: this is specific to the MDHIM in the source tree and not + * portable to other KV-stores. This needs to be revisited to + * utilize some other mechanism to retrieve all relevant KV + * pairs from the KV-store. + */ + key_lens[ndx] = sizeof(unifyfs_key_t); + key_lens[ndx + 1] = sizeof(unifyfs_key_t); + + /* create key to describe first byte we'll read */ + unifyfs_keys[ndx]->fid = fid; + unifyfs_keys[ndx]->offset = eoff; + + /* create key to describe last byte we'll read */ + unifyfs_keys[ndx + 1]->fid = fid; + unifyfs_keys[ndx + 1]->offset = eoff + elen - 1; + + ndx += 2; + last_fid = fid; + } + + // create requests for all extents of last_fid + num_keys = ndx; + rc = create_gfid_chunk_reads(thrd_ctrl, last_fid, app_id, + client_id, num_keys, + unifyfs_keys, key_lens); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("Error creating chunk reads for gfid=%d", last_fid); + } + + // cleanup + free_key_array(unifyfs_keys); + free(key_lens); + + return rc; +} + +/* function called by main thread to instruct + * resource manager thread to exit, + * returns UNIFYFS_SUCCESS on success */ +int rm_cmd_exit(reqmgr_thrd_t* thrd_ctrl) +{ + /* grab the lock */ + RM_LOCK(thrd_ctrl); + + if (thrd_ctrl->exited) { + /* already done */ + RM_UNLOCK(thrd_ctrl); + return UNIFYFS_SUCCESS; + } + + /* if delegator thread is not waiting in critical + * section, let's wait on it to come back */ + if (!thrd_ctrl->has_waiting_delegator) { + /* delegator thread is not in critical section, + * tell it we've got something and signal it */ + thrd_ctrl->has_waiting_dispatcher = 1; + pthread_cond_wait(&thrd_ctrl->thrd_cond, &thrd_ctrl->thrd_lock); + + /* we're no longer waiting */ + thrd_ctrl->has_waiting_dispatcher = 0; + } + + /* inform delegator thread that it's time to exit */ + thrd_ctrl->exit_flag = 1; + + /* signal delegator thread */ + pthread_cond_signal(&thrd_ctrl->thrd_cond); + + /* release the lock */ + RM_UNLOCK(thrd_ctrl); + + /* wait for delegator thread to exit */ + void* status; + pthread_join(thrd_ctrl->thrd, &status); + thrd_ctrl->exited = 1; + + /* free storage holding shared data structures */ + free(thrd_ctrl->del_req_set); + free(thrd_ctrl->del_req_stat->req_stat); + free(thrd_ctrl->del_req_stat); + + return UNIFYFS_SUCCESS; +} + +/* + * synchronize all the indices and file attributes + * to the key-value store + * + * @param app_id: the application id + * @param client_side_id: client rank in app + * @param gfid: global file id + * @return success/error code + */ +int rm_cmd_fsync(int app_id, int client_side_id, int gfid) +{ + int ret = 0; + + unifyfs_key_t** unifyfs_keys; + unifyfs_val_t** unifyfs_vals; + int* unifyfs_key_lens = NULL; + int* unifyfs_val_lens = NULL; + + fattr_key_t** fattr_keys = NULL; + unifyfs_file_attr_t** fattr_vals = NULL; + int* fattr_key_lens = NULL; + int* fattr_val_lens = NULL; + size_t i, attr_num_entries, extent_num_entries; + + app_config_t* app_config = (app_config_t*) + arraylist_get(app_config_list, app_id); + + extent_num_entries = *(size_t*) + (app_config->shm_superblocks[client_side_id] + + app_config->meta_offset); + + /* + * indices are stored in the superblock shared memory + * created by the client + */ + int page_sz = getpagesize(); + unifyfs_index_t* meta_payload = (unifyfs_index_t*) + (app_config->shm_superblocks[client_side_id] + + app_config->meta_offset + page_sz); + + // allocate storage for values + // TODO: possibly get this from memory pool + unifyfs_keys = alloc_key_array(extent_num_entries); + unifyfs_vals = alloc_value_array(extent_num_entries); + unifyfs_key_lens = calloc(extent_num_entries, sizeof(int)); + unifyfs_val_lens = calloc(extent_num_entries, sizeof(int)); + if ((NULL == unifyfs_keys) || + (NULL == unifyfs_vals) || + (NULL == unifyfs_key_lens) || + (NULL == unifyfs_val_lens)) { + return (int)UNIFYFS_ERROR_NOMEM; + } + + // file extents + for (i = 0; i < extent_num_entries; i++) { + unifyfs_keys[i]->fid = meta_payload[i].fid; + unifyfs_keys[i]->offset = meta_payload[i].file_pos; + + unifyfs_vals[i]->addr = meta_payload[i].mem_pos; + unifyfs_vals[i]->len = meta_payload[i].length; + unifyfs_vals[i]->delegator_rank = glb_mpi_rank; + unifyfs_vals[i]->app_id = app_id; + unifyfs_vals[i]->rank = client_side_id; + + LOGDBG("extent - fid:%d, offset:%zu, length:%zu, app:%d, clid:%d", + unifyfs_keys[i]->fid, unifyfs_keys[i]->offset, + unifyfs_vals[i]->len, unifyfs_vals[i]->app_id, + unifyfs_vals[i]->rank); + + unifyfs_key_lens[i] = sizeof(unifyfs_key_t); + unifyfs_val_lens[i] = sizeof(unifyfs_val_t); + } + + ret = unifyfs_set_file_extents((int)extent_num_entries, + unifyfs_keys, unifyfs_key_lens, + unifyfs_vals, unifyfs_val_lens); + if (ret != UNIFYFS_SUCCESS) { + // TODO: need proper error handling + LOGERR("unifyfs_set_file_extents() failed"); + goto rm_cmd_fsync_exit; + } + + // file attributes + attr_num_entries = *(size_t*) + (app_config->shm_superblocks[client_side_id] + + app_config->fmeta_offset); + + /* + * file attributes are stored in the superblock shared memory + * created by the client + */ + unifyfs_file_attr_t* attr_payload = (unifyfs_file_attr_t*) + (app_config->shm_superblocks[client_side_id] + + app_config->fmeta_offset + page_sz); + + // allocate storage for values + // TODO: possibly get this from memory pool + fattr_keys = alloc_attr_key_array(attr_num_entries); + fattr_vals = calloc(attr_num_entries, sizeof(unifyfs_file_attr_t*)); + fattr_key_lens = calloc(attr_num_entries, sizeof(int)); + fattr_val_lens = calloc(attr_num_entries, sizeof(int)); + if ((NULL == fattr_keys) || + (NULL == fattr_vals) || + (NULL == fattr_key_lens) || + (NULL == fattr_val_lens)) { + ret = (int)UNIFYFS_ERROR_NOMEM; + goto rm_cmd_fsync_exit; + } + + for (i = 0; i < attr_num_entries; i++) { + *fattr_keys[i] = attr_payload[i].gfid; + fattr_vals[i] = &(attr_payload[i]); + fattr_key_lens[i] = sizeof(fattr_key_t); + fattr_val_lens[i] = sizeof(fattr_val_t); + } + + ret = unifyfs_set_file_attributes((int)attr_num_entries, + fattr_keys, fattr_key_lens, + fattr_vals, fattr_val_lens); + if (ret != UNIFYFS_SUCCESS) { + // TODO: need proper error handling + goto rm_cmd_fsync_exit; + } + +rm_cmd_fsync_exit: + // clean up memory + + if (NULL != unifyfs_keys) { + free_key_array(unifyfs_keys); + } + + if (NULL != unifyfs_vals) { + free_value_array(unifyfs_vals); + } + + if (NULL != unifyfs_key_lens) { + free(unifyfs_key_lens); + } + + if (NULL != unifyfs_val_lens) { + free(unifyfs_val_lens); + } + + if (NULL != fattr_keys) { + free_attr_key_array(fattr_keys); + } + + if (NULL != fattr_vals) { + free(fattr_vals); + } + + if (NULL != fattr_key_lens) { + free(fattr_key_lens); + } + + if (NULL != fattr_val_lens) { + free(fattr_val_lens); + } + + return ret; +} + +/************************ + * These functions define the logic of the request manager thread + ***********************/ + +/* pack the the requests to be sent to the same + * delegator. + * ToDo: pack and send multiple rounds if the + * total request sizes is larger than REQ_BUF_LEN + * @param rank: source rank that sends the requests + * @param req_msg_buf: request buffer + * @param req_num: number of read requests + * @param *tot_sz: the total data size to read in these + * packed read requests + * @return success/error code */ +static int rm_pack_send_requests( + char* req_msg_buf, /* pointer to buffer to pack requests into */ + send_msg_t* send_metas, /* request objects to be packed */ + int req_cnt, /* number of requests */ + size_t* tot_sz) /* total data payload size we're requesting */ +{ + /* tot_sz records the aggregate data size + * requested in this transfer */ + + /* send format: + * (int) cmd - specifies type of message (SVC_CMD_RDREQ_MSG) + * (int) req_num - number of requests in message + * {sequence of send_meta_t requests} */ + size_t packed_size = (2 * sizeof(int)) + (req_cnt * sizeof(send_msg_t)); + + /* get pointer to start of send buffer */ + char* ptr = req_msg_buf; + memset(ptr, 0, packed_size); + + /* pack command */ + int cmd = (int)SVC_CMD_RDREQ_MSG; + *((int*)ptr) = cmd; + ptr += sizeof(int); + + /* pack request count */ + *((int*)ptr) = req_cnt; + ptr += sizeof(int); + + /* pack each request into the send buffer, + * total up incoming bytes as we go */ + int i; + size_t bytes = 0; + for (i = 0; i < req_cnt; i++) { + /* accumulate data size of this request */ + bytes += send_metas[i].length; + } + + /* copy requests into buffer */ + memcpy(ptr, send_metas, (req_cnt * sizeof(send_msg_t))); + ptr += (req_cnt * sizeof(send_msg_t)); + + /* increment running total size of data bytes */ + (*tot_sz) += bytes; + + /* return number of bytes used to pack requests */ + assert(packed_size == (ptr - req_msg_buf)); + return (int)packed_size; +} + +/* pack the chunk read requests for a single remote delegator. + * + * @param req_msg_buf: request buffer used for packing + * @param req_num: number of read requests + * @return size of packed buffer (or error code) + */ +static size_t rm_pack_chunk_requests(char* req_msg_buf, + remote_chunk_reads_t* remote_reads) +{ + /* send format: + * (int) cmd - specifies type of message (SVC_CMD_RDREQ_CHK) + * (int) req_cnt - number of requests in message + * {sequence of chunk_read_req_t} */ + int req_cnt = remote_reads->num_chunks; + size_t reqs_sz = req_cnt * sizeof(chunk_read_req_t); + size_t packed_size = (2 * sizeof(int)) + sizeof(size_t) + reqs_sz; + + /* get pointer to start of send buffer */ + char* ptr = req_msg_buf; + memset(ptr, 0, packed_size); + + /* pack command */ + int cmd = (int)SVC_CMD_RDREQ_CHK; + *((int*)ptr) = cmd; + ptr += sizeof(int); + + /* pack request count */ + *((int*)ptr) = req_cnt; + ptr += sizeof(int); + + /* pack total requested data size */ + *((size_t*)ptr) = remote_reads->total_sz; + ptr += sizeof(size_t); + + /* copy requests into buffer */ + memcpy(ptr, remote_reads->reqs, reqs_sz); + ptr += reqs_sz; + + /* return number of bytes used to pack requests */ + return packed_size; +} + +/* send the read requests to the remote delegator service managers + * @param thrd_ctrl : reqmgr thread control structure + * @param tot_sz : output parameter for total size of data to read + * @return success/error code */ +static int rm_send_remote_requests(reqmgr_thrd_t* thrd_ctrl, + size_t* tot_sz) +{ + // NOTE: this fn assumes thrd_ctrl->thrd_lock is locked + + int rc; + int i = 0; + + /* ToDo: Transfer the message in multiple + * rounds when total size > the size of + * send_msg_buf + * */ + + /* use this variable to total up number of incoming data bytes */ + *tot_sz = 0; + + /* get pointer to send buffer */ + char* sendbuf = thrd_ctrl->del_req_msg_buf; + + /* get pointer to start of read request array, + * and initialize index to point to first element */ + send_msg_t* msgs = thrd_ctrl->del_req_set->msg_meta; + int msg_cursor = 0; + + /* iterate over each delegator we need to send requests to */ + for (i = 0; i < thrd_ctrl->del_req_stat->del_cnt; i++) { + /* pointer to start of requests for this delegator */ + send_msg_t* reqs = msgs + msg_cursor; + + /* number of requests for this delegator */ + int req_num = thrd_ctrl->del_req_stat->req_stat[i].req_cnt; + + /* pack requests into send buffer, get size of packed data, + * increase total number of data payload we will get back */ + int packed_size = rm_pack_send_requests(sendbuf, reqs, + req_num, tot_sz); + + /* get rank of target delegator */ + int del_rank = thrd_ctrl->del_req_stat->req_stat[i].del_id; + + /* send requests */ + //MPI_Send(sendbuf, packed_size, MPI_BYTE, + // del_rank, (int)READ_REQUEST_TAG, MPI_COMM_WORLD); + rc = invoke_server_request_rpc(del_rank, 0, (int)READ_REQUEST_TAG, + (void*)sendbuf, (size_t)packed_size); + if (rc != (int)UNIFYFS_SUCCESS) { + LOGERR("server request rpc to %d failed - %s", del_rank, + unifyfs_error_enum_str((unifyfs_error_e)rc)); + } + + /* advance to requests for next delegator */ + msg_cursor += req_num; + } + + return UNIFYFS_SUCCESS; +} + +/* send the chunk read requests to remote delegators + * + * @param thrd_ctrl : reqmgr thread control structure + * @return success/error code + */ +static int rm_request_remote_chunks(reqmgr_thrd_t* thrd_ctrl) +{ + // NOTE: this fn assumes thrd_ctrl->thrd_lock is locked + + int i, j, rc; + int ret = (int)UNIFYFS_SUCCESS; + + /* get pointer to send buffer */ + char* sendbuf = thrd_ctrl->del_req_msg_buf; + + /* iterate over each active read request */ + for (i = 0; i < RM_MAX_ACTIVE_REQUESTS; i++) { + server_read_req_t* req = thrd_ctrl->read_reqs + i; + if (req->num_remote_reads > 0) { + LOGDBG("read req %d is active", i); + debug_print_read_req(req); + if (req->status == READREQ_INIT) { + req->status = READREQ_STARTED; + /* iterate over each delegator we need to send requests to */ + remote_chunk_reads_t* remote_reads; + size_t packed_sz; + for (j = 0; j < req->num_remote_reads; j++) { + remote_reads = req->remote_reads + j; + remote_reads->status = READREQ_STARTED; + + /* pack requests into send buffer, get packed size */ + packed_sz = rm_pack_chunk_requests(sendbuf, remote_reads); + + /* get rank of target delegator */ + int del_rank = remote_reads->rank; + + /* send requests */ + LOGDBG("[%d of %d] sending %d chunk requests to server %d", + j, req->num_remote_reads, + remote_reads->num_chunks, del_rank); + rc = invoke_chunk_read_request_rpc(del_rank, req, + remote_reads->num_chunks, + sendbuf, packed_sz); + if (rc != (int)UNIFYFS_SUCCESS) { + ret = rc; + LOGERR("server request rpc to %d failed - %s", + del_rank, + unifyfs_error_enum_str((unifyfs_error_e)rc)); + } + } + } else { + /* already started */ + LOGDBG("read req %d already processed", i); + } + } + } + + return ret; +} + +/* send the chunk read requests to remote delegators + * + * @param thrd_ctrl : reqmgr thread control structure + * @return success/error code + */ +static int rm_process_remote_chunk_responses(reqmgr_thrd_t* thrd_ctrl) +{ + // NOTE: this fn assumes thrd_ctrl->thrd_lock is locked + + int i, j, rc; + int ret = (int)UNIFYFS_SUCCESS; + + /* iterate over each active read request */ + for (i = 0; i < RM_MAX_ACTIVE_REQUESTS; i++) { + server_read_req_t* req = thrd_ctrl->read_reqs + i; + if ((req->num_remote_reads > 0) && + (req->status == READREQ_STARTED)) { + /* iterate over each delegator we need to send requests to */ + remote_chunk_reads_t* rcr; + for (j = 0; j < req->num_remote_reads; j++) { + rcr = req->remote_reads + j; + if (NULL == rcr->resp) { + continue; + } + LOGDBG("found read req %d responses from delegator %d", + i, rcr->rank); + rc = rm_handle_chunk_read_responses(thrd_ctrl, req, rcr); + if (rc != (int)UNIFYFS_SUCCESS) { + LOGERR("failed to handle chunk read responses"); + ret = rc; + } + } + } + } + + return ret; +} + +/* signal the client process for it to start processing read + * data in shared memory */ +static int client_signal(shm_header_t* hdr, + shm_region_state_e flag) +{ + if (flag == SHMEM_REGION_DATA_READY) { + LOGDBG("setting data-ready"); + } else if (flag == SHMEM_REGION_DATA_COMPLETE) { + LOGDBG("setting data-complete"); + } + hdr->state = flag; + /* TODO: MEM_FLUSH */ + return UNIFYFS_SUCCESS; +} + +/* wait until client has processed all read data in shared memory */ +static int client_wait(shm_header_t* hdr) +{ + int rc = (int)UNIFYFS_SUCCESS; + + /* specify time to sleep between checking flag in shared + * memory indicating client has processed data */ + struct timespec shm_wait_tm; + shm_wait_tm.tv_sec = 0; + shm_wait_tm.tv_nsec = SHM_WAIT_INTERVAL; + + /* wait for client to set flag to 0 */ + int max_sleep = 10000000; // 10s + volatile int* vip = (volatile int*)&(hdr->state); + while (*vip != SHMEM_REGION_EMPTY) { + /* not there yet, sleep for a while */ + nanosleep(&shm_wait_tm, NULL); + /* TODO: MEM_FETCH */ + max_sleep--; + if (0 == max_sleep) { + LOGERR("timed out waiting for empty"); + rc = (int)UNIFYFS_ERROR_SHMEM; + break; + } + } + + /* reset header to reflect empty state */ + hdr->meta_cnt = 0; + hdr->bytes = 0; + return rc; +} + +static shm_meta_t* reserve_shmem_meta(app_config_t* app_config, + shm_header_t* hdr, + size_t data_sz) +{ + shm_meta_t* meta = NULL; + if (NULL == hdr) { + LOGERR("invalid header"); + } else { + pthread_mutex_lock(&(hdr->sync)); + LOGDBG("shm_header(cnt=%zu, bytes=%zu)", hdr->meta_cnt, hdr->bytes); + size_t remain_size = app_config->recv_buf_sz - + (sizeof(shm_header_t) + hdr->bytes); + size_t meta_size = sizeof(shm_meta_t) + data_sz; + if (meta_size > remain_size) { + /* client-side receive buffer is full, + * inform client to start reading */ + LOGDBG("need more space in client recv buffer"); + client_signal(hdr, SHMEM_REGION_DATA_READY); + + /* wait for client to read data */ + int rc = client_wait(hdr); + if (rc != (int)UNIFYFS_SUCCESS) { + LOGERR("wait for client recv buffer space failed"); + return NULL; + } + } + size_t shm_offset = hdr->bytes; + char* shm_buf = ((char*)hdr) + sizeof(shm_header_t); + meta = (shm_meta_t*)(shm_buf + shm_offset); + LOGDBG("reserved shm_meta[%zu] and %zu payload bytes", + hdr->meta_cnt, data_sz); + hdr->meta_cnt++; + hdr->bytes += meta_size; + pthread_mutex_unlock(&(hdr->sync)); + } + return meta; +} + +/* parse the read replies from message received from service manager, + * deliver replies back to client + * + * @param app_id : application id + * @param client_id : local client rank within app + * @param recv_msg_buf : message buffer containing packed read requests + * @param ptr_tot_sz : pointer to total processed data size + * @return success/error code */ +static int rm_process_received_msg(int app_id, + int client_id, + char* recv_msg_buf, + size_t* ptr_tot_sz) +{ + /* assume we'll succeed in processing the message */ + int rc = UNIFYFS_SUCCESS; + + /* look up client app config based on client id */ + app_config_t* app_config = + (app_config_t*)arraylist_get(app_config_list, app_id); + + /* format of read replies in shared memory + * shm_header_t - shared memory region header + * {sequence of shm_meta_t + data payload} */ + + /* get pointer to shared memory buffer for this client */ + size_t header_size = sizeof(shm_header_t); + shm_header_t* hdr = (shm_header_t*)app_config->shm_recv_bufs[client_id]; + + /* format of recv_msg_buf: + * (int) num - number of read replies packed in message + * {sequence of recv_msg_t containing read replies} */ + + /* get pointer to start of receive buffer */ + char* msgptr = recv_msg_buf; + + /* extract number of read requests in this message */ + int num = *(int*)msgptr; + msgptr += sizeof(int); + + /* unpack each read reply */ + int j; + for (j = 0; j < num; j++) { + /* point to first read reply in message */ + recv_msg_t* msg = (recv_msg_t*)msgptr; + msgptr += sizeof(recv_msg_t); + + /* get pointer in shared memory for next read reply */ + shm_meta_t* meta = reserve_shmem_meta(app_config, hdr, msg->length); + if (NULL == meta) { + LOGERR("failed to reserve space for read reply"); + rc = UNIFYFS_FAILURE; + break; + } + char* shmbuf = ((char*)meta) + sizeof(shm_meta_t); + + /* copy in header for this read request */ + meta->gfid = msg->src_fid; + meta->offset = msg->src_offset; + meta->length = msg->length; + meta->errcode = msg->errcode; + + /* copy data for this read request */ + memcpy(shmbuf, msgptr, msg->length); + + /* advance to next read reply in message buffer */ + msgptr += msg->length; + + /* decrement number of bytes processed from total */ + if (NULL != ptr_tot_sz) { + *ptr_tot_sz -= msg->length; + LOGDBG("processed message of size %zu, %zu left to receive", + msg->length, *ptr_tot_sz); + } + } + + return rc; +} + +/* receive the requested data returned from service managers + * as a result of the read requests we sent to them + * + * @param thrd_ctrl: request manager thread state + * @param tot_sz: total data size to receive (excludes header bytes) + * @return success/error code */ +static int rm_receive_remote_message(reqmgr_thrd_t* thrd_ctrl, + size_t tot_sz) +{ + // NOTE: this fn assumes thrd_ctrl->thrd_lock is locked + + /* assume we'll succeed */ + int rc = UNIFYFS_SUCCESS; + + /* get app id and client id that we'll be serving, + * app id associates thread with a namespace (mountpoint) + * the client id associates the thread with a particular + * client process id */ + int app_id = thrd_ctrl->app_id; + int client_id = thrd_ctrl->client_id; + + /* lookup our data structure for this app id */ + app_config_t* app_config = + (app_config_t*)arraylist_get(app_config_list, app_id); + + /* get thread id for this client (used for MPI tags) */ + int thrd_id = app_config->thrd_idxs[client_id]; + + /* service manager will incorporate our thread id in tag, + * to distinguish between target request manager threads */ + int tag = (int)READ_RESPONSE_TAG + thrd_id; + + /* array of MPI_Request objects for window of posted receives */ + MPI_Request recv_req[RECV_BUF_CNT] = {MPI_REQUEST_NULL}; + + /* get number of receives to post and size of each buffer */ + int recv_buf_cnt = RECV_BUF_CNT; + int recv_buf_len = (int) SENDRECV_BUF_LEN; + + /* post a window of receive buffers for incoming data */ + int i; + for (i = 0; i < recv_buf_cnt; i++) { + /* post buffer for incoming receive */ + MPI_Irecv(thrd_ctrl->del_recv_msg_buf[i], recv_buf_len, MPI_BYTE, + MPI_ANY_SOURCE, tag, MPI_COMM_WORLD, &recv_req[i]); + } + + /* spin until we have received all incoming data */ + while (tot_sz > 0) { + /* wait for any receive to come in */ + int index; + MPI_Status status; + MPI_Waitany(recv_buf_cnt, recv_req, &index, &status); + + /* got a new message, get pointer to message buffer */ + char* buf = thrd_ctrl->del_recv_msg_buf[index]; + + /* unpack the data into client shared memory, + * this will internally signal client and wait + * for data to be processed if shared memory + * buffer is filled */ + int tmp_rc = rm_process_received_msg(app_id, client_id, buf, &tot_sz); + if (tmp_rc != UNIFYFS_SUCCESS) { + rc = tmp_rc; + } + + /* done processing, repost this receive buffer */ + MPI_Irecv(thrd_ctrl->del_recv_msg_buf[index], recv_buf_len, MPI_BYTE, + MPI_ANY_SOURCE, tag, MPI_COMM_WORLD, &recv_req[index]); + } + + /* cancel posted MPI receives */ + for (i = 0; i < recv_buf_cnt; i++) { + MPI_Status status; + MPI_Cancel(&recv_req[i]); + MPI_Wait(&recv_req[i], &status); + } + + /* signal client that we're now done writing data */ + shm_header_t* hdr = (shm_header_t*)app_config->shm_recv_bufs[client_id]; + client_signal(hdr, SHMEM_REGION_DATA_COMPLETE); + + /* wait for client to read data */ + client_wait(hdr); + + return rc; +} + +int rm_post_chunk_read_responses(int app_id, + int client_id, + int src_rank, + int req_id, + int num_chks, + size_t bulk_sz, + char* resp_buf) +{ + int rc, thrd_id; + app_config_t* app_config = NULL; + reqmgr_thrd_t* thrd_ctrl = NULL; + server_read_req_t* rdreq = NULL; + remote_chunk_reads_t* del_reads = NULL; + + /* lookup RM thread control structure for this app id */ + app_config = (app_config_t*) arraylist_get(app_config_list, app_id); + assert(NULL != app_config); + thrd_id = app_config->thrd_idxs[client_id]; + thrd_ctrl = rm_get_thread(thrd_id); + assert(NULL != thrd_ctrl); + + RM_LOCK(thrd_ctrl); + + /* find read req associated with req_id */ + rdreq = thrd_ctrl->read_reqs + req_id; + for (int i = 0; i < rdreq->num_remote_reads; i++) { + if (rdreq->remote_reads[i].rank == src_rank) { + del_reads = rdreq->remote_reads + i; + break; + } + } + if (NULL != del_reads) { + LOGDBG("posting chunk responses for req %d from delegator %d", + req_id, src_rank); + del_reads->resp = (chunk_read_resp_t*)resp_buf; + if (del_reads->num_chunks != num_chks) { + LOGERR("mismatch on request vs. response chunks"); + del_reads->num_chunks = num_chks; + } + del_reads->total_sz = bulk_sz; + rc = (int)UNIFYFS_SUCCESS; + } else { + LOGERR("failed to find matching chunk-reads request"); + rc = (int)UNIFYFS_FAILURE; + } + + /* inform the request manager thread we added responses */ + signal_new_responses(thrd_ctrl); + + RM_UNLOCK(thrd_ctrl); + + return rc; +} + +/* process the requested chunk data returned from service managers + * + * @param thrd_ctrl : request manager thread state + * @param rdreq : server read request + * @param del_reads : remote server chunk reads + * @return success/error code + */ +int rm_handle_chunk_read_responses(reqmgr_thrd_t* thrd_ctrl, + server_read_req_t* rdreq, + remote_chunk_reads_t* del_reads) +{ + int errcode, gfid, i, num_chks, rc, thrd_id; + int ret = (int)UNIFYFS_SUCCESS; + app_config_t* app_config = NULL; + chunk_read_resp_t* responses = NULL; + shm_header_t* client_shm = NULL; + shm_meta_t* shm_meta = NULL; + void* shm_buf = NULL; + char* data_buf = NULL; + size_t data_sz, offset; + + assert((NULL != thrd_ctrl) && + (NULL != rdreq) && + (NULL != del_reads) && + (NULL != del_reads->resp)); + + /* look up client shared memory region */ + app_config = (app_config_t*) arraylist_get(app_config_list, rdreq->app_id); + assert(NULL != app_config); + client_shm = (shm_header_t*) app_config->shm_recv_bufs[rdreq->client_id]; + + RM_LOCK(thrd_ctrl); + + num_chks = del_reads->num_chunks; + gfid = rdreq->extent.gfid; + if (del_reads->status != READREQ_STARTED) { + LOGERR("chunk read response for non-started req @ index=%d", + rdreq->req_ndx); + ret = (int32_t)UNIFYFS_ERROR_INVAL; + } else if (0 == del_reads->total_sz) { + LOGERR("empty chunk read response for gfid=%d", gfid); + ret = (int32_t)UNIFYFS_ERROR_INVAL; + } else { + LOGDBG("handling chunk read responses from server %d: " + "gfid=%d num_chunks=%d buf_size=%zu", + del_reads->rank, gfid, num_chks, + del_reads->total_sz); + responses = del_reads->resp; + data_buf = (char*)(responses + num_chks); + for (i = 0; i < num_chks; i++) { + chunk_read_resp_t* resp = responses + i; + if (resp->read_rc < 0) { + errcode = (int)-(resp->read_rc); + data_sz = 0; + } else { + errcode = 0; + data_sz = resp->nbytes; + } + offset = resp->offset; + LOGDBG("chunk response for offset=%zu: sz=%zu", offset, data_sz); + + /* allocate and register local target buffer for bulk access */ + shm_meta = reserve_shmem_meta(app_config, client_shm, data_sz); + if (NULL != shm_meta) { + shm_meta->offset = offset; + shm_meta->length = data_sz; + shm_meta->gfid = gfid; + shm_meta->errcode = errcode; + shm_buf = (void*)((char*)shm_meta + sizeof(shm_meta_t)); + if (data_sz) { + memcpy(shm_buf, data_buf, data_sz); + } + } else { + LOGERR("failed to reserve shmem space for read reply") + ret = (int32_t)UNIFYFS_ERROR_SHMEM; + } + data_buf += data_sz; + } + /* cleanup */ + free((void*)responses); + del_reads->resp = NULL; + + /* update request status */ + del_reads->status = READREQ_COMPLETE; + if (rdreq->status == READREQ_STARTED) { + rdreq->status = READREQ_PARTIAL_COMPLETE; + } + int completed_remote_reads = 0; + for (i = 0; i < rdreq->num_remote_reads; i++) { + if (rdreq->remote_reads[i].status != READREQ_COMPLETE) { + break; + } + completed_remote_reads++; + } + if (completed_remote_reads == rdreq->num_remote_reads) { + rdreq->status = READREQ_COMPLETE; + + /* signal client that we're now done writing data */ + client_signal(client_shm, SHMEM_REGION_DATA_COMPLETE); + + /* wait for client to read data */ + client_wait(client_shm); + + rc = release_read_req(thrd_ctrl, rdreq); + if (rc != (int)UNIFYFS_SUCCESS) { + LOGERR("failed to release server_read_req_t"); + } + } + } + + RM_UNLOCK(thrd_ctrl); + + return ret; +} + +/* entry point for request manager thread, one thread is created + * for each client process, client informs thread of a set of read + * requests, thread retrieves data for client and notifies client + * when data is ready + * + * delegate the read requests for the delegator thread's client. Each + * delegator thread handles one connection to one client-side rank. + * + * @param arg: pointer to control structure for the delegator thread + * + * @return NULL */ +void* rm_delegate_request_thread(void* arg) +{ + /* get pointer to our thread control structure */ + reqmgr_thrd_t* thrd_ctrl = (reqmgr_thrd_t*) arg; + + LOGDBG("I am request manager thread!"); + + /* loop forever to handle read requests from the client, + * new requests are added to a list on a shared data structure + * with main thread, new items inserted by the rpc handler */ + int rc; + while (1) { + /* grab lock */ + RM_LOCK(thrd_ctrl); + + /* process any chunk read responses */ + rc = rm_process_remote_chunk_responses(thrd_ctrl); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("failed to process remote chunk responses"); + } + + /* inform dispatcher that we're waiting for work + * inside the critical section */ + thrd_ctrl->has_waiting_delegator = 1; + + /* if dispatcher is waiting on us, signal it to go ahead, + * this coordination ensures that we'll be the next thread + * to grab the lock after the dispatcher has assigned us + * some work (rather than the dispatcher grabbing the lock + * and assigning yet more work) */ + if (thrd_ctrl->has_waiting_dispatcher == 1) { + pthread_cond_signal(&thrd_ctrl->thrd_cond); + } + + /* release lock and wait to be signaled by dispatcher */ + LOGDBG("RM[%d] waiting for work", thrd_ctrl->thrd_ndx); + pthread_cond_wait(&thrd_ctrl->thrd_cond, &thrd_ctrl->thrd_lock); + + /* set flag to indicate we're no longer waiting */ + thrd_ctrl->has_waiting_delegator = 0; + + /* go do work ... */ + LOGDBG("RM[%d] got work", thrd_ctrl->thrd_ndx); + + /* release lock and bail out if we've been told to exit */ + if (thrd_ctrl->exit_flag == 1) { + RM_UNLOCK(thrd_ctrl); + break; + } + + /* send chunk read requests to remote servers */ + rc = rm_request_remote_chunks(thrd_ctrl); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("failed to request remote chunks"); + } + + /* tot_sz tracks the total bytes we expect to receive. + * size is computed during send, decremented during receive */ + size_t tot_sz = 0; + rc = rm_send_remote_requests(thrd_ctrl, &tot_sz); + if (rc != UNIFYFS_SUCCESS) { + /* release lock and exit if we hit an error */ + RM_UNLOCK(thrd_ctrl); + return NULL; + } + if (tot_sz > 0) { + /* wait for data to come back from servers */ + rc = rm_receive_remote_message(thrd_ctrl, tot_sz); + if (rc != UNIFYFS_SUCCESS) { + /* release lock and exit if we hit an error */ + RM_UNLOCK(thrd_ctrl); + return NULL; + } + } + + /* release lock */ + RM_UNLOCK(thrd_ctrl); + } + + LOGDBG("request manager thread exiting"); + + return NULL; +} + +/* BEGIN MARGO SERVER-SERVER RPC INVOCATION FUNCTIONS */ + +/* invokes the server_hello rpc */ +int invoke_server_hello_rpc(int dst_srvr_rank) +{ + int rc = (int)UNIFYFS_SUCCESS; + hg_handle_t handle; + server_hello_in_t in; + server_hello_out_t out; + hg_return_t hret; + hg_addr_t dst_srvr_addr; + char hello_msg[UNIFYFS_MAX_HOSTNAME]; + + assert(dst_srvr_rank < (int)glb_num_servers); + dst_srvr_addr = glb_servers[dst_srvr_rank].margo_svr_addr; + + hret = margo_create(unifyfsd_rpc_context->svr_mid, dst_srvr_addr, + unifyfsd_rpc_context->rpcs.hello_id, &handle); + assert(hret == HG_SUCCESS); + + /* fill in input struct */ + snprintf(hello_msg, sizeof(hello_msg), "hello from %s", glb_host); + in.src_rank = (int32_t)glb_mpi_rank; + in.message_str = strdup(hello_msg); + + LOGDBG("invoking the server-hello rpc function"); + hret = margo_forward(handle, &in); + if (hret != HG_SUCCESS) { + rc = (int)UNIFYFS_FAILURE; + } else { + /* decode response */ + hret = margo_get_output(handle, &out); + if (hret == HG_SUCCESS) { + int32_t ret = out.ret; + LOGDBG("Got hello rpc response from %d - ret=%" PRIi32, + dst_srvr_rank, ret); + margo_free_output(handle, &out); + } else { + rc = (int)UNIFYFS_FAILURE; + } + } + + free((void*)in.message_str); + margo_destroy(handle); + + return rc; +} + +/* invokes the server_request rpc */ +int invoke_server_request_rpc(int dst_srvr_rank, int req_id, int tag, + void* data_buf, size_t buf_sz) +{ + int rc = (int)UNIFYFS_SUCCESS; + hg_handle_t handle; + server_request_in_t in; + server_request_out_t out; + hg_return_t hret; + hg_addr_t dst_srvr_addr; + hg_size_t bulk_sz = buf_sz; + + if (dst_srvr_rank == glb_mpi_rank) { + // short-circuit for local requests + if (tag == (int)READ_REQUEST_TAG) { + return sm_decode_msg((char*)data_buf); + } + } + + assert(dst_srvr_rank < (int)glb_num_servers); + dst_srvr_addr = glb_servers[dst_srvr_rank].margo_svr_addr; + + hret = margo_create(unifyfsd_rpc_context->svr_mid, dst_srvr_addr, + unifyfsd_rpc_context->rpcs.request_id, &handle); + assert(hret == HG_SUCCESS); + + /* fill in input struct */ + in.src_rank = (int32_t)glb_mpi_rank; + in.req_id = (int32_t)req_id; + in.req_tag = (int32_t)tag; + in.bulk_size = bulk_sz; + + /* register request buffer for bulk remote access */ + hret = margo_bulk_create(unifyfsd_rpc_context->svr_mid, 1, + &data_buf, &bulk_sz, + HG_BULK_READ_ONLY, &in.bulk_handle); + assert(hret == HG_SUCCESS); + + LOGDBG("invoking the server-request rpc function"); + hret = margo_forward(handle, &in); + if (hret != HG_SUCCESS) { + rc = (int)UNIFYFS_FAILURE; + } else { + /* decode response */ + hret = margo_get_output(handle, &out); + if (hret == HG_SUCCESS) { + rc = (int)out.ret; + LOGDBG("Got request rpc response from %d - ret=%d", + dst_srvr_rank, rc); + margo_free_output(handle, &out); + } else { + rc = (int)UNIFYFS_FAILURE; + } + } + + margo_bulk_free(in.bulk_handle); + margo_destroy(handle); + + return rc; +} + +/* invokes the server_request rpc */ +int invoke_chunk_read_request_rpc(int dst_srvr_rank, + server_read_req_t* rdreq, + int num_chunks, + void* data_buf, size_t buf_sz) +{ + int rc = (int)UNIFYFS_SUCCESS; + hg_handle_t handle; + chunk_read_request_in_t in; + chunk_read_request_out_t out; + hg_return_t hret; + hg_addr_t dst_srvr_addr; + hg_size_t bulk_sz = buf_sz; + + if (dst_srvr_rank == glb_mpi_rank) { + // short-circuit for local requests + return sm_issue_chunk_reads(glb_mpi_rank, + rdreq->app_id, + rdreq->client_id, + rdreq->req_ndx, + num_chunks, + (char*)data_buf); + } + + assert(dst_srvr_rank < (int)glb_num_servers); + dst_srvr_addr = glb_servers[dst_srvr_rank].margo_svr_addr; + + hret = margo_create(unifyfsd_rpc_context->svr_mid, dst_srvr_addr, + unifyfsd_rpc_context->rpcs.chunk_read_request_id, + &handle); + assert(hret == HG_SUCCESS); + + /* fill in input struct */ + in.src_rank = (int32_t)glb_mpi_rank; + in.app_id = (int32_t)rdreq->app_id; + in.client_id = (int32_t)rdreq->client_id; + in.req_id = (int32_t)rdreq->req_ndx; + in.num_chks = (int32_t)num_chunks; + in.bulk_size = bulk_sz; + + /* register request buffer for bulk remote access */ + hret = margo_bulk_create(unifyfsd_rpc_context->svr_mid, 1, + &data_buf, &bulk_sz, + HG_BULK_READ_ONLY, &in.bulk_handle); + assert(hret == HG_SUCCESS); + + LOGDBG("invoking the chunk-read-request rpc function"); + hret = margo_forward(handle, &in); + if (hret != HG_SUCCESS) { + rc = (int)UNIFYFS_FAILURE; + } else { + /* decode response */ + hret = margo_get_output(handle, &out); + if (hret == HG_SUCCESS) { + rc = (int)out.ret; + LOGDBG("Got request rpc response from %d - ret=%d", + dst_srvr_rank, rc); + margo_free_output(handle, &out); + } else { + rc = (int)UNIFYFS_FAILURE; + } + } + + margo_bulk_free(in.bulk_handle); + margo_destroy(handle); + + return rc; +} + +/* BEGIN MARGO SERVER-SERVER RPC HANDLER FUNCTIONS */ + +/* handler for remote read request response */ +static void chunk_read_response_rpc(hg_handle_t handle) +{ + int rc, src_rank, req_id; + int app_id, client_id, thrd_id; + int i, num_chks; + int32_t ret; + hg_return_t hret; + hg_bulk_t bulk_handle; + size_t bulk_sz; + chunk_read_response_in_t in; + chunk_read_response_out_t out; + void* resp_buf = NULL; + + /* get input params */ + rc = margo_get_input(handle, &in); + assert(rc == HG_SUCCESS); + src_rank = (int)in.src_rank; + app_id = (int)in.app_id; + client_id = (int)in.client_id; + req_id = (int)in.req_id; + num_chks = (int)in.num_chks; + bulk_sz = (size_t)in.bulk_size; + + if (0 == bulk_sz) { + LOGERR("empty response buffer"); + ret = (int32_t)UNIFYFS_ERROR_INVAL; + } else { + resp_buf = malloc(bulk_sz); + if (NULL == resp_buf) { + LOGERR("failed to allocate chunk read responses buffer"); + ret = (int32_t)UNIFYFS_ERROR_NOMEM; + } else { + /* pull response data */ + ret = (int32_t)UNIFYFS_SUCCESS; + const struct hg_info* hgi = margo_get_info(handle); + assert(NULL != hgi); + margo_instance_id mid = margo_hg_info_get_instance(hgi); + assert(mid != MARGO_INSTANCE_NULL); + hret = margo_bulk_create(mid, 1, &resp_buf, &in.bulk_size, + HG_BULK_WRITE_ONLY, &bulk_handle); + assert(hret == HG_SUCCESS); + hret = margo_bulk_transfer(mid, HG_BULK_PULL, hgi->addr, + in.bulk_handle, 0, + bulk_handle, 0, in.bulk_size); + assert(hret == HG_SUCCESS); + + rc = rm_post_chunk_read_responses(app_id, client_id, + src_rank, req_id, num_chks, + bulk_sz, (char*)resp_buf); + if (rc != (int)UNIFYFS_SUCCESS) { + LOGERR("failed to handle chunk read responses") + ret = rc; + } + margo_bulk_free(bulk_handle); + } + } + + /* fill output structure and return to caller */ + out.ret = ret; + hret = margo_respond(handle, &out); + assert(hret == HG_SUCCESS); + + /* free margo resources */ + margo_free_input(handle, &in); + margo_destroy(handle); +} +DEFINE_MARGO_RPC_HANDLER(chunk_read_response_rpc) diff --git a/server/src/unifyfs_request_manager.h b/server/src/unifyfs_request_manager.h new file mode 100644 index 000000000..464a4a18c --- /dev/null +++ b/server/src/unifyfs_request_manager.h @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2017-2019, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +/* + * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * Copyright (c) 2017, Florida State University. Contributions from + * the Computer Architecture and Systems Research Laboratory (CASTL) + * at the Department of Computer Science. + * + * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror + * LLNL-CODE-728877. All rights reserved. + * + * This file is part of burstfs. + * For details, see https://github.com/llnl/burstfs + * Please read https://github.com/llnl/burstfs/LICENSE for full license text. + */ + +#ifndef UNIFYFS_REQUEST_MANAGER_H +#define UNIFYFS_REQUEST_MANAGER_H + +#include "unifyfs_global.h" + +/* one entry per delegator for which we have active read requests, + * records rank of delegator and request count */ +typedef struct { + int req_cnt; /* number of requests to this delegator */ + int del_id; /* rank of delegator */ +} per_del_stat_t; + +/* records list of delegator information (rank, req count) for + * set of delegators we have active read requests for */ +typedef struct { + per_del_stat_t* req_stat; /* delegator rank and request count */ + int del_cnt; /* number of delegators we have read requests for */ +} del_req_stat_t; + +// NEW READ REQUEST STRUCTURES + +typedef struct { + readreq_status_e status; /* aggregate request status */ + int req_ndx; /* index in reqmgr read_reqs array */ + int app_id; /* app id of requesting client process */ + int client_id; /* client id of requesting client process */ + int num_remote_reads; /* size of remote_reads array */ + client_read_req_t extent; /* client read extent, includes gfid */ + chunk_read_req_t* chunks; /* array of chunk-reads */ + remote_chunk_reads_t* remote_reads; /* per-delegator remote reads array */ +} server_read_req_t; + +/* this structure is created by the main thread for each request + * manager thread, contains shared data structures where main thread + * issues read requests and request manager processes them, contains + * condition variable and lock for coordination between threads */ +typedef struct { + /* request manager thread */ + pthread_t thrd; + + /* condition variable to synchronize request manager thread + * and main thread delivering work */ + pthread_cond_t thrd_cond; + + /* lock for shared data structures (variables below) */ + pthread_mutex_t thrd_lock; + + /* flag indicating that request manager thread is waiting + * for work inside of critical region */ + int has_waiting_delegator; + + /* flag indicating main thread is in critical section waiting + * for request manager thread */ + int has_waiting_dispatcher; + + int num_read_reqs; + int next_rdreq_ndx; + server_read_req_t read_reqs[RM_MAX_ACTIVE_REQUESTS]; + + /* a list of read requests to be sent to each delegator, + * main thread adds items to this list, request manager + * processes them */ + msg_meta_t* del_req_set; + + /* statistics of read requests to be sent to each delegator */ + del_req_stat_t* del_req_stat; + + /* buffer to build read request messages */ + char del_req_msg_buf[REQ_BUF_LEN]; + + /* memory for posting receives for incoming read reply messages + * from the service threads */ + char del_recv_msg_buf[RECV_BUF_CNT][SENDRECV_BUF_LEN]; + + /* flag set to indicate request manager thread should exit */ + int exit_flag; + + /* flag set after thread has exited and join completed */ + int exited; + + /* app_id this thread is serving */ + int app_id; + + /* client_id this thread is serving */ + int client_id; + + /* index within rm_thrd_list */ + int thrd_ndx; +} reqmgr_thrd_t; + + +/* create Request Manager thread for application client */ +reqmgr_thrd_t* unifyfs_rm_thrd_create(int app_id, + int client_id); + +/* lookup Request Manager thread by index */ +reqmgr_thrd_t* rm_get_thread(int thrd_id); + +/* Request Manager pthread main */ +void* rm_delegate_request_thread(void* arg); + +/* functions called by rpc handlers to assign work + * to request manager threads */ +int rm_cmd_mread(int app_id, int client_id, + size_t req_num, void* reqbuf); + +int rm_cmd_read(int app_id, int client_id, int gfid, + size_t offset, size_t length); + +int rm_cmd_filesize(int app_id, int client_id, int gfid, size_t* outsize); + +/* function called by main thread to instruct + * resource manager thread to exit, + * returns UNIFYFS_SUCCESS on success */ +int rm_cmd_exit(reqmgr_thrd_t* thrd_ctrl); + +/* + * synchronize all the indices and file attributes + * to the key-value store + * @param sock_id: the connection id in poll_set of the delegator + * @return success/error code + */ +int rm_cmd_fsync(int app_id, int client_side_id, int gfid); + +/* update state for remote chunk reads with received response data */ +int rm_post_chunk_read_responses(int app_id, + int client_id, + int src_rank, + int req_id, + int num_chks, + size_t bulk_sz, + char* resp_buf); + +/* process the requested chunk data returned from service managers */ +int rm_handle_chunk_read_responses(reqmgr_thrd_t* thrd_ctrl, + server_read_req_t* rdreq, + remote_chunk_reads_t* del_reads); + +/* MARGO SERVER-SERVER RPC INVOCATION FUNCTIONS */ + +int invoke_server_hello_rpc(int dst_srvr_rank); + +int invoke_server_request_rpc(int dst_srvr_rank, + int req_id, + int tag, + void* data_buf, size_t buf_sz); + +int invoke_chunk_read_request_rpc(int dst_srvr_rank, + server_read_req_t* rdreq, + int num_chunks, + void* data_buf, size_t buf_sz); + +#endif diff --git a/server/src/unifyfs_service_manager.c b/server/src/unifyfs_service_manager.c new file mode 100644 index 000000000..c3ed6689c --- /dev/null +++ b/server/src/unifyfs_service_manager.c @@ -0,0 +1,2004 @@ +/* + * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2017, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +/* + * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * Copyright (c) 2017, Florida State University. Contributions from + * the Computer Architecture and Systems Research Laboratory (CASTL) + * at the Department of Computer Science. + * + * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror + * LLNL-CODE-728877. All rights reserved. + * + * This file is part of burstfs. + * For details, see https://github.com/llnl/burstfs + * Please read https://github.com/llnl/burstfs/LICENSE for full license text. + */ + +#include +#include +#include + +#include "unifyfs_global.h" +#include "unifyfs_request_manager.h" +#include "unifyfs_service_manager.h" +#include "unifyfs_server_rpcs.h" +#include "margo_server.h" + +/* The service manager thread runs in a loop waiting for incoming + * requests. When it receives a message, it unpacks all read + * requests and appends them to the sm->service_msgs list. It delays + * for some time in acting on requests in the hopes of buffering + * read bursts to make I/O more efficient. If no read request + * has come in, and the delay time out has expired, and there are + * pening read requests to service, then it services all requests. + * + * It first creates a set of read tasks based on the set of read + * requests. The read requests are sorted by source file and then + * by offset, and read requests that refer to contiguous data + * regions are merged into a large read task. Read tasks are + * added to a read_task list. + * + * The read tasks are executed to copy data into a read buffer. + * Data that is copied from shared memory simply uses memcpy(). + * Data that spans shared memory and the spillover file uses + * memcpy with pread. Data that is fully in the spillover file + * are read with async I/O (aio) operations. The aio operations + * are added to a pending_reads queue that is later waited on. + * + * After all data has been read, results are packed into send + * buffers. This is done by matching read tasks with corresponding + * read requests. When send buffers are filled with read replies + * (acks), they are sent back to requesting delegators with + * MPI_Isend calls. The MPI_Request objects for those isends are + * added to a pending_sends list, which is later waited on. + * + * After replying to all sm->service_msgs, the service manager + * thread again starts listening for more incoming requests */ + +/* records info needed to build read reply to send back to + * requesting delegator, these are appened in an ack list + * which records a set of read replies before being sent */ +typedef struct { + recv_msg_t msg; /* header information for read reply */ + char* addr; /* address of data in read buffer */ +} ack_meta_t; + +/* this records info about outstanding sends to delegators, + * including the MPI request that must be waited on to + * determine when send has completed, these are added to + * an pending sends list when sent and then iterated over + * while waiting for all pending sends to complete */ +typedef struct { + MPI_Request req; /* MPI request for outstanding isend */ + MPI_Status stat; /* status for test call */ + int src_rank; /* target delegator rank */ + int src_thrd; /* target delegator thread */ +} ack_stat_t; + +/* records info about read tasks, which are generated by + * unpacking incoming read requests from remote delegators, + * it encodes information about a range of read requests + * that generated the read task, read requests that refer + * to contiguous data may be merged into a single read task, + * and the app_id and client_id are used to determine + * the memory/files holding the source data */ +typedef struct { + size_t size; /* size of read operation */ + int start_idx; /* service_msgs_t.msg[] index of starting request */ + int end_idx; /* service_msgs_t.msg[] index of ending request */ + int app_id; /* app id holding requested data */ + int cli_id; /* client id holding requested data */ + int arrival_time; /* time stamp when read request arrived */ +} read_task_t; + +/* defines an array of read tasks */ +typedef struct { + read_task_t* read_tasks; /* list of read tasks */ + int num; /* number of active read tasks */ + int cap; /* total capacity of read task list */ +} task_set_t; + +/* defines an array of read requests, generated by unpacking + * read requests from requesting delegators */ +typedef struct { + send_msg_t* msg; /* buffer of read requests */ + int num; /* number of active read requests in buffer */ + int cap; /* total capacity of read request list */ +} service_msgs_t; + +/* tracks an outstanding read operation, these are generated + * from read tasks and added to a list of pending read operations, + * which are later waited on before sending data back to requesting + * delegators */ +typedef struct { + int err_submit; /* indicates whether read was submitted */ + struct aiocb read_cb; /* structure for outstanding aio read */ + int index; /* index in read task list for this read */ + int start_pos; /* starting byte offset into read request */ + int end_pos; /* ending byte offset into read request */ + char* mem_pos; /* buffer holding read data */ +} pended_read_t; + +/* defines a list of read replies to be sent to a delegator */ +typedef struct { + arraylist_t* ack_list; /* list of read replies for delegator */ + int rank_id; /* rank of remote delegator to send to */ + int thrd_id; /* thread id of remote delegator */ + int src_cli_rank; /* rank of client that initiated read */ + size_t src_sz; /* total data size in read replies */ + int start_cursor; /* offset within ack_list */ +} rank_ack_meta_t; + +/* defines an list of reply data for different delegators, + * list is ordered by (rank,thread) of delegator for fast lookup */ +typedef struct { + rank_ack_meta_t* ack_metas; /* read reply data for a delegator */ + int num; /* number of items in list */ +} rank_ack_task_t; + +/* Service Manager state */ +typedef struct { + /* the SM thread */ + pthread_t thrd; + + /* state synchronization mutex */ + pthread_mutex_t sync; + + /* thread status */ + int initialized; + volatile int time_to_exit; + + /* thread return status code */ + int sm_exit_rc; + + /* list of chunk read requests from remote delegators */ + arraylist_t* chunk_reads; + + /* records list of read requests from requesting delegators */ + service_msgs_t service_msgs; + + /* list of read tasks that must be executed, + * generated from read requests */ + task_set_t read_task_set; + + /* list of read reply data for each delegator */ + rank_ack_task_t rank_ack_task; + + /* list of outstanding read operations */ + arraylist_t* pended_reads; + + /* list of outstanding send operations */ + arraylist_t* pended_sends; + + /* list of buffers to be used in send operations */ + arraylist_t* send_buf_list; + + /* tracks running total of bytes in current read burst */ + long burst_data_sz; + + /* buffer to hold read data while gathering it from source + * memory/files and before being packed into send buffers + * for read replies */ + char* read_buf; +} svcmgr_state_t; +svcmgr_state_t* sm; // = NULL + +#define SM_LOCK() \ +do { \ + LOGDBG("locking service manager state"); \ + pthread_mutex_lock(&(sm->sync)); \ +} while (0) + +#define SM_UNLOCK() \ +do { \ + LOGDBG("unlocking service manager state"); \ + pthread_mutex_unlock(&(sm->sync)); \ +} while (0) + + +/* sort read requests into log files by + * app id, then client id, then log offset */ +static int compare_send_msg(const void* a, const void* b) +{ + const send_msg_t* ptr_a = a; + const send_msg_t* ptr_b = b; + + if (ptr_a->dest_app_id > ptr_b->dest_app_id) { + return 1; + } + + if (ptr_a->dest_app_id < ptr_b->dest_app_id) { + return -1; + } + + if (ptr_a->dest_client_id > ptr_b->dest_client_id) { + return 1; + } + + if (ptr_a->dest_client_id < ptr_b->dest_client_id) { + return -1; + } + + if (ptr_a->dest_offset > ptr_b->dest_offset) { + return 1; + } + + if (ptr_a->dest_offset < ptr_b->dest_offset) { + return -1; + } + + return 0; +} + +/* starts a new read task based on read request in sm->service_msgs + * at given index */ +static void create_msg_read_task(int index) +{ + /* get pointer to service message at given index */ + send_msg_t* msg = &sm->service_msgs.msg[index]; + + /* get pointer to current read task */ + int idx = sm->read_task_set.num; + read_task_t* read_task = &sm->read_task_set.read_tasks[idx]; + + /* copy fields from message to read task */ + read_task->start_idx = index; + read_task->end_idx = index; + read_task->size = msg->length; + read_task->app_id = msg->dest_app_id; + read_task->cli_id = msg->dest_client_id; + read_task->arrival_time = msg->arrival_time; +} + +/* + * Cluster read requests based on file offset and age. + * Each log file is uniquely identified by client-side app_id + * and client_id, so we can locate the target log file + * (generated by the client-side program). + * + * @return success/error + */ +static int sm_cluster_reads(void) +{ + /* sort service messages by log file (app_id, client_id) + * and then by offset within each log file */ + qsort(sm->service_msgs.msg, sm->service_msgs.num, + sizeof(send_msg_t), compare_send_msg); + + /* initialize count of read tasks */ + sm->read_task_set.num = 0; + + /* create read task given first service message */ + create_msg_read_task(0); + sm->read_task_set.num++; + + /* iterate over each service message and create read tasks, + * will merge multiple read requests into read tasks + * when two requests refer to contiguous data in a log file */ + int i; + for (i = 1; i < sm->service_msgs.num; i++) { + /* get pointer to current service message */ + send_msg_t* msg = &sm->service_msgs.msg[i]; + + /* get pointer to preivous read task */ + read_task_t* last_read = + &sm->read_task_set.read_tasks[sm->read_task_set.num - 1]; + + /* check whether current message reads from the same log file, + * as our last read task */ + if ((last_read->app_id != msg->dest_app_id) || + (last_read->cli_id != msg->dest_client_id)) { + /* reading on a different local log file, + * so create a new read task for this message */ + create_msg_read_task(i); + sm->read_task_set.num++; + } else { + /* this message reads from the same log file as our last + * read request */ + + /* get pointer to previous read request */ + send_msg_t* last_msg = &sm->service_msgs.msg[i - 1]; + + /* see if we can tack current message on to + * previous read request */ + size_t last_offset = last_msg->dest_offset + last_msg->length; + if (last_offset == msg->dest_offset) { + /* current message starts where last read request + * ends, so append it to last read request if no larger + * than read_block_size */ + + /* the size of individual read should be smaller + * than read_block_size, if read size is larger it + * needs to be split into the unit of READ_BLOCK_SIZE */ + if ((last_read->size + msg->length) <= READ_BLOCK_SIZE) { + /* tack current message on previous read request */ + last_read->end_idx = i; + last_read->size += msg->length; + + /* update minimum arrival time */ + if (msg->arrival_time < last_read->arrival_time) { + last_read->arrival_time = msg->arrival_time; + } + } else { + /* if larger than read block size, start a new + * read task, here the data size requested by + * individual read request should be smaller + * than read_block_size. The larger one + * has already been split by the initiator */ + create_msg_read_task(i); + sm->read_task_set.num++; + } + } else { + /* not contiguous from the last offset, + * start a new read request */ + create_msg_read_task(i); + sm->read_task_set.num++; + } + } + } + + return UNIFYFS_SUCCESS; +} + +/* compare by rank and then thread in increasing order */ +static int compare_rank_thrd(int src_rank, int src_thrd, + int cmp_rank, int cmp_thrd) +{ + if (src_rank > cmp_rank) { + return 1; + } + if (src_rank < cmp_rank) { + return -1; + } + if (src_thrd > cmp_thrd) { + return 1; + } + if (src_thrd < cmp_thrd) { + return -1; + } + return 0; +} + +/* Returns index where delegator (rank, thread), + * should be in list, caller must check whether + * delegator at that position matches */ +static int find_ack_meta(int src_rank, int src_thrd, + int* found) +{ + /* assume we won't find item */ + *found = 0; + + /* if nothing in list, place this as first item */ + if (sm->rank_ack_task.num == 0) { + return 0; + } + + rank_ack_meta_t* metas = sm->rank_ack_task.ack_metas; + + /* if list has one item, compare to that item */ + if (sm->rank_ack_task.num == 1) { + /* compare to first item */ + int cmp = compare_rank_thrd(src_rank, src_thrd, + metas[0].rank_id, metas[0].thrd_id); + if (cmp < 0) { + /* item is smaller than first element */ + return 0; + } else if (cmp > 0) { + /* item is smaller than first element */ + return 1; + } else { + /* item matches first element */ + *found = 1; + return 0; + } + } + + /* execute binary search for item location in list */ + int left = 0; + int right = sm->rank_ack_task.num - 1; + int mid = (left + right) / 2; + while (right > left + 1) { + /* compare item to middle element */ + int cmp = compare_rank_thrd(src_rank, src_thrd, + metas[mid].rank_id, metas[mid].thrd_id); + if (cmp > 0) { + /* item is larger than middle item, + * so bump left range up */ + left = mid; + } else if (cmp < 0) { + /* item is smaller than middle item, + * so move right down to middle */ + right = mid; + } else { + /* found an exact match with middle element */ + *found = 1; + return mid; + } + + /* update middle */ + mid = (left + right) / 2; + } + + /* two elements left in the list, compare to left element */ + int cmp_left = compare_rank_thrd(src_rank, src_thrd, + metas[left].rank_id, + metas[left].thrd_id); + if (cmp_left < 0) { + /* item should come before left element */ + return left; + } else if (cmp_left == 0) { + /* item matches left item */ + *found = 1; + return left; + } + + /* item is larger than left element, so compare to right */ + int cmp_right = compare_rank_thrd(src_rank, src_thrd, + metas[right].rank_id, + metas[right].thrd_id); + if (cmp_right < 0) { + /* item should come before right element */ + return right; + } else if (cmp_right == 0) { + /* item matches right element */ + *found = 1; + return right; + } + + /* otherwise, item must be larger than right element */ + return (right + 1); +} + +/* Insert read reply list for specified (rank_id, thrd_id) delegator + * into ack list, keep ordered by rank,thread for fast lookup */ +static int insert_ack_meta(ack_meta_t* ack, + int pos, + int rank_id, + int thrd_id, + int src_cli_rank) +{ + /* get pointer to array of read reply data for delegators */ + rank_ack_meta_t* metas = sm->rank_ack_task.ack_metas; + + /* check whether insert location is in middle of the list */ + if (pos < sm->rank_ack_task.num) { + /* need to insert in the middle, bump all entries + * past pos up a slot */ + int i; + for (i = sm->rank_ack_task.num - 1; i >= pos; i--) { + metas[i + 1] = metas[i]; + } + } + + /* get pointer to ack meta data structure */ + rank_ack_meta_t* ack_meta = &(metas[pos]); + + /* initialize with values */ + ack_meta->ack_list = arraylist_create(); + ack_meta->rank_id = rank_id; + ack_meta->thrd_id = thrd_id; + ack_meta->src_cli_rank = src_cli_rank; + ack_meta->src_sz = ack->msg.length; + ack_meta->start_cursor = 0; + + /* check that we were able to create a new ack_list */ + if (ack_meta->ack_list == NULL) { + return (int)UNIFYFS_ERROR_NOMEM; + } + + /* insert ack_meta into our list */ + arraylist_add(ack_meta->ack_list, ack); + + /* increment the number of entries in our list */ + sm->rank_ack_task.num++; + + return UNIFYFS_SUCCESS; +} + +/* Send back ack to the remote delegator. + * packs read replies into a send buffer, sends data with isend, + * adds record of send to pending sends list */ +static int sm_ack_remote_delegator(rank_ack_meta_t* ack_meta) +{ + int i; + + /* allocate more send buffers if we're at capacity */ + if (sm->send_buf_list->size == sm->send_buf_list->cap) { + /* at capacity in our list, allocate more space, + * double capacity in array */ + size_t new_cap = 2 * sm->send_buf_list->cap; + sm->send_buf_list->elems = (void**)realloc(sm->send_buf_list->elems, + (new_cap * sizeof(void*))); + + /* initialize pointers in new portion of array */ + for (i = sm->send_buf_list->cap; i < new_cap; i++) { + sm->send_buf_list->elems[i] = NULL; + } + + /* record new capacity */ + sm->send_buf_list->cap = new_cap; + } + + /* attempt to reuse allocated buffer if we can */ + if (sm->send_buf_list->elems[sm->send_buf_list->size] == NULL) { + /* need to allocate a new buffer */ + sm->send_buf_list->elems[sm->send_buf_list->size] = + malloc(SENDRECV_BUF_LEN); + } + + /* get pointer to send buffer */ + char* send_msg_buf = sm->send_buf_list->elems[sm->send_buf_list->size]; + sm->send_buf_list->size++; + + /* running total number of bytes we'll send */ + int send_sz = 0; + + /* compute number of read replies we'll send */ + size_t ack_count = arraylist_size(ack_meta->ack_list); + size_t ack_start = ack_meta->start_cursor; + int len = ack_count - ack_start; + + /* copy in number of read replies to message */ + memcpy(send_msg_buf + send_sz, &len, sizeof(int)); + send_sz += sizeof(int); + + /* pack read replies into send buffer */ + for (i = ack_start; i < ack_count; i++) { + /* get pointer to read reply header */ + ack_meta_t* meta = + (ack_meta_t*)arraylist_get(ack_meta->ack_list, i); + + /* copy read reply header to send buffer */ + memcpy(send_msg_buf + send_sz, &(meta->msg), sizeof(recv_msg_t)); + send_sz += sizeof(recv_msg_t); + + /* copy file data to send buffer */ + size_t length = (size_t) meta->msg.length; + memcpy(send_msg_buf + send_sz, meta->addr, length); + send_sz += (int) length; + } + + /* get rank and thread id of remote delegator */ + int del_rank = ack_meta->rank_id; + int del_thread = ack_meta->thrd_id; + + /* allocate a new ack stat structure to track details + * of pending send */ + ack_stat_t* ack_stat = (ack_stat_t*)malloc(sizeof(ack_stat_t)); + ack_stat->src_rank = del_rank; + ack_stat->src_thrd = del_thread; + + /* send read replies to delegator (rank and thread), + * record MPI request in ack stat structure to wait later */ + MPI_Isend(send_msg_buf, send_sz, MPI_BYTE, + del_rank, (int)READ_RESPONSE_TAG + del_thread, + MPI_COMM_WORLD, &(ack_stat->req)); + + /* add item to our list of pending sends */ + arraylist_add(sm->pended_sends, ack_stat); + + return UNIFYFS_SUCCESS; +} + +/* + * Insert a message to an entry of ack (read reply) list corresponding + * to its destination delegator. + * + * @param mem_addr : address of data to be acked in mem_pool + * @param index : identifies msg to be inserted to ack_lst + * @param src_offset: offset of the requested segment on the logical + * file (not the physical log file on SSD). + * e.g., for N-1 pattern, logical offset + * is the offset in the shared file + * @param len : length of the message + */ +static int insert_to_ack_list(char* mem_addr, + int index, + size_t src_offset, + size_t len, + int errcode) +{ + int rc = 0; + + /* get pointer to read request we are replying to */ + send_msg_t* msg = &sm->service_msgs.msg[index]; + + /* allocate a new structure to record ack meta data */ + ack_meta_t* ack = (ack_meta_t*)malloc(sizeof(ack_meta_t)); + + /* the src_offset might start from any position in the message, so + * make it a separate parameter */ + ack->msg.src_fid = msg->src_fid; /* global file id */ + ack->msg.src_offset = src_offset; /* offset in file */ + ack->msg.length = len; /* length of data */ + ack->msg.errcode = errcode; /* error code for read (pass/fail) */ + ack->addr = mem_addr; /* pointer to data in read buffer */ + + /* after setting the ack for this message, link it + * to a ack list based on its destination. */ + int src_rank = msg->src_delegator_rank; + int src_thrd = msg->src_thrd; + int src_cli_rank = msg->src_dbg_rank; + + /* find the position in the list for target delegator */ + int found = 0; + int pos = find_ack_meta(src_rank, src_thrd, &found); + + /* check whether delegator at this position is the target */ + if (!found) { + /* it's not, so insert new entry for the target into the + * list at this position */ + rc = insert_ack_meta(ack, pos, src_rank, src_thrd, src_cli_rank); + } else { + /* found the corresponding entry for target delegator */ + rank_ack_meta_t* ack_meta = &(sm->rank_ack_task.ack_metas[pos]); + + /* compute number of read replies waiting to be sent */ + int num_entries = arraylist_size(ack_meta->ack_list); + int num_to_ack = num_entries - ack_meta->start_cursor; + + /* number of bytes needed to pack existing read replies */ + size_t curr_bytes = (num_to_ack * sizeof(ack_meta_t)) + + ack_meta->src_sz; + + /* number of bytes to pack this read reply */ + size_t bytes = sizeof(ack_meta_t) + ack->msg.length; + + /* check whether we can fit this data into the + * existing send block */ + if (curr_bytes + bytes > SENDRECV_BUF_LEN) { + /* not enough room, send the current list of read replies */ + LOGDBG("early ack due to full send buffer"); + rc = sm_ack_remote_delegator(ack_meta); + + /* start a new list */ + ack_meta->src_sz = ack->msg.length; + arraylist_add(ack_meta->ack_list, ack); + + /* start_cursor records the starting ack + * for the next send */ + ack_meta->start_cursor += num_to_ack; + + /* check that our reads and sends completed ok */ + if (rc < 0) { + return (int)UNIFYFS_ERROR_SEND; + } + } else { + /* current read reply fits in send buffer, + * add it to the list */ + ack_meta->src_sz += ack->msg.length; + arraylist_add(ack_meta->ack_list, ack); + } + } + + return UNIFYFS_SUCCESS; +} + +/* Insert the data read for each element in read task list to read + * reply list, data will be sent later + * @param read_task : data returned from read task + * @param mem_addr : memory loc to copy read data + * @param start_offset: first offset in read task + * @param end_offset : last offset in read task + * @param errcode : error code on read (pass/fail) + */ +static int batch_insert_to_ack_list(read_task_t* read_task, + char* mem_addr, + int start_offset, + int end_offset, + int errcode) +{ + /* search for the starting read request in service msgs for + * a given region of read_task_t identified by start_offset + * and size of read task */ + + /* read_task + * start_offset end_offset + * read_task_t: -----------******************------------ + * service_msgs:[ ],[*********],[***], [******],[ ] + * + */ + + /* find index in read requests such that it contains starting + * offset for read task */ + int idx = read_task->start_idx; + int cur_offset = 0; + while (1) { + /* get pointer to current read request */ + send_msg_t* msg = &sm->service_msgs.msg[idx]; + + /* check whether end offset of this read request comes + * at or after starting offset of read task */ + if (cur_offset + msg->length >= start_offset) { + /* starting offset of read task falls between start + * and end offset of this read request */ + break; + } + + /* move on to next read request */ + cur_offset += msg->length; + idx++; + } + + /* compute leading bytes in read request that this read task + * does not overlap with */ + int skip_offset = start_offset - cur_offset; + + /* insert read replies for leading read request and all middle + * read requests */ + int first = 1; + int mem_pos = 0; + while (1) { + /* get pointer to read request */ + send_msg_t* msg = &sm->service_msgs.msg[idx]; + + /* stop if we reached the read request that contains the last + * byte of our read task */ + if (cur_offset + msg->length >= end_offset + 1) { + /* ending byte in read task comes before ending byte + * in read request */ + break; + } + + /* compute length and offset of read request that we cover with + * this read task, assume it's the whole read request */ + long length = msg->length; + long offset = msg->src_offset; + if (first == 1) { + /* in the first read request, the read task may start + * partway through, so skip any leading bytes */ + length = msg->length - skip_offset; + offset = msg->src_offset + skip_offset; + first = 0; + } + + /* add entry to read reply list */ + insert_to_ack_list(mem_addr + mem_pos, idx, offset, length, errcode); + + /* update our offset into read reply buffer */ + mem_pos += length; + + /* update offset into read task data */ + cur_offset += length; + + /* move on to next read request */ + idx++; + } + + /* the read task may not fully fill the ending read request */ + if (mem_pos < end_offset - start_offset + 1) { + /* compute remaining length of read task */ + long length = (end_offset - start_offset + 1) - mem_pos; + + /* starting offset for read request */ + long offset = sm->service_msgs.msg[idx].src_offset; + + /* add entry to read reply list */ + insert_to_ack_list(mem_addr + mem_pos, idx, offset, length, errcode); + } + + return UNIFYFS_SUCCESS; +} + +/* + * Wait until all data are read and sent. + * @return success/error + */ +static int sm_wait_until_digested(void) +{ + int i, rc, counter; + read_task_t* read_task; + + /* pointer to array of boolean flags for whether we need to test + * outstanding operations */ + int* flags = NULL; + + /* get number of pending read operations */ + int num_pended_reads = arraylist_size(sm->pended_reads); + if (num_pended_reads > 0) { + /* allocate space for pending flags, setting all to 0 */ + flags = (int*) calloc(num_pended_reads, sizeof(int)); + + /* wait for all pending read operations to complete */ + counter = 0; + while (1) { + /* check whether we have processed all reads */ + if (counter == num_pended_reads) { + LOGDBG("all pending reads completed"); + break; + } + LOGDBG("waiting for %d pending reads", + (num_pended_reads - counter)); + + /* iterate over pending reads */ + for (i = 0; i < num_pended_reads; i++) { + if (flags[i] != 1) { + int errcode = UNIFYFS_SUCCESS; + + /* get meta data for this pending read */ + pended_read_t* pr = + (pended_read_t*)arraylist_get(sm->pended_reads, i); + + read_task = NULL; + if (pr->index != -1) { + read_task = &(sm->read_task_set.read_tasks[pr->index]); + } + + if (pr->err_submit) { /* failed to submit */ + /* mark that this read operation failed */ + errcode = (int)UNIFYFS_ERROR_READ; + flags[i] = 1; + counter++; + + if (NULL != read_task) { + /* add read reply to ack_list as failed */ + batch_insert_to_ack_list(read_task, + pr->mem_pos, + pr->start_pos, + pr->end_pos, + errcode); + } + } else if (aio_error(&pr->read_cb) != EINPROGRESS) { + /* mark that this read operation has completed */ + flags[i] = 1; + counter++; + + /* check that read completed without error */ + ssize_t readrc = aio_return(&pr->read_cb); + if (readrc == -1) { + errcode = errno; + } else if (readrc != (ssize_t)pr->read_cb.aio_nbytes) { + /* short read considered as error */ + errcode = (int)UNIFYFS_ERROR_READ; + } + + if (NULL != read_task) { + /* add read reply to ack_list */ + batch_insert_to_ack_list(read_task, + pr->mem_pos, + pr->start_pos, + pr->end_pos, + errcode); + } + } + } + } + } + + free(flags); + flags = NULL; + + /* reset our list of pending read operations */ + arraylist_reset(sm->pended_reads); + } + + /* read operations have completed, + * send data to delegators */ + + /* iterate over list of delegators and send remaining acks */ + for (i = 0; i < sm->rank_ack_task.num; i++) { + /* get data structure for this delegator */ + rank_ack_meta_t* ack_meta = &(sm->rank_ack_task.ack_metas[i]); + + /* get total number of read replies in this list */ + int tmp_sz = arraylist_size(ack_meta->ack_list); + + /* if we have read replies we have yet to send, + * send them now */ + if (ack_meta->start_cursor < tmp_sz) { + /* got some read replies, send them */ + rc = sm_ack_remote_delegator(ack_meta); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("failed to ack delegator"); + } + } + } + + /* sends issued, now wait for them to complete */ + + /* get number of outstanding sends + * initiated in sm_ack_remote_delegator */ + int num_pended_sends = arraylist_size(sm->pended_sends); + if (num_pended_sends > 0) { + /* allocate space for flags, setting all to 0 */ + flags = (int*)calloc(num_pended_sends, sizeof(int)); + + /* wait until all acks are sent */ + counter = 0; + while (1) { + /* check whether we have waited on all sends */ + if (counter == num_pended_sends) { + LOGDBG("all pending sends completed"); + break; + } + LOGDBG("waiting for %d pending sends", + (num_pended_sends - counter)); + + /* iterate over each send we issued */ + for (i = 0; i < num_pended_sends; i++) { + /* if send is still outstanding, check whether it's done */ + if (flags[i] == 0) { + /* still outstanding, get data for this send */ + ack_stat_t* ack_stat = arraylist_get(sm->pended_sends, i); + + /* test whether send is complete */ + rc = MPI_Test(&ack_stat->req, &flags[i], + &ack_stat->stat); + if (rc != MPI_SUCCESS) { + LOGERR("MPI_Test() for pending send failed"); + } + + /* if send completed, bump our counter */ + if (flags[i] != 0) { + counter++; + } + } + } + } + + free(flags); + flags = NULL; + + /* clear our list of sends */ + arraylist_reset(sm->pended_sends); + } + + /* reset our list of read replies */ + for (i = 0; i < sm->rank_ack_task.num; i++) { + rank_ack_meta_t* ack_meta = &(sm->rank_ack_task.ack_metas[i]); + arraylist_reset(ack_meta->ack_list); + ack_meta->rank_id = -1; + ack_meta->thrd_id = -1; + ack_meta->src_sz = 0; + ack_meta->start_cursor = 0; + } + sm->rank_ack_task.num = 0; + + /* TODO: might be nice to free some send buffers here */ + + /* set active send buffers back to 0, + * we do not free send buffers so that we do not have to + * allocate them again */ + sm->send_buf_list->size = 0; + + return UNIFYFS_SUCCESS; +} + +static int compare_read_task(const void* a, const void* b) +{ + const read_task_t* ptr_a = a; + const read_task_t* ptr_b = b; + + if (ptr_a->size > ptr_b->size) { + return 1; + } + + if (ptr_a->size < ptr_b->size) { + return -1; + } + + if (ptr_a->arrival_time > ptr_b->arrival_time) { + return 1; + } + + if (ptr_a->arrival_time < ptr_b->arrival_time) { + return -1; + } + + return 0; +} + +/* + * Read and send the data via pipelined read, copy and send + * @return success/error + */ +static int sm_read_send_pipe(void) +{ + LOGDBG("processing %d reads", sm->read_task_set.num); + + /* sort read tasks by size and then by arrival time */ + qsort(sm->read_task_set.read_tasks, sm->read_task_set.num, + sizeof(read_task_t), compare_read_task); + + /* points to offset in read reply buffer */ + size_t buf_cursor = 0; + + /* iterate over read tasks and pack read replies into send buffer */ + int i; + for (i = 0; i < sm->read_task_set.num; i++) { + /* get pointer to current read task */ + read_task_t* read_task = &(sm->read_task_set.read_tasks[i]); + + /* get size of data we are to read */ + size_t size = read_task->size; + + /* check whether we have room in the read buffer to hold data */ + if ((buf_cursor + size) > READ_BUF_SZ) { + /* no room, wait until reads complete and send + * out replies */ + LOGDBG("read buf exhausted"); + sm_wait_until_digested(); + + /* reset to start of read buffer */ + buf_cursor = 0; + } + + /* get app id and client id for this read task, + * defines log files holding data */ + int app_id = read_task->app_id; + int cli_id = read_task->cli_id; + + /* look up app config for given app id */ + app_config_t* app_config = (app_config_t*) + arraylist_get(app_config_list, app_id); + assert(app_config); + + /* get index of starting service message */ + int start_idx = read_task->start_idx; + + /* get offset in log file */ + send_msg_t* msg = &sm->service_msgs.msg[start_idx]; + size_t offset = msg->dest_offset; + + /* prepare read opertions based on data location */ + if ((offset + read_task->size) <= app_config->data_size) { + /* requested data in read_task is totally in shared memory, + * get pointer to position in shared memory */ + char* log_ptr = app_config->shm_superblocks[cli_id] + + app_config->data_offset + offset; + + /* copy data into read reply buffer */ + char* buf_ptr = sm->read_buf + buf_cursor; + memcpy(buf_ptr, log_ptr, size); + buf_cursor += size; + + /* we assume memcpy worked */ + int errcode = UNIFYFS_SUCCESS; + + /* prepare read reply meta data */ + batch_insert_to_ack_list(read_task, buf_ptr, + 0, size - 1, errcode); + } else if (offset < app_config->data_size) { + /* part of the requested data is in shared memory, + * compute size in shared memory */ + size_t sz_from_mem = app_config->data_size - offset; + + /* get pointer to position in shared memory */ + char* log_ptr = app_config->shm_superblocks[cli_id] + + app_config->data_offset + offset; + + /* copy data into read reply buffer */ + char* buf_ptr = sm->read_buf + buf_cursor; + memcpy(buf_ptr, log_ptr, sz_from_mem); + buf_cursor += sz_from_mem; + + /* we assume memcpy worked */ + int errcode = UNIFYFS_SUCCESS; + + /* compute size in spillover file */ + long sz_from_ssd = size - sz_from_mem; + + /* read data from spillover file */ + int fd = app_config->spill_log_fds[cli_id]; + ssize_t nread = pread(fd, sm->read_buf + buf_cursor, + sz_from_ssd, 0); + if (nread != (ssize_t)sz_from_ssd) { + /* read error or short read, + * consider either case to be an error */ + errcode = (int)UNIFYFS_ERROR_READ; + } + buf_cursor += sz_from_ssd; + + /* prepare read reply meta data */ + batch_insert_to_ack_list(read_task, buf_ptr, + 0, size - 1, errcode); + } else { + /* all requested data in the current read task + * are in spillover files */ + int fd = app_config->spill_log_fds[cli_id]; + + /* allocate empty pending read structure */ + pended_read_t* ptr = + (pended_read_t*)malloc(sizeof(pended_read_t)); + + /* fill in aio fields */ + memset(&ptr->read_cb, 0, sizeof(struct aiocb)); + ptr->read_cb.aio_fildes = fd; + ptr->read_cb.aio_buf = sm->read_buf + buf_cursor; + ptr->read_cb.aio_offset = offset - app_config->data_size; + ptr->read_cb.aio_nbytes = size; + + /* index of read task for this pending read */ + ptr->index = i; + + /* offset locations in generating read request */ + ptr->start_pos = 0; + ptr->end_pos = size - 1; + + /* send buffer location to copy data when complete */ + ptr->mem_pos = sm->read_buf + buf_cursor; + + /* submit read as aio operation */ + ptr->err_submit = 0; + int rc = aio_read(&ptr->read_cb); + if (rc < 0) { + /* remember that we failed to submit this read */ + ptr->err_submit = 1; + } + buf_cursor += size; + + /* enqueue entry in our list of pending reads */ + arraylist_add(sm->pended_reads, ptr); + } + + /* update accounting for burst size */ + sm->burst_data_sz += size; + } + + /* have initiated all read tasks, wait for them to finish + * and send results to delegators */ + sm_wait_until_digested(); + + return UNIFYFS_SUCCESS; +} + +/* Decode the read-request message received from request_manager + * + * @param msg_buf: message buffer containing request(s) + * @return success/error code + */ +int sm_decode_msg(char* msg_buf) +{ + /* get pointer to start of receive buffer */ + char* ptr = msg_buf; + + /* advance past command */ + ptr += sizeof(int); + + /* extract number of read requests in message */ + int num = *((int*)ptr); + ptr += sizeof(int); + + /* get pointer to read request */ + send_msg_t* msg = (send_msg_t*)ptr; + + assert(NULL != sm); + SM_LOCK(); + + /* get current timestamp as integer */ + int now = (int)time(NULL); + + LOGDBG("decoding %d requests", num); + + /* allocate a larger array of service requests if needed */ + if (num + sm->service_msgs.num >= sm->service_msgs.cap) { + /* get a larger buffer (2x what is currently needed) */ + size_t count = 2 * (num + sm->service_msgs.num); + + /* allocate larger buffer (2x what is currently needed) */ + size_t bytes = count * sizeof(send_msg_t); + sm->service_msgs.msg = + (send_msg_t*)realloc(sm->service_msgs.msg, bytes); + if (sm->service_msgs.msg == NULL) { + /* failed to allocate memory */ + SM_UNLOCK(); + return (int)UNIFYFS_ERROR_NOMEM; + } + + /* got the memory, increase the capacity */ + sm->service_msgs.cap = count; + + /* allocate corresponding space for read tasks */ + bytes = count * sizeof(read_task_t); + sm->read_task_set.read_tasks = + (read_task_t*)realloc(sm->read_task_set.read_tasks, bytes); + if (sm->read_task_set.read_tasks == NULL) { + /* failed to allocate memory */ + SM_UNLOCK(); + return (int)UNIFYFS_ERROR_NOMEM; + } + + /* got the memory, increase the capacity */ + sm->read_task_set.cap = count; + } + + /* unpack read requests to fill in service messages */ + int iter; + for (iter = 0; iter < num; iter++) { + /* copy values from read request */ + sm->service_msgs.msg[sm->service_msgs.num] = msg[iter]; + + /* set time stamp on when we received this request */ + sm->service_msgs.msg[sm->service_msgs.num].arrival_time = now; + + /* increment the number of service requests + * and go to next read request */ + sm->service_msgs.num++; + } + + SM_UNLOCK(); + + return UNIFYFS_SUCCESS; +} + +/* Decode and issue chunk-reads received from request manager + * + * @param src_rank : source delegator rank + * @param src_app_id : app id at source delegator + * @param src_client_id : client id at source delegator + * @param src_req_id : request id at source delegator + * @param num_chks : number of chunk requests + * @param msg_buf : message buffer containing request(s) + * @return success/error code + */ +int sm_issue_chunk_reads(int src_rank, + int src_app_id, + int src_client_id, + int src_req_id, + int num_chks, + char* msg_buf) +{ + /* get pointer to start of receive buffer */ + char* ptr = msg_buf; + + /* advance past command */ + ptr += sizeof(int); + + /* extract number of chunk read requests */ + int num = *((int*)ptr); + ptr += sizeof(int); + assert(num == num_chks); + + size_t total_data_sz = *((size_t*)ptr); + ptr += sizeof(size_t); + + /* get pointer to read request */ + chunk_read_req_t* reqs = (chunk_read_req_t*)ptr; + + remote_chunk_reads_t* rcr = (remote_chunk_reads_t*) + calloc(1, sizeof(remote_chunk_reads_t)); + if (NULL == rcr) { + LOGERR("failed to allocate remote_chunk_reads"); + return UNIFYFS_ERROR_NOMEM; + } + rcr->rank = src_rank; + rcr->app_id = src_app_id; + rcr->client_id = src_client_id; + rcr->rdreq_id = src_req_id; + rcr->num_chunks = num_chks; + rcr->reqs = NULL; + + size_t resp_sz = sizeof(chunk_read_resp_t) * num_chks; + size_t buf_sz = resp_sz + total_data_sz; + rcr->total_sz = buf_sz; + + // NOTE: calloc() is required here, don't use malloc + char* crbuf = (char*) calloc(1, buf_sz); + if (NULL == crbuf) { + LOGERR("failed to allocate chunk_read_reqs"); + free(rcr); + return UNIFYFS_ERROR_NOMEM; + } + chunk_read_resp_t* resp = (chunk_read_resp_t*)crbuf; + rcr->resp = resp; + + char* databuf = crbuf + resp_sz; + + LOGDBG("issuing %d requests, total data size = %zu", + num_chks, total_data_sz); + + /* points to offset in read reply buffer */ + size_t buf_cursor = 0; + + int i; + int last_app = -1; + app_config_t* app_config = NULL; + for (i = 0; i < num_chks; i++) { + chunk_read_req_t* rreq = reqs + i; + chunk_read_resp_t* rresp = resp + i; + + /* get size of data we are to read */ + size_t size = rreq->nbytes; + size_t offset = rreq->log_offset; + + /* record request metadata in response */ + rresp->nbytes = size; + rresp->offset = rreq->offset; + LOGDBG("reading chunk(offset=%zu, size=%zu)", rreq->offset, size); + + /* get app id and client id for this read task, + * defines log files holding data */ + int app_id = rreq->log_app_id; + int cli_id = rreq->log_client_id; + if (app_id != last_app) { + /* look up app config for given app id */ + app_config = (app_config_t*) + arraylist_get(app_config_list, app_id); + assert(app_config); + last_app = app_id; + } + int spillfd = app_config->spill_log_fds[cli_id]; + char* log_ptr = app_config->shm_superblocks[cli_id] + + app_config->data_offset + offset; + + char* buf_ptr = databuf + buf_cursor; + + /* prepare read opertions based on data location */ + size_t sz_from_mem = 0; + size_t sz_from_spill = 0; + if ((offset + size) <= app_config->data_size) { + /* requested data is totally in shared memory */ + sz_from_mem = size; + } else if (offset < app_config->data_size) { + /* part of the requested data is in shared memory */ + sz_from_mem = app_config->data_size - offset; + sz_from_spill = size - sz_from_mem; + } else { + /* all requested data is in spillover file */ + sz_from_spill = size; + } + if (sz_from_mem > 0) { + /* read data from shared memory */ + memcpy(buf_ptr, log_ptr, sz_from_mem); + rresp->read_rc = sz_from_mem; + } + if (sz_from_spill > 0) { + /* read data from spillover file */ + ssize_t nread = pread(spillfd, (buf_ptr + sz_from_mem), + sz_from_spill, 0); + if (-1 == nread) { + rresp->read_rc = (ssize_t)(-errno); + } else { + rresp->read_rc += nread; + } + } + buf_cursor += size; + + /* update accounting for burst size */ + sm->burst_data_sz += size; + } + + if (src_rank != glb_mpi_rank) { + /* add chunk_reads to svcmgr response list */ + LOGDBG("adding to svcmgr chunk_reads"); + assert(NULL != sm); + SM_LOCK(); + arraylist_add(sm->chunk_reads, rcr); + SM_UNLOCK(); + LOGDBG("done adding to svcmgr chunk_reads"); + return UNIFYFS_SUCCESS; + } else { /* response is for myself */ + LOGDBG("responding to myself"); + int rc = rm_post_chunk_read_responses(src_app_id, src_client_id, + src_rank, src_req_id, + num_chks, buf_sz, crbuf); + if (rc != (int)UNIFYFS_SUCCESS) { + LOGERR("failed to handle chunk read responses"); + } + /* clean up allocated buffers */ + free(rcr); + return rc; + } +} + +/* initialize and launch service manager thread */ +int svcmgr_init(void) +{ + sm = (svcmgr_state_t*)calloc(1, sizeof(svcmgr_state_t)); + if (NULL == sm) { + LOGERR("failed to allocate service manager state!"); + return (int)UNIFYFS_ERROR_NOMEM; + } + + /* allocate a buffer to hold read data before packing + * into send buffers */ + sm->read_buf = malloc(READ_BUF_SZ); + if (sm->read_buf == NULL) { + LOGERR("failed to allocate service manager read buffer!"); + svcmgr_fini(); + return (int)UNIFYFS_ERROR_NOMEM; + } + + /* tracks how much data we process in each burst */ + sm->burst_data_sz = 0; + + /* allocate a list to track chunk read requests */ + sm->chunk_reads = arraylist_create(); + if (sm->chunk_reads == NULL) { + LOGERR("failed to allocate service manager chunk_reads!"); + svcmgr_fini(); + return (int)UNIFYFS_ERROR_NOMEM; + } + + /* initialize our data structure for holding read requests */ + size_t bytes = MAX_META_PER_SEND * sizeof(send_msg_t); + sm->service_msgs.msg = (send_msg_t*)malloc(bytes); + sm->service_msgs.num = 0; + sm->service_msgs.cap = MAX_META_PER_SEND; + + /* initialize our data structure for holding read tasks */ + bytes = MAX_META_PER_SEND * sizeof(read_task_t); + sm->read_task_set.read_tasks = (read_task_t*)malloc(bytes); + sm->read_task_set.num = 0; + sm->read_task_set.cap = MAX_META_PER_SEND; + + /* allocate a list to track pending data read operations */ + sm->pended_reads = arraylist_create(); + if (sm->pended_reads == NULL) { + LOGERR("failed to allocate service manager pended_reads!"); + svcmgr_fini(); + return (int)UNIFYFS_ERROR_NOMEM; + } + + /* allocate a list to track outstanding sends back to + * requesting delegators */ + sm->pended_sends = arraylist_create(); + if (sm->pended_sends == NULL) { + LOGERR("failed to allocate service manager pended_sends!"); + svcmgr_fini(); + return (int)UNIFYFS_ERROR_NOMEM; + } + + /* allocate a list to track pending send operations + * that need to be initiated back to requesting delegators */ + sm->send_buf_list = arraylist_create(); + if (sm->send_buf_list == NULL) { + LOGERR("failed to allocate service manager send_buf_list!"); + svcmgr_fini(); + return (int)UNIFYFS_ERROR_NOMEM; + } + + /* allocate memory to hold meta data for read replies */ + bytes = glb_mpi_size * MAX_NUM_CLIENTS * sizeof(rank_ack_meta_t); + sm->rank_ack_task.num = 0; + sm->rank_ack_task.ack_metas = (rank_ack_meta_t*)malloc(bytes); + + /* initialize each ack_meta structure, keep one for each potential + * application client + * (num delegaotrs * max num clients per delegator) */ + int i; + for (i = 0; i < glb_mpi_size * MAX_NUM_CLIENTS; i++) { + /* get pointer to current structure */ + rank_ack_meta_t* ack_meta = &(sm->rank_ack_task.ack_metas[i]); + ack_meta->ack_list = NULL; + ack_meta->rank_id = -1; + ack_meta->thrd_id = -1; + ack_meta->src_sz = 0; + ack_meta->start_cursor = 0; + } + + int rc = pthread_mutex_init(&(sm->sync), NULL); + if (0 != rc) { + LOGERR("failed to initialize service manager mutex!"); + svcmgr_fini(); + return (int)UNIFYFS_ERROR_THRDINIT; + } + + sm->initialized = 1; + + rc = pthread_create(&(sm->thrd), NULL, + sm_service_reads, (void*)sm); + if (rc != 0) { + LOGERR("failed to create service manager thread"); + svcmgr_fini(); + return (int)UNIFYFS_ERROR_THRDINIT; + } + + return (int)UNIFYFS_SUCCESS; +} + +/* join service manager thread (if created) and clean up state */ +int svcmgr_fini(void) +{ + if (NULL != sm) { + if (sm->thrd) { + // int exit_cmd = (int)SVC_CMD_EXIT; + // MPI_Send(&exit_cmd, sizeof(int), MPI_CHAR, glb_mpi_rank, + // (int)READ_REQUEST_TAG, MPI_COMM_WORLD); + sm->time_to_exit = 1; + pthread_join(sm->thrd, NULL); + } + if (sm->initialized) { + SM_LOCK(); + } + + arraylist_free(sm->chunk_reads); + arraylist_free(sm->pended_reads); + arraylist_free(sm->pended_sends); + arraylist_free(sm->send_buf_list); + + if (NULL != sm->service_msgs.msg) { + free(sm->service_msgs.msg); + } + if (NULL != sm->read_task_set.read_tasks) { + free(sm->read_task_set.read_tasks); + } + + int i; + for (i = 0; i < sm->rank_ack_task.num; i++) { + arraylist_free(sm->rank_ack_task.ack_metas[i].ack_list); + } + + if (sm->initialized) { + SM_UNLOCK(); + pthread_mutex_destroy(&(sm->sync)); + } + + free(sm); + sm = NULL; + } + return (int)UNIFYFS_SUCCESS; +} + +/* iterate over list of chunk reads and send responses */ +static int send_chunk_read_responses(void) +{ + int rc = (int)UNIFYFS_SUCCESS; + pthread_mutex_lock(&(sm->sync)); + int num_chunk_reads = arraylist_size(sm->chunk_reads); + if (num_chunk_reads) { + LOGDBG("processing %d chunk read responses", num_chunk_reads); + for (int i = 0; i < num_chunk_reads; i++) { + /* get data structure */ + remote_chunk_reads_t* rcr = (remote_chunk_reads_t*) + arraylist_get(sm->chunk_reads, i); + rc = invoke_chunk_read_response_rpc(rcr); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("failed to send chunk read responses"); + } + } + arraylist_reset(sm->chunk_reads); + } + pthread_mutex_unlock(&(sm->sync)); + return rc; +} + +/* entry point for the service thread, service the read requests + * received from the requesting delegators, executes a loop constantly + * waiting on incoming message, for each message that comes in, + * appends read requests to sm->service_msgs list, if no message has + * arrived, and after some wait time (to catch bursty reads), + * then convert read requests into read tasks, execute read tasks + * to build read replies, and send read replies back to delegators */ +void* sm_service_reads(void* ctx) +{ + int rc; + + LOGDBG("I am service manager thread!"); + assert(sm == (svcmgr_state_t*)ctx); + + /* received message format: + * cmd, req_num, a list of read requests */ + + /* buffer to hold received msg, + * since this is large, malloc it instead of declare it on stack + * (mitigate problems thread stack size limits) */ + char* req_msg_buf = (char*)malloc(REQ_BUF_LEN); + + /* counter for timed wait before creating read tasks, + * used to buffer multiple read requests before responding + * with the idea that reads come in bursts */ + int wait_time = 0; + + /* initialize value on how long to wait before processing + * incoming read requests */ + long bursty_interval = MAX_BURSTY_INTERVAL / 10; + + /* listen and server incoming requests until signaled to exit */ + MPI_Status status; + while (!sm->time_to_exit) { + /* post a receive for incoming request */ + MPI_Request request; + MPI_Irecv(req_msg_buf, REQ_BUF_LEN, MPI_BYTE, + MPI_ANY_SOURCE, (int)READ_REQUEST_TAG, + MPI_COMM_WORLD, &request); + + /* test whether we received anything */ + int irecv_flag = 0; + int mpi_rc = MPI_Test(&request, &irecv_flag, &status); + if (mpi_rc != MPI_SUCCESS) { + sm->sm_exit_rc = (int)UNIFYFS_ERROR_RECV; + return NULL; + } + + send_chunk_read_responses(); + + /* as long as we keep receiving requests, we'll keep skipping + * the while loop below (and its sleep) and keep appending + * items to our read request queue */ + + /* + * keep receiving the read request + * until the end of a anticipated + * bursty behavior + * */ + while (!irecv_flag) { + + if (sm->time_to_exit) { + break; + } + + send_chunk_read_responses(); + + /* if we have not received anything, sleep */ + if (bursty_interval > MIN_SLEEP_INTERVAL) { + usleep(SLEEP_INTERVAL); /* wait an interval */ + wait_time += SLEEP_INTERVAL; + } + + /* a bursty behavior is considered end when + * wait time is larger than BURSTY_INTERVAL + * */ + if ((wait_time >= bursty_interval) || + (bursty_interval <= MIN_SLEEP_INTERVAL)) { + /* if time to wait has expired, and if we have some + * queued read requests, do some work */ + + pthread_mutex_lock(&(sm->sync)); + if (sm->service_msgs.num > 0) { + /* run through list of read requests and generate + * read tasks, merge requests for contiguous data + * into a single read task */ + rc = sm_cluster_reads(); + if (rc != 0) { + sm->sm_exit_rc = rc; + pthread_mutex_unlock(&(sm->sync)); + return NULL; + } + + /* execute read tasks, wait for them to complete, + * then pack read replies, send to delegators, and + * finally wait for sends to complete */ + rc = sm_read_send_pipe(); + if (rc != 0) { + sm->sm_exit_rc = rc; + pthread_mutex_unlock(&(sm->sync)); + return NULL; + } + + /* have processed all read requests and read tasks, + * prep them for next message */ + sm->service_msgs.num = 0; + sm->read_task_set.num = 0; + + /* determine how long to wait next time based on + * how much data we just processed in this burst */ + if (sm->burst_data_sz >= LARGE_BURSTY_DATA) { + /* for large bursts above a threshold, + * wait for a fixed amount of time */ + bursty_interval = MAX_BURSTY_INTERVAL; + } else { + /* for smaller bursts, set delay proportionally + * to burst size we just processed */ + bursty_interval = + (SLEEP_SLICE_PER_UNIT * sm->burst_data_sz) / MIB; + } + + /* reset our burst size counter */ + sm->burst_data_sz = 0; + } + pthread_mutex_unlock(&(sm->sync)); + + /* reset our timer */ + wait_time = 0; + } + + /* test for receive again, will break while loop + * if we find something */ + mpi_rc = MPI_Test(&request, &irecv_flag, &status); + if (mpi_rc != MPI_SUCCESS) { + sm->sm_exit_rc = (int)UNIFYFS_ERROR_RECV; + return NULL; + } + } + + if (irecv_flag) { + /* got a message, reset wait time */ + wait_time = 0; + + /* first value of receive buffer is integer holding command */ + int reqcmd = *((int*)req_msg_buf); + if (reqcmd == (int)SVC_CMD_RDREQ_MSG) { + /* got a request for data, append read requests in message + * to our sm->service_msgs list */ + sm_decode_msg(req_msg_buf); + } else if (reqcmd == (int)SVC_CMD_EXIT) { + /* time to exit */ + sm->time_to_exit = 1; + } + } + } + + /* free receive buffer */ + free(req_msg_buf); + req_msg_buf = NULL; + + LOGDBG("service manager thread exiting"); + + sm->sm_exit_rc = (int)UNIFYFS_SUCCESS; + return NULL; +} + +/* BEGIN MARGO SERVER-SERVER RPC INVOCATION FUNCTIONS */ + +/* invokes the chunk_read_response rpc */ +int invoke_chunk_read_response_rpc(remote_chunk_reads_t* rcr) +{ + int rc = (int)UNIFYFS_SUCCESS; + int dst_srvr_rank = rcr->rank; + hg_handle_t handle; + chunk_read_response_in_t in; + chunk_read_response_out_t out; + hg_return_t hret; + hg_addr_t dst_srvr_addr; + hg_size_t bulk_sz = rcr->total_sz; + void* data_buf = (void*)rcr->resp; + + assert(dst_srvr_rank < (int)glb_num_servers); + dst_srvr_addr = glb_servers[dst_srvr_rank].margo_svr_addr; + + hret = margo_create(unifyfsd_rpc_context->svr_mid, dst_srvr_addr, + unifyfsd_rpc_context->rpcs.chunk_read_response_id, + &handle); + assert(hret == HG_SUCCESS); + + /* fill in input struct */ + in.src_rank = (int32_t)glb_mpi_rank; + in.app_id = (int32_t)rcr->app_id; + in.client_id = (int32_t)rcr->client_id; + in.req_id = (int32_t)rcr->rdreq_id; + in.num_chks = (int32_t)rcr->num_chunks; + in.bulk_size = bulk_sz; + + /* register request buffer for bulk remote access */ + hret = margo_bulk_create(unifyfsd_rpc_context->svr_mid, 1, + &data_buf, &bulk_sz, + HG_BULK_READ_ONLY, &in.bulk_handle); + assert(hret == HG_SUCCESS); + + LOGDBG("invoking the chunk-read-response rpc function"); + hret = margo_forward(handle, &in); + if (hret != HG_SUCCESS) { + rc = (int)UNIFYFS_FAILURE; + } else { + /* decode response */ + hret = margo_get_output(handle, &out); + if (hret == HG_SUCCESS) { + rc = (int)out.ret; + LOGDBG("chunk-read-response rpc to %d - ret=%d", + dst_srvr_rank, rc); + margo_free_output(handle, &out); + } else { + rc = (int)UNIFYFS_FAILURE; + } + } + + margo_bulk_free(in.bulk_handle); + margo_destroy(handle); + + /* free response data buffer */ + free(data_buf); + rcr->resp = NULL; + + return rc; +} + +/* BEGIN MARGO SERVER-SERVER RPC HANDLERS */ + +/* handler for server-server hello + * + * print the message, and return my rank */ +static void server_hello_rpc(hg_handle_t handle) +{ + int rc, src_rank; + hg_return_t hret; + char* msg; + server_hello_in_t in; + server_hello_out_t out; + + /* get input params */ + rc = margo_get_input(handle, &in); + assert(rc == HG_SUCCESS); + src_rank = (int)in.src_rank; + msg = strdup(in.message_str); + if (NULL != msg) { + LOGDBG("got message '%s' from server %d", msg, src_rank); + free(msg); + } + + /* fill output structure to return to caller */ + out.ret = (int32_t)glb_mpi_rank; + + /* send output back to caller */ + hret = margo_respond(handle, &out); + assert(hret == HG_SUCCESS); + + /* free margo resources */ + margo_free_input(handle, &in); + margo_destroy(handle); +} +DEFINE_MARGO_RPC_HANDLER(server_hello_rpc) + +/* handler for server-server request + * + * decode payload based on tag, and call appropriate svcmgr routine */ +static void server_request_rpc(hg_handle_t handle) +{ + int rc, req_id, src_rank, tag; + int32_t ret; + hg_return_t hret; + hg_bulk_t bulk_handle; + size_t bulk_sz; + server_request_in_t in; + server_request_out_t out; + + /* get input params */ + rc = margo_get_input(handle, &in); + assert(rc == HG_SUCCESS); + src_rank = (int)in.src_rank; + req_id = (int)in.req_id; + tag = (int)in.req_tag; + bulk_sz = (size_t)in.bulk_size; + + LOGDBG("handling request from server %d: tag=%d req=%d sz=%zu", + src_rank, tag, req_id, bulk_sz); + + /* get margo info */ + const struct hg_info* hgi = margo_get_info(handle); + assert(NULL != hgi); + margo_instance_id mid = margo_hg_info_get_instance(hgi); + assert(mid != MARGO_INSTANCE_NULL); + + int reqcmd = 0; + void* reqbuf = NULL; + if (bulk_sz) { + /* allocate and register local target buffer for bulk access */ + reqbuf = malloc(bulk_sz); + if (NULL == reqbuf) { + ret = (int32_t)UNIFYFS_ERROR_NOMEM; + goto request_out; + } + hret = margo_bulk_create(mid, 1, &reqbuf, &in.bulk_size, + HG_BULK_WRITE_ONLY, &bulk_handle); + assert(hret == HG_SUCCESS); + + /* pull request data */ + hret = margo_bulk_transfer(mid, HG_BULK_PULL, hgi->addr, + in.bulk_handle, 0, + bulk_handle, 0, in.bulk_size); + assert(hret == HG_SUCCESS); + reqcmd = *(int*)reqbuf; + } + + switch (tag) { + case (int)READ_REQUEST_TAG: { + /* verify this is a request for data */ + if (reqcmd == (int)SVC_CMD_RDREQ_MSG) { + LOGDBG("request command: SVC_CMD_RDREQ_MSG"); + /* got a request for data, append read requests in message + * to our sm->service_msgs list */ + sm_decode_msg((char*)reqbuf); + ret = (int32_t)UNIFYFS_SUCCESS; + } else { + LOGERR("invalid request command %d from server %d", + reqcmd, src_rank); + ret = (int32_t)UNIFYFS_ERROR_INVAL; + } + break; + } + default: { + LOGERR("invalid request tag %d", tag); + ret = (int32_t)UNIFYFS_ERROR_INVAL; + break; + } + } + +request_out: + + /* fill output structure and return to caller */ + out.ret = ret; + hret = margo_respond(handle, &out); + assert(hret == HG_SUCCESS); + + /* free margo resources */ + margo_free_input(handle, &in); + if (NULL != reqbuf) { + margo_bulk_free(bulk_handle); + free(reqbuf); + } + margo_destroy(handle); +} +DEFINE_MARGO_RPC_HANDLER(server_request_rpc) + +/* handler for server-server request + * + * decode payload based on tag, and call appropriate svcmgr routine */ +static void chunk_read_request_rpc(hg_handle_t handle) +{ + int rc, req_id, num_chks; + int src_rank, app_id, client_id; + int32_t ret; + hg_return_t hret; + hg_bulk_t bulk_handle; + size_t bulk_sz; + chunk_read_request_in_t in; + chunk_read_request_out_t out; + + /* get input params */ + rc = margo_get_input(handle, &in); + assert(rc == HG_SUCCESS); + src_rank = (int)in.src_rank; + app_id = (int)in.app_id; + client_id = (int)in.client_id; + req_id = (int)in.req_id; + num_chks = (int)in.num_chks; + bulk_sz = (size_t)in.bulk_size; + + LOGDBG("handling chunk read request from server %d: " + "req=%d num_chunks=%d bulk_sz=%zu", + src_rank, req_id, num_chks, bulk_sz); + + /* get margo info */ + const struct hg_info* hgi = margo_get_info(handle); + assert(NULL != hgi); + margo_instance_id mid = margo_hg_info_get_instance(hgi); + assert(mid != MARGO_INSTANCE_NULL); + + int reqcmd = (int)SVC_CMD_INVALID; + void* reqbuf = NULL; + if (bulk_sz) { + /* allocate and register local target buffer for bulk access */ + reqbuf = malloc(bulk_sz); + if (NULL != reqbuf) { + hret = margo_bulk_create(mid, 1, &reqbuf, &in.bulk_size, + HG_BULK_WRITE_ONLY, &bulk_handle); + assert(hret == HG_SUCCESS); + + /* pull request data */ + hret = margo_bulk_transfer(mid, HG_BULK_PULL, hgi->addr, + in.bulk_handle, 0, + bulk_handle, 0, in.bulk_size); + assert(hret == HG_SUCCESS); + reqcmd = *(int*)reqbuf; + } + } + /* verify this is a request for data */ + if (reqcmd == (int)SVC_CMD_RDREQ_CHK) { + LOGDBG("request command: SVC_CMD_RDREQ_CHK"); + /* chunk read request */ + sm_issue_chunk_reads(src_rank, app_id, client_id, req_id, + num_chks, (char*)reqbuf); + ret = (int32_t)UNIFYFS_SUCCESS; + } else { + LOGERR("invalid chunk read request command %d from server %d", + reqcmd, src_rank); + ret = (int32_t)UNIFYFS_ERROR_INVAL; + } + + /* fill output structure and return to caller */ + out.ret = ret; + hret = margo_respond(handle, &out); + assert(hret == HG_SUCCESS); + + /* free margo resources */ + margo_free_input(handle, &in); + if (NULL != reqbuf) { + margo_bulk_free(bulk_handle); + free(reqbuf); + } + margo_destroy(handle); +} +DEFINE_MARGO_RPC_HANDLER(chunk_read_request_rpc) diff --git a/server/src/unifyfs_service_manager.h b/server/src/unifyfs_service_manager.h new file mode 100644 index 000000000..2e055c37c --- /dev/null +++ b/server/src/unifyfs_service_manager.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2017, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +/* + * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * Copyright (c) 2017, Florida State University. Contributions from + * the Computer Architecture and Systems Research Laboratory (CASTL) + * at the Department of Computer Science. + * + * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror + * LLNL-CODE-728877. All rights reserved. + * + * This file is part of burstfs. + * For details, see https://github.com/llnl/burstfs + * Please read https://github.com/llnl/burstfs/LICENSE for full license text. + */ + +#ifndef UNIFYFS_SERVICE_MANAGER_H +#define UNIFYFS_SERVICE_MANAGER_H + +#include "unifyfs_global.h" + +/* service manager pthread routine */ +void* sm_service_reads(void* ctx); + +/* initialize and launch service manager */ +int svcmgr_init(void); + +/* join service manager thread and cleanup its state */ +int svcmgr_fini(void); + +/* process service request message */ +int sm_decode_msg(char* msg_buf); + +/* decode and issue chunk reads contained in message buffer */ +int sm_issue_chunk_reads(int src_rank, + int src_app_id, + int src_client_id, + int src_req_id, + int num_chks, + char* msg_buf); + +/* MARGO SERVER-SERVER RPC INVOCATION FUNCTIONS */ +int invoke_chunk_read_response_rpc(remote_chunk_reads_t* rcr); + +#endif // UNIFYFS_SERVICE_MANAGER_H diff --git a/server/src/unifyfs_sock.c b/server/src/unifyfs_sock.c new file mode 100644 index 000000000..27937fc93 --- /dev/null +++ b/server/src/unifyfs_sock.c @@ -0,0 +1,289 @@ +/* + * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2017, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +/* + * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * Copyright (c) 2017, Florida State University. Contributions from + * the Computer Architecture and Systems Research Laboratory (CASTL) + * at the Department of Computer Science. + * + * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror + * LLNL-CODE-728877. All rights reserved. + * + * This file is part of burstfs. + * For details, see https://github.com/llnl/burstfs + * Please read https://github.com/llnl/burstfs/LICENSE for full license text. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "arraylist.h" +#include "unifyfs_const.h" +#include "unifyfs_global.h" +#include "unifyfs_keyval.h" +#include "unifyfs_log.h" +#include "unifyfs_sock.h" + +char sock_path[UNIFYFS_MAX_FILENAME]; +int server_sockfd = -1; +int num_fds; +struct pollfd poll_set[MAX_NUM_CLIENTS]; +struct sockaddr_un server_address; + +int detached_sock_idx = -1; +int cur_sock_idx = -1; + +char cmd_buf[MAX_NUM_CLIENTS][CMD_BUF_SIZE]; +char ack_buf[MAX_NUM_CLIENTS][CMD_BUF_SIZE]; +int ack_msg[3] = {0}; + +/* initialize the listening socket on this delegator + * @return success/error code */ +int sock_init_server(int srvr_id) +{ + int i, rc; + + for (i = 0; i < MAX_NUM_CLIENTS; i++) { + poll_set[i].fd = -1; + } + + num_fds = 0; + + snprintf(sock_path, sizeof(sock_path), "%s.%d.%d", + SOCKET_PATH, getuid(), srvr_id); + LOGDBG("domain socket path is %s", sock_path); + unlink(sock_path); // remove domain socket leftover from prior run + + server_sockfd = socket(AF_UNIX, SOCK_STREAM, 0); + + memset(&server_address, 0, sizeof(server_address)); + server_address.sun_family = AF_UNIX; + strcpy(server_address.sun_path, sock_path); + rc = bind(server_sockfd, (struct sockaddr*)&server_address, + (socklen_t)sizeof(server_address)); + if (rc != 0) { + close(server_sockfd); + return -1; + } + + rc = listen(server_sockfd, MAX_NUM_CLIENTS); + if (rc != 0) { + close(server_sockfd); + return -1; + } + + sock_add(server_sockfd); // puts server fd at index 0 of poll_set + LOGDBG("completed sock init server"); + + // publish domain socket path + unifyfs_keyval_publish_local(key_unifyfsd_socket, sock_path); + + return 0; +} + +void sock_sanitize_client(int client_idx) +{ + /* close socket for this client id + * and set fd back to -1 */ + if (poll_set[client_idx].fd != -1) { + close(poll_set[client_idx].fd); + poll_set[client_idx].fd = -1; + } +} + +int sock_sanitize(void) +{ + int i; + for (i = 0; i < num_fds; i++) { + sock_sanitize_client(i); + } + + if (server_sockfd != -1) { + server_sockfd = -1; + unlink(sock_path); + } + + return 0; +} + +int sock_add(int fd) +{ + if (num_fds == MAX_NUM_CLIENTS) { + LOGERR("exceeded MAX_NUM_CLIENTS"); + return -1; + } + + int flag = fcntl(fd, F_GETFL); + fcntl(fd, F_SETFL, flag | O_NONBLOCK); + + LOGDBG("sock_adding fd: %d", fd); + poll_set[num_fds].fd = fd; + poll_set[num_fds].events = POLLIN | POLLHUP; + poll_set[num_fds].revents = 0; + num_fds++; + return 0; +} + +void sock_reset(void) +{ + int i; + cur_sock_idx = -1; + detached_sock_idx = -1; + for (i = 0; i < num_fds; i++) { + poll_set[i].events = POLLIN | POLLHUP; + poll_set[i].revents = 0; + } +} + +int sock_remove(int client_idx) +{ + /* in this case, we simply disable the disconnected + * file descriptor. */ + poll_set[client_idx].fd = -1; + return 0; +} + +/* + * wait for the client-side command + * */ + +int sock_wait_cmd(int poll_timeout) +{ + int rc, i, client_fd; + + sock_reset(); + rc = poll(poll_set, num_fds, poll_timeout); + if (rc < 0) { + return (int)UNIFYFS_ERROR_POLL; + } else if (rc == 0) { // timeout + return (int)UNIFYFS_SUCCESS; + } else { + LOGDBG("poll detected socket activity"); + for (i = 0; i < num_fds; i++) { + if (poll_set[i].fd == -1) { + continue; + } + if (i == 0) { // listening socket + if (poll_set[i].revents & POLLIN) { + int client_len = sizeof(struct sockaddr_un); + struct sockaddr_un client_address; + client_fd = accept(server_sockfd, + (struct sockaddr*)&client_address, + (socklen_t*)&client_len); + LOGDBG("accepted client on socket %d", client_fd); + rc = sock_add(client_fd); + if (rc < 0) { + return (int)UNIFYFS_ERROR_SOCKET_FD_EXCEED; + } + } else if (poll_set[i].revents & POLLERR) { + // unknown error on listening socket + return (int)UNIFYFS_ERROR_SOCK_LISTEN; + } + } else { // (i != 0) client sockets + rc = 0; + if (poll_set[i].revents & POLLIN) { + ssize_t bytes_read = read(poll_set[i].fd, + cmd_buf[i], CMD_BUF_SIZE); + if (bytes_read == 0) { + rc = (int)UNIFYFS_ERROR_SOCK_DISCONNECT; + } else { // read a client command + cur_sock_idx = i; + return UNIFYFS_SUCCESS; + } + } else if (poll_set[i].revents & POLLHUP) { + rc = (int)UNIFYFS_ERROR_SOCK_DISCONNECT; + } else if (poll_set[i].revents & POLLERR) { + // unknown error on client socket + rc = (int)UNIFYFS_ERROR_SOCK_OTHER; + } + if (rc) { + if (rc == (int)UNIFYFS_ERROR_SOCK_DISCONNECT) { + sock_remove(i); + detached_sock_idx = i; + } + return rc; + } + } + } + } + + return UNIFYFS_SUCCESS; + +} + +#if 0 // DEPRECATED DUE TO MARGO +/* + * send command to the client to let the client digest the + * data in the shared receive buffer + * @param: sock_id: socket index in poll_set + * @param: cmd: command type + * + * */ +int sock_notify_client(int client_idx, int cmd) +{ + LOGDBG("sock notifying fd: %d", client_sockfd); + + memset(ack_buf[client_idx], 0, sizeof(ack_buf[client_idx])); + memcpy(ack_buf[client_idx], &cmd, sizeof(int)); + + ssize_t rc = write(client_sockfd, ack_buf[client_idx], + sizeof(ack_buf[client_idx])); + if (rc < 0) { + return (int)UNIFYFS_ERROR_SOCK_OTHER; + } + return UNIFYFS_SUCCESS; +} + +int sock_ack_client(int client_idx, int ret_sz) +{ + ssize_t rc = write(poll_set[client_idx].fd, ack_buf[client_idx], ret_sz); + if (rc < 0) { + return (int)UNIFYFS_ERROR_SOCK_OTHER; + } + return UNIFYFS_SUCCESS; +} + +int sock_handle_error(int sock_error_no) +{ + return UNIFYFS_SUCCESS; +} + +char* sock_get_cmd_buf(int client_idx) +{ + return (char*) cmd_buf[client_idx]; +} + +char* sock_get_ack_buf(int client_idx) +{ + return (char*) ack_buf[client_idx]; +} + +int sock_get_id(void) +{ + return cur_sock_idx; +} + +int sock_get_error_id(void) +{ + return detached_sock_idx; +} +#endif // DEPRECATED DUE TO MARGO diff --git a/server/src/unifycr_sock.h b/server/src/unifyfs_sock.h similarity index 60% rename from server/src/unifycr_sock.h rename to server/src/unifyfs_sock.h index ce4de0d12..794f02e9f 100644 --- a/server/src/unifycr_sock.h +++ b/server/src/unifyfs_sock.h @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -27,30 +27,32 @@ * Please read https://github.com/llnl/burstfs/LICENSE for full license text. */ -#ifndef UNIFYCR_SOCK_H -#define UNIFYCR_SOCK_H +#ifndef UNIFYFS_SOCK_H +#define UNIFYFS_SOCK_H #include -#include "unifycr_const.h" +#include "unifyfs_const.h" extern int server_sockfd; extern int client_sockfd; extern struct pollfd poll_set[MAX_NUM_CLIENTS]; int sock_init_server(int srvr_id); +void sock_sanitize_client(int client_idx); +int sock_sanitize(void); int sock_add(int fd); -void sock_reset(); -int sock_wait_cli_cmd(); -char* sock_get_cmd_buf(int sock_id); +int sock_remove(int client_idx); +void sock_reset(void); +int sock_wait_cmd(int poll_timeout); + +#if 0 // DEPRECATED DUE TO MARGO int sock_handle_error(int sock_error_no); -int sock_get_id(); -int sock_get_error_id(); -int sock_ack_cli(int sock_id, int ret_sz); -int sock_sanitize(); -void sock_sanitize_cli(int client_id); -char* sock_get_ack_buf(int sock_id); -int sock_remove(int idx); -int sock_notify_cli(int sock_id, int cmd); -char* sock_get_cmd_buf(int sock_id); +int sock_get_id(void); +int sock_get_error_id(void); +char* sock_get_cmd_buf(int client_idx); +char* sock_get_ack_buf(int client_idx); +int sock_ack_client(int client_idx, int ret_sz); +int sock_notify_client(int client_idx, int cmd); +#endif // DEPRECATED DUE TO MARGO #endif diff --git a/t/0001-setup.t b/t/0001-setup.t index 43d7fa766..e9fd500a0 100755 --- a/t/0001-setup.t +++ b/t/0001-setup.t @@ -2,7 +2,7 @@ # # Perform some initial setup for the test suite. This is not # implemented as a sharness test because leaving a process running -# behind (i.e. unifycrd) causes tap-driver.sh to hang waiting for +# behind (i.e. unifyfsd) causes tap-driver.sh to hang waiting for # the process to exit. # @@ -14,55 +14,55 @@ echo 1..1 # common metadata directory across multiple tests. Save the value to a # script in known location that later test scripts can source. # -export UNIFYCR_MOUNT_POINT=$(mktemp -d) -export UNIFYCR_META_DB_PATH=$(mktemp -d) +export UNIFYFS_MOUNT_POINT=$(mktemp -d) +export UNIFYFS_META_DB_PATH=$(mktemp -d) # -# Source test environment first to pick up UNIFYCR_TEST_RUN_SCRIPT +# Source test environment first to pick up UNIFYFS_TEST_RUN_SCRIPT # . $(dirname $0)/sharness.d/00-test-env.sh -cat >"$UNIFYCR_TEST_RUN_SCRIPT" <<-EOF -export UNIFYCR_MOUNT_POINT=$UNIFYCR_MOUNT_POINT -export UNIFYCR_META_DB_PATH=$UNIFYCR_META_DB_PATH +cat >"$UNIFYFS_TEST_RUN_SCRIPT" <<-EOF +export UNIFYFS_MOUNT_POINT=$UNIFYFS_MOUNT_POINT +export UNIFYFS_META_DB_PATH=$UNIFYFS_META_DB_PATH EOF -. $(dirname $0)/sharness.d/01-unifycr-settings.sh +. $(dirname $0)/sharness.d/01-unifyfs-settings.sh . $(dirname $0)/sharness.d/02-functions.sh # -# Start the UnifyCR daemon after killing and cleanup up after any previously +# Start the UnifyFS daemon after killing and cleanup up after any previously # running instance. # -unifycrd_stop_daemon -unifycrd_cleanup -unifycrd_start_daemon +unifyfsd_stop_daemon +unifyfsd_cleanup +unifyfsd_start_daemon # -# Make sure the unifycrd process starts. +# Make sure the unifyfsd process starts. # -if ! process_is_running unifycrd 5 ; then - echo not ok 1 - unifycrd started +if ! process_is_running unifyfsd 5 ; then + echo not ok 1 - unifyfsd started exit 1 fi # -# Make sure unifycrd stays running for 5 more seconds to catch cases where +# Make sure unifyfsd stays running for 5 more seconds to catch cases where # it dies during initialization. # -if process_is_not_running unifycrd 5; then - echo not ok 1 - unifycrd running +if process_is_not_running unifyfsd 5; then + echo not ok 1 - unifyfsd running exit 1 fi # -# Make sure unifycrd successfully generated client runstate file +# Make sure unifyfsd successfully generated client runstate file # uid=$(id -u) -if ! test -f $UNIFYCR_META_DB_PATH/unifycr-runstate.conf.$uid ; then - echo not ok 1 - unifycrd runstate +if ! test -f $UNIFYFS_META_DB_PATH/unifyfs-runstate.conf.$uid ; then + echo not ok 1 - unifyfsd runstate exit 1 fi -echo ok 1 - unifycrd running +echo ok 1 - unifyfsd running exit 0 diff --git a/t/0100-sysio-gotcha.t b/t/0100-sysio-gotcha.t index 90ed93a4a..9a2795f5c 100755 --- a/t/0100-sysio-gotcha.t +++ b/t/0100-sysio-gotcha.t @@ -1,8 +1,8 @@ #!/bin/bash # # Source sharness environment scripts to pick up test environment -# and UnifyCR runtime settings. +# and UnifyFS runtime settings. # . $(dirname $0)/sharness.d/00-test-env.sh -. $(dirname $0)/sharness.d/01-unifycr-settings.sh -$UNIFYCR_BUILD_DIR/t/sys/sysio-gotcha.t +. $(dirname $0)/sharness.d/01-unifyfs-settings.sh +$UNIFYFS_BUILD_DIR/t/sys/sysio-gotcha.t diff --git a/t/0200-stdio-gotcha.t b/t/0200-stdio-gotcha.t index f8015c576..df7cad09f 100755 --- a/t/0200-stdio-gotcha.t +++ b/t/0200-stdio-gotcha.t @@ -1,8 +1,8 @@ #!/bin/bash # # Source sharness environment scripts to pick up test environment -# and UnifyCR runtime settings. +# and UnifyFS runtime settings. # . $(dirname $0)/sharness.d/00-test-env.sh -. $(dirname $0)/sharness.d/01-unifycr-settings.sh -$UNIFYCR_BUILD_DIR/t/std/stdio-gotcha.t +. $(dirname $0)/sharness.d/01-unifyfs-settings.sh +$UNIFYFS_BUILD_DIR/t/std/stdio-gotcha.t diff --git a/t/0500-sysio-static.t b/t/0500-sysio-static.t index b508a83bf..3772ab8ac 100755 --- a/t/0500-sysio-static.t +++ b/t/0500-sysio-static.t @@ -1,8 +1,8 @@ #!/bin/bash # # Source sharness environment scripts to pick up test environment -# and UnifyCR runtime settings. +# and UnifyFS runtime settings. # . $(dirname $0)/sharness.d/00-test-env.sh -. $(dirname $0)/sharness.d/01-unifycr-settings.sh -$UNIFYCR_BUILD_DIR/t/sys/sysio-static.t +. $(dirname $0)/sharness.d/01-unifyfs-settings.sh +$UNIFYFS_BUILD_DIR/t/sys/sysio-static.t diff --git a/t/0600-stdio-static.t b/t/0600-stdio-static.t index 79015e3de..e1322b635 100755 --- a/t/0600-stdio-static.t +++ b/t/0600-stdio-static.t @@ -1,8 +1,8 @@ #!/bin/bash # # Source sharness environment scripts to pick up test environment -# and UnifyCR runtime settings. +# and UnifyFS runtime settings. # . $(dirname $0)/sharness.d/00-test-env.sh -. $(dirname $0)/sharness.d/01-unifycr-settings.sh -$UNIFYCR_BUILD_DIR/t/std/stdio-static.t +. $(dirname $0)/sharness.d/01-unifyfs-settings.sh +$UNIFYFS_BUILD_DIR/t/std/stdio-static.t diff --git a/t/9005-unifycr-unmount.t b/t/9005-unifyfs-unmount.t similarity index 50% rename from t/9005-unifycr-unmount.t rename to t/9005-unifyfs-unmount.t index 8bb2a9619..31e5bdd7f 100755 --- a/t/9005-unifycr-unmount.t +++ b/t/9005-unifyfs-unmount.t @@ -1,8 +1,8 @@ #!/bin/bash # # Source sharness environment scripts to pick up test environment -# and UnifyCR runtime settings. +# and UnifyFS runtime settings. # . $(dirname $0)/sharness.d/00-test-env.sh -. $(dirname $0)/sharness.d/01-unifycr-settings.sh -$UNIFYCR_BUILD_DIR/t/unifycr_unmount.t +. $(dirname $0)/sharness.d/01-unifyfs-settings.sh +$UNIFYFS_BUILD_DIR/t/unifyfs_unmount.t diff --git a/t/9010-stop-unifycrd.t b/t/9010-stop-unifycrd.t deleted file mode 100755 index 771854023..000000000 --- a/t/9010-stop-unifycrd.t +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -test_description="Shut down unifycrd" - -. $(dirname $0)/sharness.sh - -test_expect_success "Stop unifycrd" ' - unifycrd_stop_daemon - process_is_not_running unifycrd 5 -' - -test_done diff --git a/t/9010-stop-unifyfsd.t b/t/9010-stop-unifyfsd.t new file mode 100755 index 000000000..70112f67b --- /dev/null +++ b/t/9010-stop-unifyfsd.t @@ -0,0 +1,12 @@ +#!/bin/bash + +test_description="Shut down unifyfsd" + +. $(dirname $0)/sharness.sh + +test_expect_success "Stop unifyfsd" ' + unifyfsd_stop_daemon + process_is_not_running unifyfsd 5 +' + +test_done diff --git a/t/9020-mountpoint-empty.t b/t/9020-mountpoint-empty.t index b24d61e36..54c20b59e 100755 --- a/t/9020-mountpoint-empty.t +++ b/t/9020-mountpoint-empty.t @@ -1,18 +1,18 @@ #!/bin/bash # -# This test checks that I/O to the UnifyCR mount point was properly -# intercepted and redirected to the UnifyCR daemon. +# This test checks that I/O to the UnifyFS mount point was properly +# intercepted and redirected to the UnifyFS daemon. # -# If this test fails, then at least one file made it to the UnifyCr mountpoint. +# If this test fails, then at least one file made it to the UnifyFS mountpoint. # This most likely means the corresponding wrapped function is failing in some # way and the call is falling through to the operating system. -test_description="Verify UnifyCR intercepted mount point is empty" +test_description="Verify UnifyFS intercepted mount point is empty" . $(dirname $0)/sharness.sh -test_expect_success "Intercepted mount point $UNIFYCR_MOUNT_POINT is empty" ' - test_dir_is_empty $UNIFYCR_MOUNT_POINT +test_expect_success "Intercepted mount point $UNIFYFS_MOUNT_POINT is empty" ' + test_dir_is_empty $UNIFYFS_MOUNT_POINT ' test_done diff --git a/t/9100-metadata-api.t b/t/9100-metadata-api.t new file mode 100755 index 000000000..7ae09a671 --- /dev/null +++ b/t/9100-metadata-api.t @@ -0,0 +1,10 @@ +#!/bin/bash + +test_description="Test Metadata API" + +# +# Source sharness environment scripts to pick up test environment +# and UnifyFS runtime settings. +# +. $(dirname $0)/sharness.d/00-test-env.sh +$UNIFYFS_BUILD_DIR/t/server/metadata.t diff --git a/t/Makefile.am b/t/Makefile.am index f4046b005..8394d9e74 100644 --- a/t/Makefile.am +++ b/t/Makefile.am @@ -9,9 +9,10 @@ TESTS = \ 0200-stdio-gotcha.t \ 0500-sysio-static.t \ 0600-stdio-static.t \ - 9005-unifycr-unmount.t \ - 9010-stop-unifycrd.t \ - 9020-mountpoint-empty.t + 9005-unifyfs-unmount.t \ + 9010-stop-unifyfsd.t \ + 9020-mountpoint-empty.t \ + 9100-metadata-api.t check_SCRIPTS = \ 0001-setup.t \ @@ -19,9 +20,10 @@ check_SCRIPTS = \ 0200-stdio-gotcha.t \ 0500-sysio-static.t \ 0600-stdio-static.t \ - 9005-unifycr-unmount.t \ - 9010-stop-unifycrd.t \ - 9020-mountpoint-empty.t + 9005-unifyfs-unmount.t \ + 9010-stop-unifyfsd.t \ + 9020-mountpoint-empty.t \ + 9100-metadata-api.t EXTRA_DIST = \ $(check_SCRIPTS) \ @@ -39,31 +41,55 @@ libexec_PROGRAMS = \ std/stdio-gotcha.t \ sys/sysio-static.t \ std/stdio-static.t \ - unifycr_unmount.t + server/metadata.t \ + unifyfs_unmount.t test_ldadd = \ $(top_builddir)/t/lib/libtap.la \ $(top_builddir)/t/lib/libtestutil.la \ - $(top_builddir)/client/src/libunifycr_gotcha.la \ + $(top_builddir)/client/src/libunifyfs_gotcha.la \ $(MPI_CLDFLAGS) $(FLATCC_LDFLAGS) $(FLATCC_LIBS) test_static_ldadd = \ $(top_builddir)/t/lib/libtap.la \ $(top_builddir)/t/lib/libtestutil.la \ - $(top_builddir)/client/src/libunifycr.la + $(top_builddir)/client/src/libunifyfs.la test_static_ldflags = \ -static \ $(CP_WRAPPERS) $(AM_LDFLAGS) $(MPI_CLDFLAGS) $(FLATCC_LDFLAGS) \ $(FLATCC_LIBS) +test_metadata_ldadd = \ + $(top_builddir)/t/lib/libtap.la \ + $(top_builddir)/t/lib/libtestutil.la \ + $(top_builddir)/common/src/libunifyfs_common.la \ + $(top_builddir)/server/src/libunifyfsd.a \ + $(top_builddir)/meta/src/libmdhim.a \ + $(LEVELDB_LDFLAGS) $(LEVELDB_LIBS) \ + $(MPI_CLDFLAGS) \ + $(MERCURY_LDFLAGS) $(MERCURY_LIBS) \ + $(ARGOBOTS_LDFLAGS) $(ARGOBOTS_LIBS) \ + $(MARGO_LDFLAGS) $(MARGO_LIBS) \ + $(FLATCC_LDFLAGS) $(FLATCC_LIBS) \ + -lpthread -lm -lstdc++ -lrt + +test_meta_cppflags = \ + -I$(top_srcdir) \ + -I$(top_srcdir)/server/src \ + -I$(top_srcdir)/common/src \ + -D_GNU_SOURCE \ + $(AM_CPPFLAGS) \ + $(MARGO_CFLAGS) \ + $(MPI_CFLAGS) + test_cppflags = \ -I$(top_srcdir) \ -I$(top_srcdir)/client/src \ -I$(top_srcdir)/common/src \ -D_GNU_SOURCE \ $(AM_CPPFLAGS) \ - $(MPI_CFLAGS) + $(MPI_CFLAGS) sys_sysio_gotcha_t_SOURCES = sys/sysio_suite.h \ sys/sysio_suite.c \ @@ -101,7 +127,16 @@ std_stdio_static_t_CPPFLAGS = $(test_cppflags) std_stdio_static_t_LDADD = $(test_static_ldadd) std_stdio_static_t_LDFLAGS = $(test_static_ldflags) -unifycr_unmount_t_SOURCES = unifycr_unmount.c -unifycr_unmount_t_CPPFLAGS = $(test_cppflags) -unifycr_unmount_t_LDADD = $(test_ldadd) -unifycr_unmount_t_LDFLAGS = $(AM_LDFLAGS) +server_metadata_t_SOURCES = \ + server/metadata_suite.h \ + server/metadata_suite.c \ + server/unifyfs_meta_get_test.c + +server_metadata_t_CPPFLAGS = $(test_meta_cppflags) +server_metadata_t_LDADD = $(test_metadata_ldadd) +server_metadata_t_LDFLAGS = $(AM_LDFLAGS) + +unifyfs_unmount_t_SOURCES = unifyfs_unmount.c +unifyfs_unmount_t_CPPFLAGS = $(test_cppflags) +unifyfs_unmount_t_LDADD = $(test_ldadd) +unifyfs_unmount_t_LDFLAGS = $(AM_LDFLAGS) diff --git a/t/README.md b/t/README.md index c5c4cee0a..8d08b376b 100644 --- a/t/README.md +++ b/t/README.md @@ -1,4 +1,30 @@ -#Testing +# Testing -Our [Testing Guide](https://unifycr.readthedocs.io/en/dev/testing.html) has our +UnifyFS's unit testing and continuous integration testing suites. + +Our [Testing Guide](https://unifyfs.readthedocs.io/en/dev/testing.html) has our complete testing documentation. + +## Unit Tests + +The UnifyFS Unit Test Suite uses the Test Anything Protocol (TAP) and the +Automake test harness. By convention, test scripts and programs that output TAP +are named with a “.t” extension. + +Test cases in shell scripts are implemented with +[sharness](https://github.com/chriscool/sharness), which is included in +the UnifyFS source distribution. See the file `sharness.sh` for all available +test interfaces. UnifyFS-specific sharness code is implemented in scripts in the +directory `sharness.d/`. Scripts in `sharness.d/` are primarily used to set +environment variables and define convenience functions. All scripts in +`sharness.d/` are automatically included when your script sources `sharness.sh`. + +C programs use the [libtap library](https://github.com/zorgnax/libtap) +to implement test cases. Convenience functions common to test cases written in +C are implemmented in the library `lib/testutil.c`. + +## Continuous Integration Tests + +The UnifyFS Continuous Integration (CI) Test Suite is found in `ci/` and also uses [sharness](https://github.com/chriscool/sharness). +Additional sharness convenience functions and variables needed for the CI tests +are also found in `ci/` and are sourced when running `ci/001-setup.sh`. diff --git a/t/ci/001-setup.sh b/t/ci/001-setup.sh new file mode 100755 index 000000000..bdead8817 --- /dev/null +++ b/t/ci/001-setup.sh @@ -0,0 +1,257 @@ +#!/bin/sh + +# This script checks for an installation of UnifyFS (either with Spack or in +# $HOME/UnifyFS/install) and then sets up variables needed for testing. +# +# All of this is done in this script so that tests can be run individually if +# desired. To run all tests simply run the RUN_TESTS.sh script. If Individual +# tests are desired to be run, source the 001-setup.sh script first, followed by +# 002-start-server.sh. Then source each desired script after that preceded by +# `$CI_DIR`. When finished, source the 990-stop-server.sh script last. +# +# E.g.: +# $ . full/path/to/001-setup.sh +# $ . $CI_DIR/002-start-server.sh +# $ . $CI_DIR/100-writeread-tests.sh +# $ . $CI_DIR/990-stop-server.sh +# +# To run all of the tests, simply run RUN_CI_TESTS.sh +# +# E.g.: +# $ ./RUN_CI_TESTS.sh +# or +# $ prove -v RUN_CI_TESTS.sh +# +# Before doing either of these, make sure you have interactively allocated nodes +# or are submitting a batch job. + +test_description="Set up UnifyFS testing environment" + +SETUP_USAGE="$(cat </dev/null) ]]; + then + # Might have a problem with variants and arch + UNIFYFS_INSTALL="$(spack location -i unifyfs)" + # Else search for unifyfsd starting in $CI_PROJDIR and omitting spack_root + elif [[ -x $(find_executable $CI_PROJDIR "*/bin/unifyfsd" $SPACK_ROOT) ]]; + then + # Set UNIFYFS_INSTALL to the dir containing bin/ and libexec/ + UNIFYFS_INSTALL="$(dirname "$(dirname \ + "$(find_executable $CI_PROJDIR "*/bin/unifyfsd" $SPACK_ROOT)")")" + else + echo >&2 "$errmsg Unable to find UnifyFS install directory" + echo >&2 "$errmsg \`spack install unifyfs\`, set the" \ + "\$UNIFYFS_INSTALL envar to the directory containing bin/" \ + "and libexec/, or manually install to \$CI_PROJDIR/*" + exit 1 + fi +fi + +# Make sure UNIFYFS_INSTALL, bin/, and libexec/ exist +if [[ -d $UNIFYFS_INSTALL && -d ${UNIFYFS_INSTALL}/bin && + -d ${UNIFYFS_INSTALL}/libexec ]]; then + echo "$infomsg Found UnifyFS install directory: $UNIFYFS_INSTALL" + + UNIFYFS_BIN="$UNIFYFS_INSTALL/bin" + UNIFYFS_EXAMPLES="$UNIFYFS_INSTALL/libexec" + echo "$infomsg Found UnifyFS bin directory: $UNIFYFS_BIN" + echo "$infomsg Found UnifyFS examples directory: $UNIFYFS_EXAMPLES" +else + echo >&2 "$errmsg Ensure \$UNIFYFS_INSTALL exists and is the directory" \ + "containing bin/ and libexec/" +fi + +# Check for necessary Spack modules if Spack is detected +# Since GitLab Runners don't like this, just warn users running this by hand but +# don't fail out +if [[ -n $(which spack 2>/dev/null) ]]; then + loaded_modules=$(module list 2>&1) + modules="gotcha leveldb flatcc argobots mercury margo" + for mod in $modules; do + if ! [[ $(echo "$loaded_modules" | fgrep "$mod") ]]; then + echo "$errmsg $mod not detected. Please 'spack load $mod'" + fi + done +fi + + +########## Determine job launcher and source associated setup ########## + +# Source envar, functions, and set up JOB_RUN_COMMAND if lsf, slurm, or mpirun +# TODO: mpirun compatibility +echo "$infomsg Finding job launcher" +if [[ -n $(which jsrun 2>/dev/null) ]]; then + source $CI_DIR/setup-lsf.sh +elif [[ -n $(which srun 2>/dev/null) ]]; then + source $CI_DIR/setup-slurm.sh +else + echo >&2 "$errmsg Failed to find a suitable parallel job launcher" + exit 1 +fi +echo "$infomsg JOB_RUN_COMMAND established: $JOB_RUN_COMMAND" + + +########## Set up CI and UNIFYFS configuration variables ########## + +# Turn up log verbosity +export UNIFYFS_LOG_VERBOSITY=${UNIFYFS_LOG_VERBOSITY:-5} + +# Set up location for logs and potentially auto cleanup if user didn't provide +# an alternate location for the logs +if [[ -z $UNIFYFS_LOG_DIR ]]; then + # User can choose to not cleanup logs on success + export CI_LOG_CLEANUP=${CI_LOG_CLEANUP:-yes} + # If no log cleanup, move logs to $CI_DIR + if [[ $CI_LOG_CLEANUP =~ ^(no|NO)$ || $CI_CLEANUP =~ ^(no|NO)$ ]]; then + logdir=$CI_DIR/${SYSTEM_NAME}_${JOB_ID}_logs + else # else put logs in sharness trash dir that sharness deletes + logdir=$SHARNESS_TRASH_DIRECTORY/${SYSTEM_NAME}_${JOB_ID}_logs + echo "$infomsg Set CI_LOG_CLEANUP=no to keep logs when all tests pass" + fi + mkdir -p $logdir +fi +export UNIFYFS_LOG_DIR=${UNIFYFS_LOG_DIR:-$logdir} +echo "$infomsg Logs are in UNIFYFS_LOG_DIR: $UNIFYFS_LOG_DIR" + +export UNIFYFS_SHAREDFS_DIR=${UNIFYFS_SHAREDFS_DIR:-$UNIFYFS_LOG_DIR} +echo "$infomsg UNIFYFS_SHAREDFS_DIR set as $UNIFYFS_SHAREDFS_DIR" + +# daemonize +export UNIFYFS_DAEMONIZE=${UNIFYFS_DAEMONIZE:-off} + +# temp +nlt=${TMPDIR}/unifyfs.${USER}.${SYSTEM_NAME}.${JOB_ID} +export CI_TEMP_DIR=${CI_TEMP_DIR:-$nlt} +export UNIFYFS_RUNSTATE_DIR=${UNIFYFS_RUNSTATE_DIR:-$CI_TEMP_DIR} +export UNIFYFS_META_DB_PATH=${UNIFYFS_META_DB_PATH:-$CI_TEMP_DIR} +echo "$infomsg UNIFYFS_RUNSTATE_DIR set as $UNIFYFS_RUNSTATE_DIR" +echo "$infomsg UNIFYFS_META_DB_PATH set as $UNIFYFS_META_DB_PATH" +echo "$infomsg Set CI_TEMP_DIR to change both of these to same path" + +# storage +nls=$nlt +export CI_STORAGE_DIR=${CI_STORAGE_DIR:-$nls} +export UNIFYFS_SPILLOVER_SIZE=${UNIFYFS_SPILLOVER_SIZE:-$GB} +export UNIFYFS_SPILLOVER_ENABLED=${UNIFYFS_SPILLOVER_ENABLED:-yes} +export UNIFYFS_SPILLOVER_DATA_DIR=${UNIFYFS_SPILLOVER_DATA_DIR:-$CI_STORAGE_DIR} +export UNIFYFS_SPILLOVER_META_DIR=${UNIFYFS_SPILLOVER_META_DIR:-$CI_STORAGE_DIR} +echo "$infomsg UNIFYFS_SPILLOVER_DATA_DIR set as $UNIFYFS_SPILLOVER_DATA_DIR" +echo "$infomsg UNIFYFS_SPILLOVER_META_DIR set as $UNIFYFS_SPILLOVER_META_DIR" +echo "$infomsg Set CI_STORAGE_DIR to change both of these to same path" + + +########## Set up mountpoints and sharness testing prereqs ########## + +# Running tests with UNIFYFS_MOUNTPOINT set to a real dir will disable posix +# tests unless user sets CI_TEST_POSIX=yes +export UNIFYFS_MP=${UNIFYFS_MOUNTPOINT:-/unifyfs} +# If UNIFYFS_MOUNTPOINT is real dir, disable posix tests (unless user wants it) +# and set REAL_MP prereq to enable test that checks if UNIFYFS_MOUNTPOINT is +# empty +if [[ -d $UNIFYFS_MP ]]; then + export CI_TEST_POSIX=no + test_set_prereq REAL_MP +fi +echo "$infomsg UNIFYFS_MOUNTPOINT established: $UNIFYFS_MP" + +export CI_TEST_POSIX=${CI_TEST_POSIX:-yes} +# Set up a real mountpoint for posix tests to write files to and allow tests to +# check that those files exist +if [[ ! $CI_TEST_POSIX =~ ^(no|NO)$ ]]; then + if [[ -z $CI_POSIX_MP ]]; then + # needs to be a shared file system + pmp=${SHARNESS_TRASH_DIRECTORY}/unify_posix_mp.${SYSTEM_NAME}.${JOB_ID} + mkdir $pmp + fi + export CI_POSIX_MP=${CI_POSIX_MP:-$pmp} + echo "$infomsg CI_POSIX_MP established: $CI_POSIX_MP" + + # Set test_posix prereq + test_set_prereq TEST_POSIX +fi + +# prereq for pdsh for cleaning hosts +[[ -n $(which pdsh 2>/dev/null) ]] && test_set_prereq PDSH + +# skip cleanup_hosts test in 990-stop_server.sh if cleanup is not desired +export CI_HOST_CLEANUP=${CI_HOST_CLEANUP:-yes} +if ! [[ $CI_HOST_CLEANUP =~ ^(no|NO)$ || $CI_CLEANUP =~ ^(no|NO)$ ]]; then + test_set_prereq CLEAN +fi + +# capture environment after all job setup completed +env &> ${UNIFYFS_LOG_DIR}/job.environ diff --git a/t/ci/002-start-server.sh b/t/ci/002-start-server.sh new file mode 100755 index 000000000..8d9d0a4a5 --- /dev/null +++ b/t/ci/002-start-server.sh @@ -0,0 +1,97 @@ +#!/bin/sh + +# This test check the mountpoints and then starts the unifyfsd server. It then +# checks that the server is still running for all subsequent tests. If the +# servers fails to start, cleanup_hosts is called and testing is exited. + +test_description="Start the UnifyFS server" + +while [[ $# -gt 0 ]] +do + case $1 in + -h|--help) + ci_dir=$(dirname "$(readlink -fm $BASH_SOURCE)") + $ci_dir/001-setup.sh -h + exit + ;; + *) + echo "usage ./002-start-server.sh -h|--help" + exit 1 + ;; + esac +done + +# Verify we have the mountpoints needed for testing +if test_have_prereq REAL_MP; then + test_expect_success "UNIFYFS_MOUNTPOINT ($UNIFYFS_MP) is real directory" ' + test_path_is_dir $UNIFYFS_MP || + { test_set_prereq FAIL; return 1; } + ' + + # If FAIL prereq is set, then UNIFYFS_MOUNTPOINT was set to a real dir, but + # it is not a shared directory. If this is the case, no point in starting + # servers to proceed with tests. + if test_have_prereq FAIL; then + say "UNIFYFS_MOUNTPOINT set to real dir; dir not found on all hosts." + say "Exiting." + test_done + fi +else # ensure mountpoint is a non-real directory + test_expect_success "UNIFYFS_MOUNTPOINT ($UNIFYFS_MP) is fake directory" ' + test_must_fail test_path_is_dir $UNIFYFS_MP + ' +fi + +# If running posix tests, posix mountpoint needs to be a real, shared dir +# If it's not, prereq will not be set and posix tests will be skipped +test_expect_success TEST_POSIX "CI_POSIX_MP ($CI_POSIX_MP) is shared dir" ' + test_path_is_shared_dir $CI_POSIX_MP && + test_set_prereq POSIX +' + +# Start the server +test_expect_success "unifyfsd hasn't started yet" ' + process_is_not_running unifyfsd 10 +' + +$UNIFYFS_BIN/unifyfs start -c -d -S $UNIFYFS_SHAREDFS_DIR \ + -e $UNIFYFS_BIN/unifyfsd &> ${UNIFYFS_LOG_DIR}/unifyfs.start.out & + +test_expect_success "unifyfsd started" ' + process_is_running unifyfsd 10 || + { test_set_prereq FAIL; return 1; } +' + +# If FAIL prereq is set, then unifyfsd failed to start. +# No point in proceeding with testing, so cleanup and exit early +if test_have_prereq FAIL; then + say "unifyfsd failed to start." + say "Try setting longer wait time. Cleaning up and exiting." + cleanup_hosts + test_done +fi + +# If unifyfsd doesn't stay running running, set fail test prereq +test_expect_success "unifyfsd hasn't died" ' + test_must_fail process_is_not_running unifyfsd 10 || + { test_set_prereq FAIL; return 1; } +' + +# If FAIL prereq is set, then unifyfsd failed to stay running. +# No point in proceeding with testing, so cleanup and exit early +if test_have_prereq FAIL; then + say "unifyfsd failed to stay running. Cleaning up and exiting." + cleanup_hosts + test_done +fi + +# unifyfsd has started by this point. Set up a trap to cleanup the hosts in the +# event of an early/unexpected non-zero exit. This trap gets removed after the +# final cleanup_hosts is called in 990-stop-server.sh. This is to prevent the +# trap from triggering when the overall test suite exits with 1 if/when any +# tests have failed. +clean_fail() { + echo >&2 "Error in $1. Cleaning hosts" + cleanup_hosts +} +trap 'clean_fail $BASH_SOURCE' EXIT diff --git a/t/ci/100-writeread-tests.sh b/t/ci/100-writeread-tests.sh new file mode 100755 index 000000000..6969dd607 --- /dev/null +++ b/t/ci/100-writeread-tests.sh @@ -0,0 +1,246 @@ +#!/bin/bash + +# This contains all the tests for the writeread example. +# +# There are convenience functions such as `unify_run_test()` and get_filename() +# in the ci-functions.sh script that can make adding new tests easier. See the +# full UnifyFS documentatation for more info. +# +# There are multiple ways to to run an example using `unify_run_test()` +# 1. unify_run_test $app_name "$app_args" app_output +# 2. app_output=$(unify_run_test $app_name "app_args") +# +# ---- Method 1 +# app_output=$(unify_run_test $app_name "$app_args") +# rc=$? + +# echo "$app_output" +# lcount=$(printf "%s" "$app_output" | wc -l) +# ---- + +# ---- Method 2 +# unify_run_test $app_name "$app_args" app_output +# rc=$? +# +# lcount=$(echo "$app_output" | wc -l) +# ---- +# +# The output of an example can then be tested with sharness, for example: +# +# test_expect_success "$app_name $app_args: (line_count=$lcount, rc=$rc)" ' +# test $rc = 0 && +# test $lcount = 8 +# ' +# +# For these tests, always include -b -c -n and -p in the app_args +# Then for the necessary tests, include -k -M -P -S -V or -x. + + +test_description="Writeread Tests" + +while [[ $# -gt 0 ]] +do + case $1 in + -h|--help) + ci_dir=$(dirname "$(readlink -fm $BASH_SOURCE)") + $ci_dir/001-setup.sh -h + exit + ;; + *) + echo "usage ./100-writeread-tests.sh -h|--help" + exit 1 + ;; + esac +done + +# These two functions are simply to prevent code duplication since testing the +# output of each example with sharness is the same process. These do not need to +# be used, especially if wanting to test for something specific when running an +# example. +unify_test_writeread() { + app_name=writeread-${1} + + # Run the test and get output + unify_run_test $app_name "$2" app_output + rc=$? + lcount=$(echo "$app_output" | wc -l) + + # Evaluate output + test_expect_success "$app_name $app_args: (line_count=${lcount}, rc=$rc)" ' + test $rc = 0 && + test $lcount = 8 + ' +} + +unify_test_writeread_posix() { + app_name=writeread-posix + + # Run the test and get output + if test_have_prereq POSIX; then + unify_run_test $app_name "$1" app_output + rc=$? + lcount=$(echo "$app_output" | wc -l) + filename=$(get_filename $app_name "$1" ".app") + fi + + # Evaluate output + test_expect_success POSIX "$app_name $1: (line_count=${lcount}, rc=$rc)" ' + test $rc = 0 && + test $lcount = 8 && + if [[ $io_pattern =~ (n1)$ ]]; then + test_path_is_file ${CI_POSIX_MP}/$filename + else + test_path_has_file_per_process $CI_POSIX_MP $filename + fi + ' +} + +# writeread-static -p n1 -n 2 -c 4KB -b 16KB +runmode=static +io_pattern="-p n1" +io_sizes="-n 2 -c $((4 * $KB)) -b $((16 * $KB))" +app_args="$io_pattern $io_sizes" +unify_test_writeread $runmode "$app_args" + +# writeread-gotcha -p n1 -n 2 -c 4KB -b 16KB +runmode=gotcha +unify_test_writeread $runmode "$app_args" + +# writeread-posix -p n1 -n 2 -c 4KB -b 16KB +runmode=posix +unify_test_writeread_posix "$app_args" + +# Switch to -p nn +io_pattern="-p nn" +app_args="$io_pattern $io_sizes" + +# writeread-posix -p nn -n 2 -c 4KB -b 16KB +unify_test_writeread_posix "$app_args" + +# writeread-gotcha -p nn -n 2 -c 4KB -b 16KB +runmode=gotcha +unify_test_writeread $runmode "$app_args" + +# writeread-static -p nn -n 2 -c 4KB -b 16KB +runmode=static +unify_test_writeread $runmode "$app_args" + +# Increase sizes: -n 16 -c 32KB -b 1MB + +# writeread-static -p nn -n 16 -c 32KB -b 1MB +io_sizes="-n 16 -c $((32 * $KB)) -b $MB" +app_args="$io_pattern $io_sizes" +unify_test_writeread $runmode "$app_args" + +# writeread-gotcha -p nn -n 16 -c 32KB -b 1MB +runmode=gotcha +unify_test_writeread $runmode "$app_args" + +# writeread-posix -p nn -n 16 -c 32KB -b 1MB +runmode=posix +unify_test_writeread_posix "$app_args" + +# Switch back to -p n1 +io_pattern="-p n1" +app_args="$io_pattern $io_sizes" + +# writeread-posix -p n1 -n 16 -c 32KB -b 1MB +unify_test_writeread_posix "$app_args" + +# writeread-gotcha -p n1 -n 16 -c 32KB -b 1MB +runmode=gotcha +unify_test_writeread $runmode "$app_args" + +# writeread-static -p n1 -n 16 -c 32KB -b 1MB +runmode=static +unify_test_writeread $runmode "$app_args" + +# Increase sizes: -n 32 -c 64KB -b 1MB + +# writeread-static -p n1 -n 32 -c 64KB -b 1MB +io_sizes="-n 32 -c $((64 * $KB)) -b $MB" +app_args="$io_pattern $io_sizes" +unify_test_writeread $runmode "$app_args" + +# writeread-gotcha -p n1 -n 32 -c 64KB -b 1MB +runmode=gotcha +unify_test_writeread $runmode "$app_args" + +# writeread-posix -p n1 -n 32 -c 64KB -b 1MB +runmode=posix +unify_test_writeread_posix "$app_args" + +# Switch to -p nn +io_pattern="-p nn" +app_args="$io_pattern $io_sizes" + +# writeread-posix -p nn -n 32 -c 64KB -b 1MB +unify_test_writeread_posix "$app_args" + +# writeread-gotcha -p nn -n 32 -c 64KB -b 1MB +runmode=gotcha +unify_test_writeread $runmode "$app_args" + +# writeread-static -p nn -n 32 -c 64KB -b 1MB +runmode=static +unify_test_writeread $runmode "$app_args" + +# Increase sizes: -n 64 -c 1MB -b 4MB + +# writeread-static -p nn -n 64 -c 1MB -b 4MB +io_sizes="-n 64 -c $MB -b $((4 * $MB))" +app_args="$io_pattern $io_sizes" +unify_test_writeread $runmode "$app_args" + +# writeread-gotcha -p nn -n 64 -c 1MB -b 4MB +runmode=gotcha +unify_test_writeread $runmode "$app_args" + +# writeread-posix -p nn -n 64 -c 1MB -b 4MB +runmode=posix +unify_test_writeread_posix "$app_args" + +# Switch back to -p n1 +io_pattern="-p n1" +app_args="$io_pattern $io_sizes" + +# writeread-posix -p n1 -n 64 -c 1MB -b 4MB +unify_test_writeread_posix "$app_args" + +# writeread-gotcha -p n1 -n 64 -c 1MB -b 4MB +runmode=gotcha +unify_test_writeread $runmode "$app_args" + +# writeread-static -p n1 -n 64 -c 1MB -b 4MB +runmode=static +unify_test_writeread $runmode "$app_args" + +# Increase sizes: -n 32 -c 1MB -b 16MB + +# writeread-static -p n1 -n 32 -c 1MB -b 16MB +io_sizes="-n 32 -c $MB -b $((16 * $MB))" +app_args="$io_pattern $io_sizes" +unify_test_writeread $runmode "$app_args" + +# writeread-gotcha -p n1 -n 32 -c 1MB -b 16MB +runmode=gotcha +unify_test_writeread $runmode "$app_args" + +# writeread-posix -p n1 -n 32 -c 1MB -b 16MB +runmode=posix +unify_test_writeread_posix "$app_args" + +# Switch to -p nn +io_pattern="-p nn" +app_args="$io_pattern $io_sizes" + +# writeread-posix -p nn -n 32 -c 1MB -b 16MB +unify_test_writeread_posix "$app_args" + +# writeread-gotcha -p nn -n 32 -c 1MB -b 16MB +runmode=gotcha +unify_test_writeread $runmode "$app_args" + +# writeread-static -p nn -n 32 -c 1MB -b 16MB +runmode=static +unify_test_writeread $runmode "$app_args" diff --git a/t/ci/110-write-tests.sh b/t/ci/110-write-tests.sh new file mode 100755 index 000000000..bba5d1ea9 --- /dev/null +++ b/t/ci/110-write-tests.sh @@ -0,0 +1,246 @@ +#!/bin/bash + +# This contains all the tests for the write example. +# +# There are convenience functions such as `unify_run_test()` and get_filename() +# in the ci-functions.sh script that can make adding new tests easier. See the +# full UnifyFS documentatation for more info. +# +# There are multiple ways to to run an example using `unify_run_test()` +# 1. unify_run_test $app_name "$app_args" app_output +# 2. app_output=$(unify_run_test $app_name "app_args") +# +# ---- Method 1 +# app_output=$(unify_run_test $app_name "$app_args") +# rc=$? + +# echo "$app_output" +# lcount=$(printf "%s" "$app_output" | wc -l) +# ---- + +# ---- Method 2 +# unify_run_test $app_name "$app_args" app_output +# rc=$? +# +# lcount=$(echo "$app_output" | wc -l) +# ---- +# +# The output of an example can then be tested with sharness, for example: +# +# test_expect_success "$app_name $app_args: (line_count=$lcount, rc=$rc)" ' +# test $rc = 0 && +# test $lcount = 8 +# ' +# +# For these tests, always include -b -c -n and -p in the app_args +# Then for the necessary tests, include -M -P -S -V or -x. + + +test_description="Write Tests" + +while [[ $# -gt 0 ]] +do + case $1 in + -h|--help) + ci_dir=$(dirname "$(readlink -fm $BASH_SOURCE)") + $ci_dir/001-setup.sh -h + exit + ;; + *) + echo "usage ./200-write-tests.sh -h|--help" + exit 1 + ;; + esac +done + +# These two functions are simply to prevent code duplication since testing the +# output of each example with sharness is the same process. These do not need to +# be used, especially if wanting to test for something specific when running an +# example. +unify_test_write() { + app_name=write-${1} + + # Run the test and get output + unify_run_test $app_name "$2" app_output + rc=$? + lcount=$(echo "$app_output" | wc -l) + + # Evaluate output + test_expect_success "$app_name $app_args: (line_count=${lcount}, rc=$rc)" ' + test $rc = 0 && + test $lcount = 11 + ' +} + +unify_test_write_posix() { + app_name=write-posix + + # Run the test and get output + if test_have_prereq POSIX; then + unify_run_test $app_name "$1" app_output + rc=$? + lcount=$(echo "$app_output" | wc -l) + filename=$(get_filename $app_name "$1" ".app") + fi + + # Evaluate output + test_expect_success POSIX "$app_name $1: (line_count=${lcount}, rc=$rc)" ' + test $rc = 0 && + test $lcount = 11 && + if [[ $io_pattern =~ (n1)$ ]]; then + test_path_is_file ${CI_POSIX_MP}/$filename + else + test_path_has_file_per_process $CI_POSIX_MP $filename + fi + ' +} + +# write-static -p n1 -n 2 -c 4KB -b 16KB +runmode=static +io_pattern="-p n1" +io_sizes="-n 2 -c $((4 * $KB)) -b $((16 * $KB))" +app_args="$io_pattern $io_sizes" +unify_test_write $runmode "$app_args" + +# write-gotcha -p n1 -n 2 -c 4KB -b 16KB +runmode=gotcha +unify_test_write $runmode "$app_args" + +# write-posix -p n1 -n 2 -c 4KB -b 16KB +runmode=posix +unify_test_write_posix "$app_args" + +# Switch to -p nn +io_pattern="-p nn" +app_args="$io_pattern $io_sizes" + +# write-posix -p nn -n 2 -c 4KB -b 16KB +unify_test_write_posix "$app_args" + +# write-gotcha -p nn -n 2 -c 4KB -b 16KB +runmode=gotcha +unify_test_write $runmode "$app_args" + +# write-static -p nn -n 2 -c 4KB -b 16KB +runmode=static +unify_test_write $runmode "$app_args" + +# Increase sizes: -n 16 -c 32KB -b 1MB + +# write-static -p nn -n 16 -c 32KB -b 1MB +io_sizes="-n 16 -c $((32 * $KB)) -b $MB" +app_args="$io_pattern $io_sizes" +unify_test_write $runmode "$app_args" + +# write-gotcha -p nn -n 16 -c 32KB -b 1MB +runmode=gotcha +unify_test_write $runmode "$app_args" + +# write-posix -p nn -n 16 -c 32KB -b 1MB +runmode=posix +unify_test_write_posix "$app_args" + +# Switch back to -p n1 +io_pattern="-p n1" +app_args="$io_pattern $io_sizes" + +# write-posix -p n1 -n 16 -c 32KB -b 1MB +unify_test_write_posix "$app_args" + +# write-gotcha -p n1 -n 16 -c 32KB -b 1MB +runmode=gotcha +unify_test_write $runmode "$app_args" + +# write-static -p n1 -n 16 -c 32KB -b 1MB +runmode=static +unify_test_write $runmode "$app_args" + +# Increase sizes: -n 32 -c 64KB -b 1MB + +# write-static -p n1 -n 32 -c 64KB -b 1MB +io_sizes="-n 32 -c $((64 * $KB)) -b $MB" +app_args="$io_pattern $io_sizes" +unify_test_write $runmode "$app_args" + +# write-gotcha -p n1 -n 32 -c 64KB -b 1MB +runmode=gotcha +unify_test_write $runmode "$app_args" + +# write-posix -p n1 -n 32 -c 64KB -b 1MB +runmode=posix +unify_test_write_posix "$app_args" + +# Switch to -p nn +io_pattern="-p nn" +app_args="$io_pattern $io_sizes" + +# write-posix -p nn -n 32 -c 64KB -b 1MB +unify_test_write_posix "$app_args" + +# write-gotcha -p nn -n 32 -c 64KB -b 1MB +runmode=gotcha +unify_test_write $runmode "$app_args" + +# write-static -p nn -n 32 -c 64KB -b 1MB +runmode=static +unify_test_write $runmode "$app_args" + +# Increase sizes: -n 64 -c 1MB -b 4MB + +# write-static -p nn -n 64 -c 1MB -b 4MB +io_sizes="-n 64 -c $MB -b $((4 * $MB))" +app_args="$io_pattern $io_sizes" +unify_test_write $runmode "$app_args" + +# write-gotcha -p nn -n 64 -c 1MB -b 4MB +runmode=gotcha +unify_test_write $runmode "$app_args" + +# write-posix -p nn -n 64 -c 1MB -b 4MB +runmode=posix +unify_test_write_posix "$app_args" + +# Switch back to -p n1 +io_pattern="-p n1" +app_args="$io_pattern $io_sizes" + +# write-posix -p n1 -n 64 -c 1MB -b 4MB +unify_test_write_posix "$app_args" + +# write-gotcha -p n1 -n 64 -c 1MB -b 4MB +runmode=gotcha +unify_test_write $runmode "$app_args" + +# write-static -p n1 -n 64 -c 1MB -b 4MB +runmode=static +unify_test_write $runmode "$app_args" + +# Increase sizes: -n 32 -c 1MB -b 16MB + +# write-static -p n1 -n 32 -c 1MB -b 16MB +io_sizes="-n 32 -c $MB -b $((16 * $MB))" +app_args="$io_pattern $io_sizes" +unify_test_write $runmode "$app_args" + +# write-gotcha -p n1 -n 32 -c 1MB -b 16MB +runmode=gotcha +unify_test_write $runmode "$app_args" + +# write-posix -p n1 -n 32 -c 1MB -b 16MB +runmode=posix +unify_test_write_posix "$app_args" + +# Switch to -p nn +io_pattern="-p nn" +app_args="$io_pattern $io_sizes" + +# write-posix -p nn -n 32 -c 1MB -b 16MB +unify_test_write_posix "$app_args" + +# write-gotcha -p nn -n 32 -c 1MB -b 16MB +runmode=gotcha +unify_test_write $runmode "$app_args" + +# write-static -p nn -n 32 -c 1MB -b 16MB +runmode=static +unify_test_write $runmode "$app_args" diff --git a/t/ci/120-read-tests.sh b/t/ci/120-read-tests.sh new file mode 100755 index 000000000..686c4e94f --- /dev/null +++ b/t/ci/120-read-tests.sh @@ -0,0 +1,240 @@ +#!/bin/bash + +# This contains all the tests for the read example. +# +# There are convenience functions such as `unify_run_test()` and get_filename() +# in the ci-functions.sh script that can make adding new tests easier. See the +# full UnifyFS documentatation for more info. +# +# There are multiple ways to to run an example using `unify_run_test()` +# 1. unify_run_test $app_name "$app_args" app_output +# 2. app_output=$(unify_run_test $app_name "app_args") +# +# ---- Method 1 +# app_output=$(unify_run_test $app_name "$app_args") +# rc=$? + +# echo "$app_output" +# lcount=$(printf "%s" "$app_output" | wc -l) +# ---- + +# ---- Method 2 +# unify_run_test $app_name "$app_args" app_output +# rc=$? +# +# lcount=$(echo "$app_output" | wc -l) +# ---- +# +# The output of an example can then be tested with sharness, for example: +# +# test_expect_success "$app_name $app_args: (line_count=$lcount, rc=$rc)" ' +# test $rc = 0 && +# test $lcount = 8 +# ' +# +# For these tests, always include -b -c -n and -p in the app_args +# Then for the necessary tests, include -M -P -S -V or -x. + + +test_description="Read Tests" + +while [[ $# -gt 0 ]] +do + case $1 in + -h|--help) + ci_dir=$(dirname "$(readlink -fm $BASH_SOURCE)") + $ci_dir/001-setup.sh -h + exit + ;; + *) + echo "usage ./300-read-tests.sh -h|--help" + exit 1 + ;; + esac +done + +# These two functions are simply to prevent code duplication since testing the +# output of each example with sharness is the same process. These do not need to +# be used, especially if wanting to test for something specific when running an +# example. +unify_test_read() { + app_name=read-${1} + + # Run the test and get output + unify_run_test $app_name "$2" app_output + rc=$? + lcount=$(echo "$app_output" | wc -l) + + # Evaluate output + test_expect_success "$app_name $app_args: (line_count=${lcount}, rc=$rc)" ' + test $rc = 0 && + test $lcount = 11 + ' +} + +unify_test_read_posix() { + app_name=read-posix + + # Run the test and get output + if test_have_prereq POSIX; then + unify_run_test $app_name "$1" app_output + rc=$? + lcount=$(echo "$app_output" | wc -l) + fi + + # Evaluate output + test_expect_success POSIX "$app_name $1: (line_count=${lcount}, rc=$rc)" ' + test $rc = 0 && + test $lcount = 11 + ' +} + +# read-static -p n1 -n 2 -c 4KB -b 16KB +runmode=static +io_pattern="-p n1" +io_sizes="-n 2 -c $((4 * $KB)) -b $((16 * $KB))" +app_args="$io_pattern $io_sizes" +unify_test_read $runmode "$app_args" + +# read-gotcha -p n1 -n 2 -c 4KB -b 16KB +runmode=gotcha +unify_test_read $runmode "$app_args" + +# read-posix -p n1 -n 2 -c 4KB -b 16KB +runmode=posix +unify_test_read_posix "$app_args" + +# Switch to -p nn +io_pattern="-p nn" +app_args="$io_pattern $io_sizes" + +# read-posix -p nn -n 2 -c 4KB -b 16KB +unify_test_read_posix "$app_args" + +# read-gotcha -p nn -n 2 -c 4KB -b 16KB +runmode=gotcha +unify_test_read $runmode "$app_args" + +# read-static -p nn -n 2 -c 4KB -b 16KB +runmode=static +unify_test_read $runmode "$app_args" + +# Increase sizes: -n 16 -c 32KB -b 1MB + +# read-static -p nn -n 16 -c 32KB -b 1MB +io_sizes="-n 16 -c $((32 * $KB)) -b $MB" +app_args="$io_pattern $io_sizes" +unify_test_read $runmode "$app_args" + +# read-gotcha -p nn -n 16 -c 32KB -b 1MB +runmode=gotcha +unify_test_read $runmode "$app_args" + +# read-posix -p nn -n 16 -c 32KB -b 1MB +runmode=posix +unify_test_read_posix "$app_args" + +# Switch back to -p n1 +io_pattern="-p n1" +app_args="$io_pattern $io_sizes" + +# read-posix -p n1 -n 16 -c 32KB -b 1MB +unify_test_read_posix "$app_args" + +# read-gotcha -p n1 -n 16 -c 32KB -b 1MB +runmode=gotcha +unify_test_read $runmode "$app_args" + +# read-static -p n1 -n 16 -c 32KB -b 1MB +runmode=static +unify_test_read $runmode "$app_args" + +# Increase sizes: -n 32 -c 64KB -b 1MB + +# read-static -p n1 -n 32 -c 64KB -b 1MB +io_sizes="-n 32 -c $((64 * $KB)) -b $MB" +app_args="$io_pattern $io_sizes" +unify_test_read $runmode "$app_args" + +# read-gotcha -p n1 -n 32 -c 64KB -b 1MB +runmode=gotcha +unify_test_read $runmode "$app_args" + +# read-posix -p n1 -n 32 -c 64KB -b 1MB +runmode=posix +unify_test_read_posix "$app_args" + +# Switch to -p nn +io_pattern="-p nn" +app_args="$io_pattern $io_sizes" + +# read-posix -p nn -n 32 -c 64KB -b 1MB +unify_test_read_posix "$app_args" + +# read-gotcha -p nn -n 32 -c 64KB -b 1MB +runmode=gotcha +unify_test_read $runmode "$app_args" + +# read-static -p nn -n 32 -c 64KB -b 1MB +runmode=static +unify_test_read $runmode "$app_args" + +# Increase sizes: -n 64 -c 1MB -b 4MB + +# read-static -p nn -n 64 -c 1MB -b 4MB +io_sizes="-n 64 -c $MB -b $((4 * $MB))" +app_args="$io_pattern $io_sizes" +unify_test_read $runmode "$app_args" + +# read-gotcha -p nn -n 64 -c 1MB -b 4MB +runmode=gotcha +unify_test_read $runmode "$app_args" + +# read-posix -p nn -n 64 -c 1MB -b 4MB +runmode=posix +unify_test_read_posix "$app_args" + +# Switch back to -p n1 +io_pattern="-p n1" +app_args="$io_pattern $io_sizes" + +# read-posix -p n1 -n 64 -c 1MB -b 4MB +unify_test_read_posix "$app_args" + +# read-gotcha -p n1 -n 64 -c 1MB -b 4MB +runmode=gotcha +unify_test_read $runmode "$app_args" + +# read-static -p n1 -n 64 -c 1MB -b 4MB +runmode=static +unify_test_read $runmode "$app_args" + +# Increase sizes: -n 32 -c 1MB -b 16MB + +# read-static -p n1 -n 32 -c 1MB -b 16MB +io_sizes="-n 32 -c $MB -b $((16 * $MB))" +app_args="$io_pattern $io_sizes" +unify_test_read $runmode "$app_args" + +# read-gotcha -p n1 -n 32 -c 1MB -b 16MB +runmode=gotcha +unify_test_read $runmode "$app_args" + +# read-posix -p n1 -n 32 -c 1MB -b 16MB +runmode=posix +unify_test_read_posix "$app_args" + +# Switch to -p nn +io_pattern="-p nn" +app_args="$io_pattern $io_sizes" + +# read-posix -p n1 -n 32 -c 1MB -b 16MB +unify_test_read_posix "$app_args" + +# read-gotcha -p n1 -n 32 -c 1MB -b 16MB +runmode=gotcha +unify_test_read $runmode "$app_args" + +# read-static -p n1 -n 32 -c 1MB -b 16MB +runmode=static +unify_test_read $runmode "$app_args" diff --git a/t/ci/990-stop-server.sh b/t/ci/990-stop-server.sh new file mode 100755 index 000000000..def649c87 --- /dev/null +++ b/t/ci/990-stop-server.sh @@ -0,0 +1,58 @@ +#!/bin/sh + +# This test checks that the server terminated successfully, checks the +# mountpoints, and then cleans up the hosts (if desired). + +test_description="Stopping the UnifyFS server" + +while [[ $# -gt 0 ]] +do + case $1 in + -h|--help) + ci_dir=$(dirname "$(readlink -fm $BASH_SOURCE)") + $ci_dir/001-setup.sh -h + exit + ;; + *) + echo "usage ./990-stop-server.sh -h|--help" + exit 1 + ;; + esac +done + +test_expect_success "unifyfsd is still running" ' + process_is_running unifyfsd 10 +' + +$UNIFYFS_BIN/unifyfs terminate -d &> ${UNIFYFS_LOG_DIR}/unifyfs.terminate.out + +test_expect_success "unifyfsd has stopped" ' + process_is_not_running unifyfsd 10 +' + +test_expect_success "verify unifyfsd has stopped" ' + test_must_fail process_is_running unifyfsd 10 +' + +# If UNIFYFS_MOUNTPOINT is an existing dir, verify that is it empty +test_expect_success REAL_MP "Verify UNIFYFS_MOUNTPOINT ($UNIFYFS_MP) is empty" ' + test_dir_is_empty $UNIFYFS_MP +' + +# Cleanup posix mountpoint +test_expect_success POSIX "Cleanup CI_POSIX_MP: $CI_POSIX_MP" ' + rm -rf $CI_POSIX_MP/*posix* +' + +# cleanup_hosts +test_expect_success PDSH,CLEAN "Cleanup hosts" ' + cleanup_hosts +' +# Remove trap +# If any tests failed, the suite will exit with 1 which will trigger the trap. +# Since the hosts were already cleaned at this point, can remove trap to prevent +# cleanup_hosts from being called again. +trap - EXIT + +# end here if running tests individually +[[ -z $full_run ]] && test_done diff --git a/t/ci/README.md b/t/ci/README.md new file mode 100644 index 000000000..5aa441012 --- /dev/null +++ b/t/ci/README.md @@ -0,0 +1,77 @@ +# Integration Testing + +The UnifyFS [examples](https://github.com/LLNL/UnifyFS/tree/dev/examples) are +being used as integration tests with continuous integration tools. These scripts +depend on [sharness](https://github.com/chriscool/sharness) which is set up in +the containing directory (`UnifyFS/t/`) and may not function properly if moved. + +The numbered scripts are the setup and tests being run by `RUN_CI_TESTS.sh`. + +`001-setup.sh` +: Checks for an installation of UnifyFS and sets up variables needed for +testing. + +`002-start-server.sh` +: Starts unifyfsd and tests to ensure it is running. + +`100-900` +: Scripts for testing the examples. + +`990-stop-server.sh` +: Stops unifyfsd and cleans up. + +The other scripts (`ci-functions.sh`, `setup-lsf.sh`, and `setup-slurm.sh`) +contain helper functions and variables used by the tests. Full details can be +found in [UnifyFS testing documentation](https://unifyfs.readthedocs.io/en/dev/testing.html#integration-tests). + +## Quickstart + +In order to run these scripts, make sure you are either submitting a batch job +to run them, or are first in an interactive allocation. + +> **Note:** `mpirun` is currently not supported but coming soon. + +To run all of the tests, simply run `./RUN_CI_TESTS.sh`. + +E.g.: + +```shell +$ ./RUN_CI_TESTS.sh +``` + +or + +```shell +$ prove -v RUN_CI_TESTS.sh +``` + +In order to run individual tests, source the `001-setup.sh` script first, +followed by `002-start-server.sh`. Then source each desired script after that +preceded by `$CI_DIR`. When finished, source the `990-stop-server.sh` script +last. + +E.g.: + +```shell +$ . full/path/to/001-setup.sh +$ . $CI_DIR/002-start-server.sh +$ . $CI_DIR/100-writeread-tests.sh +$ . $CI_DIR/990-stop-server.sh +``` + +If additional tests are desired, create a script after the fashion of +`100-writeread-tests.sh` where the prefixed number indicates the desired order +for running the tests. Then source that script in `RUN_CI_TESTS.sh` in the +desired order. + +### Environment Variables + +There are environment variables that can be set to change the default behavior +of this testing suite. See our [testing +guide](https://unifyfs.readthedocs.io/en/dev/testing.html#configuration-variables) +for a full list. + +## Developers + +See our [Testing Guide](https://unifyfs.readthedocs.io/en/dev/testing.html#integration-tests) +for full documentation on writing and adding additional tests. diff --git a/t/ci/RUN_CI_TESTS.sh b/t/ci/RUN_CI_TESTS.sh new file mode 100755 index 000000000..48a6910a9 --- /dev/null +++ b/t/ci/RUN_CI_TESTS.sh @@ -0,0 +1,127 @@ +#!/bin/sh + +# This script is to run the entire integration test suite of TAP tests for +# Unify. +# In order to run individual tests, run `./001-setup.sh -h`. +# +# To run all of the tests, simply run RUN_CI_TESTS.sh. +# +# E.g.: +# $ ./RUN_CI_TESTS.sh +# or +# $ prove -v RUN_CI_TESTS.sh +# +# If individual tests are desired to be run, source the 001-setup.sh script +# first, followed by 002-start-server.sh. Then source each desired script after +# that preceded by `$CI_DIR`. When finished, source the 990-stop-server.sh +# script last. +# +# E.g.: +# $ . full/path/to/001-setup.sh +# $ . $CI_DIR/002-start-server.sh +# $ . $CI_DIR/100-writeread-tests.sh +# $ . $CI_DIR/990-stop-server.sh +# +# Before doing either of these, make sure you have interactively allocated nodes +# or are submitting a batch job. +# +# If additional tests are desired, create a script after the fashion of +# 100-writeread-tests.sh where the prefixed number indicates the desired order +# for running the tests. Then source that script in this script below, in the +# desired order. + +test_description="Unify Integration Testing Suite" + +RUN_CI_TESTS_USAGE="$(cat <&2 "$errmsg USAGE: $FUNCNAME abs_path *file|*path/file" \ + "[prune_path]" + return 1 + fi + + # If dir provided in $3, set it as prune + [[ -n $3 ]] && local l_prune="-ipath $3 -prune -o" + local l_target="-path $2 -print -quit" + + local l_ret="$(find $1 -executable $l_prune $l_target)" + echo $l_ret + return 0 +} + +# Calculate the elapsed time between the two given times. +# $2 should be >= $1 +# +# $1 - The initial of the two times (in seconds) +# $2 - The latter of the two times (in seconds) +# +# Returns the elapsed time formated as HH:MM:SS +elapsed_time() +{ + # USAGE: elapsed_time start_time_in_seconds end_time_in_seconds + if [[ $# -ne 2 || $2 -lt $1 ]]; then + echo >&2 "$errmsg USAGE: $FUNCNAME start_time_in_sec end_time_in_sec" + return 1 + else + local l_start_time=$1 + local l_end_time=$2 + local l_diff=$(( l_end_time - l_start_time )) + # Determining the time $diff is since EPOC allows for it to auto format + local l_elap=$(date -u --date="@$l_diff" +'%X') + echo $l_elap + return 0 + fi +} + +# Format $1 bytes to KB, MB, or GB (e.g., format_bytes "1024" becomes 1KB) +# +# $1 - The positive whole number of bytes to format as KB, MB, or GB +# +# Returns $1 formatted as KB, MB, or GB +format_bytes() +{ + # USAGE: format_bytes int + if [[ -z $1 || $1 -lt 0 ]]; then + echo >&2 "$errmsg USAGE: $FUNCNAME int" + return 1 + fi + + if [[ $1 -lt $MB ]]; then # less than 1MB + if !(($1 % $KB)); then # divisible by 1KB + echo $(($1/$KB))KB + else # not divisible by 1KB + echo $(bc -l <<< "scale=2;$1/(2^10)")KB + fi + elif [[ $1 -ge $MB && $1 -lt $GB ]]; then # between 1MB and 1GB + if !(($1 % $MB)); then # divisible by 1MB + echo $(($1/$MB))MB + else # not divisible by 1MB + echo $(bc -l <<< "scale=2;$1/(2^20)")MB + fi + else # greater than or equal to 1GB + if !(($1 % $GB)); then # divisible by 1GB + echo $(($1/$GB))GB + else # not divisible by 1GB + echo $(bc -l <<< "scale=2;$1/(2^30)")GB + fi + fi + return 0 +} + +# Build the filename for an example so that if it shows up in the +# $UNIFYFS_MOUNTPOINT, it can be tracked to it's originating test +# +# Also allows testers to get what the filename will be in advance if called +# from test suite. This could be used for posix tests to ensure the file showed +# up in the mount point, as well as for cp/stat tests that potentially need the +# filename from a previous test. +# +# Bear in mind, the filename created in unify_run_test will have a .app suffix. +# +# $1 - The app_name that will be prepended to the formated app_args in the +# resulting filename +# $2 - The app_args that will be formated and appended to the app_name +# $3 - Optional suffix to append to the end of the file +# +# Returns a string with the spaces removed and hyphens replaced by underscores +# E.g.,: get_filename write-gotcha "-p n1 -n 32 -c 1024 -b 1048576" ".app" +# becomes +# write-gotcha_pn1_n32_c1KB_b1MB.app +get_filename() +{ + # USAGE: get_filename app_name app_args [app_suffix] + if [[ $# -lt 2 || $# -gt 3 || -z $1 || -z $2 ]]; then + echo >&2 "$errmsg USAGE: $FUNCNAME app_name app_args [app_suffix]" + return 1 + fi + + # Remove any blank spaces + local l_remove_spaces=${2//[[:blank:]]/} + # Replace hyphen(-) with underscore(_) + local l_replace_hyphens=${l_remove_spaces//-/_} + + # Parse out chunksize and blocksize values + local l_cs=$(echo $l_replace_hyphens | sed -r 's/.*c([0-9]{3,}).*/\1/') + local l_bs=$(echo $l_replace_hyphens | sed -r 's/.*b([0-9]{3,}).*/\1/') + + # Format chunksize and blocksize to KB, MB, or GB and replace them in the + # original string + local l_replace_chunk=${l_replace_hyphens//$l_cs/$(format_bytes "$l_cs")} + local l_replace_block=${l_replace_chunk//$l_bs/$(format_bytes "$l_bs")} + + # Finally build the filename + if [[ -n $3 ]]; then + # Append suffix if provided + local l_filename="${1}${l_replace_block}${3}" + else + local l_filename="${1}${l_replace_block}" + fi + echo $l_filename +} + +# Builds the test command that will be executed. Automatically sets any options +# that are always wanted (-vkf and the appropriate -m if posix test or not). +# +# Automatically builds the filename for -f based on the input app_name and +# app_args and has .app appended to the end. This filename then also has .err +# appended and is used for the stderr output file with JOB_RUN_COMMAND. +# +# Args that can be passed in are ([-pncbx][-A|-M|-P|-S|-V]). All other args are +# set automatically. +# +# $1 - Name of the example application to be tested (basetest-runmode) +# $2 - Args for $1 consisting of ([-pncbx][-A|-M|-P|-S|-V]). Encase in quotes. +# $3 - The runmode of test, used to determine if posix and set correct args +# +# Returns the full test command ready to be executed. +build_test_command() +{ + # USAGE: build_test_command app_exe_name app_args([-pncbx][-A|-M|-P|-S|-V]) + if [[ $# -ne 3 ]]; then + echo >&2 "$errmsg USAGE: $FUNCNAME app_name" \ + "app_args([-pncbx][-A|-M|-P|-S|-V]) runmode" + return 1 + fi + + # Autogenerate and format the filename based on app_name and app_args + local l_filename="$(get_filename $1 "$2")" + + # Add stderr output file to finish building JOB_RUN_COMMAND + local l_err_filename="$app_err ${UNIFYFS_LOG_DIR}/${l_filename}.err" + local l_job_run_command="$JOB_RUN_COMMAND $l_err_filename" + + # Build example_command with options that are always wanted. Might need to + # adjust for other tests (i.e., app-mpiio), or write new functions + local l_verbose="-v" + local l_app_id="-a $app_id" + + # Filename needs to be the write file if testing the read example + local l_app_name=$(echo $1 | sed -r 's/(\w)-.*/\1/') + if [[ $l_app_name = "read" ]]; then + local l_app_filename="-f $(get_filename write-$3 "$2").app" + else + #local l_check="-k" # not reliable atm + local l_app_filename="-f ${l_filename}.app" + fi + + # Set mountpoint to an existing one if running posix test + if [[ $3 = "posix" ]]; then + local l_mount="-U -m $CI_POSIX_MP" + else + local l_mount="-m $UNIFYFS_MP" + fi + + # Assemble full example_command + local l_app_args="$2 $l_app_id $l_check $l_verbose $l_mount $l_app_filename" + local l_full_app_name="${UNIFYFS_EXAMPLES}/${1} $l_app_args" + + # Assemble full test_command + local l_test_command="$l_job_run_command $l_full_app_name" + echo $l_test_command +} + +# Given a example application name and application args, run the example with +# the appropriate MPI runner and args. This function is meant to make running +# the cr, write, read, and writeread examples as easy as possible from the +# testing files. +# +# The build_test_command is called which automatically sets any options that +# are always wanted (-vkf and appropriate -m if posix test or not). The stderr +# output file is also created (based on the filename that is autogenerated) and +# the appropriate option is set for the JOB_RUN_COMMAND. +# +# Args that can be passed in are ([-pncbx][-A|-M|-P|-S|-V]). All other args are +# set automatically, including the filename (which is generated based on the +# input app_name and app_args). +# +# The third parameter is an optional "pass-by-reference" parameter that can +# contain the variable name for the resulting output to be stored in. +# Thus this function can be called in two different way: +# 1. unify_run_test $app_name "$app_args" app_output +# 2. app_output=$(unify_run_test $app_name "app_args") +# +# $1 - Name and runmode of the example application to be tested +# $2 - Args for $1 consisting of ([-pncbx][-A|-M|-P|-S|-V]). Encase in quotes. +# $3 - Optional output variable that is "passed by reference". +# +# Returns the return code of the executed example as well as the output +# produced by running the example. +unify_run_test() +{ + # USAGE: unify_run_test app_name app_args([-pncbx][-A|-M|-P|-S|-V]) + # [output_variable_name] + if [[ $# -lt 2 || $# -gt 3 ]]; then + echo >&2 "$errmsg USAGE: $FUNCNAME app_name" \ + "app_args([-pncbx][-A|-M|-P|-S|-V]) [output_variable_name]" + return 1 + fi + + # Parse out the runmode and check if valid + local l_runmode=$(echo $1 | sed -r 's/.*-(\w)/\1/') + if [[ ! $l_runmode =~ ^(static|gotcha|posix)$ ]]; then + echo >&2 "$errmsg In $FUNCNAME, runmode not valid in app_name ($1)" + return 1 + fi + + # Skip this test if posix test and CI_TEST_POSIX=no|NO + if ! test_have_prereq POSIX && [[ $l_runmode = "posix" ]]; then + return 42 + fi + + # Fail if user passed in filename, mountpoint, verbose or disable + # UnifyFS since these are auto added + local opt='(-f|--file|-m|--mount|-v|--verbose|-U|--disable-unifyfs)' + for s in $2; do + if [[ $s =~ $opt ]]; then + echo >&2 "$errmsg Call $FUNCNAME without $opt. Found $s" + return 1 + fi + done + + # Finally build and run the test + local l_test_command=$(build_test_command $1 "$2" $l_runmode) + say "Results for unifyfs_run_test: $l_test_command:" + + # Uncomment to change app_id (-a) for each test. Comment to leave as 0. + #app_id=$(echo $(($app_id + 1))) + + # Get resulting output and rc of running the test + local l_app_output; l_app_output="$($l_test_command)" + local l_rc=$? + + # Put the resulting output in the optional reference parameter + local l_input_var=$3 + if [[ "$l_input_var" ]]; then + eval $l_input_var="'$l_app_output'" + fi + + echo "$l_app_output" + return $l_rc + +} + +# Does some post-testing cleanup to include checking if any unifyfsd is still +# running and kills them after creating a stack trace. Also removes any files +# that were leftover on the hosts. +cleanup_hosts() +{ + + # Capture all output from cleanup in a log + exec 3>&1 4>&2 + exec &> ${UNIFYFS_LOG_DIR}/hosts.cleanup + + # Get the list of hosts in this allocation + local l_hl=$(get_hostlist) + echo "Hostlist: $l_hl" + local l_app=unifyfsd + + echo "+++++ cleaning processes +++++" + echo " --- collecting stacks ---" + # unifyfsd should have already been terminated at this point, so for each + # host, check if unifyfsd is still running. If so, export the pid for + # convenience, echo a message, and generate a stack. If not, echo it's not. + pdsh -w $l_hl '[[ -n $(pgrep "'$l_app'") ]] && \ + (export upid=$(pgrep "'$l_app'") && \ + echo "'$l_app' (pid $upid) still running - creating stack..." && \ + gstack $upid > \ + "'${UNIFYFS_LOG_DIR}'"/"'${l_app}'".pid-${upid}.stack) || \ + echo "'$l_app' not running"' + + echo " --- killing processes ---" + pdsh -w $l_hl 'pkill -e "'$l_app'"' + + echo "+++++ cleaning files +++++" + pdsh -w $l_hl 'test -f /dev/shm/svr_id && /bin/cat /dev/shm/svr_id' + pdsh -w $l_hl 'test -f /dev/shm/unifyfsd_id && /bin/cat \ + /dev/shm/unifyfsd_id' + pdsh -w $l_hl '/bin/rm -rfv /tmp/na_sm /tmp/*unifyfs* /var/tmp/*unifyfs* \ + /dev/shm/unifyfsd_id /dev/shm/svr_id /dev/shm/*na_sm* \ + "'${UNIFYFS_SPILLOVER_DATA_DIR}'"/spill*.log \ + "'${UNIFYFS_SPILLOVER_META_DIR}'"/spill*.log \ + /dev/shm/*-recv-* /dev/shm/*-req-* /dev/shm/*-super-*' + + # Reset capturing all output + exec 1>&3 2>&4 +} diff --git a/t/ci/setup-lsf.sh b/t/ci/setup-lsf.sh new file mode 100755 index 000000000..028483fde --- /dev/null +++ b/t/ci/setup-lsf.sh @@ -0,0 +1,82 @@ +#!/bin/sh + +[[ -z $LSB_DJOB_HOSTFILE ]] && \ + { echo >&2 "$errmsg Found jsrun; no allocated nodes detected"; exit 1; } + +# Functions specific to LSF + +# Get the compute hosts being used in the current allocation. +# +# Returns the unique hosts being used on LSF, exluding the launch host. +get_lsf_hosts() +{ + # NOTE: There's potential that some versions of LSF may change to where + # they put the user on the first compute node rather than a launch node. + local l_hosts=$(uniq $LSB_DJOB_HOSTFILE | tail -n +2) + echo $l_hosts +} + +# Gets the compute hosts being used in the current allocation and returns them +# as a list so they are usable by commands like pdsh. +# +# Returns the unique hosts being used on LSF as a list. +# (e.g., host1,host2,...,hostn) +get_hostlist() +{ + local l_hosts=$(get_lsf_hosts) + #replace spaces with commas + local l_hostlist=${l_hosts//[[:blank:]]/,} + echo $l_hostlist +} + +# Variables specific to LSF +nnodes=$(get_lsf_hosts | wc -w) + +# Define each resource set +nprocs=${CI_NPROCS:-$nnodes} +ncores=${CI_NCORES:-20} + +# Total resource sets and how many per host +nrs_per_node=${CI_NRS_PER_NODE:-1} +nres_sets=${CI_NRES_SETS:-$(($nnodes * $nrs_per_node))} + +if [[ $ncores -gt 20 ]]; then + echo >&2 "$errmsg Number of cores-per-resource-set (\$CI_NCORES=$ncores)" \ + "needs to be <= 20." + exit 1 +fi + +if (($nres_sets % $nrs_per_node)); then + echo >&2 "$errmsg Total number of resource sets ($nres_sets) must be" \ + "divisible by resource-sets-per-node ($nrs_per_node). Set" \ + "\$CI_NRES_SETS and/or \$CI_NRS_PER_NODE accordingly." + exit 1 +fi + +if [ $(($nrs_per_node * $ncores)) -gt 40 ]; then + echo >&2 "$errmsg Number of cores-per-resource-set ($ncores) *"\ + "resource-sets-per-node ($nrs_per_node) = $(($nrs_per_node*$ncores))" \ + "needs to be <= 40. Set \$CI_NCORES and/or \$CI_NRS_PER_NODE" \ + "accordingly." + exit 1 +fi + +if [ $(($nres_sets / $nrs_per_node)) -gt $nnodes ]; then + echo >&2 "$errmsg At least $(($nres_sets/$nrs_per_node)) allocated nodes" \ + "required for $nres_sets total resource sets with $nrs_per_node" \ + "resource-sets-per-node. Only $nnodes node(s) detected." + exit 1 +fi + +jsargs="-a${nprocs} -c${ncores} -r${nrs_per_node} -n${nres_sets}" + +app_out="-o" +app_err="-k" +JOB_RUN_COMMAND="jsrun $jsargs" +JOB_RUN_ONCE_PER_NODE="jsrun -r1" +JOB_ID=${JOB_ID:-$LSB_JOBID} + +echo "$infomsg ====================== LSF Job Info ======================" +echo "$infomsg ----------------------- Job Status -----------------------" +bjobs -l $JOB_ID +echo "$infomsg ----------------------------------------------------------" diff --git a/t/ci/setup-slurm.sh b/t/ci/setup-slurm.sh new file mode 100755 index 000000000..48749ef35 --- /dev/null +++ b/t/ci/setup-slurm.sh @@ -0,0 +1,34 @@ +#!/bin/sh + +[[ -z $SLURM_NNODES ]] && \ + { echo >&2 "$errmsg Found srun; no allocated nodes detected"; exit 1; } + +# Functions specific to SLURM + +# Get the compute hosts being used in the current allocation. Used with pdsh +# during cleanup after tests are done. +# +# Returns the unique hosts being used on SLURM. +get_hostlist() +{ + # In slurm, can do `pdsh -j $JOBID` to target all nodes in allocation, + # however using nodelist for consistency of cleanup_hosts function when + # using pdsh + echo $SLURM_NODELIST +} + +# Variables specific to SLURM +nnodes=$SLURM_NNODES +nres_sets=$SLURM_NNODES +nprocs=${CI_NPROCS:-$nnodes} + +app_out="-o" +app_err="-e" +JOB_RUN_COMMAND="srun -N${nnodes} --ntasks-per-node=${nprocs}" +JOB_RUN_ONCE_PER_NODE="srun -n${nnodes}" +JOB_ID=${JOB_ID:-$SLURM_JOBID} + +echo "$infomsg ====================== SLURM Job Info ======================" +echo "$infomsg ------------------------ Job Status ------------------------" +checkjob -v $JOB_ID +echo "$infomsg ------------------------------------------------------------" diff --git a/t/lib/testutil.c b/t/lib/testutil.c index a03a63a46..75ec78a2a 100644 --- a/t/lib/testutil.c +++ b/t/lib/testutil.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ #include @@ -79,15 +79,15 @@ void testutil_rand_path(char* buf, size_t len, const char* pfx) } /* - * Return a pointer to the path name of the UnifyCR mount point. Use the - * value of the environment variable UNIFYCR_MOUNT_POINT if it exists, + * Return a pointer to the path name of the UnifyFS mount point. Use the + * value of the environment variable UNIFYFS_MOUNT_POINT if it exists, * otherwise use P_tmpdir which is defined in stdio.h and is typically * /tmp. */ char* testutil_get_mount_point(void) { char* path; - char* env = getenv("UNIFYCR_MOUNT_POINT"); + char* env = getenv("UNIFYFS_MOUNT_POINT"); if (env != NULL) { path = env; diff --git a/t/lib/testutil.h b/t/lib/testutil.h index d5fc6bab3..001885914 100644 --- a/t/lib/testutil.h +++ b/t/lib/testutil.h @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -25,8 +25,8 @@ void testutil_rand_string(char* buf, size_t len); void testutil_rand_path(char* buf, size_t len, const char* pfx); /* - * Return a pointer to the path name of the UnifyCR mount point. Use the - * value of the environment variable UNIFYCR_MOUNT_POINT if it exists, + * Return a pointer to the path name of the UnifyFS mount point. Use the + * value of the environment variable UNIFYFS_MOUNT_POINT if it exists, * otherwise use P_tmpdir which is defined in stdio.h and is typically * /tmp. */ diff --git a/t/server/metadata_suite.c b/t/server/metadata_suite.c new file mode 100644 index 000000000..88d7c31c3 --- /dev/null +++ b/t/server/metadata_suite.c @@ -0,0 +1,80 @@ +#include + +#include "unifyfs_configurator.h" +#include "unifyfs_metadata.h" +#include "unifyfs_log.h" +#include "unifyfs_runstate.h" + +#include "t/lib/tap.h" + +#include "metadata_suite.h" + +int main(int argc, char* argv[]) +{ + /* need to initialize enougth of the server to use the metadata API */ + unifyfs_cfg_t server_cfg; + int rc, provided, glb_rank, glb_size; + + /* get the configuration */ + rc = unifyfs_config_init(&server_cfg, argc, argv); + if (rc != 0) { + exit(1); + } + + rc = unifyfs_write_runstate(&server_cfg); + if (rc != (int)UNIFYFS_SUCCESS) { + exit(1); + } + + rc = MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); + + if (rc != MPI_SUCCESS) { + exit(1); + } + + rc = MPI_Comm_rank(MPI_COMM_WORLD, &glb_rank); + if (rc != MPI_SUCCESS) { + exit(1); + } + + rc = MPI_Comm_size(MPI_COMM_WORLD, &glb_size); + if (rc != MPI_SUCCESS) { + exit(1); + } + + rc = meta_init_store(&server_cfg); + if (rc != 0) { + LOG(LOG_ERR, "%s", + unifyfs_error_enum_description(UNIFYFS_ERROR_MDINIT)); + exit(1); + } + + /* + * necessary infrastructure is initialized + * running tests + */ + + plan(NO_PLAN); + + // keep the order + + unifyfs_set_file_attribute_test(); + unifyfs_get_file_attribute_test(); + + + /* + * shut down infrastructure + */ + + // shutdown the metadata service + meta_sanitize(); + + // finalize mpi + MPI_Finalize(); + + // finish the testing + // needs to be last call + done_testing(); + + return EXIT_SUCCESS; +} diff --git a/t/server/metadata_suite.h b/t/server/metadata_suite.h new file mode 100644 index 000000000..3afcce2c2 --- /dev/null +++ b/t/server/metadata_suite.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2019, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + + +/* This is the collection of metadata tests to be run inside of + * metadata_suite.c. These tests are testing the wrapper functions found in + * server/src/unifyfs_metadata.c. + */ + + +#ifndef METADATA_SUITE_H +#define METADATA_SUITE_H + +int unifyfs_set_file_attribute_test(void); +int unifyfs_get_file_attribute_test(void); +int unifyfs_get_file_extents_test(void); + +#endif /* METADATA_SUITE_H */ diff --git a/t/server/unifyfs_meta_get_test.c b/t/server/unifyfs_meta_get_test.c new file mode 100644 index 000000000..db2c304e6 --- /dev/null +++ b/t/server/unifyfs_meta_get_test.c @@ -0,0 +1,60 @@ +#include + +#include "metadata_suite.h" +#include "unifyfs_meta.h" +#include "unifyfs_metadata.h" +#include "t/lib/tap.h" + +#define TEST_META_GFID_VALUE 0xbeef +#define TEST_META_FID_VALUE 0xfeed +#define TEST_META_FILE "/unifyfs/filename/to/nowhere" + +int unifyfs_set_file_attribute_test(void) +{ + int rc; + + /* create dummy file attribute */ + unifyfs_file_attr_t fattr = {0}; + + fattr.gfid = TEST_META_GFID_VALUE; + fattr.fid = TEST_META_FID_VALUE; + snprintf(fattr.filename, sizeof(fattr.filename), TEST_META_FILE); + fflush(NULL); + + rc = unifyfs_set_file_attribute(&fattr); + ok(UNIFYFS_SUCCESS == rc, "Stored file attribute"); + fflush(NULL); + return 0; +} + +int unifyfs_get_file_attribute_test(void) +{ + int rc; + unifyfs_file_attr_t fattr; + + rc = unifyfs_get_file_attribute(TEST_META_GFID_VALUE, &fattr); + ok(UNIFYFS_SUCCESS == rc && + TEST_META_GFID_VALUE == fattr.gfid && + TEST_META_FID_VALUE == fattr.fid && + (0 == strcmp(fattr.filename, TEST_META_FILE)), + "Retrieve file attributes (rc = %d, gfid = 0x%02X, fid = 0x%02X)", + rc, fattr.gfid, fattr.fid + ); + return 0; +} + +// this test is not run right now +int unifyfs_get_file_extents_test(void) +{ + int rc, num_values, num_keys; + int key_lens[16]; + unifyfs_key_t keys[16]; + unifyfs_keyval_t keyval[16]; + + rc = unifyfs_get_file_extents(num_keys, &keys, key_lens, + &num_values, &keyval); + ok(UNIFYFS_SUCCESS == rc, + "Retrieved file extents (rc = %d)", rc + ); + return 0; +} diff --git a/t/sharness.d/00-test-env.sh b/t/sharness.d/00-test-env.sh index 9f42bb129..5ab26fa28 100644 --- a/t/sharness.d/00-test-env.sh +++ b/t/sharness.d/00-test-env.sh @@ -1,15 +1,15 @@ # # Export some variables used by the test suite. # -# UNIFYCR_BUILD_DIR is set to build path (based on current directory) +# UNIFYFS_BUILD_DIR is set to build path (based on current directory) # -if test -z "$UNIFYCR_BUILD_DIR"; then +if test -z "$UNIFYFS_BUILD_DIR"; then if test -z "${builddir}"; then - UNIFYCR_BUILD_DIR="$(cd .. && pwd)" + UNIFYFS_BUILD_DIR="$(cd .. && pwd)" else - UNIFYCR_BUILD_DIR="$(cd ${builddir}/.. && pwd))" + UNIFYFS_BUILD_DIR="$(cd ${builddir}/.. && pwd))" fi - export UNIFYCR_BUILD_DIR + export UNIFYFS_BUILD_DIR fi # @@ -17,15 +17,17 @@ fi # to store dynamically generated paths for mountpoints and # metadata directories. # -export UNIFYCR_TEST_RUN_SCRIPT=$UNIFYCR_BUILD_DIR/t/test_run_env.sh +export UNIFYFS_TEST_RUN_SCRIPT=$UNIFYFS_BUILD_DIR/t/test_run_env.sh # # Find MPI job launcher. # -if test -n "$(which srun 2>/dev/null)"; then +if test -n "$(which jsrun 2>/dev/null)"; then + JOB_RUN_COMMAND="jsrun -r1 -n1" +elif test -n "$(which srun 2>/dev/null)"; then JOB_RUN_COMMAND="srun -n1 -N1" elif test -n "$(which mpirun 2>/dev/null)"; then - JOB_RUN_COMMAND="mpirun -wd $UNIFYCR_BUILD_DIR -np 1" + JOB_RUN_COMMAND="mpirun -wd $UNIFYFS_BUILD_DIR -np 1" fi if test -z "$JOB_RUN_COMMAND"; then @@ -39,6 +41,6 @@ export JOB_RUN_COMMAND # # Set paths to executables # -export UNIFYCRD=$UNIFYCR_BUILD_DIR/server/src/unifycrd -export TEST_WRITE_GOTCHA=$UNIFYCR_BUILD_DIR/client/tests/test_write_gotcha -export TEST_READ_GOTCHA=$UNIFYCR_BUILD_DIR/client/tests/test_read_gotcha +export UNIFYFSD=$UNIFYFS_BUILD_DIR/server/src/unifyfsd +export TEST_WRITE_GOTCHA=$UNIFYFS_BUILD_DIR/client/tests/test_write_gotcha +export TEST_READ_GOTCHA=$UNIFYFS_BUILD_DIR/client/tests/test_read_gotcha diff --git a/t/sharness.d/01-unifycr-settings.sh b/t/sharness.d/01-unifycr-settings.sh deleted file mode 100644 index c1decb5d1..000000000 --- a/t/sharness.d/01-unifycr-settings.sh +++ /dev/null @@ -1,35 +0,0 @@ -# -# Export variables to control UNIFYCR runtime settings. -# - -# -# Source a script that is dynamically generated by 0001-setup.t. -# -. $UNIFYCR_TEST_RUN_SCRIPT - - -# Common settings -UNIFYCR_MOUNTPOINT=${UNIFYCR_MOUNT_POINT:-$(mktemp -d)} -export UNIFYCR_MOUNTPOINT - -# Server settings -UNIFYCR_META_DB_PATH=${UNIFYCR_META_DB_PATH:-$(mktemp -d)} -UNIFYCR_META_DB_NAME=${UNIFYCR_META_DB_NAME:-unifycr_db} -UNIFYCR_META_SERVER_RATIO=${UNIFYCR_META_SERVER_RATIO:-1} -UNIFYCR_LOG_DIR=${UNIFYCR_LOG_DIRECTORY:-$UNIFYCR_META_DB_PATH} -UNIFYCR_LOG_FILE=${UNIFYCR_LOG_FILE:-unifycrd_debuglog} -UNIFYCR_RUNSTATE_DIR=${UNIFYCR_RUNSTATE_DIR:-$UNIFYCR_META_DB_PATH} -export UNIFYCR_LOG_DIR -export UNIFYCR_LOG_FILE -export UNIFYCR_META_DB_NAME -export UNIFYCR_META_DB_PATH -export UNIFYCR_META_SERVER_RATIO -export UNIFYCR_RUNSTATE_DIR - -# Client settings -UNIFYCR_SPILLOVER_ENABLED=${UNIFYCR_SPILLOVER_ENABLED:-"Y"} -UNIFYCR_SPILLOVER_DATA_DIR=${UNIFYCR_SPILLOVER_DATA_DIR:-$UNIFYCR_META_DB_PATH} -UNIFYCR_SPILLOVER_META_DIR=${UNIFYCR_SPILLOVER_META_DIR:-$UNIFYCR_META_DB_PATH} -export UNIFYCR_SPILLOVER_DATA_DIR -export UNIFYCR_SPILLOVER_META_DIR -export UNIFYCR_SPILLOVER_ENABLED diff --git a/t/sharness.d/01-unifyfs-settings.sh b/t/sharness.d/01-unifyfs-settings.sh new file mode 100644 index 000000000..a3155f3cc --- /dev/null +++ b/t/sharness.d/01-unifyfs-settings.sh @@ -0,0 +1,37 @@ +# +# Export variables to control UNIFYFS runtime settings. +# + +# +# Source a script that is dynamically generated by 0001-setup.t. +# +. $UNIFYFS_TEST_RUN_SCRIPT + + +# Common settings +UNIFYFS_MOUNTPOINT=${UNIFYFS_MOUNT_POINT:-$(mktemp -d)} +export UNIFYFS_MOUNTPOINT + +# Server settings +UNIFYFS_META_DB_PATH=${UNIFYFS_META_DB_PATH:-$(mktemp -d)} +UNIFYFS_META_DB_NAME=${UNIFYFS_META_DB_NAME:-unifyfs_db} +UNIFYFS_META_SERVER_RATIO=${UNIFYFS_META_SERVER_RATIO:-1} +UNIFYFS_LOG_DIR=${UNIFYFS_LOG_DIRECTORY:-$UNIFYFS_META_DB_PATH} +UNIFYFS_LOG_FILE=${UNIFYFS_LOG_FILE:-unifyfsd_debuglog} +UNIFYFS_RUNSTATE_DIR=${UNIFYFS_RUNSTATE_DIR:-$UNIFYFS_META_DB_PATH} +UNIFYFS_SHAREDFS_DIR=${UNIFYFS_SHAREDFS_DIR:-$UNIFYFS_META_DB_PATH} +export UNIFYFS_LOG_DIR +export UNIFYFS_LOG_FILE +export UNIFYFS_META_DB_NAME +export UNIFYFS_META_DB_PATH +export UNIFYFS_META_SERVER_RATIO +export UNIFYFS_RUNSTATE_DIR +export UNIFYFS_SHAREDFS_DIR + +# Client settings +UNIFYFS_SPILLOVER_ENABLED=${UNIFYFS_SPILLOVER_ENABLED:-"Y"} +UNIFYFS_SPILLOVER_DATA_DIR=${UNIFYFS_SPILLOVER_DATA_DIR:-$UNIFYFS_META_DB_PATH} +UNIFYFS_SPILLOVER_META_DIR=${UNIFYFS_SPILLOVER_META_DIR:-$UNIFYFS_META_DB_PATH} +export UNIFYFS_SPILLOVER_DATA_DIR +export UNIFYFS_SPILLOVER_META_DIR +export UNIFYFS_SPILLOVER_ENABLED diff --git a/t/sharness.d/02-functions.sh b/t/sharness.d/02-functions.sh index 53a280756..e4bb0b256 100755 --- a/t/sharness.d/02-functions.sh +++ b/t/sharness.d/02-functions.sh @@ -1,5 +1,5 @@ # -# Project-local sharness code for UnifyCR +# Project-local sharness code for UnifyFS # # Run command with a timeout @@ -20,7 +20,7 @@ run_timeout() # Returns 0 if the named process is found, otherwise returns 1. process_is_running() { - local proc=${1:-"unifycrd"} + local proc=${1:-"unifyfsd"} local secs_to_wait=${2:-15} local max_loops=$(($secs_to_wait * 2)) local i=0 @@ -45,7 +45,7 @@ process_is_running() # Returns 0 if the named process is not found, otherwise returns 1. process_is_not_running() { - local proc=${1:-"unifycrd"} + local proc=${1:-"unifyfsd"} local secs_to_wait=${2:-15} local max_loops=$(($secs_to_wait * 2)) local i=0 @@ -62,27 +62,44 @@ process_is_not_running() } # Create metadata directory if needed and start daemon. -unifycrd_start_daemon() +unifyfsd_start_daemon() { - if test -z "$UNIFYCR_META_DB_PATH"; then + # Make sure metadata directory exists + if test -z "$UNIFYFS_META_DB_PATH"; then + return 1 + elif ! test -d "$UNIFYFS_META_DB_PATH" && + ! mkdir $UNIFYFS_META_DB_PATH; then return 1 fi - if ! test -d "$UNIFYCR_META_DB_PATH" && - ! mkdir $UNIFYCR_META_DB_PATH; then - return 1 - fi - $UNIFYCRD + # Generate servers hostfile + # if test -z "$UNIFYFS_SHAREDFS_DIR"; then + # return 1 + # elif ! test -d "$UNIFYFS_SHAREDFS_DIR" && + # ! mkdir $UNIFYFS_SHAREDFS_DIR; then + # return 1 + # fi + # srvr_hosts=$UNIFYFS_SHAREDFS_DIR/unifyfsd.hosts + # if [ ! -f $srvr_hosts ]; then + # touch $srvr_hosts + # echo "1" >> $srvr_hosts + # hostname >> $srvr_hosts + # fi + # export UNIFYFS_SERVER_HOSTFILE=$srvr_hosts + + # run server daemon + $UNIFYFSD } -# Kill UnifyCR daemon. -unifycrd_stop_daemon() +# Kill UnifyFS daemon. +unifyfsd_stop_daemon() { - while killall -q -s TERM unifycrd 2>/dev/null; do :; done + while killall -q -s TERM unifyfsd 2>/dev/null; do :; done } # Remove the metadata directory. -unifycrd_cleanup() +unifyfsd_cleanup() { - rm -rf $UNIFYCR_META_DB_PATH + test -d "$UNIFYFS_META_DB_PATH" && rm -rf $UNIFYFS_META_DB_PATH + # test -d "$UNIFYFS_SHAREDFS_DIR" && rm -rf $UNIFYFS_SHAREDFS_DIR } diff --git a/t/std/fopen-fclose.c b/t/std/fopen-fclose.c index 23b23d3c5..3e1995714 100644 --- a/t/std/fopen-fclose.c +++ b/t/std/fopen-fclose.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ #include @@ -20,16 +20,16 @@ #include "t/lib/tap.h" #include "t/lib/testutil.h" -/* This function contains the tests for UNIFYCR_WRAP(fopen) and - * UNIFYCR_WRAP(fclose) found in client/src/unifycr-stdio.c. +/* This function contains the tests for UNIFYFS_WRAP(fopen) and + * UNIFYFS_WRAP(fclose) found in client/src/unifyfs-stdio.c. * * Notice the tests are ordered in a logical testing order. Changing the order * or adding new tests in between two others could negatively affect the * desired results. */ -int fopen_fclose_test(char* unifycr_root) +int fopen_fclose_test(char* unifyfs_root) { /* Diagnostic message for reading and debugging output */ - diag("Starting UNIFYCR_WRAP(fopen/fclose) tests"); + diag("Starting UNIFYFS_WRAP(fopen/fclose) tests"); char path[64]; char path2[64]; @@ -37,8 +37,8 @@ int fopen_fclose_test(char* unifycr_root) int rc; /* Generate a random file name in the mountpoint path to test on */ - testutil_rand_path(path, sizeof(path), unifycr_root); - testutil_rand_path(path2, sizeof(path2), unifycr_root); + testutil_rand_path(path, sizeof(path), unifyfs_root); + testutil_rand_path(path2, sizeof(path2), unifyfs_root); /* Verify we can create a new file. */ errno = 0; @@ -80,7 +80,7 @@ int fopen_fclose_test(char* unifycr_root) "fclose already closed file %s should fail (rc=%d, errno=%d): %s", path, rc, errno, strerror(errno)); - diag("Finished UNIFYCR_WRAP(fopen/fclose) tests"); + diag("Finished UNIFYFS_WRAP(fopen/fclose) tests"); return 0; } diff --git a/t/std/stdio_suite.c b/t/std/stdio_suite.c index acda26dc2..2a1dd4e8c 100644 --- a/t/std/stdio_suite.c +++ b/t/std/stdio_suite.c @@ -7,20 +7,20 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ #include #include -#include +#include #include "t/lib/tap.h" #include "t/lib/testutil.h" #include "stdio_suite.h" -/* The test suite for stdio wrappers found in client/src/unifycr-stdio.c. +/* The test suite for stdio wrappers found in client/src/unifyfs-stdio.c. * * * To add new tests to existing stdio tests: @@ -28,9 +28,9 @@ * .c file. * * - * When a new wrapper in unifycr-stdio.c needs to be tested: + * When a new wrapper in unifyfs-stdio.c needs to be tested: * 1. Create a .c file with a function called - * _test(char* unifycr_root) that contains all the TAP + * _test(char* unifyfs_root) that contains all the TAP * tests specific to that wrapper. * 2. Add the _test to stdio_suite.h. * 3. Add the .c file to the /t/Makefile.am under the @@ -41,7 +41,7 @@ int main(int argc, char* argv[]) { int rank_num; int rank; - char* unifycr_root; + char* unifyfs_root; int rc; MPI_Init(&argc, &argv); @@ -50,14 +50,14 @@ int main(int argc, char* argv[]) plan(NO_PLAN); - unifycr_root = testutil_get_mount_point(); + unifyfs_root = testutil_get_mount_point(); - /* Verify unifycr_mount succeeds. */ - rc = unifycr_mount(unifycr_root, rank, rank_num, 0); - ok(rc == 0, "unifycr_mount at %s (rc=%d)", unifycr_root, rc); + /* Verify unifyfs_mount succeeds. */ + rc = unifyfs_mount(unifyfs_root, rank, rank_num, 0); + ok(rc == 0, "unifyfs_mount at %s (rc=%d)", unifyfs_root, rc); if (rc != 0) { - BAIL_OUT("unifycr_mount in stdio_suite failed"); + BAIL_OUT("unifyfs_mount in stdio_suite failed"); } /* Add tests for new functions below in the order desired for testing. @@ -69,11 +69,11 @@ int main(int argc, char* argv[]) * tests that ran first to break as that is likely to cause subsequent * failures to start passing. */ - fopen_fclose_test(unifycr_root); - - done_testing(); + fopen_fclose_test(unifyfs_root); MPI_Finalize(); + done_testing(); + return 0; } diff --git a/t/std/stdio_suite.h b/t/std/stdio_suite.h index f4d8cf635..df5033a7b 100644 --- a/t/std/stdio_suite.h +++ b/t/std/stdio_suite.h @@ -7,20 +7,20 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* This is the collection of stdio wrapper tests to be run inside of * stdio_suite.c. These tests are testing the wrapper functions found in - * client/src/unifycr-stdio.c. + * client/src/unifyfs-stdio.c. * * - * When a new wrapper in unifycr-stdio.c needs to be tested: + * When a new wrapper in unifyfs-stdio.c needs to be tested: * 1. Create a .c file with a function called - * _test(char* unifycr_root) that contains all the TAP + * _test(char* unifyfs_root) that contains all the TAP * tests for that wrapper. * 2. Add the function name to this file, with comments. * 3. Add the .c file to the /t/Makefile.am under the @@ -30,7 +30,7 @@ #ifndef STDIO_SUITE_H #define STDIO_SUITE_H -/* Tests for UNIFYCR_WRAP(fopen) and UNIFYCR_WRAP(fclose) */ -int fopen_fclose_test(char* unifycr_root); +/* Tests for UNIFYFS_WRAP(fopen) and UNIFYFS_WRAP(fclose) */ +int fopen_fclose_test(char* unifyfs_root); #endif /* STDIO_SUITE_H */ diff --git a/t/sys/creat-close.c b/t/sys/creat-close.c index 82f43d819..16367036b 100644 --- a/t/sys/creat-close.c +++ b/t/sys/creat-close.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ #include @@ -20,16 +20,16 @@ #include "t/lib/tap.h" #include "t/lib/testutil.h" -/* This function contains the tests for UNIFYCR_WRAP(creat) and - * UNIFYCR_WRAP(close) found in client/src/unifycr-sysio.c. +/* This function contains the tests for UNIFYFS_WRAP(creat) and + * UNIFYFS_WRAP(close) found in client/src/unifyfs-sysio.c. * * Notice the tests are ordered in a logical testing order. Changing the order * or adding new tests in between two others could negatively affect the * desired results. */ -int creat_close_test(char* unifycr_root) +int creat_close_test(char* unifyfs_root) { /* Diagnostic message for reading and debugging output */ - diag("Starting UNIFYCR_WRAP(creat/close) tests"); + diag("Starting UNIFYFS_WRAP(creat/close) tests"); char path[64]; int mode = 0600; @@ -37,7 +37,7 @@ int creat_close_test(char* unifycr_root) int rc = -1; /* Create a random file name at the mountpoint path to test on */ - testutil_rand_path(path, sizeof(path), unifycr_root); + testutil_rand_path(path, sizeof(path), unifyfs_root); /* Verify closing a non-existent file fails with errno=EBADF */ errno = 0; @@ -81,7 +81,7 @@ int creat_close_test(char* unifycr_root) * checks if anything files are found in the mountpoint, meaning creat * wasn't wrapped properly. */ - diag("Finished UNIFYCR_WRAP(creat/close) tests"); + diag("Finished UNIFYFS_WRAP(creat/close) tests"); return 0; } diff --git a/t/sys/creat64.c b/t/sys/creat64.c index c3c492215..f2a6ab126 100644 --- a/t/sys/creat64.c +++ b/t/sys/creat64.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ #include @@ -21,16 +21,16 @@ #include "t/lib/tap.h" #include "t/lib/testutil.h" -/* This function contains the tests for UNIFYCR_WRAP(creat64) found in - * client/src/unifycr-sysio.c. +/* This function contains the tests for UNIFYFS_WRAP(creat64) found in + * client/src/unifyfs-sysio.c. * * Notice the tests are ordered in a logical testing order. Changing the order * or adding new tests in between two others could negatively affect the * desired results. */ -int creat64_test(char* unifycr_root) +int creat64_test(char* unifyfs_root) { /* Diagnostic message for reading and debugging output */ - diag("Starting UNIFYCR_WRAP(creat64) tests"); + diag("Starting UNIFYFS_WRAP(creat64) tests"); char path[64]; int mode = 0600; @@ -38,9 +38,9 @@ int creat64_test(char* unifycr_root) int rc; /* Create a random file name at the mountpoint path to test on */ - testutil_rand_path(path, sizeof(path), unifycr_root); + testutil_rand_path(path, sizeof(path), unifyfs_root); - skip(1, 2, "remove when UNIFYCR(create64) has been implemented"); + skip(1, 2, "remove when UNIFYFS(create64) has been implemented"); /* Verify we can create a non-existent file. */ errno = 0; fd = creat64(path, mode); @@ -58,7 +58,7 @@ int creat64_test(char* unifycr_root) rc = close(fd); end_skip; - diag("Finished UNIFYCR_WRAP(creat64) tests"); + diag("Finished UNIFYFS_WRAP(creat64) tests"); return 0; } diff --git a/t/sys/mkdir-rmdir.c b/t/sys/mkdir-rmdir.c index 3c1a01848..e3ac88622 100644 --- a/t/sys/mkdir-rmdir.c +++ b/t/sys/mkdir-rmdir.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ #include @@ -20,16 +20,16 @@ #include "t/lib/tap.h" #include "t/lib/testutil.h" -/* This function contains the tests for UNIFYCR_WRAP(mkdir) and - * UNIFYCR_WRAP(rmdir) found in client/src/unifycr-sysio.c. +/* This function contains the tests for UNIFYFS_WRAP(mkdir) and + * UNIFYFS_WRAP(rmdir) found in client/src/unifyfs-sysio.c. * * Notice the tests are ordered in a logical testing order. Changing the order * or adding new tests in between two others could negatively affect the * desired results. */ -int mkdir_rmdir_test(char* unifycr_root) +int mkdir_rmdir_test(char* unifyfs_root) { /* Diagnostic message for reading and debugging output */ - diag("Starting UNIFYCR_WRAP(mkdir/rmdir) tests"); + diag("Starting UNIFYFS_WRAP(mkdir/rmdir) tests"); char dir_path[64]; char file_path[64]; @@ -42,8 +42,8 @@ int mkdir_rmdir_test(char* unifycr_root) int rc; /* Create random dir and file path names at the mountpoint to test on */ - testutil_rand_path(dir_path, sizeof(dir_path), unifycr_root); - testutil_rand_path(file_path, sizeof(file_path), unifycr_root); + testutil_rand_path(dir_path, sizeof(dir_path), unifyfs_root); + testutil_rand_path(file_path, sizeof(file_path), unifyfs_root); /* Create path of a subdirectory under dir_path */ strcpy(subdir_path, dir_path); @@ -124,7 +124,7 @@ int mkdir_rmdir_test(char* unifycr_root) rc = close(fd); /* todo_mkdir_4: Remove when issue is resolved */ - todo("mkdir_4: unifycr currently creates all paths as separate entities"); + todo("mkdir_4: unifyfs currently creates all paths as separate entities"); /* Verify creating a directory whose parent is a file fails with * errno=ENOTDIR */ fd = creat(file_path, file_mode); @@ -145,7 +145,7 @@ int mkdir_rmdir_test(char* unifycr_root) file_path, rc, errno, strerror(errno)); /* todo_mkdir_5: Remove when issue is resolved */ - todo("mkdir_5: unifycr currently creates all paths as separate entities"); + todo("mkdir_5: unifyfs currently creates all paths as separate entities"); /* Verify rmdir a non-empty directory fails with errno=ENOTEMPTY */ errno = 0; rc = rmdir(dir_path); @@ -169,10 +169,10 @@ int mkdir_rmdir_test(char* unifycr_root) /* Verify trying to rmdir the mount point fails with errno=EBUSY */ errno = 0; - rc = rmdir(unifycr_root); + rc = rmdir(unifyfs_root); ok(rc < 0 && errno == EBUSY, "rmdir mount point %s should fail (rc=%d, errno=%d): %s", - unifycr_root, rc, errno, strerror(errno)); + unifyfs_root, rc, errno, strerror(errno)); /* CLEANUP * @@ -180,7 +180,7 @@ int mkdir_rmdir_test(char* unifycr_root) * if anything actually ended up in the mountpoint, meaning a function * wasn't wrapped properly. */ - diag("Finished UNIFYCR_WRAP(mkdir/rmdir) tests"); + diag("Finished UNIFYFS_WRAP(mkdir/rmdir) tests"); return 0; } diff --git a/t/sys/open.c b/t/sys/open.c index 0e86ee79f..d88a5d4a1 100644 --- a/t/sys/open.c +++ b/t/sys/open.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ #include @@ -21,16 +21,16 @@ #include "t/lib/tap.h" #include "t/lib/testutil.h" -/* This function contains the tests for UNIFYCR_WRAP(open) found in - * client/src/unifycr-sysio.c. +/* This function contains the tests for UNIFYFS_WRAP(open) found in + * client/src/unifyfs-sysio.c. * * Notice the tests are ordered in a logical testing order. Changing the order * or adding new tests in between two others could negatively affect the * desired results. */ -int open_test(char* unifycr_root) +int open_test(char* unifyfs_root) { /* Diagnostic message for reading and debugging output */ - diag("Starting UNIFYCR_WRAP(open) tests"); + diag("Starting UNIFYFS_WRAP(open) tests"); char path[64]; char dir_path[64]; @@ -40,8 +40,8 @@ int open_test(char* unifycr_root) int rc; /* Create a random file and dir name at the mountpoint path to test on */ - testutil_rand_path(path, sizeof(path), unifycr_root); - testutil_rand_path(dir_path, sizeof(dir_path), unifycr_root); + testutil_rand_path(path, sizeof(path), unifyfs_root); + testutil_rand_path(dir_path, sizeof(dir_path), unifyfs_root); /* Verify opening a non-existent file without O_CREAT fails with * errno=ENOENT */ @@ -94,7 +94,7 @@ int open_test(char* unifycr_root) * properly. */ rc = rmdir(dir_path); - diag("Finished UNIFYCR_WRAP(open) tests"); + diag("Finished UNIFYFS_WRAP(open) tests"); return 0; } diff --git a/t/sys/open64.c b/t/sys/open64.c index d1d2436d7..f831e72ae 100644 --- a/t/sys/open64.c +++ b/t/sys/open64.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ #include @@ -19,20 +19,20 @@ #include #include #include -#include +#include #include "t/lib/tap.h" #include "t/lib/testutil.h" -/* This function contains the tests for UNIFYCR_WRAP(open64) found in - * client/src/unifycr-sysio.c. +/* This function contains the tests for UNIFYFS_WRAP(open64) found in + * client/src/unifyfs-sysio.c. * * Notice the tests are ordered in a logical testing order. Changing the order * or adding new tests in between two others could negatively affect the * desired results. */ -int open64_test(char* unifycr_root) +int open64_test(char* unifyfs_root) { /* Diagnostic message for reading and debugging output */ - diag("Starting UNIFYCR_WRAP(open64) tests"); + diag("Starting UNIFYFS_WRAP(open64) tests"); char path[64]; int mode = 0600; @@ -40,7 +40,7 @@ int open64_test(char* unifycr_root) int rc; /* Create a random file name at the mountpoint path to test on */ - testutil_rand_path(path, sizeof(path), unifycr_root); + testutil_rand_path(path, sizeof(path), unifyfs_root); /* Verify opening a non-existent file without O_CREAT fails with * errno=ENOENT */ @@ -74,7 +74,7 @@ int open64_test(char* unifycr_root) rc = close(fd); - diag("Finished UNIFYCR_WRAP(open64) tests"); + diag("Finished UNIFYFS_WRAP(open64) tests"); return 0; } diff --git a/t/sys/sysio_suite.c b/t/sys/sysio_suite.c index ba3f47e18..e3db87859 100644 --- a/t/sys/sysio_suite.c +++ b/t/sys/sysio_suite.c @@ -7,20 +7,20 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ #include #include -#include +#include #include "t/lib/tap.h" #include "t/lib/testutil.h" #include "sysio_suite.h" -/* The test suite for sysio wrappers found in client/src/unifycr-sysio.c. +/* The test suite for sysio wrappers found in client/src/unifyfs-sysio.c. * * * To add new tests to existing sysio tests: @@ -28,9 +28,9 @@ * .c file. * * - * When a new wrapper in unifycr-sysio.c needs to be tested: + * When a new wrapper in unifyfs-sysio.c needs to be tested: * 1. Create a .c file with a function called - * _test(char* unifycr_root) that contains all the TAP + * _test(char* unifyfs_root) that contains all the TAP * tests specific to that wrapper. * 2. Add the _test to sysio_suite.h. * 3. Add the .c file to the /t/Makefile.am under the @@ -41,7 +41,7 @@ int main(int argc, char* argv[]) { int rank_num; int rank; - char* unifycr_root; + char* unifyfs_root; int rc; MPI_Init(&argc, &argv); @@ -50,15 +50,15 @@ int main(int argc, char* argv[]) plan(NO_PLAN); - unifycr_root = testutil_get_mount_point(); + unifyfs_root = testutil_get_mount_point(); - /* Verify unifycr_mount succeeds. */ - rc = unifycr_mount(unifycr_root, rank, rank_num, 0); - ok(rc == 0, "unifycr_mount at %s (rc=%d)", unifycr_root, rc); + /* Verify unifyfs_mount succeeds. */ + rc = unifyfs_mount(unifyfs_root, rank, rank_num, 0); + ok(rc == 0, "unifyfs_mount at %s (rc=%d)", unifyfs_root, rc); /* If the mount fails, bailout, as there is no point in running the tests */ if (rc != 0) { - BAIL_OUT("unifycr_mount in sysio_suite failed"); + BAIL_OUT("unifyfs_mount in sysio_suite failed"); } /* Add tests for new functions below in the order desired for testing. @@ -71,19 +71,19 @@ int main(int argc, char* argv[]) * break as that is likely to cause subsequent failures to start passing. */ - creat_close_test(unifycr_root); + creat_close_test(unifyfs_root); - creat64_test(unifycr_root); + creat64_test(unifyfs_root); - mkdir_rmdir_test(unifycr_root); + mkdir_rmdir_test(unifyfs_root); - open_test(unifycr_root); + open_test(unifyfs_root); - open64_test(unifycr_root); - - done_testing(); + open64_test(unifyfs_root); MPI_Finalize(); + done_testing(); + return 0; } diff --git a/t/sys/sysio_suite.h b/t/sys/sysio_suite.h index ff7c3dddf..01f4e87f6 100644 --- a/t/sys/sysio_suite.h +++ b/t/sys/sysio_suite.h @@ -7,20 +7,20 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* This is the collection of sysio wrapper tests to be run inside of * sysio_suite.c. These tests are testing the wrapper functions found in - * client/src/unifycr-sysio.c. + * client/src/unifyfs-sysio.c. * * - * When a new wrapper in unifycr-sysio.c needs to be tested: + * When a new wrapper in unifyfs-sysio.c needs to be tested: * 1. Create a .c file with a function called - * _test(char *unifycr_root) that contains all the TAP + * _test(char *unifyfs_root) that contains all the TAP * tests for that wrapper. * 2. Add the function name to this file, with comments. * 3. Add the .c file to the /t/Makefile.am under the @@ -30,19 +30,19 @@ #ifndef SYSIO_SUITE_H #define SYSIO_SUITE_H -/* Tests for UNIFYCR_WRAP(creat) and UNIFYCR_WRAP(close) */ -int creat_close_test(char* unifycr_root); +/* Tests for UNIFYFS_WRAP(creat) and UNIFYFS_WRAP(close) */ +int creat_close_test(char* unifyfs_root); -/* Tests for UNIFYCR_WRAP(creat64) */ -int creat64_test(char* unifycr_root); +/* Tests for UNIFYFS_WRAP(creat64) */ +int creat64_test(char* unifyfs_root); -/* Tests for UNIFYCR_WRAP(mkdir) and UNIFYCR_WRAP(rmdir) */ -int mkdir_rmdir_test(char* unifycr_root); +/* Tests for UNIFYFS_WRAP(mkdir) and UNIFYFS_WRAP(rmdir) */ +int mkdir_rmdir_test(char* unifyfs_root); -/* Tests for UNIFYCR_WRAP(open) */ -int open_test(char* unifycr_root); +/* Tests for UNIFYFS_WRAP(open) */ +int open_test(char* unifyfs_root); -/* Tests for UNIFYCR_WRAP(open64) */ -int open64_test(char* unifycr_root); +/* Tests for UNIFYFS_WRAP(open64) */ +int open64_test(char* unifyfs_root); #endif /* SYSIO_SUITE_H */ diff --git a/t/unifycr_unmount.c b/t/unifyfs_unmount.c similarity index 53% rename from t/unifycr_unmount.c rename to t/unifyfs_unmount.c index 2fec9c375..a90d363df 100644 --- a/t/unifycr_unmount.c +++ b/t/unifyfs_unmount.c @@ -1,12 +1,12 @@ #include #include -#include +#include #include "t/lib/tap.h" #include "t/lib/testutil.h" int main(int argc, char* argv[]) { - char* unifycr_root; + char* unifyfs_root; int rank_num; int rank; int rc; @@ -17,20 +17,20 @@ int main(int argc, char* argv[]) plan(NO_PLAN); - unifycr_root = testutil_get_mount_point(); + unifyfs_root = testutil_get_mount_point(); /* - * Verify unifycr_mount succeeds. + * Verify unifyfs_mount succeeds. */ - rc = unifycr_mount(unifycr_root, rank, rank_num, 0); - ok(rc == 0, "unifycr_mount at %s (rc=%d)", unifycr_root, rc); + rc = unifyfs_mount(unifyfs_root, rank, rank_num, 0); + ok(rc == 0, "unifyfs_mount at %s (rc=%d)", unifyfs_root, rc); - rc = unifycr_unmount(); - ok(rc == 0, "unifycr_unmount succeeds (rc=%d)", rc); - - done_testing(); + rc = unifyfs_unmount(); + ok(rc == 0, "unifyfs_unmount succeeds (rc=%d)", rc); MPI_Finalize(); + done_testing(); + return 0; } diff --git a/util/Makefile.am b/util/Makefile.am index 0f758272a..ca451f459 100644 --- a/util/Makefile.am +++ b/util/Makefile.am @@ -1 +1 @@ -SUBDIRS = unifycr +SUBDIRS = scripts unifyfs diff --git a/util/scripts/Makefile.am b/util/scripts/Makefile.am new file mode 100644 index 000000000..b6ea9d28d --- /dev/null +++ b/util/scripts/Makefile.am @@ -0,0 +1 @@ +SUBDIRS = lsfcsm diff --git a/util/scripts/lsfcsm/Makefile.am b/util/scripts/lsfcsm/Makefile.am new file mode 100644 index 000000000..14e9a22b8 --- /dev/null +++ b/util/scripts/lsfcsm/Makefile.am @@ -0,0 +1,3 @@ +sbin_SCRIPTS = unifyfs_lsfcsm_prolog unifyfs_lsfcsm_epilog + +CLEANFILES = $(sbin_SCRIPTS) diff --git a/util/scripts/lsfcsm/README.md b/util/scripts/lsfcsm/README.md new file mode 100644 index 000000000..f58e12f6a --- /dev/null +++ b/util/scripts/lsfcsm/README.md @@ -0,0 +1,52 @@ +### Integration with IBM Platform LSF with Cluster System Manager (CSM) + +To enable job-level integration of UnifyFS, where the UnifyFS daemon +is automatically launched on each node during job startup and +terminate once the job completes, additions must be made +to the CSM prologue and epilogue Python scripts. Typically, +these CSM scripts are found somewhere within the `/opt/ibm/csm` +directory and named `privileged_prolog` and `privileged_epilog`. +Consult with your LSF system administrator for more precise location +information, and to make the changes described below. + +We provide helper scripts that launch and terminate a UnifyFS daemon +(i.e., `unifyfsd`) on a node. The scripts are: +* launch: `/sbin/unifyfs_lsfcsm_prolog` +* terminate: `/sbin/unifyfs_lsfcsm_epilog` + +Integration of the helper scripts requires editing the CSM prologue and +epilogue scripts to support a new "unifyfs" job allocation flag. +Users may specify the allocation flag as an option to the `bsub` +command: +```shell +[prompt]$ bsub -alloc_flags unifyfs jobscript.lsf +``` +Alternatively, the job script may contain the flag as a `#BSUB` +directive: +```shell +#! /bin/bash -l +#BSUB -alloc_flags UNIFYFS +``` + +Supporting the new flag is as simple as adding a new `elif` block to +the flag parsing loop, as shown below for an excerpt from +`privileged_prolog`. Note that the CSM scripts use uppercase strings +for flag comparison. + +```python +flags = args.user_flags.split(" ") +regex = re.compile ("(\D+)(\d*)$") +for feature in flags: + fparse = regex.match (feature.upper()) + if hasattr(fparse, 'group'): + if fparse.group(0) == "HUGEPAGES": + os.chmod("/dev/hugepages", 1777) + elif fparse.group(0) == "UNIFYFS": + cmd = ['%s/unifyfs_lsfcsm_prolog' % JOB_SCRIPT_DIR] + runcmd (cmd) +``` + +The provided helper scripts should be installed as root to the +`JOB_SCRIPT_DIR` used by the CSM scripts (e.g., `/opt/ibm/csm/prologs`). + + diff --git a/util/scripts/lsfcsm/unifyfs_lsfcsm_epilog.in b/util/scripts/lsfcsm/unifyfs_lsfcsm_epilog.in new file mode 100644 index 000000000..ac62adbfb --- /dev/null +++ b/util/scripts/lsfcsm/unifyfs_lsfcsm_epilog.in @@ -0,0 +1,20 @@ +#!/bin/bash + +bindir=@unifyfs_bin_path@ +unifyfsd=$bindir/unifyfsd + +exec &>> /var/log/unifyfs-lsfcsm.$CSM_PRIMARY_JOB_ID + +echo "Stopping UnifyFS daemon" + +if [[ -x $(type -p pkill) ]]; then + echo "running: pkill unifyfsd" + pkill unifyfsd +elif [[ -x $(type -p pidof) ]]; then + pid=$(pidof -s $unifyfsd) + if [[ -n $pid ]]; then + echo "running: kill $pid" + kill $pid + fi +fi + diff --git a/util/scripts/lsfcsm/unifyfs_lsfcsm_prolog.in b/util/scripts/lsfcsm/unifyfs_lsfcsm_prolog.in new file mode 100644 index 000000000..8860c22af --- /dev/null +++ b/util/scripts/lsfcsm/unifyfs_lsfcsm_prolog.in @@ -0,0 +1,25 @@ +#!/bin/bash + +bindir=@unifyfs_bin_path@ +unifyfsd=$bindir/unifyfsd + +exec &>> /var/log/unifyfs-lsfcsm.$CSM_PRIMARY_JOB_ID + +echo "Starting UnifyFS daemon" + +if [[ -f /etc/rc.d/init.d/functions ]]; then + source /etc/rc.d/init.d/functions + echo "running: daemon --user=$CSM_USER_NAME $unifyfsd" + daemon --user=$CSM_USER_NAME $unifyfsd & +elif [[ -x $(type -p runuser) ]]; then + echo "running: runuser -s /bin/bash $CSM_USER_NAME -c $unifyfsd" + runuser -s /bin/bash $CSM_USER_NAME -c $unifyfsd & +elif [[ -x $(type -p su) ]]; then + echo "running: su -s /bin/bash $CSM_USER_NAME -c $unifyfsd" + su -s /bin/bash $CSM_USER_NAME -c $unifyfsd & +else + # WHAT KIND OF SYSTEM IS THIS? + echo "ERROR: $0 - unable to run unifyfsd as user $CSM_USER_NAME" + exit 1 +fi + diff --git a/util/unifycr/Makefile.am b/util/unifyfs/Makefile.am similarity index 100% rename from util/unifycr/Makefile.am rename to util/unifyfs/Makefile.am diff --git a/util/unifycr/src/Makefile.am b/util/unifyfs/src/Makefile.am similarity index 50% rename from util/unifycr/src/Makefile.am rename to util/unifyfs/src/Makefile.am index f6bd4d657..9ae1304f1 100644 --- a/util/unifycr/src/Makefile.am +++ b/util/unifyfs/src/Makefile.am @@ -1,11 +1,11 @@ -bin_PROGRAMS = unifycr +bin_PROGRAMS = unifyfs -unifycr_SOURCES = unifycr.c \ - unifycr-rm.c +unifyfs_SOURCES = unifyfs.c \ + unifyfs-rm.c -noinst_HEADERS = unifycr.h +noinst_HEADERS = unifyfs.h -unifycr_LDADD = $(top_builddir)/common/src/libunifycr_common.la +unifyfs_LDADD = $(top_builddir)/common/src/libunifyfs_common.la AM_CPPFLAGS = -I$(top_srcdir)/common/src \ -DBINDIR=\"$(bindir)\" \ diff --git a/util/unifycr/src/unifycr-rm.c b/util/unifyfs/src/unifyfs-rm.c similarity index 75% rename from util/unifycr/src/unifycr-rm.c rename to util/unifyfs/src/unifyfs-rm.c index 84168efcb..46f030a3a 100644 --- a/util/unifycr/src/unifycr-rm.c +++ b/util/unifyfs/src/unifyfs-rm.c @@ -7,9 +7,9 @@ * LLNL-CODE-741539 * All rights reserved. * - * This is the license for UnifyCR. - * For details, see https://github.com/LLNL/UnifyCR. - * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ /* @@ -46,29 +46,29 @@ #include #include -#include "unifycr.h" +#include "unifyfs.h" -typedef int (*unifycr_rm_read_resource_t)(unifycr_resource_t* resource); +typedef int (*unifyfs_rm_read_resource_t)(unifyfs_resource_t* resource); -typedef int (*unifycr_rm_launch_t)(unifycr_resource_t* resource, - unifycr_args_t* args); +typedef int (*unifyfs_rm_launch_t)(unifyfs_resource_t* resource, + unifyfs_args_t* args); -typedef int (*unifycr_rm_terminate_t)(unifycr_resource_t* resource, - unifycr_args_t* args); +typedef int (*unifyfs_rm_terminate_t)(unifyfs_resource_t* resource, + unifyfs_args_t* args); struct _ucr_resource_manager { const char* type; - unifycr_rm_read_resource_t read_resource; - unifycr_rm_launch_t launch; - unifycr_rm_terminate_t terminate; + unifyfs_rm_read_resource_t read_resource; + unifyfs_rm_launch_t launch; + unifyfs_rm_terminate_t terminate; }; typedef struct _ucr_resource_manager _ucr_resource_manager_t; /* * TODO: currently, we cannot launch if no resource manager is detected - * (UNIFYCR_RM_INVALID). + * (UNIFYFS_RM_INVALID). */ /** @@ -78,13 +78,12 @@ typedef struct _ucr_resource_manager _ucr_resource_manager_t; * * @return -ENOSYS */ -static int invalid_read_resource(unifycr_resource_t* resource) +static int invalid_read_resource(unifyfs_resource_t* resource) { return -ENOSYS; } - /** * @brief Parse a hostfile containing one host per line * @@ -94,7 +93,7 @@ static int invalid_read_resource(unifycr_resource_t* resource) * * @return 0 on success, negative errno otherwise */ -static int parse_hostfile(unifycr_resource_t* resource, +static int parse_hostfile(unifyfs_resource_t* resource, char* hostfile, size_t n_nodes) { @@ -147,6 +146,44 @@ static int parse_hostfile(unifycr_resource_t* resource, return ret; } +/** + * @brief Write a server-shared hostfile + * + * @param resource The job resource record + * @param args The command-line options + * + * @return 0 on success, negative errno otherwise + */ +static int write_hostfile(unifyfs_resource_t* resource, + unifyfs_args_t* args) +{ + int ret = 0; + size_t i; + FILE* fp = NULL; + char hostfile[UNIFYFS_MAX_FILENAME]; + + if (NULL == args->share_dir) { + return -EINVAL; + } + + snprintf(hostfile, sizeof(hostfile), "%s/unifyfsd.hosts", + args->share_dir); + fp = fopen(hostfile, "w"); + if (!fp) { + return -errno; + } + + // first line: number of hosts + fprintf(fp, "%zu\n", resource->n_nodes); + for (i = 0; i < resource->n_nodes; i++) { + fprintf(fp, "%s\n", resource->nodes[i]); + } + fclose(fp); + args->share_hostfile = strdup(hostfile); + + return ret; +} + /** * @brief Get node list from $LSB_HOSTS or $LSB_MCPU_HOSTS. * @@ -154,7 +191,7 @@ static int parse_hostfile(unifycr_resource_t* resource, * * @return 0 on success, negative errno otherwise */ -static int lsf_read_resource(unifycr_resource_t* resource) +static int lsf_read_resource(unifyfs_resource_t* resource) { size_t i, n_nodes; char* val; @@ -182,10 +219,11 @@ static int lsf_read_resource(unifycr_resource_t* resource) lsb_hosts = strdup(val); // replace spaces with zeroes - for (pos = lsb_hosts; *pos; pos++) + for (pos = lsb_hosts; *pos; pos++) { if (isspace(*pos)) { *pos = '\0'; } + } // count nodes, skipping first pos = lsb_hosts + (strlen(lsb_hosts) + 1); // skip launch node @@ -249,7 +287,7 @@ static int lsf_read_resource(unifycr_resource_t* resource) * * @return 0 on success, negative errno otherwise */ -static int pbs_read_resource(unifycr_resource_t* resource) +static int pbs_read_resource(unifyfs_resource_t* resource) { size_t n_nodes = 0; char* num_nodes_str = NULL; @@ -277,7 +315,7 @@ static int pbs_read_resource(unifycr_resource_t* resource) * * @return 0 on success, negative errno otherwise */ -static int slurm_read_resource(unifycr_resource_t* resource) +static int slurm_read_resource(unifyfs_resource_t* resource) { int ret; size_t len = 0; @@ -287,7 +325,7 @@ static int slurm_read_resource(unifycr_resource_t* resource) char* hostfile = NULL; char* tmpdir = NULL; const char* cmd_fmt = "scontrol show hostnames > %s"; - const char* hostfile_fmt = "%s/unifycr-hosts.%d"; + const char* hostfile_fmt = "%s/unifyfs-hosts.%d"; const char* tmp = "/tmp"; // get num nodes @@ -333,7 +371,7 @@ static int slurm_read_resource(unifycr_resource_t* resource) * * @return number of server arguments */ -static size_t construct_server_argv(unifycr_args_t* args, +static size_t construct_server_argv(unifyfs_args_t* args, char** server_argv) { size_t argc; @@ -343,19 +381,18 @@ static size_t construct_server_argv(unifycr_args_t* args, if (args->server_path != NULL) { server_argv[0] = strdup(args->server_path); } else { - server_argv[0] = strdup(BINDIR "/unifycrd"); + server_argv[0] = strdup(BINDIR "/unifyfsd"); } } argc = 1; if (args->debug) { if (server_argv != NULL) { - server_argv[argc] = strdup("-d"); - server_argv[argc + 1] = strdup("-v"); + server_argv[argc] = strdup("-v"); snprintf(number, sizeof(number), "%d", args->debug); - server_argv[argc + 2] = strdup(number); + server_argv[argc + 1] = strdup(number); } - argc += 3; + argc += 2; } if (args->cleanup) { @@ -365,11 +402,11 @@ static size_t construct_server_argv(unifycr_args_t* args, argc++; } - if (args->consistency != UNIFYCR_CM_LAMINATED) { + if (args->consistency != UNIFYFS_CM_LAMINATED) { if (server_argv != NULL) { server_argv[argc] = strdup("-c"); server_argv[argc + 1] = - strdup(unifycr_cm_enum_str(args->consistency)); + strdup(unifyfs_cm_enum_str(args->consistency)); } argc += 2; } @@ -382,6 +419,14 @@ static size_t construct_server_argv(unifycr_args_t* args, argc += 2; } + if (server_argv != NULL) { + server_argv[argc] = strdup("-S"); + server_argv[argc + 1] = strdup(args->share_dir); + server_argv[argc + 2] = strdup("-H"); + server_argv[argc + 3] = strdup(args->share_hostfile); + } + argc += 4; + return argc; } @@ -393,8 +438,8 @@ static size_t construct_server_argv(unifycr_args_t* args, * * @return -ENOSYS */ -static int invalid_launch(unifycr_resource_t* resource, - unifycr_args_t* args) +static int invalid_launch(unifyfs_resource_t* resource, + unifyfs_args_t* args) { return -ENOSYS; } @@ -407,8 +452,8 @@ static int invalid_launch(unifycr_resource_t* resource, * * @return -ENOSYS */ -static int invalid_terminate(unifycr_resource_t* resource, - unifycr_args_t* args) +static int invalid_terminate(unifyfs_resource_t* resource, + unifyfs_args_t* args) { return -ENOSYS; } @@ -421,8 +466,8 @@ static int invalid_terminate(unifycr_resource_t* resource, * * @return */ -static int jsrun_launch(unifycr_resource_t* resource, - unifycr_args_t* args) +static int jsrun_launch(unifyfs_resource_t* resource, + unifyfs_args_t* args) { size_t argc, jsrun_argc, server_argc; char** argv = NULL; @@ -449,7 +494,7 @@ static int jsrun_launch(unifycr_resource_t* resource, construct_server_argv(args, argv + jsrun_argc); execvp(argv[0], argv); - perror("failed to execvp() jsrun to launch unifycrd"); + perror("failed to execvp() jsrun to launch unifyfsd"); return -errno; } @@ -461,15 +506,15 @@ static int jsrun_launch(unifycr_resource_t* resource, * * @return */ -static int jsrun_terminate(unifycr_resource_t* resource, - unifycr_args_t* args) +static int jsrun_terminate(unifyfs_resource_t* resource, + unifyfs_args_t* args) { size_t argc, jsrun_argc; char** argv = NULL; char n_nodes[16]; - // full command: jsrun pkill unifycrd - jsrun_argc = 11; + // full command: jsrun pkill -n unifyfsd + jsrun_argc = 12; snprintf(n_nodes, sizeof(n_nodes), "%zu", resource->n_nodes); // setup full command argv @@ -485,10 +530,11 @@ static int jsrun_terminate(unifycr_resource_t* resource, argv[7] = strdup("-c1"); argv[8] = strdup("-a1"); argv[9] = strdup("pkill"); - argv[10] = strdup("unifycrd"); + argv[10] = strdup("-n"); + argv[11] = strdup("unifyfsd"); execvp(argv[0], argv); - perror("failed to execvp() jsrun to pkill unifycrd"); + perror("failed to execvp() jsrun to pkill unifyfsd"); return -errno; } @@ -500,8 +546,8 @@ static int jsrun_terminate(unifycr_resource_t* resource, * * @return */ -static int mpirun_launch(unifycr_resource_t* resource, - unifycr_args_t* args) +static int mpirun_launch(unifyfs_resource_t* resource, + unifyfs_args_t* args) { size_t argc, mpirun_argc, server_argc; char** argv = NULL; @@ -525,7 +571,7 @@ static int mpirun_launch(unifycr_resource_t* resource, construct_server_argv(args, argv + mpirun_argc); execvp(argv[0], argv); - perror("failed to execvp() mpirun to launch unifycrd"); + perror("failed to execvp() mpirun to launch unifyfsd"); return -errno; } @@ -537,15 +583,15 @@ static int mpirun_launch(unifycr_resource_t* resource, * * @return */ -static int mpirun_terminate(unifycr_resource_t* resource, - unifycr_args_t* args) +static int mpirun_terminate(unifyfs_resource_t* resource, + unifyfs_args_t* args) { size_t argc, mpirun_argc; char** argv = NULL; char n_nodes[16]; - // full command: mpirun pkill unifycrd - mpirun_argc = 7; + // full command: mpirun pkill -n unifyfsd + mpirun_argc = 8; snprintf(n_nodes, sizeof(n_nodes), "%zu", resource->n_nodes); // setup full command argv @@ -557,10 +603,11 @@ static int mpirun_terminate(unifycr_resource_t* resource, argv[3] = strdup("--map-by"); argv[4] = strdup("ppr:1:node"); argv[5] = strdup("pkill"); - argv[6] = strdup("unifycrd"); + argv[6] = strdup("-n"); + argv[7] = strdup("unifyfsd"); execvp(argv[0], argv); - perror("failed to execvp() mpirun to pkill unifycrd"); + perror("failed to execvp() mpirun to pkill unifyfsd"); return -errno; } @@ -573,8 +620,8 @@ static int mpirun_terminate(unifycr_resource_t* resource, * * @return */ -static int srun_launch(unifycr_resource_t* resource, - unifycr_args_t* args) +static int srun_launch(unifyfs_resource_t* resource, + unifyfs_args_t* args) { size_t argc, srun_argc, server_argc; char** argv = NULL; @@ -593,12 +640,12 @@ static int srun_launch(unifycr_resource_t* resource, argv[0] = strdup("srun"); argv[1] = strdup("-N"); argv[2] = strdup(n_nodes); - argv[3] = strdup("-n"); - argv[4] = strdup(n_nodes); + argv[3] = strdup("--ntasks-per-node"); + argv[4] = strdup("1"); construct_server_argv(args, argv + srun_argc); execvp(argv[0], argv); - perror("failed to execvp() srun to launch unifycrd"); + perror("failed to execvp() srun to launch unifyfsd"); return -errno; } @@ -610,15 +657,15 @@ static int srun_launch(unifycr_resource_t* resource, * * @return */ -static int srun_terminate(unifycr_resource_t* resource, - unifycr_args_t* args) +static int srun_terminate(unifyfs_resource_t* resource, + unifyfs_args_t* args) { size_t argc, srun_argc; char** argv = NULL; char n_nodes[16]; - // full command: srun pkill unifycrd - srun_argc = 7; + // full command: srun pkill -n unifyfsd + srun_argc = 8; snprintf(n_nodes, sizeof(n_nodes), "%zu", resource->n_nodes); // setup full command argv @@ -630,10 +677,11 @@ static int srun_terminate(unifycr_resource_t* resource, argv[3] = strdup("-n"); argv[4] = strdup(n_nodes); argv[5] = strdup("pkill"); - argv[6] = strdup("unifycrd"); + argv[6] = strdup("-n"); + argv[7] = strdup("unifyfsd"); execvp(argv[0], argv); - perror("failed to execvp() srun to pkill unifycrd"); + perror("failed to execvp() srun to pkill unifyfsd"); return -errno; } @@ -645,8 +693,8 @@ static int srun_terminate(unifycr_resource_t* resource, * * @return */ -static int script_launch(unifycr_resource_t* resource, - unifycr_args_t* args) +static int script_launch(unifyfs_resource_t* resource, + unifyfs_args_t* args) { size_t argc, script_argc, server_argc; char** argv = NULL; @@ -679,15 +727,15 @@ static int script_launch(unifycr_resource_t* resource, * * @return */ -static int script_terminate(unifycr_resource_t* resource, - unifycr_args_t* args) +static int script_terminate(unifyfs_resource_t* resource, + unifyfs_args_t* args) { size_t argc, script_argc; char** argv = NULL; char n_nodes[16]; - // full command: