Merge pull request #83 from ddavila0/xcache-consistency-check

Xcache consistency check

.travis.yml

@@ -1,6 +1,7 @@
 sudo: required
 env:
   - REPO_NAME=${TRAVIS_REPO_SLUG#*/}
+    GET_FILES=true
 
 git:
   depth: false
@@ -17,5 +18,5 @@ before_install:
   - sudo docker pull opensciencegrid/osg-build
 
 script:
-  - docker run -v $(pwd):/$REPO_NAME -e REPO_NAME=$REPO_NAME --cap-add=SYS_ADMIN opensciencegrid/osg-build build-from-github
+  - docker run -v $(pwd):/$REPO_NAME -e REPO_NAME=$REPO_NAME -e GET_FILES=$GET_FILES --cap-add=SYS_ADMIN opensciencegrid/osg-build build-from-github
 

Makefile

@@ -34,7 +34,8 @@ XROOTD_CONFIGD := $(wildcard configs/atlas-xcache/config.d/*) \
 
 SYSTEMD_UNITS := $(wildcard configs/stash-cache/systemd/*) \
                  $(wildcard configs/xcache/systemd/*) \
-                 $(wildcard configs/stash-origin/systemd/*)
+                 $(wildcard configs/stash-origin/systemd/*) \
+                 $(wildcard configs/xcache-consistency-check/systemd/*)
 
 TMPFILES_D := configs/stash-cache/tmpfiles/stash-cache.conf \
               configs/stash-origin/tmpfiles/stash-origin.conf \
@@ -90,6 +91,11 @@ install:
 	ln -srf $(DESTDIR)/$(INSTALL_XROOTD_DIR)/xrootd-stash-origin.cfg $(DESTDIR)/$(INSTALL_XROOTD_DIR)/xrootd-stash-origin-auth.cfg
 	mkdir -p $(DESTDIR)/$(INSTALL_XROOTD_DIR)/config.d
 	install -p -m 0644 $(XROOTD_CONFIGD) $(DESTDIR)/$(INSTALL_XROOTD_DIR)/config.d
+	# XCache Consistency Check
+	mkdir -p $(DESTDIR)/usr/bin
+	mkdir -p $(DESTDIR)/var/lib/xcache-consistency-check
+	install -p -m 0755 src/xcache-consistency-check $(DESTDIR)/usr/bin/xcache-consistency-check
+	install -p -m 0644 configs/xcache-consistency-check/xrootd/xcache-consistency-check.cfg $(DESTDIR)/etc/xrootd/xcache-consistency-check.cfg
 	# systemd unit files
 	mkdir -p $(DESTDIR)/$(INSTALL_SYSTEMD_UNITDIR)
 	install -p -m 0644 $(SYSTEMD_UNITS) $(DESTDIR)/$(INSTALL_SYSTEMD_UNITDIR)
@@ -153,4 +159,3 @@ endif
 
 check:
 	pylint -E $(LIBEXEC_FILES) $(PYTHON_LIB)
-

configs/xcache-consistency-check/systemd/xcache-consistency-check.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=XCache consistency check
+
+[Service]
+User=xrootd
+Group=xrootd
+Type=simple
+Environment=PYTHONPATH=/usr/lib/xcache-consistency-check/usr/lib/python2.7/site-packages/:/usr/lib/xcache-consistency-check/usr/lib64/python2.7/site-packages/
+ExecStart=/usr/bin/xcache-consistency-check --config /etc/xrootd/xcache-consistency-check.cfg
+
+[Install]
+WantedBy=multi-user.target

configs/xcache-consistency-check/systemd/xcache-consistency-check.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=Analyze XCache root files periodically
+
+[Timer]
+OnBootSec=5min
+OnUnitActiveSec=24h
+Unit=xcache-consistency-check.service
+
+[Install]
+WantedBy=timers.target

configs/xcache-consistency-check/xrootd/xcache-consistency-check.cfg

@@ -0,0 +1,19 @@
+[Main]
+
+#(REQUIRED) Points to the top of the directory where the .root files to analyze are found
+# When set it to 'auto'(default), it will attempt to set the 'path' automatically to be
+# Xrootd's configuration parameter: 'oss.localroot'
+path = auto
+
+#(REQUIRED) Where to store the DataBase
+db = /var/lib/xcache-consistency-check/db.sql
+
+# Where to store the logs
+logfile = /var/log/xrootd/xcache-consistency-check.log
+
+# Num of processes used to decompress baskets in parallel
+num_procs = 1
+
+# Once a file has been analyzed it will wait for this many seconds
+# before analyzing it again, even if the file has changed
+last_check_threshold = 86400

rpm/xcache.spec

@@ -1,12 +1,20 @@
 Name:      xcache
 Summary:   XCache scripts and configurations
-Version:   1.2.1
+Version:   1.3.0
 Release:   1%{?dist}
 License:   Apache 2.0
 Group:     Grid
 URL:       https://opensciencegrid.org/docs/
-BuildArch: noarch
 Source0:   %{name}-%{version}.tar.gz
+Source1:   https://vdt.cs.wisc.edu/upstream/xcache/1.3.0/python-deps/numpy-1.16.6-cp27-cp27mu-manylinux1_x86_64.whl
+Source2:   https://vdt.cs.wisc.edu/upstream/xcache/1.3.0/python-deps/cachetools-3.1.1-py2.py3-none-any.whl
+Source3:   https://vdt.cs.wisc.edu/upstream/xcache/1.3.0/python-deps/awkward-0.12.20-py2.py3-none-any.whl
+Source4:   https://vdt.cs.wisc.edu/upstream/xcache/1.3.0/python-deps/uproot_methods-0.7.3-py2.py3-none-any.whl
+Source5:   https://vdt.cs.wisc.edu/upstream/xcache/1.3.0/python-deps/uproot-3.11.2-py2.py3-none-any.whl
+Source6:   https://vdt.cs.wisc.edu/upstream/xcache/1.3.0/python-deps/xxhash-1.4.3-cp27-cp27mu-manylinux1_x86_64.whl
+Source7:   https://vdt.cs.wisc.edu/upstream/xcache/1.3.0/python-deps/lz4-2.2.1-cp27-cp27mu-manylinux1_x86_64.whl
+Source8:   https://vdt.cs.wisc.edu/upstream/xcache/1.3.0/python-deps/pyliblzma-0.5.3.tar.bz2
+
 
 BuildRequires: systemd
 %{?systemd_requires}
@@ -40,6 +48,30 @@ Obsoletes: stashcache-daemon < 1.0.0
 %postun
 %systemd_postun_with_restart xcache-reporter.service xcache-reporter.timer xrootd-renew-proxy.service xrootd-renew-proxy.timer
 
+########################################
+%package -n xcache-consistency-check
+BuildRequires: python-pip
+BuildRequires: python-devel
+BuildRequires: xz-devel
+Summary: Consistency check for root files
+AutoReq: no
+%global __provides_exclude ^libgfortran.*\\.so.*$|^libopenblasp.*\\.so.*$
+
+Requires: xz
+Requires: xrootd-server
+
+%description -n xcache-consistency-check
+%{summary}
+
+%post -n xcache-consistency-check
+
+/bin/systemctl daemon-reload >/dev/null 2>&1 || :
+%systemd_post xcache-consistency-check.service xcache-consistency-check.timer
+%preun -n xcache-consistency-check
+%systemd_preun xcache-consistency-check.service xcache-consistency-check.timer
+%postun -n xcache-consistency-check
+%systemd_postun_with_restart xcache-consistency-check.service xcache-consistency-check.timer
+
 ########################################
 %package -n stash-origin
 Summary: The OSG Data Federation origin server
@@ -145,6 +177,12 @@ echo "*** This version does not build on EL 6 ***"
 exit 1
 %endif
 mkdir -p %{buildroot}%{_sysconfdir}/xrootd
+mkdir -p %{buildroot}/usr/lib/xcache-consistency-check
+for whl in %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} %{SOURCE5} %{SOURCE6} %{SOURCE7} %{SOURCE8} 
+do
+    pip2 install -I --no-deps "$whl" --root %{buildroot}/usr/lib/xcache-consistency-check
+done
+
 make install DESTDIR=%{buildroot}
 
 # Create xrootd certificate directory
@@ -165,6 +203,14 @@ mkdir -p %{buildroot}%{_sysconfdir}/grid-security/xrd
 %attr(0755, xrootd, xrootd) %dir /run/xcache-auth
 %{_tmpfilesdir}/xcache.conf
 
+%files -n xcache-consistency-check
+%attr(0755, xrootd, xrootd) %{_bindir}/xcache-consistency-check
+%dir %attr(0755, xrootd, xrootd) /var/lib/xcache-consistency-check
+%{_unitdir}/xcache-consistency-check.service
+%{_unitdir}/xcache-consistency-check.timer
+%config(noreplace) %{_sysconfdir}/xrootd/xcache-consistency-check.cfg
+/usr/lib/xcache-consistency-check/*
+
 %files -n stash-origin
 %config %{_sysconfdir}/xrootd/xrootd-stash-origin.cfg
 %config %{_sysconfdir}/xrootd/xrootd-stash-origin-auth.cfg
@@ -228,6 +274,9 @@ mkdir -p %{buildroot}%{_sysconfdir}/grid-security/xrd
 %config %{_sysconfdir}/xrootd/config.d/03-redir-tuning.cfg
 
 %changelog
+* Mon Jan 27 2020 Diego Davila <[email protected]> - 1.3.0-1
+- Adding subpackage for consistency check (SOFTWARE-3976)
+
 * Wed Dec 18 2019 Edgar Fajardo <[email protected]> - 1.2.1-1
 - Fixed bug in which cmsd filesystem was configured to be the cache (SOFTWARE-3952)
 
@@ -369,4 +418,3 @@ mkdir -p %{buildroot}%{_sysconfdir}/grid-security/xrd
 
 * Wed Apr 22 2015 Mátyás Selmeci <[email protected]> 0.1-1.osg
 - Created metapackages with stub config files
-