diff --git a/Makefile.am b/Makefile.am
index c58eb4d5e53..d313957722a 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1209,6 +1209,7 @@ libsss_iface_sync_la_LDFLAGS = \
     -avoid-version \
     $(NULL)
 
+if BUILD_WITH_LIBSECRET
 pkglib_LTLIBRARIES += libsss_secrets.la
 
 libsss_secrets_la_SOURCES = \
@@ -1228,6 +1229,7 @@ libsss_secrets_la_LIBADD = \
 libsss_secrets_la_LDFLAGS = \
     -avoid-version \
     $(NULL)
+endif
 
 pkglib_LTLIBRARIES += libsss_util.la
 libsss_util_la_SOURCES = \
@@ -1800,13 +1802,11 @@ sssd_kcm_SOURCES = \
     src/responder/kcm/kcmsrv_ccache_mem.c \
     src/responder/kcm/kcmsrv_ccache_json.c \
     src/responder/kcm/kcmsrv_ccache_secdb.c \
-    src/responder/kcm/kcmsrv_ccache_secrets.c \
     src/responder/kcm/kcmsrv_ops.c \
     src/responder/kcm/kcmsrv_op_queue.c \
     src/util/sss_sockets.c \
     src/util/sss_krb5.c \
     src/util/sss_iobuf.c \
-    src/util/tev_curl.c \
     $(SSSD_RESPONDER_OBJ) \
     $(NULL)
 sssd_kcm_CFLAGS = \
@@ -1818,7 +1818,6 @@ sssd_kcm_CFLAGS = \
     $(NULL)
 sssd_kcm_LDADD = \
     $(KRB5_LIBS) \
-    $(CURL_LIBS) \
     $(JANSSON_LIBS) \
     $(SSSD_LIBS) \
     $(UUID_LIBS) \
@@ -1828,6 +1827,17 @@ sssd_kcm_LDADD = \
     libsss_sbus.la \
     libsss_secrets.la \
     $(NULL)
+
+if BUILD_SECRETS
+sssd_kcm_SOURCES += \
+    src/responder/kcm/kcmsrv_ccache_secrets.c \
+    src/util/tev_curl.c \
+    $(NULL)
+sssd_kcm_LDADD += \
+    $(CURL_LIBS) \
+    $(NULL)
+endif
+
 endif
 
 sssd_be_SOURCES = \
@@ -3939,6 +3949,7 @@ intgcheck-prepare:
 	    --with-ldb-lib-dir="$$prefix"/lib/ldb \
 	    --enable-intgcheck-reqs \
 	    --without-semanage \
+	    --with-secrets \
 	    --with-session-recording-shell=/bin/false \
 	    --enable-local-provider \
 	    $(INTGCHECK_CONFIGURE_FLAGS) \
@@ -4876,8 +4887,6 @@ if HAVE_SYSTEMD_UNIT
         src/sysv/systemd/sssd-pam.socket \
         src/sysv/systemd/sssd-pam-priv.socket \
         src/sysv/systemd/sssd-pam.service \
-        src/sysv/systemd/sssd-secrets.socket \
-        src/sysv/systemd/sssd-secrets.service \
         $(NULL)
 if BUILD_AUTOFS
     systemdunit_DATA += \
@@ -4896,6 +4905,12 @@ if BUILD_PAC_RESPONDER
         src/sysv/systemd/sssd-pac.service \
         $(NULL)
 endif
+if BUILD_SECRETS
+    systemdunit_DATA += \
+        src/sysv/systemd/sssd-secrets.socket \
+        src/sysv/systemd/sssd-secrets.service \
+        $(NULL)
+endif
 if BUILD_SSH
     systemdunit_DATA += \
         src/sysv/systemd/sssd-ssh.socket \
@@ -5033,6 +5048,7 @@ src/sysv/systemd/sssd-pam.service: src/sysv/systemd/sssd-pam.service.in Makefile
 	@$(MKDIR_P) src/sysv/systemd/
 	$(replace_script)
 
+if BUILD_SECRETS
 src/sysv/systemd/sssd-secrets.socket: src/sysv/systemd/sssd-secrets.socket.in Makefile
 	@$(MKDIR_P) src/sysv/systemd/
 	$(replace_script)
@@ -5040,6 +5056,7 @@ src/sysv/systemd/sssd-secrets.socket: src/sysv/systemd/sssd-secrets.socket.in Ma
 src/sysv/systemd/sssd-secrets.service: src/sysv/systemd/sssd-secrets.service.in Makefile
 	@$(MKDIR_P) src/sysv/systemd/
 	$(replace_script)
+endif
 
 if BUILD_AUTOFS
 src/sysv/systemd/sssd-autofs.socket: src/sysv/systemd/sssd-autofs.socket.in Makefile
@@ -5088,9 +5105,25 @@ src/sysv/systemd/sssd-sudo.service: src/sysv/systemd/sssd-sudo.service.in Makefi
 endif
 
 if BUILD_KCM
+if BUILD_SECRETS
+kcm_socket_requires = Requires=sssd-secrets.socket
+else
+kcm_socket_requires =
+endif
+
+kcm_edit_cmd = $(edit_cmd) \
+        -e 's|@kcm_socket_requires[@]|$(kcm_socket_requires)|g'
+
+kcm_replace_script = \
+    @rm -f $@ $@.tmp; \
+    srcdir=''; \
+        test -f ./$@.in || srcdir=$(srcdir)/; \
+        $(kcm_edit_cmd) $${srcdir}$@.in >$@.tmp; \
+    mv $@.tmp $@
+
 src/sysv/systemd/sssd-kcm.socket: src/sysv/systemd/sssd-kcm.socket.in Makefile
 	@$(MKDIR_P) src/sysv/systemd/
-	$(replace_script)
+	$(kcm_replace_script)
 
 src/sysv/systemd/sssd-kcm.service: src/sysv/systemd/sssd-kcm.service.in Makefile
 	@$(MKDIR_P) src/sysv/systemd/
@@ -5155,7 +5188,7 @@ endif
 	$(INSTALL) -d -m 0711 $(DESTDIR)$(sssdconfdir) \
                           $(DESTDIR)$(sssdconfdir)/conf.d \
                           $(DESTDIR)$(sssdconfdir)/pki
-if BUILD_SECRETS
+if BUILD_WITH_LIBSECRET
 	$(MKDIR_P) $(DESTDIR)$(secdbpath)
 endif
 
diff --git a/configure.ac b/configure.ac
index 9df463d9c8b..1aac65f4d85 100644
--- a/configure.ac
+++ b/configure.ac
@@ -212,6 +212,7 @@ m4_include([src/external/test_ca.m4])
 
 if test x$with_secrets = xyes; then
     m4_include([src/external/libhttp_parser.m4])
+    m4_include([src/external/libcurl.m4])
 fi
 
 if test x$with_kcm = xyes; then
@@ -219,10 +220,14 @@ if test x$with_kcm = xyes; then
 fi
 
 if test x$with_kcm = xyes -o x$with_secrets = xyes; then
-    m4_include([src/external/libcurl.m4])
+    BUILD_WITH_LIBSECRET=1
+    AC_DEFINE_UNQUOTED(BUILD_WITH_LIBSECRET, 1, [libsecret will be built])
     m4_include([src/external/libjansson.m4])
 fi
 
+AM_CONDITIONAL([BUILD_WITH_LIBSECRET],
+               [test x"$BUILD_WITH_LIBSECRET" != "x"])
+
 # This variable is defined by external/libcurl.m4, but conditionals
 # must be always evaluated
 AM_CONDITIONAL([BUILD_WITH_LIBCURL],
diff --git a/contrib/sssd.spec.in b/contrib/sssd.spec.in
index 46fe693963d..5ebd51f41f6 100644
--- a/contrib/sssd.spec.in
+++ b/contrib/sssd.spec.in
@@ -118,11 +118,8 @@
     %global enable_systemtap_opt --enable-systemtap
 %endif
 
-%if (0%{?fedora} || 0%{?rhel} >= 7)
-    %global with_secrets 1
-%else
-    %global with_secret_responder --without-secrets
-%endif
+%global with_secrets 0
+%global with_secret_responder --without-secrets
 
 %if (0%{?fedora} >= 23 || 0%{?rhel} >= 7)
     %global with_kcm 1
@@ -284,13 +281,13 @@ BuildRequires: systemtap-sdt-devel
 %endif
 %if (0%{?with_secrets} == 1)
 BuildRequires: http-parser-devel
+BuildRequires: libcurl-devel
 %endif
 %if (0%{?with_kcm} == 1)
 BuildRequires: libuuid-devel
 %endif
 %if (0%{?with_secrets} == 1 || 0%{?with_kcm} == 1)
 BuildRequires: jansson-devel
-BuildRequires: libcurl-devel
 %endif
 %if (0%{?with_gdm_pam_extensions} == 1)
 BuildRequires: gdm-pam-extensions-devel
@@ -1028,7 +1025,9 @@ done
 %{_libdir}/%{name}/libsss_iface_sync.so
 %{_libdir}/%{name}/libifp_iface.so
 %{_libdir}/%{name}/libifp_iface_sync.so
+%if (0%{?with_secrets} == 1 || 0%{?with_kcm} == 1)
 %{_libdir}/%{name}/libsss_secrets.so
+%endif
 
 %{ldb_modulesdir}/memberof.so
 %{_bindir}/sss_ssh_authorizedkeys
@@ -1360,9 +1359,7 @@ done
 
 %if (0%{?with_kcm} == 1)
 %files kcm -f sssd_kcm.lang
-%if (0%{?with_secrets} == 1)
 %attr(700,root,root) %dir %{secdbpath}
-%endif
 %{_libexecdir}/%{servicename}/sssd_kcm
 %if (0%{?with_secrets} == 1)
 %{_libexecdir}/%{servicename}/sssd_secrets
@@ -1371,10 +1368,10 @@ done
 %{_datadir}/sssd-kcm/kcm_default_ccache
 %{_unitdir}/sssd-kcm.socket
 %{_unitdir}/sssd-kcm.service
-%{_unitdir}/sssd-secrets.socket
-%{_unitdir}/sssd-secrets.service
 %{_mandir}/man8/sssd-kcm.8*
 %if (0%{?with_secrets} == 1)
+%{_unitdir}/sssd-secrets.socket
+%{_unitdir}/sssd-secrets.service
 %{_mandir}/man5/sssd-secrets.5*
 %endif
 %endif
@@ -1392,7 +1389,6 @@ getent passwd sssd >/dev/null || useradd -r -g sssd -d / -s /sbin/nologin -c "Us
 %systemd_post sssd-pac.socket
 %systemd_post sssd-pam.socket
 %systemd_post sssd-pam-priv.socket
-%systemd_post sssd-secrets.socket
 %systemd_post sssd-ssh.socket
 %systemd_post sssd-sudo.socket
 
@@ -1403,7 +1399,6 @@ getent passwd sssd >/dev/null || useradd -r -g sssd -d / -s /sbin/nologin -c "Us
 %systemd_preun sssd-pac.socket
 %systemd_preun sssd-pam.socket
 %systemd_preun sssd-pam-priv.socket
-%systemd_preun sssd-secrets.socket
 %systemd_preun sssd-ssh.socket
 %systemd_preun sssd-sudo.socket
 
@@ -1418,8 +1413,6 @@ getent passwd sssd >/dev/null || useradd -r -g sssd -d / -s /sbin/nologin -c "Us
 %systemd_postun_with_restart sssd-pam.socket
 %systemd_postun_with_restart sssd-pam-priv.socket
 %systemd_postun_with_restart sssd-pam.service
-%systemd_postun_with_restart sssd-secrets.socket
-%systemd_postun_with_restart sssd-secrets.service
 %systemd_postun_with_restart sssd-ssh.socket
 %systemd_postun_with_restart sssd-ssh.service
 %systemd_postun_with_restart sssd-sudo.socket
@@ -1446,6 +1439,18 @@ getent passwd sssd >/dev/null || useradd -r -g sssd -d / -s /sbin/nologin -c "Us
 %systemd_postun_with_restart sssd-kcm.service
 %endif
 
+%if (0%{?with_secrets} == 1)
+%post secrets
+%systemd_postun_with_restart sssd-secrets.socket
+
+%preun secrets
+%systemd_preun_with_restart sssd-secrets.socket
+
+%postun secrets
+%systemd_postun_with_restart sssd-secrets.socket
+%systemd_postun_with_restart sssd-secrets.service
+%endif
+
 %else
 # sysv
 %post common
diff --git a/src/conf_macros.m4 b/src/conf_macros.m4
index a8171743a10..5f28c78445c 100644
--- a/src/conf_macros.m4
+++ b/src/conf_macros.m4
@@ -883,11 +883,11 @@ AC_DEFUN([SSSD_RUNSTATEDIR],
 AC_DEFUN([WITH_SECRETS],
   [ AC_ARG_WITH([secrets],
                 [AC_HELP_STRING([--with-secrets],
-                                [Whether to build with secrets support [yes]]
+                                [Whether to build with secrets support [no]]
                                )
                 ],
                 [with_secrets=$withval],
-                with_secrets=yes
+                with_secrets=no
                )
 
     if test x"$with_secrets" = xyes; then
diff --git a/src/responder/kcm/kcmsrv_ccache.c b/src/responder/kcm/kcmsrv_ccache.c
index b04a9da1142..af2bcf8bb54 100644
--- a/src/responder/kcm/kcmsrv_ccache.c
+++ b/src/responder/kcm/kcmsrv_ccache.c
@@ -247,10 +247,12 @@ struct kcm_ccdb *kcm_ccdb_init(TALLOC_CTX *mem_ctx,
         DEBUG(SSSDBG_FUNC_DATA, "KCM back end: memory\n");
         ccdb->ops = &ccdb_mem_ops;
         break;
+#ifdef BUILD_SECRETS
     case CCDB_BE_SECRETS:
         DEBUG(SSSDBG_FUNC_DATA, "KCM back end: sssd-secrets\n");
         ccdb->ops = &ccdb_sec_ops;
         break;
+#endif /* BUILD_SECRETS */
     case CCDB_BE_SECDB:
         DEBUG(SSSDBG_FUNC_DATA, "KCM back end: libsss_secrets\n");
         ccdb->ops = &ccdb_secdb_ops;
diff --git a/src/sysv/systemd/sssd-kcm.socket.in b/src/sysv/systemd/sssd-kcm.socket.in
index 8b742847dfe..e8a5f0acade 100644
--- a/src/sysv/systemd/sssd-kcm.socket.in
+++ b/src/sysv/systemd/sssd-kcm.socket.in
@@ -1,7 +1,7 @@
 [Unit]
 Description=SSSD Kerberos Cache Manager responder socket
 Documentation=man:sssd-kcm(8)
-Requires=sssd-secrets.socket
+@kcm_socket_requires@
 
 [Socket]
 ListenStream=@runstatedir@/.heim_org.h5l.kcm-socket
diff --git a/src/tests/dlopen-tests.c b/src/tests/dlopen-tests.c
index ab386fa8fbf..400810c397e 100644
--- a/src/tests/dlopen-tests.c
+++ b/src/tests/dlopen-tests.c
@@ -46,8 +46,10 @@ struct so {
     { "libsss_nss_idmap.so", { LIBPFX"libsss_nss_idmap.so", NULL } },
     { "libnss_sss.so", { LIBPFX"libnss_sss.so", NULL } },
     { "libsss_certmap.so", { LIBPFX"libsss_certmap.so", NULL } },
-    { "libsss_secrets.so", { LIBPFX"libsss_secrets.so", NULL } },
     { "pam_sss.so", { LIBPFX"pam_sss.so", NULL } },
+#ifdef BUILD_WITH_LIBSECRET
+    { "libsss_secrets.so", { LIBPFX"libsss_secrets.so", NULL } },
+#endif /* BUILD_WITH_LIBSECRET */
 #ifdef BUILD_LIBWBCLIENT
     { "libwbclient.so", { LIBPFX"libwbclient.so", NULL } },
 #endif /* BUILD_LIBWBCLIENT */
diff --git a/src/tests/intg/test_kcm.py b/src/tests/intg/test_kcm.py
index e56ed1595e8..b021125995b 100644
--- a/src/tests/intg/test_kcm.py
+++ b/src/tests/intg/test_kcm.py
@@ -179,6 +179,12 @@ def setup_for_kcm_sec(request, kdc_instance):
     Just set up the local provider for tests and enable the KCM
     responder
     """
+    sec_resp_path = os.path.join(config.LIBEXEC_PATH, "sssd", "sssd_secrets")
+    if not os.access(sec_resp_path, os.X_OK):
+        # It would be cleaner to use pytest.mark.skipif on the package level
+        # but upstream insists on supporting RHEL-6.
+        pytest.skip("No Secrets responder, skipping")
+
     kcm_path = os.path.join(config.RUNSTATEDIR, "kcm.socket")
     sssd_conf = create_sssd_conf(kcm_path, "secrets")
     return common_setup_for_kcm_mem(request, kdc_instance, kcm_path, sssd_conf)