diff --git a/pam_yubico.c b/pam_yubico.c
index 5a50250..ddbcbf8 100644
--- a/pam_yubico.c
+++ b/pam_yubico.c
@@ -473,6 +473,35 @@ do_challenge_response(pam_handle_t *pamh, struct cfg *cfg, const char *username)
 
   ret = PAM_AUTH_ERR;
 
+  pwres = getpwnam_r (username, &pass, pwbuf, pwbuflen, &p);
+  if (p == NULL) {
+      DBG ("getpwnam_r: %s", strerror(pwres));
+      goto out;
+  }
+
+  DBG("Checking for user challenge files");
+  switch(check_user_challenge_file(cfg->chalresp_path, p, cfg->debug_file)) {
+    case AUTH_FOUND:
+      DBG("Challenge files found");
+      break;
+    case AUTH_NOT_FOUND:
+      DBG("No challenge files found");
+      if (cfg->nullok) {
+        ret = PAM_IGNORE;
+      } else {
+        ret = PAM_USER_UNKNOWN;
+      }
+      goto out;
+    case AUTH_ERROR:
+      DBG ("Internal error while looking for user challenge files");
+      ret = PAM_AUTHINFO_UNAVAIL;
+      goto out;
+    default:
+      DBG ("Unhandled value while looking for user challenge files");
+      ret = PAM_AUTHINFO_UNAVAIL;
+      goto out;
+  }
+
   if (! init_yubikey(&yk)) {
     DBG("Failed initializing YubiKey");
     goto out;
@@ -483,12 +512,6 @@ do_challenge_response(pam_handle_t *pamh, struct cfg *cfg, const char *username)
     goto out;
   }
 
-  pwres = getpwnam_r (username, &pass, pwbuf, pwbuflen, &p);
-  if (p == NULL) {
-      DBG ("getpwnam_r: %s", strerror(pwres));
-      goto out;
-  }
-
   if (! get_user_challenge_file (yk, cfg->chalresp_path, p, &userfile, cfg->debug_file)) {
     DBG("Failed getting user challenge file for user %s", username);
     goto out;
diff --git a/util.c b/util.c
index 57d2e9b..e59ce96 100644
--- a/util.c
+++ b/util.c
@@ -38,6 +38,7 @@
 #include <sys/stat.h>
 #include <errno.h>
 #include <fcntl.h>
+#include <glob.h>
 #include <unistd.h>
 
 #include "util.h"
@@ -281,6 +282,66 @@ int challenge_response(YK_KEY *yk, int slot,
 	return 1;
 }
 
+int
+check_user_challenge_file(const char *chalresp_path, const struct passwd *user, FILE *debug_file)
+{
+  /*
+   * This function will look for users challenge files.
+   *
+   * Returns one of AUTH_FOUND, AUTH_NOT_FOUND, AUTH_ERROR
+   */
+  int len;
+  int r;
+  int ret = AUTH_NOT_FOUND;
+  char *userfile;
+  char *userfile_pattern;
+  glob_t userfile_glob;
+  const char *filename = NULL;
+
+  if (! chalresp_path) {
+    filename = "challenge";
+  } else {
+    filename = user->pw_name;
+  }
+
+  r = get_user_cfgfile_path(chalresp_path, filename, user, &userfile);
+  if (!r) {
+    ret = AUTH_ERROR;
+    goto out;
+  }
+
+  if (!access(userfile, F_OK)) {
+    ret = AUTH_FOUND;
+    goto clean_userfile;
+  }
+
+  len = strlen(userfile) + 2 + 1;
+  if ((userfile_pattern = malloc(len)) == NULL) {
+    goto clean_userfile;
+  }
+  snprintf(userfile_pattern, len, "%s-*", userfile);
+
+  r = glob(userfile_pattern, 0, NULL, &userfile_glob);
+  globfree(&userfile_glob);
+  switch (r) {
+    case 0:
+      ret = AUTH_FOUND;
+      goto clean_userfile_pattern;
+    case GLOB_NOMATCH:
+      break;
+    default:
+      ret = AUTH_ERROR;
+      goto clean_userfile_pattern;
+  }
+
+clean_userfile_pattern:
+  free(userfile_pattern);
+clean_userfile:
+  free(userfile);
+out:
+  return ret;
+}
+
 int
 get_user_challenge_file(YK_KEY *yk, const char *chalresp_path, const struct passwd *user, char **fn, FILE *debug_file)
 {
diff --git a/util.h b/util.h
index 37421c9..996b0c4 100644
--- a/util.h
+++ b/util.h
@@ -80,6 +80,7 @@ typedef struct chalresp_state CR_STATE;
 
 int generate_random(void *buf, int len);
 
+int check_user_challenge_file(const char *chalresp_path, const struct passwd *user, FILE *debug_file);
 int get_user_challenge_file(YK_KEY *yk, const char *chalresp_path, const struct passwd *user, char **fn, FILE *debug_file);
 
 int load_chalresp_state(FILE *f, CR_STATE *state, bool verbose, FILE *debug_file);