diff --git a/pam_yubico.c b/pam_yubico.c
index 3819642..c5031c0 100644
--- a/pam_yubico.c
+++ b/pam_yubico.c
@@ -353,25 +353,6 @@ struct cfg
   char *chalresp_path;
 };
 
-int
-get_user_challenge_file(const char *chalresp_path, const char *username, char **fn)
-{
-  /* Getting file from user home directory, i.e. ~/.yubico/challenge, or
-   * from a system wide directory.
-   *
-   * Format is hex(challenge):hex(response):slot num
-   */
-
-  /* The challenge to use is located in a file in the user's home directory,
-   * which therefor can't be encrypted. If an encrypted home directory is used,
-   * the option chalresp_path can be used to point to a system-wide directory.
-   */
-
-  char *filename = "challenge";
-
-  return get_user_cfgfile_path (chalresp_path, filename, username, fn);
-}
-
 static int
 do_challenge_response(struct cfg *cfg, const char *username)
 {
@@ -391,7 +372,18 @@ do_challenge_response(struct cfg *cfg, const char *username)
   ret = PAM_AUTH_ERR;
   flags |= YK_FLAG_MAYBLOCK;
 
-  if (! get_user_challenge_file (cfg->chalresp_path, username, &userfile)) {
+  if (! init_yubikey(&yk)) {
+    D(("Failed initializing YubiKey"));
+    goto out;
+  }
+
+  if (! check_firmware_version(yk, false, true)) {
+    D(("YubiKey does not support Challenge-Response (version 2.2 required)"));
+    goto out;
+  }
+
+
+  if (! get_user_challenge_file (yk, cfg->chalresp_path, username, &userfile)) {
     D(("Failed getting user challenge file for user %s", username));
     goto out;
   }
@@ -404,16 +396,6 @@ do_challenge_response(struct cfg *cfg, const char *username)
   if (! load_chalresp_state(f, &state))
     goto out;
 
-  if (! init_yubikey(&yk)) {
-    D(("Failed initializing YubiKey"));
-    goto out;
-  }
-
-  if (! check_firmware_version(yk, false, true)) {
-    D(("YubiKey does not support Challenge-Response (version 2.2 required)"));
-    goto out;
-  }
-
   if (! challenge_response(yk, state.slot, state.challenge, state.challenge_len,
 			   true, flags, false,
 			   buf, sizeof(buf), &response_len)) {
diff --git a/util.c b/util.c
index 30f3eaa..de763b1 100644
--- a/util.c
+++ b/util.c
@@ -185,6 +185,43 @@ int challenge_response(YK_KEY *yk, int slot,
 	return 1;
 }
 
+int
+get_user_challenge_file(YK_KEY *yk, const char *chalresp_path, const char *username, char **fn)
+{
+  /* Getting file from user home directory, i.e. ~/.yubico/challenge, or
+   * from a system wide directory.
+   *
+   * Format is hex(challenge):hex(response):slot num
+   */
+
+  /* The challenge to use is located in a file in the user's home directory,
+   * which therefor can't be encrypted. If an encrypted home directory is used,
+   * the option chalresp_path can be used to point to a system-wide directory.
+   */
+  
+  const char *filename; /* not including directory */
+  unsigned int serial = 0;
+  
+  if (! yk_get_serial(yk, 0, 0, &serial)) {
+    D (("Failed to read serial number (serial-api-visible disabled?)."));
+    if (! chalresp_path)
+      filename = "challenge";
+    else
+      filename = username;
+  } else {
+    /* We have serial number */
+    int res = asprintf (&filename, "%s-%i", chalresp_path == NULL ? "challenge" : username, serial);
+    
+    if (res < 1)
+      filename = NULL;
+  }
+  
+  if (filename == NULL)
+    return 0;
+  
+  return get_user_cfgfile_path (chalresp_path, filename, username, fn);
+}
+
 int
 load_chalresp_state(FILE *f, CR_STATE *state)
 {
@@ -201,9 +238,12 @@ load_chalresp_state(FILE *f, CR_STATE *state)
    * (twice because we hex encode the challenge and response)
    */
   r = fscanf(f, "v1:%126[0-9a-z]:%40[0-9a-z]:%d", &challenge_hex, &response_hex, &slot);
-  D(("Challenge: %s, response: %s, slot: %d", challenge_hex, response_hex, slot));
-  if (r != 3)
+  if (r != 3) {
+    D(("Could not parse contents of chalres_state file (%i)", r));
     goto out;
+  }
+
+  D(("Challenge: %s, response: %s, slot: %d", challenge_hex, response_hex, slot));
 
   if (! yubikey_hex_p(challenge_hex)) {
     D(("Invalid challenge hex input : %s", challenge_hex));
diff --git a/util.h b/util.h
index b3b1caf..796db21 100644
--- a/util.h
+++ b/util.h
@@ -31,12 +31,16 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#ifndef __PAM_YUBICO_UTIL_H_INCLUDED__
+#define __PAM_YUBICO_UTIL_H_INCLUDED__
+
+#include <stdio.h>
+
 #include <ykclient.h>
 #include <ykcore.h>
 #include <ykstatus.h>
 #include <ykdef.h>
 
-#ifndef D
 #if defined(DEBUG_PAM)
 # if defined(HAVE_SECURITY__PAM_MACROS_H)
 #  define DEBUG
@@ -49,7 +53,6 @@
   } while (0)
 # endif /* HAVE_SECURITY__PAM_MACROS_H */
 #endif /* DEBUG_PAM */
-#endif /* D */
 
 /* Challenges can be 0..63 or 64 bytes long, depending on YubiKey configuration.
  * We settle for 63 bytes to have something that works with all configurations.
@@ -68,7 +71,9 @@ struct chalresp_state {
 typedef struct chalresp_state CR_STATE;
 
 int generate_random(char *buf, int len);
+
 int get_user_cfgfile_path(const char *common_path, const char *filename, const char *username, char **fn);
+int get_user_challenge_file(YK_KEY *yk, const char *chalresp_path, const char *username, char **fn);
 
 int load_chalresp_state(FILE *f, CR_STATE *state);
 int write_chalresp_state(FILE *f, CR_STATE *state);
@@ -79,3 +84,5 @@ int challenge_response(YK_KEY *yk, int slot,
 		       unsigned char *challenge, unsigned int len,
 		       bool hmac, unsigned int flags, bool verbose,
 		       unsigned char *response, int res_size, int *res_len);
+
+#endif /* __PAM_YUBICO_UTIL_H_INCLUDED__ */