From 0142f265e52645b6098c574b6b8d29016e238dff Mon Sep 17 00:00:00 2001
From: Fredrik Thulin <fredrik@yubico.com>
Date: Mon, 14 Mar 2011 14:31:22 +0100
Subject: [PATCH] generate_challenge() only generated half as many bytes as it
 should.

Changed generate_challenge() to generating bytes instead of a hex
encoded string, to not have to decode what we just encoded - instead
just generate plain bytes of randomness and then encode them once.
---
 pam_yubico.c | 31 +++++++++++++++++--------------
 1 file changed, 17 insertions(+), 14 deletions(-)

diff --git a/pam_yubico.c b/pam_yubico.c
index 2fd4b73..00a8caf 100644
--- a/pam_yubico.c
+++ b/pam_yubico.c
@@ -383,24 +383,22 @@ struct cfg
   char *chalresp_path;
 };
 
-/* Fill buf with len/2 bytes of random data (hex-encoded) */
+/* Fill buf with len bytes of random data */
 
 static int generate_challenge(char *buf, int len)
 {
   FILE *u;
-  int i;
+  int i, res;
 
   u = fopen("/dev/urandom", "r");
   if (!u) {
     return -1;
   }
 
-  for (i = 0; i < len/2; i++) {
-    int t = getc(u);
-    sprintf(buf, "%x", t);
-    buf++;
-  }
-  return 0;
+  res = fread(buf, 1, (size_t) len, u);
+  fclose(u);
+
+  return (res != len);
 }
 
 int
@@ -448,12 +446,12 @@ do_challenge_response(struct cfg *cfg, const char *username)
 {
   char *userfile = NULL;
   FILE *f = NULL;
-  char challenge_hex[64], expected_response[64];
   char challenge[32];
+  char challenge_hex[sizeof(challenge) * 2 + 1], expected_response[64];
   int r, slot, ret, fd;
 
   unsigned char response[64];
-  unsigned char response_hex[sizeof(response) * 2];
+  unsigned char response_hex[sizeof(response) * 2 + 1];
   int yk_cmd;
   unsigned int flags = 0;
   unsigned int response_len = 0;
@@ -509,12 +507,14 @@ do_challenge_response(struct cfg *cfg, const char *username)
     goto out;
   }
 
-  D(("Got the expected response, generating new challenge."));
+  D(("Got the expected response, generating new challenge (%i bytes).", sizeof(challenge)));
 
-  if (generate_challenge(challenge_hex, 64) < 0)
+  if (generate_challenge(challenge, 20)) {
+    D(("Failed generating new challenge!"));
     goto out;
-  yubikey_hex_decode(challenge, challenge_hex, strlen(challenge_hex));
-  if (!yk_write_to_key(yk, yk_cmd, challenge, strlen(challenge_hex)/2))
+  }
+
+  if (!yk_write_to_key(yk, yk_cmd, challenge, 20))
     goto out;
 
   if (! yk_read_response_from_key(yk, slot, flags,
@@ -526,6 +526,9 @@ do_challenge_response(struct cfg *cfg, const char *username)
   /* the yk_* functions leave 'junk' in errno */
   errno = 0;
 
+  memset(challenge_hex, 0, sizeof(challenge_hex));
+  memset(response_hex, 0, sizeof(response_hex));
+  yubikey_hex_encode(challenge_hex, (char *)challenge, 20);
   yubikey_hex_encode(response_hex, (char *)response, response_len > 20 ? 20 : response_len);
   rewind(f);
   fd = fileno(f);