diff --git a/pdf.c b/pdf.c
index 1db9e37c7cf82d38735e1a13b93283cac139ca81..e130384ca2cd590b79079e4537f3d9c72a29ad3f 100644
--- a/pdf.c
+++ b/pdf.c
@@ -373,16 +373,21 @@ HParsedToken *
 act_a85partialgroup(const HParseResult *p, void *u)
 {
 	uint8_t bytes_helper[4];
-	size_t bytes_used = 1;
+	size_t bytes_used = 0;
 	uint8_t *bytes;
 
 	uint32_t fourbytes = H_CAST_UINT(p->ast);
 
-	for (size_t i; i < 4; ++i)
+	/* Scan the uint backwards to find the first non-zero byte */
+	for (size_t i = 3; i > 0; --i)
 	{
-		bytes_helper[i] = (fourbytes >> (3-i * 8)) & 0xFF;
-		bytes_used += 1;
+		/* Shift by 0, 8, 16 and 24 to get the correct byte */
+		bytes_helper[i] = (fourbytes >> ((3-i) * 8)) & 0xFF;
+		/* If we haven't set bytes_used yet, and the particular byte is nonzero */
+		if (!bytes_used && bytes_helper[i])
+			bytes_used = i;
 	}
+	assert(bytes_used > 0);
 
 	bytes = h_arena_malloc(p->arena, bytes_used);
 	return H_MAKE_BYTES(bytes, bytes_used);