Complete AuthKey generation process

It works sometimes. Probably due to unsigned mistakes (50% chance)

lib/src/main/java/io/github/lonamiwebs/overgram/crypto/AuthKey.java

@@ -0,0 +1,40 @@
+package io.github.lonamiwebs.overgram.crypto;
+
+import io.github.lonamiwebs.overgram.utils.Utils;
+
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+
+public class AuthKey {
+    public final byte[] key;
+    public final long auxHash;
+    public final long keyId;
+
+    public AuthKey(final byte[] key) {
+        this.key = key;
+
+        final ByteBuffer digest = ByteBuffer.wrap(Utils.sha1digest(key)).order(ByteOrder.LITTLE_ENDIAN);
+        auxHash = digest.getLong();
+        digest.position(digest.position() + 4);
+        keyId = digest.getLong();
+    }
+
+    public BigInteger calcNewNonceHash(final BigInteger newNonce, final int number) {
+        // Big integer is big endian but we need little endian
+        final byte[] data = Utils.sha1digest(Utils.concat(
+                Utils.reversed(newNonce.toByteArray()), Utils.concat(
+                        ByteBuffer.allocate(1).put((byte) number).array(),
+                        ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN).putLong(auxHash).array()
+                )
+        ));
+
+        // Little endian once again so read (20..4]
+        final byte[] numberBytes = new byte[16];
+        for (int i = 16; i-- != 0; ) {
+            numberBytes[i] = data[19 - i];
+        }
+
+        return new BigInteger(numberBytes);
+    }
+}

lib/src/main/java/io/github/lonamiwebs/overgram/crypto/Authenticator.java

@@ -4,16 +4,19 @@ import io.github.lonamiwebs.overgram.network.MTProtoPlainSender;
 import io.github.lonamiwebs.overgram.tl.Abstract;
 import io.github.lonamiwebs.overgram.tl.Functions;
 import io.github.lonamiwebs.overgram.tl.Types;
+import io.github.lonamiwebs.overgram.utils.AES;
+import io.github.lonamiwebs.overgram.utils.BinaryReader;
 import io.github.lonamiwebs.overgram.utils.RSA;
 import io.github.lonamiwebs.overgram.utils.Utils;
 import javafx.util.Pair;
 
 import java.io.IOException;
 import java.math.BigInteger;
+import java.nio.ByteBuffer;
 
 public class Authenticator {
 
-    public static void doAuthentication(final MTProtoPlainSender sender) throws IOException, ClassNotFoundException {
+    public static AuthKey doAuthentication(final MTProtoPlainSender sender) throws IOException, ClassNotFoundException {
         final BigInteger nonce = new BigInteger(Utils.randomBytes(16));
         final Types.ResPQ resPq = sender.send(
                 new Functions.ReqPqMulti().nonce(nonce));
@@ -56,6 +59,7 @@ public class Authenticator {
             throw new SecurityException("Step 2 could not find a known RSA key");
         }
 
+        // TODO This *sometimes* fails, huh
         final Abstract.ServerDHParams abstractDhParams = sender.send(
                 new Functions.ReqDHParams()
                         .nonce(resPq.nonce())
@@ -76,7 +80,80 @@ public class Authenticator {
             throw new SecurityException("Step 2 DH params have invalid nonce");
         }
         if (!dhParams.serverNonce().equals(resPq.serverNonce())) {
-            throw new SecurityException("Step 2 DH params have invalid nonce");
+            throw new SecurityException("Step 2 DH params have invalid server nonce");
         }
+
+        if (dhParams.encryptedAnswer().length % 16 != 0) {
+            throw new SecurityException("Step 3 AES block size mismatch");
+        }
+
+        final Pair<byte[], byte[]> keyIv = Utils.generateKeyDataFromNonce(resPq.serverNonce(), newNonce);
+        final byte[] key = keyIv.getKey();
+        final byte[] iv = keyIv.getValue();
+        final byte[] plainText = AES.decryptIge(dhParams.encryptedAnswer(), key, iv);
+
+        final BinaryReader dhReader = new BinaryReader(ByteBuffer.wrap(plainText));
+        dhReader.seek(20); // hash sum
+        final Types.ServerDHInnerData dhInnerData = (Types.ServerDHInnerData) dhReader.readTl();
+
+        if (!dhInnerData.nonce().equals(resPq.nonce())) {
+            throw new SecurityException("Step 3 DH data have invalid nonce");
+        }
+        if (!dhInnerData.serverNonce().equals(resPq.serverNonce())) {
+            throw new SecurityException("Step 3 DH data have invalid server nonce");
+        }
+
+        // Big endian once again (and unsigned, so contact with a single zero byte)
+        final BigInteger dhPrime = new BigInteger(Utils.concat(new byte[1], dhInnerData.dhPrime()));
+        final BigInteger ga = new BigInteger(Utils.concat(new byte[1], dhInnerData.gA()));
+        final long timeOffset = dhInnerData.serverTime() - (System.currentTimeMillis() / 1000);
+
+        final byte[] bBytes = Utils.randomBytes(257);
+        bBytes[0] = 0;
+        final BigInteger b = new BigInteger(bBytes);
+        final BigInteger gb = new BigInteger(Integer.toString(dhInnerData.g())).modPow(b, dhPrime);
+        final BigInteger gab = ga.modPow(b, dhPrime);
+
+        final byte[] clientDhInnerData = new Types.ClientDHInnerData()
+                .nonce(resPq.nonce())
+                .serverNonce(resPq.serverNonce())
+                .retryId(0)
+                .gB(gb.toByteArray())
+                .serializeToBytes();
+
+        final byte[] clientDhInnerEncrypted = AES.encryptIge(Utils.concat(
+                Utils.sha1digest(clientDhInnerData), clientDhInnerData), key, iv);
+
+        final Abstract.SetClientDHParamsAnswer abstractDh = sender.send(
+                new Functions.SetClientDHParams()
+                        .nonce(resPq.nonce())
+                        .serverNonce(resPq.serverNonce())
+                        .encryptedData(clientDhInnerEncrypted)
+        );
+
+        if (!(abstractDh instanceof Types.DhGenOk)) {
+            // Once again we could check the values for other types but it's invalid anyway
+            throw new SecurityException("Step 3 DH response was not OK");
+        }
+
+        final Types.DhGenOk dhGen = (Types.DhGenOk) abstractDh;
+        if (!dhGen.nonce().equals(resPq.nonce())) {
+            throw new SecurityException("Step 3 DH gen data have invalid nonce");
+        }
+        if (!dhGen.serverNonce().equals(resPq.serverNonce())) {
+            throw new SecurityException("Step 3 DH gen data have invalid server nonce");
+        }
+
+        final AuthKey authKey = new AuthKey(gab.toByteArray());
+
+        // OK, Retry and Fail have numbers 1, 2 and 3 respectively
+        final int nonceNumber = 1;
+
+        final BigInteger newNonceHash = authKey.calcNewNonceHash(newNonce, nonceNumber);
+        if (!dhGen.newNonceHash1().equals(newNonceHash)) {
+            throw new SecurityException("Step 3 DH gen data have invalid nonce hash");
+        }
+
+        return authKey; // TODO And time offset
     }
 }

lib/src/main/java/io/github/lonamiwebs/overgram/utils/AES.java

@@ -0,0 +1,96 @@
+package io.github.lonamiwebs.overgram.utils;
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.Cipher;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+import javax.crypto.spec.SecretKeySpec;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+
+public class AES {
+    public static byte[] decryptIge(final byte[] cipherText, final byte[] key, final byte[] iv) {
+        final Cipher cipher;
+        try {
+            cipher = Cipher.getInstance("AES/ECB/NoPadding");
+        } catch (NoSuchAlgorithmException | NoSuchPaddingException ignored) {
+            throw new SecurityException("AES/ECB/NoPadding is required but not available");
+        }
+        try {
+            cipher.init(Cipher.DECRYPT_MODE, new SecretKeySpec(key, "AES"));
+        } catch (InvalidKeyException ignored) {
+            throw new SecurityException("Invalid key");
+        }
+
+        final int blockSize = cipher.getBlockSize();
+        final byte[] plainText = new byte[cipherText.length];
+
+        byte[] x;
+        byte[] y;
+        byte[] xPrev = Arrays.copyOfRange(iv, 0, blockSize);
+        byte[] yPrev = Arrays.copyOfRange(iv, blockSize, iv.length);
+        try {
+            for (int i = 0; i < cipherText.length; ) {
+                x = Arrays.copyOfRange(cipherText, i, i + blockSize);
+                y = Utils.xor(cipher.doFinal(Utils.xor(x, yPrev)), xPrev);
+                xPrev = x;
+                yPrev = y;
+
+                for (int j = 0; j < blockSize; ++j) {
+                    plainText[i] = y[j];
+                    ++i;
+                }
+            }
+        } catch (IllegalBlockSizeException | BadPaddingException ignored) {
+            throw new SecurityException("Illegal block size or padding");
+        }
+
+        return plainText;
+    }
+
+    public static byte[] encryptIge(final byte[] plainText, final byte[] key, final byte[] iv) {
+        final Cipher cipher;
+        try {
+            cipher = Cipher.getInstance("AES/ECB/NoPadding");
+        } catch (NoSuchAlgorithmException | NoSuchPaddingException ignored) {
+            throw new SecurityException("AES/ECB/NoPadding is required but not available");
+        }
+        try {
+            cipher.init(Cipher.ENCRYPT_MODE, new SecretKeySpec(key, "AES"));
+        } catch (InvalidKeyException ignored) {
+            throw new SecurityException("Invalid key");
+        }
+
+        final int blockSize = cipher.getBlockSize();
+        final byte[] paddedPlain;
+        if (plainText.length % 16 == 0) {
+            paddedPlain = plainText;
+        } else {
+            paddedPlain = Utils.concat(plainText, Utils.randomBytes(16 - plainText.length % 16));
+        }
+
+        final byte[] cipherText = new byte[paddedPlain.length];
+        byte[] x;
+        byte[] y;
+        byte[] xPrev = Arrays.copyOfRange(iv, 0, blockSize);
+        byte[] yPrev = Arrays.copyOfRange(iv, blockSize, iv.length);
+        try {
+            for (int i = 0; i < paddedPlain.length; ) {
+                y = Arrays.copyOfRange(paddedPlain, i, i + blockSize);
+                x = Utils.xor(cipher.doFinal(Utils.xor(y, xPrev)), yPrev);
+                yPrev = y;
+                xPrev = x;
+
+                for (int j = 0; j < blockSize; ++j) {
+                    cipherText[i] = x[j];
+                    ++i;
+                }
+            }
+        } catch (IllegalBlockSizeException | BadPaddingException ignored) {
+            throw new SecurityException("Illegal block size or padding");
+        }
+
+        return cipherText;
+    }
+}

lib/src/main/java/io/github/lonamiwebs/overgram/utils/BinaryReader.java

@@ -125,4 +125,8 @@ public class BinaryReader {
         }
         return result;
     }
+
+    public void seek(final int delta) {
+        buffer.position(buffer.position() + delta);
+    }
 }

lib/src/main/java/io/github/lonamiwebs/overgram/utils/Utils.java

@@ -6,6 +6,7 @@ import java.math.BigInteger;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.security.SecureRandom;
+import java.util.Arrays;
 
 public class Utils {
 
@@ -44,6 +45,16 @@ public class Utils {
         return bytes;
     }
 
+    public static byte[] concat(final byte[] left, final byte[] right, final int rightLength) {
+        final byte[] result = Arrays.copyOf(left, left.length + rightLength);
+        System.arraycopy(right, 0, result, left.length, rightLength);
+        return result;
+    }
+
+    public static byte[] concat(final byte[] left, final byte[] right) {
+        return concat(left, right, right.length);
+    }
+
     public static Pair<BigInteger, BigInteger> factorize(final BigInteger pq) {
         final BigInteger two = new BigInteger("2");
 
@@ -110,4 +121,26 @@ public class Utils {
         sha1.reset();
         return result;
     }
+
+    public static Pair<byte[], byte[]> generateKeyDataFromNonce(BigInteger serverNonce, BigInteger newNonce) {
+        final byte[] serverNonceBytes = reversed(serverNonce.toByteArray());
+        final byte[] newNonceBytes = reversed(newNonce.toByteArray());
+
+        final byte[] hash1 = sha1digest(concat(newNonceBytes, serverNonceBytes));
+        final byte[] hash2 = sha1digest(concat(serverNonceBytes, newNonceBytes));
+        final byte[] hash3 = sha1digest(concat(newNonceBytes, newNonceBytes));
+
+        final byte[] key = concat(hash1, hash2, 12);
+        final byte[] iv = new byte[8];
+        System.arraycopy(hash2, 12, iv, 0, 8);
+        return new Pair<>(key, concat(iv, concat(hash3, newNonceBytes, 4)));
+    }
+
+    public static byte[] xor(final byte[] a, final byte[] b) {
+        final byte[] result = new byte[Math.min(a.length, b.length)];
+        for (int i = 0; i < result.length; ++i) {
+            result[i] = (byte) (0xff & (a[i] ^ b[i]));
+        }
+        return result;
+    }
 }