Out of memory exception when decrypt large file using Cipher

The bad news are: IMHO the error is caused by an bad implementation of AES GCM-mode in native Java. Even if you could get it to work you will find that the decryption of a large file (1 GB or so) will take a lot of time (maybe hours ?). But there are good news: you could/should use BouncyCastle as service provider for your decryption task - that way the decryption will work and it's much faster.

The following full example will create a sample file of 1 gb size, encrypts it with BouncyCastle and later decrypts it. In the end there is a file compare to show that plain and decrypted file contents are equal and the files will be deleted. You need temporary a total of more than 3 GB free space on your device to run this example.

Using a buffer of 64 KB I'm running this example with this data:

Milliseconds for Encryption: 14295 | Decryption: 16249

A buffer of 1 KB is a little bit slower on encryption side but much slower on decryption task:

Milliseconds for Encryption: 15250 | Decryption: 21952

A last word regarding your cipher - "AES/GCM/PKCS5Padding" is not existing and "available" in some implementations but the real used algorithm is "AES/GCM/NoPadding" (see Can PKCS5Padding be in AES/GCM mode? for more details).

import org.bouncycastle.jce.provider.BouncyCastleProvider;
import javax.crypto.BadPaddingException;
import javax.crypto.Cipher;
import javax.crypto.IllegalBlockSizeException;
import javax.crypto.NoSuchPaddingException;
import javax.crypto.spec.GCMParameterSpec;
import javax.crypto.spec.SecretKeySpec;
import java.io.*;
import java.nio.file.Files;
import java.security.*;
import java.util.Arrays;

public class GcmTestBouncyCastle {
    public static void main(String[] args) throws IOException, NoSuchPaddingException, InvalidAlgorithmParameterException,
            NoSuchAlgorithmException, IllegalBlockSizeException, BadPaddingException, NoSuchProviderException, InvalidKeyException {
        System.out.println("Encryption & Decryption with BouncyCastle AES-GCM-Mode");
        System.out.println("https://stackoverflow.com/questions/61792534/out-of-memory-exception-when-decrypt-large-file-using-cipher");
        // you need bouncy castle, get version 1.65 here:
        // https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk15on/1.65
        Security.addProvider(new BouncyCastleProvider());
        // setup files
        // filenames
        String filenamePlain = "plain.dat";
        String filenameEncrypt = "encrypt.dat";
        String filenameDecrypt = "decrypt.dat";
        // generate a testfile of 1024 byte | 1 gb
        //createFileWithDefinedLength(filenamePlain, 1024);
        createFileWithDefinedLength(filenamePlain, 1024 * 1024 * 1024); // 1 gb
        // time measurement
        long startMilli = 0;
        long encryptionMilli = 0;
        long decryptionMilli = 0;
        // generate nonce/iv
        int GCM_NONCE_LENGTH = 12; // for a nonce of 96 bit length
        int GCM_TAG_LENGTH = 16;
        int GCM_KEY_LENGTH = 32; // 32 = 256 bit keylength, 16 = 128 bit keylength
        SecureRandom r = new SecureRandom();
        byte[] nonce = new byte[GCM_NONCE_LENGTH];
        r.nextBytes(nonce);
        // key should be generated as random byte[]
        byte[] key = new byte[GCM_KEY_LENGTH];
        r.nextBytes(key);
        // encrypt file
        startMilli = System.currentTimeMillis();
        encryptWithGcmBc(filenamePlain, filenameEncrypt, key, nonce, GCM_TAG_LENGTH);
        encryptionMilli = System.currentTimeMillis() - startMilli;
        startMilli = System.currentTimeMillis();
        decryptWithGcmBc(filenameEncrypt, filenameDecrypt, key, nonce, GCM_TAG_LENGTH);
        decryptionMilli = System.currentTimeMillis() - startMilli;
        // check that plain and decrypted files are equal
        System.out.println("SHA256-file compare " + filenamePlain + " | " + filenameDecrypt + " : "
                + Arrays.equals(sha256File(filenamePlain), sha256File(filenameDecrypt)));
        System.out.println("Milliseconds for Encryption: " + encryptionMilli + " | Decryption: " + decryptionMilli);
        // clean up with files
        Files.deleteIfExists(new File(filenamePlain).toPath());
        Files.deleteIfExists(new File(filenameEncrypt).toPath());
        Files.deleteIfExists(new File(filenameDecrypt).toPath());
    }

    public static void encryptWithGcmBc(String filenamePlain, String filenameEnc, byte[] key, byte[] nonce, int gcm_tag_length)
            throws IOException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException,
            InvalidAlgorithmParameterException, IllegalBlockSizeException, BadPaddingException, NoSuchProviderException {
        Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding", "BC");
        SecretKeySpec keySpec = new SecretKeySpec(key, "AES");
        GCMParameterSpec gcmSpec = new GCMParameterSpec(gcm_tag_length * 8, nonce);
        cipher.init(Cipher.ENCRYPT_MODE, keySpec, gcmSpec);

        try (FileInputStream fis = new FileInputStream(filenamePlain);
             BufferedInputStream in = new BufferedInputStream(fis);
             FileOutputStream out = new FileOutputStream(filenameEnc);
             BufferedOutputStream bos = new BufferedOutputStream(out)) {
            //byte[] ibuf = new byte[1024];
            byte[] ibuf = new byte[0x10000]; // = 65536
            int len;
            while ((len = in.read(ibuf)) != -1) {
                byte[] obuf = cipher.update(ibuf, 0, len);
                if (obuf != null)
                    bos.write(obuf);
            }
            byte[] obuf = cipher.doFinal();
            if (obuf != null)
                bos.write(obuf);
        }
    }

    public static void decryptWithGcmBc(String filenameEnc, String filenameDec, byte[] key, byte[] nonce, int gcm_tag_length)
            throws IOException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException,
            InvalidAlgorithmParameterException, IllegalBlockSizeException, BadPaddingException, NoSuchProviderException {
        try (FileInputStream in = new FileInputStream(filenameEnc);
             FileOutputStream out = new FileOutputStream(filenameDec)) {
            //byte[] ibuf = new byte[1024];
            byte[] ibuf = new byte[0x10000]; // = 65536
            int len;
            Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding", "BC");
            SecretKeySpec keySpec = new SecretKeySpec(key, "AES");
            GCMParameterSpec gcmSpec = new GCMParameterSpec(gcm_tag_length * 8, nonce);
            cipher.init(Cipher.DECRYPT_MODE, keySpec, gcmSpec);
            while ((len = in.read(ibuf)) != -1) {
                byte[] obuf = cipher.update(ibuf, 0, len);
                if (obuf != null)
                    out.write(obuf);
            }
            byte[] obuf = cipher.doFinal();
            if (obuf != null)
                out.write(obuf);
        }
    }

    // just for creating a large file within seconds
    private static void createFileWithDefinedLength(String filenameString, long sizeLong) throws IOException {
        RandomAccessFile raf = new RandomAccessFile(filenameString, "rw");
        try {
            raf.setLength(sizeLong);
        } finally {
            raf.close();
        }
    }

    // just for file comparing
    public static byte[] sha256File(String filenameString) throws IOException, NoSuchAlgorithmException {
        byte[] buffer = new byte[8192];
        int count;
        MessageDigest md = MessageDigest.getInstance("SHA-256");
        BufferedInputStream bis = new BufferedInputStream(new FileInputStream(filenameString));
        while ((count = bis.read(buffer)) > 0) {
            md.update(buffer, 0, count);
        }
        bis.close();
        return md.digest();
    }
}