PartParseTests.java

package software.coley.lljzip;

import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import org.objectweb.asm.ClassReader;
import org.objectweb.asm.ClassVisitor;
import org.objectweb.asm.ClassWriter;
import org.objectweb.asm.MethodVisitor;
import org.objectweb.asm.Opcodes;
import software.coley.lljzip.format.compression.ZipCompressions;
import software.coley.lljzip.format.model.CentralDirectoryFileHeader;
import software.coley.lljzip.format.model.LocalFileHeader;
import software.coley.lljzip.format.model.ZipArchive;
import software.coley.lljzip.format.model.ZipPart;
import software.coley.lljzip.format.read.ForwardScanZipReader;
import software.coley.lljzip.util.MemorySegmentUtil;

import javax.annotation.Nonnull;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.zip.ZipFile;

import static org.junit.jupiter.api.Assertions.*;

/**
 * Parse tests for {@link ZipArchive}.
 * Ensures all the {@link ZipPart}s of the archive are read as expected.
 *
 * @author Matt Coley
 */
public class PartParseTests {
	@ParameterizedTest
	@ValueSource(strings = {
			"src/test/resources/sample-code-7z.zip", // ZIP made from 7z
			"src/test/resources/sample-code-windows.zip", // ZIP made from windows built in 'send to zip'
	})
	public void testStandardCodeZip(String path) {
		try {
			ZipArchive zip = ZipIO.readStandard(Paths.get(path));
			assertNotNull(zip);

			// Each code zip contains these files
			assertTrue(hasFile(zip, "ClassFile.java"));
			assertTrue(hasFile(zip, "ClassMember.java"));
			assertTrue(hasFile(zip, "ConstPool.java"));
			assertTrue(hasFile(zip, "Descriptor.java"));
			assertTrue(hasFile(zip, "Field.java"));
			assertTrue(hasFile(zip, "Method.java"));
		} catch (IOException ex) {
			fail(ex);
		}
	}

	@ParameterizedTest
	@ValueSource(strings = {
			"hello.jar",
			"hello-secret.jar",
			"hello-secret-0-length-locals.jar",
			"hello-secret-junkheader.jar",
	})
	public void testHello(String name) {
		try {
			Path data = Paths.get("src/test/resources/" + name);
			ZipArchive zipStd = ZipIO.readStandard(data);
			ZipArchive zipJvm = ZipIO.readJvm(data);
			assertNotNull(zipStd);
			assertNotNull(zipJvm);
			assertEquals(zipJvm, zipJvm);

			// The 'hello' jars has a manifest and single class to run itself when invoked via 'java -jar'
			assertTrue(hasFile(zipStd, "META-INF/MANIFEST.MF"));
			assertTrue(hasFile(zipStd, "Hello.class"));
		} catch (IOException ex) {
			fail(ex);
		}
	}

	@ParameterizedTest
	@ValueSource(strings = {
			"hello-concat.jar",
			"hello-concat-junkheader.jar",
			"hello-merged.jar",
			"hello-merged-junkheader.jar",
	})
	public void testConcatAndMerged(String name) {
		try {
			Path path = Paths.get("src/test/resources/" + name);
			ZipArchive zipStd = ZipIO.readStandard(path);
			ZipArchive zipJvm = ZipIO.readJvm(path);
			assertNotNull(zipStd);
			assertNotNull(zipJvm);
			assertNotEquals(zipJvm, zipStd);
			assertNotEquals(zipJvm.getEnd(), zipStd.getEnd());
			assertTrue(hasFile(zipJvm, "META-INF/MANIFEST.MF"));
			assertTrue(hasFile(zipJvm, "Hello.class"));

			// Assert that the standard ZIP reader read the 'first' version of the class
			// and the JVM reader read the 'second' version of the class.
			LocalFileHeader stdHello = zipStd.getLocalFileByName("Hello.class");
			LocalFileHeader jvmHello = zipJvm.getLocalFileByName("Hello.class");
			assertNotEquals(stdHello, jvmHello);
			String stdHelloRaw = MemorySegmentUtil.toString(ZipCompressions.decompress(stdHello));
			String jvmHelloRaw = MemorySegmentUtil.toString(ZipCompressions.decompress(jvmHello));
			assertFalse(stdHelloRaw.isEmpty());
			assertFalse(jvmHelloRaw.isEmpty());
			assertTrue(stdHelloRaw.contains("Hello world"));
			assertTrue(jvmHelloRaw.contains("The secret code is: ROSE"));
		} catch (IOException ex) {
			fail(ex);
		}
	}

	@ParameterizedTest
	@ValueSource(strings = {
			"hello-end-declares-0-entries.jar",
			"hello-end-declares-0-entries-0-offset.jar",
			"hello-junk-dir-length.jar",
			"hello-junk-eocd.jar",
			"hello-junk-local-length.jar",
			"hello-total-junk.jar",
			"hello-total-junk-large.jar",
			"hello-wrong-local-compression.jar",
			"hello-zeroed-locals.jar",
			"hello-concat.jar",
			"hello-concat-junkheader.jar",
			"hello-merged.jar",
			"hello-merged-fake-empty.jar",
			"hello-merged-junkheader.jar",
			"hello-secret-0-length-locals.jar",
			"hello-secret-junkheader.jar",
			"hello-secret-trailing-slash.jar",
			"hello-secret-trailing-slash-0-length-locals.jar",
			"hello-txt-stored.jar",
			"hello-txt-type-0.jar",
	})
	public void testJvmCanRecoverData(String name) {
		try {
			Path path = Paths.get("src/test/resources/" + name);
			ZipArchive zip = ZipIO.readJvm(path);
			List<LocalFileHeader> localFiles = zip.getNameFilteredLocalFiles(n -> n.contains(".class"));
			assertEquals(1, localFiles.size(), "More than 1 class");
			byte[] decompressed = MemorySegmentUtil.toByteArray(ZipCompressions.decompress(localFiles.get(0)));
			String decompressedStr = new String(decompressed);
			assertDoesNotThrow(() -> {
				ClassWriter cw = new ClassWriter(0);
				ClassReader cr = new ClassReader(decompressed);
				cr.accept(cw, 0);
			}, "Failed to read class, must have failed to decompress");
			assertTrue(decompressedStr.contains("Hello world") || decompressedStr.contains("The secret code is: ROSE"));
		} catch (IOException ex) {
			fail(ex);
		}
	}

	@Test
	public void testEocdMisleading() {
		try {
			// This sample has a few tricks which historically have caused some problems
			// for our EOCD detection heuristics. We expect to see the manifest and class file, even though the EOCD is misleading.
			// There are other 'files' in the jar, but they're junk.
			ZipArchive zip = ZipIO.readJvm(Paths.get("src/test/resources/sample-eocd-oob.jar"));
			assertNotNull(zip);
			assertTrue(hasFile(zip, "META-INF/MANIFEST.MF"));
			assertTrue(hasFile(zip, "Hello.class/"));
		} catch (IOException ex) {
			fail(ex);
		}
	}

	@Test
	public void testDuplicateManifest() {
		try {
			// This sample is similar in that it has a number of tricks which historically have caused some problems.
			// There's a lot of junk in here like the other sample, but we should still be able to see the manifest and class file.
			ZipArchive zip = ZipIO.readJvm(Paths.get("src/test/resources/sample-duplicate-manifest-oob.jar"));
			assertNotNull(zip);
			assertTrue(hasFile(zip, "META-INF/MANIFEST.MF"));
			assertTrue(hasFile(zip, "Hello.class/"));

			// Check for code in the class files. If we fell for the trap classes, then we won't have any code to visit.
			AtomicBoolean visitedCode = new AtomicBoolean(false);
			List<LocalFileHeader> localFiles = zip.getNameFilteredLocalFiles(n -> n.contains(".class"));
			for (LocalFileHeader localFile : localFiles) {
				try {
					byte[] byteArray = MemorySegmentUtil.toByteArray(ZipCompressions.decompress(localFile));
					ClassReader cr = new ClassReader(byteArray);
					cr.accept(new ClassVisitor(Opcodes.ASM9) {
						@Override
						public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) {
							return new MethodVisitor(Opcodes.ASM9) {
								@Override
								public void visitMethodInsn(int opcode, String owner, String name, String descriptor, boolean isInterface) {
									visitedCode.set(true);
								}
							};
						}
					}, 0);
				} catch (Throwable t) {
					// There are junk classes in here, so I expect failures when trying to read them. That's fine, just ignore those.
				}
			}
			assertTrue(visitedCode.get(), "One of the classes should have had code to visit, but none did. We fell for trap classes");

			String manifest = MemorySegmentUtil.toString(ZipCompressions.decompress(zip.getLocalFileByName("META-INF/MANIFEST.MF")));
			assertTrue(manifest.contains("Main-Class: Hello"), "Should resolve the executable manifest entry");
			assertFalse(manifest.contains("com/sun/internal/Cleanup"), "Should not resolve the decoy manifest entry");
 		} catch (IOException ex) {
			fail(ex);
		}
	}

	@Test
	public void testZip64ByEntryCount() {
		// Standard ZIP format has a limit of 65,535 entries.
		// The sample is a zipped jar with a Zip64 jar inside of it (because its easily compressible data, so why not?)
		final int entryCount = 70_000;
		try {
			// Extract the zip64 jar
			byte[] zip64bytes;
			try (ZipFile zip = new ZipFile(Paths.get("src/test/resources/there-is-a-zip64-in-here.zip").toFile())) {
				zip64bytes = zip.getInputStream(zip.getEntry("lljzip-zip64-4605877303365506557.jar")).readAllBytes();
			}

			// Both standard/jvm readers should handle all entries in the zip64 jar.
			try (ZipArchive zipStd = ZipIO.readStandard(zip64bytes);
			     ZipArchive zipJvm = ZipIO.readJvm(zip64bytes)) {
				assertEquals(entryCount, zipStd.getCentralDirectories().size());
				assertEquals(entryCount, zipStd.getLocalFiles().size());
				assertNotNull(zipStd.getLocalFileByName("E" + (entryCount - 1)));

				assertEquals(entryCount, zipJvm.getCentralDirectories().size());
				assertEquals(entryCount, zipJvm.getLocalFiles().size());
				assertNotNull(zipJvm.getLocalFileByName("E" + (entryCount - 1)));
			}
		} catch (IOException ex) {
			fail(ex);
		}
	}

	@Test
	public void testLocalHeaderDetectMismatch() {
		Path path = Paths.get("src/test/resources/hello-secret-0-length-locals.jar");

		try {
			// The 'standard' strategy does not adopt CEN values when reading local entries.
			// The 'jvm' strategy does.
			ZipArchive zipStd = ZipIO.readStandard(path);
			assertNotNull(zipStd);

			LocalFileHeader hello = zipStd.getLocalFileByName("Hello.class");
			assertNotNull(hello);
			assertEquals(0, hello.getFileData().byteSize()); // Should be empty

			// The local file header says the contents are 0 bytes, but the central header has the real length
			assertTrue(hello.hasDifferentValuesThanCentralDirectoryHeader());

			// The solution to differing values is to adopt values in the reader strategy
			ZipArchive zipStdAndAdopt = ZipIO.read(path, new ForwardScanZipReader() {
				@Override
				public void postProcessLocalFileHeader(@Nonnull LocalFileHeader file) {
					assertDoesNotThrow(() -> file.adoptLinkedCentralDirectoryValues());
				}
			});
			LocalFileHeader helloAdopted = zipStdAndAdopt.getLocalFileByName("Hello.class");
			assertFalse(helloAdopted.hasDifferentValuesThanCentralDirectoryHeader());
			assertNotEquals(0, helloAdopted.getFileData().byteSize()); // Should have data

			// The JVM strategy copies most properties, except for size.
			ZipArchive zipJvm = ZipIO.readJvm(path);
			helloAdopted = zipJvm.getLocalFileByName("Hello.class");
			assertNotEquals(0, helloAdopted.getFileData().byteSize()); // Should have data, even if not sourced from values in the CEN
		} catch (IOException ex) {
			fail(ex);
		}
	}

	@Test
	public void testEndInLocalDataDesc() {
		// The non-class files have been zeroed out in this sample, but we don't care.
		// The main point is that the "bin/3" and "bin/4" red-herring headers are properly dealt with,
		// and we can see that there are 639 class files in this jar file.
		try (ZipArchive archive = ZipIO.readJvm(Paths.get("src/test/resources/end-in-local-data-desc.jar"))) {
			assertEquals(639, archive.getNameFilteredLocalFiles(f -> f.endsWith(".class")).size());
		} catch (IOException ex) {
			fail(ex);
		}
	}

	@Test
	public void testMergedFakeEmpty() {
		try (ZipArchive zipJvm = ZipIO.readJvm(Paths.get("src/test/resources/hello-merged-fake-empty.jar"))) {
			assertNotNull(zipJvm);
			assertTrue(hasFile(zipJvm, "META-INF/MANIFEST.MF"));
			assertTrue(hasFile(zipJvm, "Hello.class/")); // has trailing slash in class name
		} catch (IOException ex) {
			fail(ex);
		}
	}

	private static boolean hasFile(ZipArchive zip, String name) {
		return !zip.getNameFilteredLocalFiles(name::equals).isEmpty();
	}
}