--- make/autoconf/libraries.m4 | 4 ++ make/autoconf/spec.gmk.in | 1 + make/modules/java.base/lib/CoreLibraries.gmk | 39 ++++++++++-- src/hotspot/share/runtime/arguments.cpp | 9 +++ .../share/classes/java/util/zip/Deflater.java | 27 ++++++++ .../java/util/zip/GZIPInputStream.java | 62 ++++++++++++++++++- .../java/util/zip/GZIPOutputStream.java | 35 ++++++++++- .../share/classes/java/util/zip/Inflater.java | 23 +++++++ .../java/util/zip/InflaterInputStream.java | 21 +++++++ src/java.base/share/conf/security/java.policy | 4 ++ src/java.base/share/native/libzip/Deflater.c | 38 ++++++++++++ src/java.base/share/native/libzip/Inflater.c | 37 ++++++++++- .../zip/GZIP/GZIPOutputStreamHeaderTest.java | 4 ++ 13 files changed, 296 insertions(+), 8 deletions(-) diff --git a/make/autoconf/libraries.m4 b/make/autoconf/libraries.m4 index 264a83189..b6a261eef 100644 --- a/make/autoconf/libraries.m4 +++ b/make/autoconf/libraries.m4 @@ -198,4 +198,8 @@ AC_DEFUN_ONCE([LIB_SETUP_MISC_LIBS], # Control if libzip can use mmap. Available for purposes of overriding. LIBZIP_CAN_USE_MMAP=true AC_SUBST(LIBZIP_CAN_USE_MMAP) + + # Control if libz can use mmap. Available for purposes of overriding. + LIBZ_CAN_USE_MMAP=true + AC_SUBST(LIBZ_CAN_USE_MMAP) ]) diff --git a/make/autoconf/spec.gmk.in b/make/autoconf/spec.gmk.in index e3f43ebf0..769c3ae3c 100644 --- a/make/autoconf/spec.gmk.in +++ b/make/autoconf/spec.gmk.in @@ -787,6 +787,7 @@ USE_EXTERNAL_LIBZ:=@USE_EXTERNAL_LIBZ@ LIBZ_CFLAGS:=@LIBZ_CFLAGS@ LIBZ_LIBS:=@LIBZ_LIBS@ LIBZIP_CAN_USE_MMAP:=@LIBZIP_CAN_USE_MMAP@ +LIBZ_CAN_USE_MMAP:=@LIBZ_CAN_USE_MMAP@ MSVCR_DLL:=@MSVCR_DLL@ VCRUNTIME_1_DLL:=@VCRUNTIME_1_DLL@ MSVCP_DLL:=@MSVCP_DLL@ diff --git a/make/modules/java.base/lib/CoreLibraries.gmk b/make/modules/java.base/lib/CoreLibraries.gmk index bb09d8cf8..8b03ea56e 100644 --- a/make/modules/java.base/lib/CoreLibraries.gmk +++ b/make/modules/java.base/lib/CoreLibraries.gmk @@ -119,9 +119,39 @@ $(BUILD_LIBJAVA): $(BUILD_LIBFDLIBM) ########################################################################################## -BUILD_LIBZIP_EXCLUDES := -ifeq ($(USE_EXTERNAL_LIBZ), true) - LIBZIP_EXCLUDES += zlib +ifeq ($(USE_EXTERNAL_LIBZ), false) + ifeq ($(OPENJDK_TARGET_OS), linux) + ifeq ($(LIBZ_CAN_USE_MMAP), true) + BUILD_LIBZ_MMAP := -DUSE_MMAP + endif + + $(eval $(call SetupJdkLibrary, BUILD_LIBZ, \ + NAME := z, \ + SRC := $(TOPDIR)/src/java.base/share/native/libzip/zlib, \ + OPTIMIZATION := LOW, \ + CFLAGS := $(filter-out -fvisibility=hidden,$(CFLAGS_JDKLIB) $(LIBZ_CFLAGS)), \ + CFLAGS_unix := $(BUILD_LIBZ_MMAP) -UDEBUG, \ + DISABLED_WARNINGS_gcc := unused-function implicit-fallthrough, \ + DISABLED_WARNINGS_gcc_gzlib.c := implicit-function-declaration, \ + DISABLED_WARNINGS_gcc_gzwrite.c := implicit-function-declaration, \ + DISABLED_WARNINGS_gcc_gzread.c := implicit-function-declaration, \ + DISABLED_WARNINGS_clang := format-nonliteral, \ + LDFLAGS := $(LDFLAGS_JDKLIB) \ + $(call SET_SHARED_LIBRARY_ORIGIN), \ + LIBS_unix := , \ + LIBS_windows := jvm.lib $(WIN_JAVA_LIB), \ + )) + + $(BUILD_LIBZ): $(BUILD_LIBJAVA) + + TARGETS += $(BUILD_LIBZ) + endif +endif + +########################################################################################## + +ifeq ($(OPENJDK_TARGET_OS), linux) + LIBZIP_EXCLUDES := zlib endif ifeq ($(LIBZIP_CAN_USE_MMAP), true) @@ -139,10 +169,11 @@ $(eval $(call SetupJdkLibrary, BUILD_LIBZIP, \ DISABLED_WARNINGS_clang := format-nonliteral deprecated-non-prototype, \ LDFLAGS := $(LDFLAGS_JDKLIB) \ $(call SET_SHARED_LIBRARY_ORIGIN), \ - LIBS_unix := -ljvm -ljava $(LIBZ_LIBS), \ + LIBS_unix := -ljvm -ljava -lz, \ LIBS_windows := jvm.lib $(WIN_JAVA_LIB), \ )) +$(BUILD_LIBZIP): $(BUILD_LIBZ) $(BUILD_LIBZIP): $(BUILD_LIBJAVA) TARGETS += $(BUILD_LIBZIP) diff --git a/src/hotspot/share/runtime/arguments.cpp b/src/hotspot/share/runtime/arguments.cpp index bc56a1322..9ef0a0db2 100644 --- a/src/hotspot/share/runtime/arguments.cpp +++ b/src/hotspot/share/runtime/arguments.cpp @@ -2617,6 +2617,15 @@ jint Arguments::parse_each_vm_init_arg(const JavaVMInitArgs* args, bool* patch_m // -D } else if (match_option(option, "-D", &tail)) { const char* value; +#ifndef AARCH64 + if (match_option(option, "-DGZIP_USE_KAE=", &value)) { + if (strcmp(value, "true") == 0) { + jio_fprintf(defaultStream::output_stream(), + "-DGZIP_USE_KAE is not supported. This system propertiy is valid only on aarch64 architecture machines.\n" + "The compression action is performed using the native compression capability of the JDK.\n"); + } + } +#endif if (match_option(option, "-Djava.endorsed.dirs=", &value) && *value!= '\0' && strcmp(value, "\"\"") != 0) { // abort if -Djava.endorsed.dirs is set diff --git a/src/java.base/share/classes/java/util/zip/Deflater.java b/src/java.base/share/classes/java/util/zip/Deflater.java index 88da99ba6..02852ab64 100644 --- a/src/java.base/share/classes/java/util/zip/Deflater.java +++ b/src/java.base/share/classes/java/util/zip/Deflater.java @@ -201,6 +201,20 @@ public class Deflater { init(level, DEFAULT_STRATEGY, nowrap)); } + /** + * Creates a new compressor using the specified compression level + * and windowBits. + * This method is mainly used to support the KAE-zip feature. + * @param level the compression level (0-9) + * @param windowBits compression format (-15~31) + */ + public Deflater(int level, int windowBits) { + this.level = level; + this.strategy = DEFAULT_STRATEGY; + this.zsRef = new DeflaterZStreamRef(this, + initKAE(level, DEFAULT_STRATEGY, windowBits)); + } + /** * Creates a new compressor using the specified compression level. * Compressed data will be generated in ZLIB format. @@ -878,6 +892,18 @@ public class Deflater { } } + /** + * Resets deflater so that a new set of input data can be processed. + * Java fields are not initialized. + * This method is mainly used to support the KAE-zip feature. + */ + public void resetKAE() { + synchronized (zsRef) { + ensureOpen(); + reset(zsRef.address()); + } + } + /** * Closes the compressor and discards any unprocessed input. * @@ -909,6 +935,7 @@ public class Deflater { } private static native long init(int level, int strategy, boolean nowrap); + private static native long initKAE(int level, int strategy, int windowBits); private static native void setDictionary(long addr, byte[] b, int off, int len); private static native void setDictionaryBuffer(long addr, long bufAddress, int len); diff --git a/src/java.base/share/classes/java/util/zip/GZIPInputStream.java b/src/java.base/share/classes/java/util/zip/GZIPInputStream.java index 4f3f9eae4..1d8afae1c 100644 --- a/src/java.base/share/classes/java/util/zip/GZIPInputStream.java +++ b/src/java.base/share/classes/java/util/zip/GZIPInputStream.java @@ -54,6 +54,23 @@ public class GZIPInputStream extends InflaterInputStream { private boolean closed = false; + /** + * The field is mainly used to support the KAE-zip feature. + */ + private static boolean GZIP_USE_KAE = false; + + private static int WINDOWBITS = 31; + + private static int FLUSHKAE = 2; + + static { + if ("aarch64".equals(System.getProperty("os.arch"))) { + GZIP_USE_KAE = Boolean.parseBoolean(System.getProperty("GZIP_USE_KAE", "false")); + WINDOWBITS = Integer.parseInt(System.getProperty("WINDOWBITS", "31")); + FLUSHKAE = Integer.parseInt(System.getProperty("FLUSHKAE", "2")); + } + } + /** * Check to make sure that this stream has not been closed */ @@ -74,8 +91,13 @@ public class GZIPInputStream extends InflaterInputStream { * @throws IllegalArgumentException if {@code size <= 0} */ public GZIPInputStream(InputStream in, int size) throws IOException { - super(in, in != null ? new Inflater(true) : null, size); + super(in, in != null ? (GZIP_USE_KAE ? new Inflater(WINDOWBITS, FLUSHKAE) : new Inflater(true)) : null, size); usesDefaultInflater = true; + + // When GZIP_USE_KAE is true, the header of the file is readed + // through the native zlib library, not in java code. + if (GZIP_USE_KAE) return; + readHeader(in); } @@ -116,13 +138,16 @@ public class GZIPInputStream extends InflaterInputStream { } int n = super.read(buf, off, len); if (n == -1) { - if (readTrailer()) + if (GZIP_USE_KAE ? readTrailerKAE() : readTrailer()) eos = true; else return this.read(buf, off, len); } else { crc.update(buf, off, n); } + if (GZIP_USE_KAE && inf.finished()) { + if (readTrailerKAE()) eos = true; + } return n; } @@ -237,6 +262,39 @@ public class GZIPInputStream extends InflaterInputStream { return false; } + /* + * Reads GZIP member trailer and returns true if the eos + * reached, false if there are more (concatenated gzip + * data set) + * + * This method is mainly used to support the KAE-zip feature. + */ + private boolean readTrailerKAE() throws IOException { + InputStream in = this.in; + int n = inf.getRemaining(); + if (n > 0) { + in = new SequenceInputStream( + new ByteArrayInputStream(buf, len - n, n), + new FilterInputStream(in) { + public void close() throws IOException {} + }); + } + // If there are more bytes available in "in" or the leftover in the "inf" is > 18 bytes: + // next.header.min(10) + next.trailer(8), try concatenated case + + if (n > 18) { + inf.reset(); + inf.setInput(buf, len - n, n); + } else { + try { + fillKAE(n); + } catch (IOException e) { + return true; + } + } + return false; + } + /* * Reads unsigned integer in Intel byte order. */ diff --git a/src/java.base/share/classes/java/util/zip/GZIPOutputStream.java b/src/java.base/share/classes/java/util/zip/GZIPOutputStream.java index cdfac329c..f9570265e 100644 --- a/src/java.base/share/classes/java/util/zip/GZIPOutputStream.java +++ b/src/java.base/share/classes/java/util/zip/GZIPOutputStream.java @@ -55,6 +55,20 @@ public class GZIPOutputStream extends DeflaterOutputStream { // Represents the default "unknown" value for OS header, per RFC-1952 private static final byte OS_UNKNOWN = (byte) 255; + /** + * The field is mainly used to support the KAE-zip feature. + */ + private static boolean GZIP_USE_KAE = false; + + private static int WINDOWBITS = 31; + + static { + if ("aarch64".equals(System.getProperty("os.arch"))) { + GZIP_USE_KAE = Boolean.parseBoolean(System.getProperty("GZIP_USE_KAE", "false")); + WINDOWBITS = Integer.parseInt(System.getProperty("WINDOWBITS", "31")); + } + } + /** * Creates a new output stream with the specified buffer size. * @@ -90,10 +104,16 @@ public class GZIPOutputStream extends DeflaterOutputStream { public GZIPOutputStream(OutputStream out, int size, boolean syncFlush) throws IOException { - super(out, out != null ? new Deflater(Deflater.DEFAULT_COMPRESSION, true) : null, + super(out, out != null ? + (GZIP_USE_KAE ? new Deflater(Deflater.DEFAULT_COMPRESSION, WINDOWBITS) : + new Deflater(Deflater.DEFAULT_COMPRESSION, true)) : null, size, syncFlush); usesDefaultDeflater = true; + + // When GZIP_USE_KAE is true, the header of the file is written + // through the native zlib library, not in java code. + if (GZIP_USE_KAE) return; writeHeader(); crc.reset(); } @@ -163,6 +183,13 @@ public class GZIPOutputStream extends DeflaterOutputStream { int len = def.deflate(buf, 0, buf.length); if (def.finished() && len <= buf.length - TRAILER_SIZE) { // last deflater buffer. Fit trailer at the end + // When GZIP_USE_KAE is true, the trailer of the file is written + // through the native zlib library, not in java code. + if (GZIP_USE_KAE) { + out.write(buf, 0, len); + def.resetKAE(); + return; + } writeTrailer(buf, len); len = len + TRAILER_SIZE; out.write(buf, 0, len); @@ -173,6 +200,12 @@ public class GZIPOutputStream extends DeflaterOutputStream { } // if we can't fit the trailer at the end of the last // deflater buffer, we write it separately + // When GZIP_USE_KAE is true, the trailer of the file is written + // through the native zlib library, not in java code. + if (GZIP_USE_KAE) { + def.resetKAE(); + return; + } byte[] trailer = new byte[TRAILER_SIZE]; writeTrailer(trailer, 0); out.write(trailer); diff --git a/src/java.base/share/classes/java/util/zip/Inflater.java b/src/java.base/share/classes/java/util/zip/Inflater.java index b6fe36010..77e4dd07f 100644 --- a/src/java.base/share/classes/java/util/zip/Inflater.java +++ b/src/java.base/share/classes/java/util/zip/Inflater.java @@ -131,6 +131,17 @@ public class Inflater { this.zsRef = new InflaterZStreamRef(this, init(nowrap)); } + /** + * Creates a new decompressor. + * This method is mainly used to support the KAE-zip feature. + * + * @param windowBits compression format (-15~31) + * @param flushKAE inflate flush type (0~6) + */ + public Inflater(int windowBits, int flushKAE) { + this.zsRef = new InflaterZStreamRef(this, initKAE(windowBits, flushKAE)); + } + /** * Creates a new decompressor. */ @@ -692,6 +703,17 @@ public class Inflater { } } + /** + * Resets inflater so that a new set of input data can be processed. + * This method is mainly used to support the KAE-zip feature. + */ + public void resetKAE() { + synchronized (zsRef) { + ensureOpen(); + reset(zsRef.address()); + } + } + /** * Closes the decompressor and discards any unprocessed input. * @@ -716,6 +738,7 @@ public class Inflater { private static native void initIDs(); private static native long init(boolean nowrap); + private static native long initKAE(int windowBits, int flushKAE); private static native void setDictionary(long addr, byte[] b, int off, int len); private static native void setDictionaryBuffer(long addr, long bufAddress, int len); diff --git a/src/java.base/share/classes/java/util/zip/InflaterInputStream.java b/src/java.base/share/classes/java/util/zip/InflaterInputStream.java index 534241d3d..88f8c5f1b 100644 --- a/src/java.base/share/classes/java/util/zip/InflaterInputStream.java +++ b/src/java.base/share/classes/java/util/zip/InflaterInputStream.java @@ -246,6 +246,27 @@ public class InflaterInputStream extends FilterInputStream { inf.setInput(buf, 0, len); } + /** + * Fills input buffer with more data to decompress. + * This method is mainly used to support the KAE-zip feature. + * @param n Maximum Read Bytes + * @throws IOException if an I/O error has occurred + */ + protected void fillKAE(int n) throws IOException { + ensureOpen(); + byte[] buftmp = new byte[buf.length]; + if (n != 0) { + System.arraycopy(buf, buf.length - n, buftmp, 0, n); + } + int kaelen = in.read(buftmp, n, buf.length - n); + if (kaelen == -1) { + throw new EOFException("Unexpected end of ZLIB input stream"); + } + System.arraycopy(buftmp, 0, buf, buf.length - n - kaelen, n + kaelen); + inf.reset(); + inf.setInput(buf, buf.length - n - kaelen, n + kaelen); + } + /** * Tests if this input stream supports the {@code mark} and * {@code reset} methods. The {@code markSupported} diff --git a/src/java.base/share/conf/security/java.policy b/src/java.base/share/conf/security/java.policy index 1554541d1..4eda61464 100644 --- a/src/java.base/share/conf/security/java.policy +++ b/src/java.base/share/conf/security/java.policy @@ -41,4 +41,8 @@ grant { permission java.util.PropertyPermission "java.vm.version", "read"; permission java.util.PropertyPermission "java.vm.vendor", "read"; permission java.util.PropertyPermission "java.vm.name", "read"; + + permission java.util.PropertyPermission "GZIP_USE_KAE", "read"; + permission java.util.PropertyPermission "WINDOWBITS", "read"; + permission java.util.PropertyPermission "FLUSHKAE", "read"; }; diff --git a/src/java.base/share/native/libzip/Deflater.c b/src/java.base/share/native/libzip/Deflater.c index 1ed1994d4..c04d5c42a 100644 --- a/src/java.base/share/native/libzip/Deflater.c +++ b/src/java.base/share/native/libzip/Deflater.c @@ -76,6 +76,44 @@ Java_java_util_zip_Deflater_init(JNIEnv *env, jclass cls, jint level, } } +JNIEXPORT jlong JNICALL +Java_java_util_zip_Deflater_initKAE(JNIEnv *env, jclass cls, jint level, + jint strategy, jint windowBits) +{ + z_stream *strm = calloc(1, sizeof(z_stream)); + + if (strm == 0) { + JNU_ThrowOutOfMemoryError(env, 0); + return jlong_zero; + } else { + const char *msg; + int ret = deflateInit2(strm, level, Z_DEFLATED, + windowBits, + DEF_MEM_LEVEL, strategy); + switch (ret) { + case Z_OK: + return ptr_to_jlong(strm); + case Z_MEM_ERROR: + free(strm); + JNU_ThrowOutOfMemoryError(env, 0); + return jlong_zero; + case Z_STREAM_ERROR: + free(strm); + JNU_ThrowIllegalArgumentException(env, 0); + return jlong_zero; + default: + msg = ((strm->msg != NULL) ? strm->msg : + (ret == Z_VERSION_ERROR) ? + "zlib returned Z_VERSION_ERROR: " + "compile time and runtime zlib implementations differ" : + "unknown error initializing zlib library"); + free(strm); + JNU_ThrowInternalError(env, msg); + return jlong_zero; + } + } +} + static void throwInternalErrorHelper(JNIEnv *env, z_stream *strm, const char *fixmsg) { const char *msg = NULL; msg = (strm->msg != NULL) ? strm->msg : fixmsg; diff --git a/src/java.base/share/native/libzip/Inflater.c b/src/java.base/share/native/libzip/Inflater.c index a41e9775b..4835bfac2 100644 --- a/src/java.base/share/native/libzip/Inflater.c +++ b/src/java.base/share/native/libzip/Inflater.c @@ -44,6 +44,7 @@ static jfieldID inputConsumedID; static jfieldID outputConsumedID; +static jint inflaterFlushType = Z_PARTIAL_FLUSH; JNIEXPORT void JNICALL Java_java_util_zip_Inflater_initIDs(JNIEnv *env, jclass cls) @@ -87,6 +88,40 @@ Java_java_util_zip_Inflater_init(JNIEnv *env, jclass cls, jboolean nowrap) } } +JNIEXPORT jlong JNICALL +Java_java_util_zip_Inflater_initKAE(JNIEnv *env, jclass cls, jint windowBits, jint flushKAE) +{ + z_stream *strm = calloc(1, sizeof(z_stream)); + inflaterFlushType = flushKAE; + + if (strm == NULL) { + JNU_ThrowOutOfMemoryError(env, 0); + return jlong_zero; + } else { + const char *msg; + int ret = inflateInit2(strm, windowBits); + switch (ret) { + case Z_OK: + return ptr_to_jlong(strm); + case Z_MEM_ERROR: + free(strm); + JNU_ThrowOutOfMemoryError(env, 0); + return jlong_zero; + default: + msg = ((strm->msg != NULL) ? strm->msg : + (ret == Z_VERSION_ERROR) ? + "zlib returned Z_VERSION_ERROR: " + "compile time and runtime zlib implementations differ" : + (ret == Z_STREAM_ERROR) ? + "inflateInit2 returned Z_STREAM_ERROR" : + "unknown error initializing zlib library"); + free(strm); + JNU_ThrowInternalError(env, msg); + return jlong_zero; + } + } +} + static void checkSetDictionaryResult(JNIEnv *env, jlong addr, int res) { switch (res) { @@ -137,7 +172,7 @@ static jint doInflate(jlong addr, strm->avail_in = inputLen; strm->avail_out = outputLen; - ret = inflate(strm, Z_PARTIAL_FLUSH); + ret = inflate(strm, inflaterFlushType); return ret; } diff --git a/test/jdk/java/util/zip/GZIP/GZIPOutputStreamHeaderTest.java b/test/jdk/java/util/zip/GZIP/GZIPOutputStreamHeaderTest.java index 93c2e91fe..4038ad9a7 100644 --- a/test/jdk/java/util/zip/GZIP/GZIPOutputStreamHeaderTest.java +++ b/test/jdk/java/util/zip/GZIP/GZIPOutputStreamHeaderTest.java @@ -35,6 +35,10 @@ import java.util.zip.GZIPOutputStream; * @bug 8244706 * @summary Verify that the OS header flag in the stream written out by java.util.zip.GZIPOutputStream * has the correct expected value + * @comment This test case is not suitable for GZIP-zip feature testing, the ninth byte in the header + * of the gzip file identifies the operating system that generated the file. By default, the byte of + * the compressed package generated by JDK is 0xff (OS_UNKNOWN). KAE-zip relies on the underlying + * zlib library to write header files, this field is set to 0x03 (OS_UNIX). Therefore, this case will fail * @run testng GZIPOutputStreamHeaderTest */ public class GZIPOutputStreamHeaderTest { -- 2.19.1