From 579a36762293fa4c9f2831e27e7af7713a0838a0 Mon Sep 17 00:00:00 2001 From: Michal Foksa Date: Fri, 14 Feb 2025 10:11:43 +0100 Subject: [PATCH 01/12] Prevent class with empty Java package being trusted by SpecificDatumReader (#3311) (cherry picked from commit 34558273491dbaede744dfb9319759125118356c) --- .../org/apache/avro/specific/SpecificDatumReader.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java b/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java index 8950f165991..e782c58c867 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java +++ b/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java @@ -141,10 +141,10 @@ private void checkSecurity(Class clazz) throws ClassNotFoundException { break; } } - if (!found) { - throw new SecurityException("Forbidden " + clazz - + "! This class is not trusted to be included in Avro schema using java-class. Please set org.apache.avro.SERIALIZABLE_PACKAGES system property with the packages you trust."); - } + } + if (!found) { + throw new SecurityException("Forbidden " + clazz + + "! This class is not trusted to be included in Avro schema using java-class. Please set org.apache.avro.SERIALIZABLE_PACKAGES system property with the packages you trust."); } } From 35389bd43753fcd21adc01cd1471439c44020fa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Mon, 19 May 2025 21:07:24 +0200 Subject: [PATCH 02/12] Remove the default serializable packages and deprecated the property to introduce org.apache.avro.SERIALIZABLE_CLASSES instead (#3376) (cherry picked from commit 90a937f28fac3e583d7efae1e59aacbf6f246d30) --- lang/java/avro/pom.xml | 3 + .../avro/specific/SpecificDatumReader.java | 63 ++++++++++++++----- .../org/apache/avro/reflect/TestReflect.java | 1 + lang/java/ipc/pom.xml | 3 + 4 files changed, 54 insertions(+), 16 deletions(-) diff --git a/lang/java/avro/pom.xml b/lang/java/avro/pom.xml index 7064f080f7f..02033f939be 100644 --- a/lang/java/avro/pom.xml +++ b/lang/java/avro/pom.xml @@ -72,6 +72,9 @@ maven-surefire-plugin none + + java.math.BigDecimal,java.math.BigInteger,java.net.URI,java.net.URL,java.io.File,java.lang.Integer,org.apache.avro.reflect.TestReflect$R10 + diff --git a/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java b/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java index e782c58c867..88c760ca16c 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java +++ b/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java @@ -23,10 +23,13 @@ import org.apache.avro.generic.GenericDatumReader; import org.apache.avro.io.ResolvingDecoder; import org.apache.avro.util.ClassUtils; + import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.HashSet; +import java.util.Set; /** * {@link org.apache.avro.io.DatumReader DatumReader} for generated Java @@ -34,14 +37,31 @@ */ public class SpecificDatumReader extends GenericDatumReader { + /** + * @deprecated prefer to use {@link #SERIALIZABLE_CLASSES} instead. + */ + @Deprecated public static final String[] SERIALIZABLE_PACKAGES; + public static final String[] SERIALIZABLE_CLASSES; + static { - SERIALIZABLE_PACKAGES = System.getProperty("org.apache.avro.SERIALIZABLE_PACKAGES", - "java.lang,java.math,java.io,java.net,org.apache.avro.reflect").split(","); + // no serializable classes by default + String serializableClassesProp = System.getProperty("org.apache.avro.SERIALIZABLE_CLASSES"); + SERIALIZABLE_CLASSES = (serializableClassesProp == null) ? new String[0] : serializableClassesProp.split(","); + + // no serializable packages by default + String serializablePackagesProp = System.getProperty("org.apache.avro.SERIALIZABLE_PACKAGES"); + SERIALIZABLE_PACKAGES = (serializablePackagesProp == null) ? new String[0] : serializablePackagesProp.split(","); } + // The primitive "class names" based on Class.isPrimitive() + private static final Set PRIMITIVES = new HashSet<>(Arrays.asList(Boolean.TYPE.getName(), + Character.TYPE.getName(), Byte.TYPE.getName(), Short.TYPE.getName(), Integer.TYPE.getName(), Long.TYPE.getName(), + Float.TYPE.getName(), Double.TYPE.getName(), Void.TYPE.getName())); + private final List trustedPackages = new ArrayList<>(); + private final List trustedClasses = new ArrayList<>(); public SpecificDatumReader() { this(null, null, SpecificData.get()); @@ -69,11 +89,14 @@ public SpecificDatumReader(Schema writer, Schema reader) { public SpecificDatumReader(Schema writer, Schema reader, SpecificData data) { super(writer, reader, data); trustedPackages.addAll(Arrays.asList(SERIALIZABLE_PACKAGES)); + trustedClasses.addAll(Arrays.asList(SERIALIZABLE_CLASSES)); } /** Construct given a {@link SpecificData}. */ public SpecificDatumReader(SpecificData data) { super(data); + trustedPackages.addAll(Arrays.asList(SERIALIZABLE_PACKAGES)); + trustedClasses.addAll(Arrays.asList(SERIALIZABLE_CLASSES)); } /** Return the contained {@link SpecificData}. */ @@ -115,8 +138,8 @@ private Class getPropAsClass(Schema schema, String prop) { if (name == null) return null; try { + checkSecurity(name); Class clazz = ClassUtils.forName(getData().getClassLoader(), name); - checkSecurity(clazz); return clazz; } catch (ClassNotFoundException e) { throw new AvroRuntimeException(e); @@ -127,31 +150,39 @@ private boolean trustAllPackages() { return (trustedPackages.size() == 1 && "*".equals(trustedPackages.get(0))); } - private void checkSecurity(Class clazz) throws ClassNotFoundException { - if (trustAllPackages() || clazz.isPrimitive()) { + private void checkSecurity(String className) throws ClassNotFoundException { + if (trustAllPackages() || PRIMITIVES.contains(className)) { return; } - boolean found = false; - Package thePackage = clazz.getPackage(); - if (thePackage != null) { - for (String trustedPackage : getTrustedPackages()) { - if (thePackage.getName().equals(trustedPackage) || thePackage.getName().startsWith(trustedPackage + ".")) { - found = true; - break; - } + for (String trustedClass : getTrustedClasses()) { + if (className.equals(trustedClass)) { + return; } } - if (!found) { - throw new SecurityException("Forbidden " + clazz - + "! This class is not trusted to be included in Avro schema using java-class. Please set org.apache.avro.SERIALIZABLE_PACKAGES system property with the packages you trust."); + + for (String trustedPackage : getTrustedPackages()) { + if (className.startsWith(trustedPackage)) { + return; + } } + + throw new SecurityException("Forbidden " + className + + "! This class is not trusted to be included in Avro schema using java-class. Please set org.apache.avro.SERIALIZABLE_CLASSES system property with the class you trust or org.apache.avro.SERIALIZABLE_PACKAGES system property with the packages you trust."); } + /** + * @deprecated Use getTrustedClasses() instead + */ + @Deprecated public final List getTrustedPackages() { return trustedPackages; } + public final List getTrustedClasses() { + return trustedClasses; + } + @Override protected Object readRecord(Object old, Schema expected, ResolvingDecoder in) throws IOException { SpecificData data = getSpecificData(); diff --git a/lang/java/avro/src/test/java/org/apache/avro/reflect/TestReflect.java b/lang/java/avro/src/test/java/org/apache/avro/reflect/TestReflect.java index e3065d59b87..5eab3551ff0 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/reflect/TestReflect.java +++ b/lang/java/avro/src/test/java/org/apache/avro/reflect/TestReflect.java @@ -570,6 +570,7 @@ void checkReadWrite(Object object) throws Exception { } void checkReadWrite(Object object, Schema s) throws Exception { + ReflectDatumWriter writer = new ReflectDatumWriter<>(s); ByteArrayOutputStream out = new ByteArrayOutputStream(); writer.write(object, factory.directBinaryEncoder(out, null)); diff --git a/lang/java/ipc/pom.xml b/lang/java/ipc/pom.xml index dc468f878ee..3202476cedd 100644 --- a/lang/java/ipc/pom.xml +++ b/lang/java/ipc/pom.xml @@ -62,6 +62,9 @@ 1 false none + + java.math.BigDecimal,java.math.BigInteger + From d269d47f150a9c25009904348945823cd2041910 Mon Sep 17 00:00:00 2001 From: Oscar WvH-K Date: Sun, 10 Aug 2025 15:53:10 +0200 Subject: [PATCH 03/12] java-[key-]class allowed packages must be packages --- .../avro/specific/SpecificDatumReader.java | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java b/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java index 88c760ca16c..510c9fed84d 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java +++ b/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.HashSet; import java.util.Set; +import java.util.stream.Stream; /** * {@link org.apache.avro.io.DatumReader DatumReader} for generated Java @@ -47,12 +48,29 @@ public class SpecificDatumReader extends GenericDatumReader { static { // no serializable classes by default - String serializableClassesProp = System.getProperty("org.apache.avro.SERIALIZABLE_CLASSES"); - SERIALIZABLE_CLASSES = (serializableClassesProp == null) ? new String[0] : serializableClassesProp.split(","); + SERIALIZABLE_CLASSES = streamPropertyEntries(System.getProperty("org.apache.avro.SERIALIZABLE_CLASSES")) + .toArray(String[]::new); // no serializable packages by default - String serializablePackagesProp = System.getProperty("org.apache.avro.SERIALIZABLE_PACKAGES"); - SERIALIZABLE_PACKAGES = (serializablePackagesProp == null) ? new String[0] : serializablePackagesProp.split(","); + SERIALIZABLE_PACKAGES = streamPropertyEntries(System.getProperty("org.apache.avro.SERIALIZABLE_PACKAGES")) + // Add a '.' suffix to ensure we'll be matching package names instead of + // arbitrary prefixes, except for the wildcard "*", which allows all + // packages (this is only safe in fully controlled environments!). + .map(entry -> "*".equals(entry) ? entry : entry + ".").toArray(String[]::new); + } + + /** + * Parse a comma separated list into non-empty entries. Leading and trailing + * whitespace is stripped. + * + * @param commaSeparatedEntries the comma separated list of entries + * @return a stream of the entries + */ + private static Stream streamPropertyEntries(String commaSeparatedEntries) { + if (commaSeparatedEntries == null) { + return Stream.empty(); + } + return Stream.of(commaSeparatedEntries.split(",")).map(String::trim).filter(s -> !s.isEmpty()); } // The primitive "class names" based on Class.isPrimitive() @@ -167,8 +185,10 @@ private void checkSecurity(String className) throws ClassNotFoundException { } } - throw new SecurityException("Forbidden " + className - + "! This class is not trusted to be included in Avro schema using java-class. Please set org.apache.avro.SERIALIZABLE_CLASSES system property with the class you trust or org.apache.avro.SERIALIZABLE_PACKAGES system property with the packages you trust."); + throw new SecurityException("Forbidden " + className + "! This class is not trusted to be included in Avro " + + "schemas using java-class. Please set the system property org.apache.avro.SERIALIZABLE_CLASSES to the comma " + + "separated list of classes you trust. You can also set the system property " + + "org.apache.avro.SERIALIZABLE_PACKAGES to the comma separated list of the packages you trust."); } /** From 335cc5aa3588e168840469ad3ee72000cd30b72b Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 25 Jul 2024 11:12:36 +0200 Subject: [PATCH 04/12] Update docs path (#3045) --- build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.sh b/build.sh index a6327b28d5c..2991995faa9 100755 --- a/build.sh +++ b/build.sh @@ -182,7 +182,8 @@ do # build docs cp -r doc/ build/staging-web/ find build/staging-web/ -type f -print0 | xargs -0 sed -r -i "s#\+\+version\+\+#${VERSION,,}#g" - mv build/staging-web/content/en/docs/++version++ build/staging-web/content/en/docs/"${VERSION,,}" + mkdir -p build/staging-web/public/docs/ + mv build/staging-web/doc/content/en/docs/++version++ build/staging-web/public/docs/"${VERSION,,}" read -n 1 -s -r -p "Build build/staging-web/ manually now. Press a key to continue..." # If it was a SNAPSHOT, it was lowercased during the build. cp -R build/staging-web/public/docs/"${VERSION,,}"/* "build/$DOC_DIR/" From 2d139882406ab3071dc1aa4e40b98545580e2678 Mon Sep 17 00:00:00 2001 From: Oscar WvH-K Date: Sun, 10 Aug 2025 15:20:40 +0200 Subject: [PATCH 05/12] Preparing for release 1.11.5 --- doc/examples/java-example/pom.xml | 8 ++++---- doc/examples/mr-example/pom.xml | 8 ++++---- lang/java/android/pom.xml | 2 +- lang/java/archetypes/avro-service-archetype/pom.xml | 2 +- lang/java/archetypes/pom.xml | 2 +- lang/java/avro/pom.xml | 2 +- lang/java/compiler/pom.xml | 2 +- lang/java/grpc/pom.xml | 2 +- lang/java/integration-test/codegen-test/pom.xml | 2 +- lang/java/integration-test/pom.xml | 2 +- .../java/integration-test/test-custom-conversions/pom.xml | 2 +- lang/java/ipc-jetty/pom.xml | 2 +- lang/java/ipc-netty/pom.xml | 2 +- lang/java/ipc/pom.xml | 2 +- lang/java/mapred/pom.xml | 2 +- lang/java/maven-plugin/pom.xml | 2 +- lang/java/perf/pom.xml | 2 +- lang/java/pom.xml | 2 +- lang/java/protobuf/pom.xml | 2 +- lang/java/thrift/pom.xml | 2 +- lang/java/tools/pom.xml | 2 +- lang/java/trevni/avro/pom.xml | 2 +- lang/java/trevni/core/pom.xml | 2 +- lang/java/trevni/doc/pom.xml | 2 +- lang/java/trevni/pom.xml | 2 +- lang/js/package-lock.json | 2 +- lang/js/package.json | 2 +- pom.xml | 4 ++-- share/VERSION.txt | 2 +- 29 files changed, 36 insertions(+), 36 deletions(-) diff --git a/doc/examples/java-example/pom.xml b/doc/examples/java-example/pom.xml index a71aab28c09..6fc2cfffc0e 100644 --- a/doc/examples/java-example/pom.xml +++ b/doc/examples/java-example/pom.xml @@ -38,7 +38,7 @@ org.apache.avro avro - 1.11.4 + 1.11.5 @@ -55,7 +55,7 @@ org.apache.avro avro-maven-plugin - 1.11.4 + 1.11.5 generate-sources @@ -72,7 +72,7 @@ org.apache.maven.plugins maven-plugin - 1.11.4 + 1.11.5 1.8 1.8 @@ -92,7 +92,7 @@ org.apache.avro avro-maven-plugin - [1.11.4,) + [1.11.5,) schema diff --git a/doc/examples/mr-example/pom.xml b/doc/examples/mr-example/pom.xml index aae8c723c1e..66fa36f2499 100644 --- a/doc/examples/mr-example/pom.xml +++ b/doc/examples/mr-example/pom.xml @@ -45,7 +45,7 @@ org.apache.avro avro-maven-plugin - 1.11.4 + 1.11.5 generate-sources @@ -73,7 +73,7 @@ org.apache.avro avro-maven-plugin - [1.11.4,) + [1.11.5,) schema @@ -94,12 +94,12 @@ org.apache.avro avro - 1.11.4 + 1.11.5 org.apache.avro avro-mapred - 1.11.4 + 1.11.5 org.apache.hadoop diff --git a/lang/java/android/pom.xml b/lang/java/android/pom.xml index 46904c52af6..49ff1832a1c 100644 --- a/lang/java/android/pom.xml +++ b/lang/java/android/pom.xml @@ -22,7 +22,7 @@ avro-parent org.apache.avro - 1.11.4 + 1.11.5 ../pom.xml diff --git a/lang/java/archetypes/avro-service-archetype/pom.xml b/lang/java/archetypes/avro-service-archetype/pom.xml index d293b0936fe..4204aad6b95 100644 --- a/lang/java/archetypes/avro-service-archetype/pom.xml +++ b/lang/java/archetypes/avro-service-archetype/pom.xml @@ -23,7 +23,7 @@ avro-archetypes-parent org.apache.avro - 1.11.4 + 1.11.5 ../pom.xml diff --git a/lang/java/archetypes/pom.xml b/lang/java/archetypes/pom.xml index a632ae7b5d5..aff41d24305 100644 --- a/lang/java/archetypes/pom.xml +++ b/lang/java/archetypes/pom.xml @@ -22,7 +22,7 @@ org.apache.avro avro-parent - 1.11.4 + 1.11.5 ../pom.xml diff --git a/lang/java/avro/pom.xml b/lang/java/avro/pom.xml index 02033f939be..3ef5d02ea8c 100644 --- a/lang/java/avro/pom.xml +++ b/lang/java/avro/pom.xml @@ -23,7 +23,7 @@ avro-parent org.apache.avro - 1.11.4 + 1.11.5 ../ diff --git a/lang/java/compiler/pom.xml b/lang/java/compiler/pom.xml index 33718dba924..cbdb5468346 100644 --- a/lang/java/compiler/pom.xml +++ b/lang/java/compiler/pom.xml @@ -23,7 +23,7 @@ avro-parent org.apache.avro - 1.11.4 + 1.11.5 ../pom.xml diff --git a/lang/java/grpc/pom.xml b/lang/java/grpc/pom.xml index 33ed8b87a96..02d0b56b91d 100644 --- a/lang/java/grpc/pom.xml +++ b/lang/java/grpc/pom.xml @@ -23,7 +23,7 @@ org.apache.avro avro-parent - 1.11.4 + 1.11.5 ../ diff --git a/lang/java/integration-test/codegen-test/pom.xml b/lang/java/integration-test/codegen-test/pom.xml index 09ce70bba71..7f8a5738bdf 100644 --- a/lang/java/integration-test/codegen-test/pom.xml +++ b/lang/java/integration-test/codegen-test/pom.xml @@ -23,7 +23,7 @@ avro-integration-test org.apache.avro - 1.11.4 + 1.11.5 ../ diff --git a/lang/java/integration-test/pom.xml b/lang/java/integration-test/pom.xml index f93feb766dd..de48110b3d8 100644 --- a/lang/java/integration-test/pom.xml +++ b/lang/java/integration-test/pom.xml @@ -23,7 +23,7 @@ avro-parent org.apache.avro - 1.11.4 + 1.11.5 ../ diff --git a/lang/java/integration-test/test-custom-conversions/pom.xml b/lang/java/integration-test/test-custom-conversions/pom.xml index 38a9f66aaf5..d99b7fd8d2c 100644 --- a/lang/java/integration-test/test-custom-conversions/pom.xml +++ b/lang/java/integration-test/test-custom-conversions/pom.xml @@ -23,7 +23,7 @@ avro-integration-test org.apache.avro - 1.11.4 + 1.11.5 ../ diff --git a/lang/java/ipc-jetty/pom.xml b/lang/java/ipc-jetty/pom.xml index cbd20f2314e..7ebe2561ba8 100644 --- a/lang/java/ipc-jetty/pom.xml +++ b/lang/java/ipc-jetty/pom.xml @@ -23,7 +23,7 @@ avro-parent org.apache.avro - 1.11.4 + 1.11.5 ../ diff --git a/lang/java/ipc-netty/pom.xml b/lang/java/ipc-netty/pom.xml index 6a5fe621a10..0f77bfcb32a 100644 --- a/lang/java/ipc-netty/pom.xml +++ b/lang/java/ipc-netty/pom.xml @@ -23,7 +23,7 @@ avro-parent org.apache.avro - 1.11.4 + 1.11.5 ../ diff --git a/lang/java/ipc/pom.xml b/lang/java/ipc/pom.xml index 3202476cedd..ef7275f88bf 100644 --- a/lang/java/ipc/pom.xml +++ b/lang/java/ipc/pom.xml @@ -23,7 +23,7 @@ avro-parent org.apache.avro - 1.11.4 + 1.11.5 ../ diff --git a/lang/java/mapred/pom.xml b/lang/java/mapred/pom.xml index e6fb35e7864..0a83cdfb31e 100644 --- a/lang/java/mapred/pom.xml +++ b/lang/java/mapred/pom.xml @@ -23,7 +23,7 @@ avro-parent org.apache.avro - 1.11.4 + 1.11.5 ../ diff --git a/lang/java/maven-plugin/pom.xml b/lang/java/maven-plugin/pom.xml index f1351a3fe63..9114dfd8f83 100644 --- a/lang/java/maven-plugin/pom.xml +++ b/lang/java/maven-plugin/pom.xml @@ -23,7 +23,7 @@ avro-parent org.apache.avro - 1.11.4 + 1.11.5 ../pom.xml diff --git a/lang/java/perf/pom.xml b/lang/java/perf/pom.xml index 73d86c702bd..b6dd4f1a069 100644 --- a/lang/java/perf/pom.xml +++ b/lang/java/perf/pom.xml @@ -23,7 +23,7 @@ avro-parent org.apache.avro - 1.11.4 + 1.11.5 ../ diff --git a/lang/java/pom.xml b/lang/java/pom.xml index 77bd5803e06..d2ac4440c20 100644 --- a/lang/java/pom.xml +++ b/lang/java/pom.xml @@ -22,7 +22,7 @@ org.apache.avro avro-toplevel - 1.11.4 + 1.11.5 ../../pom.xml diff --git a/lang/java/protobuf/pom.xml b/lang/java/protobuf/pom.xml index bc92e9042a3..22245ea6d87 100644 --- a/lang/java/protobuf/pom.xml +++ b/lang/java/protobuf/pom.xml @@ -23,7 +23,7 @@ avro-parent org.apache.avro - 1.11.4 + 1.11.5 ../ diff --git a/lang/java/thrift/pom.xml b/lang/java/thrift/pom.xml index a85f024b588..38989fb57bf 100644 --- a/lang/java/thrift/pom.xml +++ b/lang/java/thrift/pom.xml @@ -23,7 +23,7 @@ avro-parent org.apache.avro - 1.11.4 + 1.11.5 ../ diff --git a/lang/java/tools/pom.xml b/lang/java/tools/pom.xml index 436cf8e1c85..87f7895b4b0 100644 --- a/lang/java/tools/pom.xml +++ b/lang/java/tools/pom.xml @@ -23,7 +23,7 @@ avro-parent org.apache.avro - 1.11.4 + 1.11.5 ../ diff --git a/lang/java/trevni/avro/pom.xml b/lang/java/trevni/avro/pom.xml index a035f05a9f7..32c420f5e95 100644 --- a/lang/java/trevni/avro/pom.xml +++ b/lang/java/trevni/avro/pom.xml @@ -22,7 +22,7 @@ trevni-java org.apache.avro - 1.11.4 + 1.11.5 ../ diff --git a/lang/java/trevni/core/pom.xml b/lang/java/trevni/core/pom.xml index 08a5fd57fa8..089f95ab25e 100644 --- a/lang/java/trevni/core/pom.xml +++ b/lang/java/trevni/core/pom.xml @@ -22,7 +22,7 @@ trevni-java org.apache.avro - 1.11.4 + 1.11.5 ../ diff --git a/lang/java/trevni/doc/pom.xml b/lang/java/trevni/doc/pom.xml index fc6bae86a4d..28aa6ea82aa 100644 --- a/lang/java/trevni/doc/pom.xml +++ b/lang/java/trevni/doc/pom.xml @@ -22,7 +22,7 @@ trevni-java org.apache.avro - 1.11.4 + 1.11.5 .. diff --git a/lang/java/trevni/pom.xml b/lang/java/trevni/pom.xml index 064be245134..b005484aeb9 100644 --- a/lang/java/trevni/pom.xml +++ b/lang/java/trevni/pom.xml @@ -23,7 +23,7 @@ avro-parent org.apache.avro - 1.11.4 + 1.11.5 ../ diff --git a/lang/js/package-lock.json b/lang/js/package-lock.json index cb61de31f17..f2d667dff99 100644 --- a/lang/js/package-lock.json +++ b/lang/js/package-lock.json @@ -1,6 +1,6 @@ { "name": "avro-js", - "version": "1.11.4-SNAPSHOT", + "version": "1.11.5", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/lang/js/package.json b/lang/js/package.json index 84383ce0b99..4c8678bc29a 100644 --- a/lang/js/package.json +++ b/lang/js/package.json @@ -1,6 +1,6 @@ { "name": "avro-js", - "version": "1.11.4", + "version": "1.11.5", "author": "Avro Developers ", "description": "JavaScript Avro implementation", "contributors": [ diff --git a/pom.xml b/pom.xml index bf99f329c3e..7bf72f6e924 100644 --- a/pom.xml +++ b/pom.xml @@ -27,7 +27,7 @@ org.apache.avro avro-toplevel - 1.11.4 + 1.11.5 pom Apache Avro Toplevel @@ -64,7 +64,7 @@ 3.2.5 - 1723820671 + 1754831786 diff --git a/share/VERSION.txt b/share/VERSION.txt index 3d0e62313ce..e6dbb7c238f 100644 --- a/share/VERSION.txt +++ b/share/VERSION.txt @@ -1 +1 @@ -1.11.4 +1.11.5 From 33b855a07d28c9cd541880ad8de4934a8364154c Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 3 Jun 2025 21:45:17 +0200 Subject: [PATCH 06/12] Fix mypy test (#3397) * Fix mypy test And also run the Docker tests on pull-requests * Thanks Martin (cherry picked from commit 49fca82579ce865c4332108ad4557ec742afe8db) --- lang/py/avro/io.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lang/py/avro/io.py b/lang/py/avro/io.py index 7b5576697eb..15427c48302 100644 --- a/lang/py/avro/io.py +++ b/lang/py/avro/io.py @@ -644,9 +644,10 @@ def readers_schema(self, readers_schema: avro.schema.Schema) -> None: def read(self, decoder: "BinaryDecoder") -> object: if self.writers_schema is None: raise avro.errors.IONotReadyException("Cannot read without a writer's schema.") - if self.readers_schema is None: - self.readers_schema = self.writers_schema - return self.read_data(self.writers_schema, self.readers_schema, decoder) + reader_schema = self.readers_schema + if reader_schema is None: + reader_schema = self.writers_schema + return self.read_data(self.writers_schema, reader_schema, decoder) def read_data(self, writers_schema: avro.schema.Schema, readers_schema: avro.schema.Schema, decoder: "BinaryDecoder") -> object: # schema matching From 4c08651762c684b32d0cb88731eaf8f2f3f4b253 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Sat, 12 Oct 2024 22:20:20 +0800 Subject: [PATCH 07/12] ci: Remove deprecated pypy-3.9 test (#3211) (cherry picked from commit 86fde6980445de641f97e1f329b00d2bd14aa668) --- .github/workflows/test-lang-py.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-lang-py.yml b/.github/workflows/test-lang-py.yml index 00485b7103e..1ebae370943 100644 --- a/.github/workflows/test-lang-py.yml +++ b/.github/workflows/test-lang-py.yml @@ -43,11 +43,6 @@ jobs: - '3.11' - '3.10' - '3.9' - - '3.8' - - '3.7' - - '3.6' - - 'pypy-3.7' - - 'pypy-3.6' steps: - uses: actions/checkout@v4 @@ -92,9 +87,13 @@ jobs: - '3.9' - '3.8' - '3.7' +<<<<<<< HEAD - '3.6' - 'pypy-3.7' - 'pypy-3.6' +======= + - 'pypy-3.10' +>>>>>>> 982ad792ce (ci: Remove deprecated pypy-3.9 test (#3211)) steps: - uses: actions/checkout@v4 From a49a87f14678f075878c162893fbd51d1dce1e49 Mon Sep 17 00:00:00 2001 From: Oscar WvH-K Date: Sun, 29 Jun 2025 18:20:12 +0200 Subject: [PATCH 08/12] Drop apparently unsupported pypy version --- .github/workflows/test-lang-py.yml | 5 +++++ lang/py/tox.ini | 4 ++-- share/docker/Dockerfile | 18 ++++++++++++++++-- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-lang-py.yml b/.github/workflows/test-lang-py.yml index 1ebae370943..08e319a99fe 100644 --- a/.github/workflows/test-lang-py.yml +++ b/.github/workflows/test-lang-py.yml @@ -43,6 +43,8 @@ jobs: - '3.11' - '3.10' - '3.9' + - '3.8' + - 'pypy-3.10' steps: - uses: actions/checkout@v4 @@ -86,12 +88,15 @@ jobs: - '3.10' - '3.9' - '3.8' +<<<<<<< HEAD - '3.7' <<<<<<< HEAD - '3.6' - 'pypy-3.7' - 'pypy-3.6' ======= +======= +>>>>>>> 6e5e37876b (Drop apparently unsupported pypy version) - 'pypy-3.10' >>>>>>> 982ad792ce (ci: Remove deprecated pypy-3.9 test (#3211)) diff --git a/lang/py/tox.ini b/lang/py/tox.ini index e7bba45baf1..6a3c5d2d752 100644 --- a/lang/py/tox.ini +++ b/lang/py/tox.ini @@ -26,8 +26,8 @@ envlist = py38 py39 py310 - pypy3.6 - pypy3.7 + py311 + pypy3.10 [coverage:run] diff --git a/share/docker/Dockerfile b/share/docker/Dockerfile index c1c2565e593..4ee1a007037 100644 --- a/share/docker/Dockerfile +++ b/share/docker/Dockerfile @@ -167,10 +167,24 @@ RUN curl -sSL https://cpanmin.us \ Module::Install::Repository \ && rm -rf .cpanm -# Install Python packages -ENV PIP_NO_CACHE_DIR=off +# Install Python3 +ENV PATH="${PATH}:/opt/pypy3.10/bin" \ + PIP_NO_CACHE_DIR=off + +# https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope +ARG BUILDARCH +RUN case "${BUILDARCH:?}" in \ + arm64) pypyarch=aarch64;; \ + *) pypyarch=linux64;; \ + esac \ + && cd /opt \ + && for url in https://downloads.python.org/pypy/pypy3.10-v7.3.12-"$pypyarch".tar.bz2; \ + do curl -fsSL "$url" | tar -xvjpf -; \ + done \ + && ln -s pypy3.10* pypy3.10 # Install Python3 packages +ENV PIP_NO_CACHE_DIR=off RUN python3 -m pip install --upgrade pip setuptools wheel \ && python3 -m pip install tox zstandard From 811b3a862205b4bf0670e98590b6eb7ef3e292d4 Mon Sep 17 00:00:00 2001 From: Oscar WvH-K Date: Mon, 18 Aug 2025 10:14:00 +0200 Subject: [PATCH 09/12] Avoid 20 second sleep installing node. --- share/docker/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/share/docker/Dockerfile b/share/docker/Dockerfile index 4ee1a007037..b976f9772fa 100644 --- a/share/docker/Dockerfile +++ b/share/docker/Dockerfile @@ -114,8 +114,11 @@ RUN set -eux; \ ENV PATH="/opt/maven/bin:${PATH}" # Install nodejs +# The node deprecation warnings cause a 20 second sleep. +# But mom, I'm not even tired! RUN curl -sSL https://deb.nodesource.com/setup_14.x \ - | bash - \ + | sed 's/sleep/echo "But mom!"/' \ + | bash \ && apt-get -qqy install nodejs \ && apt-get -qqy clean \ && npm install -g grunt-cli \ From a50134bed746f6234283950687e98098e706ba1d Mon Sep 17 00:00:00 2001 From: Oscar WvH-K Date: Mon, 18 Aug 2025 10:20:08 +0200 Subject: [PATCH 10/12] Fix perl build --- lang/perl/Makefile.PL | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lang/perl/Makefile.PL b/lang/perl/Makefile.PL index ffac5e9d11b..13508b67197 100644 --- a/lang/perl/Makefile.PL +++ b/lang/perl/Makefile.PL @@ -26,10 +26,13 @@ license 'apache'; version $version; readme_from 'lib/Avro.pm'; all_from 'lib/Avro.pm'; -build_requires 'Test::More', 0.88; +configure_requires 'Module::Install'; +configure_requires 'Module::Install::ReadmeFromPod'; test_requires 'Math::BigInt'; test_requires 'Perl::Critic'; test_requires 'Test::Exception'; +test_requires 'Test::More', 0.88; +test_requires 'Test::Pod'; requires 'Compress::Zlib'; requires 'Compress::Zstd'; requires 'Encode'; From a0d0130aea75b8319f251c3805f18a1776efa563 Mon Sep 17 00:00:00 2001 From: Oscar WvH-K Date: Wed, 20 Aug 2025 01:41:30 +0200 Subject: [PATCH 11/12] Fix docs path in build.sh --- build.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 2991995faa9..d30c6b142c7 100755 --- a/build.sh +++ b/build.sh @@ -183,7 +183,7 @@ do cp -r doc/ build/staging-web/ find build/staging-web/ -type f -print0 | xargs -0 sed -r -i "s#\+\+version\+\+#${VERSION,,}#g" mkdir -p build/staging-web/public/docs/ - mv build/staging-web/doc/content/en/docs/++version++ build/staging-web/public/docs/"${VERSION,,}" + mv build/staging-web/content/en/docs/++version++ build/staging-web/public/docs/"${VERSION,,}" read -n 1 -s -r -p "Build build/staging-web/ manually now. Press a key to continue..." # If it was a SNAPSHOT, it was lowercased during the build. cp -R build/staging-web/public/docs/"${VERSION,,}"/* "build/$DOC_DIR/" @@ -218,7 +218,6 @@ do else gpg --pinentry-mode loopback --local-user="$GPG_LOCAL_USER" --passphrase "$password" --armor --output "$f.asc" --detach-sig "$f" fi - done set -x From 257db287e4cf3f3831013780e709226d4aa188d9 Mon Sep 17 00:00:00 2001 From: Oscar Westra van Holthe - Kind Date: Fri, 4 Oct 2024 10:49:56 +0200 Subject: [PATCH 12/12] AVRO-4053: doc consistency in velocity templates (#3150) * AVRO-4053: Improve consistency in javadoc comments * AVRO-4053: Add test case * AVRO-4053: rename test * AVRO-4053: Fix omission * AVRO-4053: spotless * AVRO-4053: Restore old name in public API * AVRO-4053: Static imports on top (cherry picked from commit 84bc7322ca1c04ab4a8e4e708acf1e271541aac4) --- .../compiler/specific/SpecificCompiler.java | 43 +++++++++++++---- .../specific/templates/java/classic/enum.vm | 4 +- .../specific/templates/java/classic/fixed.vm | 4 +- .../templates/java/classic/protocol.vm | 8 ++-- .../specific/templates/java/classic/record.vm | 38 +++++++-------- .../specific/TestSpecificCompiler.java | 48 +++++++++++++++++++ .../src/test/resources/simple_record.avsc | 5 +- .../specific/TestSpecificCompiler.java | 2 +- 8 files changed, 114 insertions(+), 38 deletions(-) diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java index 03e8761d8ce..746f629d08c 100644 --- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java @@ -36,6 +36,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.avro.Conversion; @@ -1012,27 +1013,43 @@ public String conversionInstance(Schema schema) { */ public String[] javaAnnotations(JsonProperties props) { final Object value = props.getObjectProp("javaAnnotation"); - if (value == null) - return new String[0]; - if (value instanceof String) + if (value instanceof String && isValidAsAnnotation((String) value)) return new String[] { value.toString() }; if (value instanceof List) { final List list = (List) value; final List annots = new ArrayList<>(list.size()); for (Object o : list) { - annots.add(o.toString()); + if (isValidAsAnnotation(o.toString())) + annots.add(o.toString()); } return annots.toArray(new String[0]); } return new String[0]; } + private static final String PATTERN_IDENTIFIER_PART = "\\p{javaJavaIdentifierStart}\\p{javaJavaIdentifierPart}*"; + private static final String PATTERN_IDENTIFIER = String.format("(?:%s(?:\\.%s)*)", PATTERN_IDENTIFIER_PART, + PATTERN_IDENTIFIER_PART); + private static final String PATTERN_STRING = "\"(?:\\\\[\\\\\"ntfb]|(?", ">"); } /** diff --git a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/enum.vm b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/enum.vm index c15b7ecd124..1e34e85610a 100644 --- a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/enum.vm +++ b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/enum.vm @@ -19,7 +19,7 @@ package $this.mangle($schema.getNamespace()); #end #if ($schema.getDoc()) -/** $schema.getDoc() */ +/** $this.escapeForJavadoc($schema.getDoc()) */ #end #foreach ($annotation in $this.javaAnnotations($schema)) @$annotation @@ -28,7 +28,7 @@ package $this.mangle($schema.getNamespace()); public enum ${this.mangleTypeIdentifier($schema.getName())} implements org.apache.avro.generic.GenericEnumSymbol<${this.mangleTypeIdentifier($schema.getName())}> { #foreach ($symbol in ${schema.getEnumSymbols()})${this.mangle($symbol)}#if ($foreach.hasNext), #end#end ; - public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("${this.javaEscape($schema.toString())}"); + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("${this.escapeForJavaString($schema.toString())}"); public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } @Override diff --git a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/fixed.vm b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/fixed.vm index dbbef6ffb12..0f305af3190 100644 --- a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/fixed.vm +++ b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/fixed.vm @@ -19,7 +19,7 @@ package $this.mangle($schema.getNamespace()); #end #if ($schema.getDoc()) -/** $schema.getDoc() */ +/** $this.escapeForJavadoc($schema.getDoc()) */ #end #foreach ($annotation in $this.javaAnnotations($schema)) @$annotation @@ -28,7 +28,7 @@ package $this.mangle($schema.getNamespace()); @org.apache.avro.specific.AvroGenerated public class ${this.mangleTypeIdentifier($schema.getName())} extends org.apache.avro.specific.SpecificFixed { private static final long serialVersionUID = ${this.fingerprint64($schema)}L; - public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("${this.javaEscape($schema.toString())}"); + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("${this.escapeForJavaString($schema.toString())}"); public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } public org.apache.avro.Schema getSchema() { return SCHEMA$; } diff --git a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/protocol.vm b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/protocol.vm index f9a4d1aeccd..46ac443ac8d 100644 --- a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/protocol.vm +++ b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/protocol.vm @@ -20,7 +20,7 @@ package $this.mangle($protocol.getNamespace()); #end #if ($protocol.getDoc()) -/** $protocol.getDoc() */ +/** $this.escapeForJavadoc($protocol.getDoc()) */ #end #foreach ($annotation in $this.javaAnnotations($protocol)) @$annotation @@ -37,7 +37,7 @@ public interface $this.mangleTypeIdentifier($protocol.getName()) { * $this.escapeForJavadoc($message.getDoc()) #end #foreach ($p in $message.getRequest().getFields())## -#if ($p.doc()) * @param ${this.mangle($p.name())} $p.doc() +#if ($p.doc()) * @param ${this.mangle($p.name())} $this.escapeForJavadoc($p.doc()) #end #end */ @@ -62,7 +62,7 @@ ${this.mangle($error.getFullName())}## ## Generate nested callback API #if ($protocol.getDoc()) - /** $protocol.getDoc() */ + /** $this.escapeForJavadoc($protocol.getDoc()) */ #end @org.apache.avro.specific.AvroGenerated public interface Callback extends $this.mangleTypeIdentifier($protocol.getName()) { @@ -78,7 +78,7 @@ ${this.mangle($error.getFullName())}## * $this.escapeForJavadoc($message.getDoc()) #end #foreach ($p in $message.getRequest().getFields())## -#if ($p.doc()) * @param ${this.mangle($p.name())} $p.doc() +#if ($p.doc()) * @param ${this.mangle($p.name())} $this.escapeForJavadoc($p.doc()) #end #end * @throws java.io.IOException The async call could not be completed. diff --git a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm index 971cfa67758..fa89ebb7845 100755 --- a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm +++ b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm @@ -30,7 +30,7 @@ import org.apache.avro.message.SchemaStore; #if (${this.gettersReturnOptional} || ${this.createOptionalGetters})import java.util.Optional;#end #if ($schema.getDoc()) -/** $schema.getDoc() */ +/** $this.escapeForJavadoc($schema.getDoc()) */ #end #foreach ($annotation in $this.javaAnnotations($schema)) @$annotation @@ -117,7 +117,7 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError #foreach ($field in $schema.getFields()) #if ($field.doc()) - /** $field.doc() */ + /** $this.escapeForJavadoc($field.doc()) */ #end #foreach ($annotation in $this.javaAnnotations($field)) @$annotation @@ -156,7 +156,7 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError /** * All-args constructor. #foreach ($field in $schema.getFields()) -#if ($field.doc()) * @param ${this.mangle($field.name())} $field.doc() +#if ($field.doc()) * @param ${this.mangle($field.name())} $this.escapeForJavadoc($field.doc()) #else * @param ${this.mangle($field.name())} The new value for ${field.name()} #end #end @@ -228,8 +228,8 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError #foreach ($field in $schema.getFields()) #if (${this.gettersReturnOptional} && (!${this.optionalGettersForNullableFieldsOnly} || ${field.schema().isNullable()})) /** - * Gets the value of the '${this.mangle($field.name(), $schema.isError())}' field as an Optional<${this.javaType($field.schema())}>. -#if ($field.doc()) * $field.doc() + * Gets the value of the '${this.mangle($field.name(), $schema.isError())}' field as an Optional<${this.escapeForJavadoc(${this.javaType($field.schema())})}>. +#if ($field.doc()) * $this.escapeForJavadoc($field.doc()) #end * @return The value wrapped in an Optional<${this.javaType($field.schema())}>. */ @@ -239,7 +239,7 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError #else /** * Gets the value of the '${this.mangle($field.name(), $schema.isError())}' field. -#if ($field.doc()) * @return $field.doc() +#if ($field.doc()) * @return $this.escapeForJavadoc($field.doc()) #else * @return The value of the '${this.mangle($field.name(), $schema.isError())}' field. #end */ @@ -250,8 +250,8 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError #if (${this.createOptionalGetters}) /** - * Gets the value of the '${this.mangle($field.name(), $schema.isError())}' field as an Optional<${this.javaType($field.schema())}>. -#if ($field.doc()) * $field.doc() + * Gets the value of the '${this.mangle($field.name(), $schema.isError())}' field as an Optional<${this.escapeForJavadoc(${this.javaType($field.schema())})}>. +#if ($field.doc()) * $this.escapeForJavadoc($field.doc()) #end * @return The value wrapped in an Optional<${this.javaType($field.schema())}>. */ @@ -263,7 +263,7 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError #if ($this.createSetters) /** * Sets the value of the '${this.mangle($field.name(), $schema.isError())}' field. -#if ($field.doc()) * $field.doc() +#if ($field.doc()) * $this.escapeForJavadoc($field.doc()) #end * @param value the value to set. */ @@ -317,7 +317,7 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError #foreach ($field in $schema.getFields()) #if ($field.doc()) - /** $field.doc() */ + /** $this.escapeForJavadoc($field.doc()) */ #end private ${this.javaUnbox($field.schema(), false)} ${this.mangle($field.name(), $schema.isError())}; #if (${this.hasBuilder($field.schema())}) @@ -396,7 +396,7 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError #foreach ($field in $schema.getFields()) /** * Gets the value of the '${this.mangle($field.name(), $schema.isError())}' field. -#if ($field.doc()) * $field.doc() +#if ($field.doc()) * $this.escapeForJavadoc($field.doc()) #end * @return The value. */ @@ -406,8 +406,8 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError #if (${this.createOptionalGetters}) /** - * Gets the value of the '${this.mangle($field.name(), $schema.isError())}' field as an Optional<${this.javaType($field.schema())}>. -#if ($field.doc()) * $field.doc() + * Gets the value of the '${this.mangle($field.name(), $schema.isError())}' field as an Optional<${this.escapeForJavadoc(${this.javaType($field.schema())})}>. +#if ($field.doc()) * $this.escapeForJavadoc($field.doc()) #end * @return The value wrapped in an Optional<${this.javaType($field.schema())}>. */ @@ -418,7 +418,7 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError /** * Sets the value of the '${this.mangle($field.name(), $schema.isError())}' field. -#if ($field.doc()) * $field.doc() +#if ($field.doc()) * $this.escapeForJavadoc($field.doc()) #end * @param value The value of '${this.mangle($field.name(), $schema.isError())}'. * @return This builder. @@ -435,7 +435,7 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError /** * Checks whether the '${this.mangle($field.name(), $schema.isError())}' field has been set. -#if ($field.doc()) * $field.doc() +#if ($field.doc()) * $this.escapeForJavadoc($field.doc()) #end * @return True if the '${this.mangle($field.name(), $schema.isError())}' field has been set, false otherwise. */ @@ -446,7 +446,7 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError #if (${this.hasBuilder($field.schema())}) /** * Gets the Builder instance for the '${this.mangle($field.name(), $schema.isError())}' field and creates one if it doesn't exist yet. -#if ($field.doc()) * $field.doc() +#if ($field.doc()) * $this.escapeForJavadoc($field.doc()) #end * @return This builder. */ @@ -463,7 +463,7 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError /** * Sets the Builder instance for the '${this.mangle($field.name(), $schema.isError())}' field -#if ($field.doc()) * $field.doc() +#if ($field.doc()) * $this.escapeForJavadoc($field.doc()) #end * @param value The builder instance that must be set. * @return This builder. @@ -477,7 +477,7 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError /** * Checks whether the '${this.mangle($field.name(), $schema.isError())}' field has an active Builder instance -#if ($field.doc()) * $field.doc() +#if ($field.doc()) * $this.escapeForJavadoc($field.doc()) #end * @return True if the '${this.mangle($field.name(), $schema.isError())}' field has an active Builder instance */ @@ -488,7 +488,7 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError /** * Clears the value of the '${this.mangle($field.name(), $schema.isError())}' field. -#if ($field.doc()) * $field.doc() +#if ($field.doc()) * $this.escapeForJavadoc($field.doc()) #end * @return This builder. */ diff --git a/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java b/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java index 1d075bda7ce..6a102cfa06e 100644 --- a/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java +++ b/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java @@ -36,6 +36,8 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import javax.tools.Diagnostic; import javax.tools.DiagnosticListener; import javax.tools.JavaCompiler; @@ -50,6 +52,7 @@ import org.apache.avro.LogicalType; import org.apache.avro.LogicalTypes; +import org.apache.avro.Protocol; import org.apache.avro.Schema; import org.apache.avro.SchemaBuilder; import org.apache.avro.generic.GenericData.StringType; @@ -951,4 +954,49 @@ public LogicalType fromSchema(Schema schema) { } } + @Test + public void docsAreEscaped_avro4053() { + String jsonSchema = "{\n" + " \"protocol\": \"DummyProtocol\",\n" + + " \"doc\": \"*/\\nTest escaping \\n/*\",\n" + " \"types\" : [\n" + + " {\"type\": \"fixed\", \"name\": \"Hash\", \"size\": 16, \"doc\": \"*/\\nTest escaping \\n/*\"" + + "},\n" + + " {\"type\": \"enum\", \"name\": \"Status\", \"symbols\": [\"ON\", \"OFF\"], \"doc\": \"*/\\nTest escaping \\n/*\"},\n" + + " " + " {\"type\": \"record\", \"name\": \"Message\", \"fields\" : [\n" + + " {\"name\": \"content\", \"type\": \"string\", \"doc\": \"*/\\nTest escaping \\n/*\"},\n" + + " {\"name\": \"status\", \"type\": \"Status\", \"doc\": \"*/\\nTest escaping \\n/*\"},\n" + + " {\"name\": \"signature\", \"type\": \"Hash\", \"doc\": \"*/\\nTest escaping \\n/*\"}\n" + + " ]}\n" + " ],\n" + " \"messages\" : {\n" + " \"echo\": {\"request\": [" + + "{\"name\": \"msg\", \"type\": \"Message\"}" + + "], \"response\": \"Message\", \"doc\": \"*/\\nTest escaping \\n/*\"}\n" + " },\n" + + " \"javaAnnotation\": [\n" + " \"Deprecated(forRemoval = true, since = \\\"forever\\\")\",\n" + + " \"SuppressWarnings(\\\"ALL\\\")\",\n" + " \"SuppressWarnings(\\\"CodeInjection\\\")/*\",\n" + + " \" This is inside a comment as each line is prefixed with @\",\n" + + " \" and the next bit is really dangerous... */ static { System.exit(); }\"\n" + " ]\n" + "}"; + Collection outputs = new SpecificCompiler(Protocol.parse(jsonSchema)).compile(); + for (SpecificCompiler.OutputFile outputFile : outputs) { + assertFalse("Threats present?", outputFile.contents.contains("*/\\nTest escaping \\n/*")); + + int expectedEscapeCount = countOccurrences(Pattern.compile("Test escaping", Pattern.LITERAL), + outputFile.contents); + int escapedJavaDocCount = countOccurrences(Pattern.compile("\\*/\\s*Test escaping <threats>\\s*/\\*"), + outputFile.contents); + // noinspection RegExpRedundantEscape + int escapedDocStringCount = countOccurrences( + Pattern.compile("\\\"doc\\\":\\\"*/\\\\nTest escaping \\\\n/*\\\"", Pattern.LITERAL), + outputFile.contents); + assertEquals("Escaped threats in " + outputFile.path, expectedEscapeCount, + escapedJavaDocCount + escapedDocStringCount); + + assertFalse("Code injection present? " + outputFile.contents, + Pattern.compile("\\{ System.exit\\(\\); }(?!\\\\\")").matcher(outputFile.contents).find()); + } + } + + private int countOccurrences(Pattern pattern, String textToSearch) { + int count = 0; + for (Matcher matcher = pattern.matcher(textToSearch); matcher.find();) { + count++; + } + return count; + } } diff --git a/lang/java/compiler/src/test/resources/simple_record.avsc b/lang/java/compiler/src/test/resources/simple_record.avsc index d78fd17e64e..e6e7cb79564 100644 --- a/lang/java/compiler/src/test/resources/simple_record.avsc +++ b/lang/java/compiler/src/test/resources/simple_record.avsc @@ -1,8 +1,9 @@ { - "type": "record", + "type": "record", "name": "SimpleRecord", + "doc": ",*/\n hoping the compiler won't barf on strange comments\n/*", "fields" : [ {"name": "value", "type": "int"}, {"name": "nullableValue", "type": ["null","int"], "doc" : "doc"} ] -} \ No newline at end of file +} diff --git a/lang/java/ipc/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java b/lang/java/ipc/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java index 7c42ca9d9fc..c59d9e7a73f 100644 --- a/lang/java/ipc/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java +++ b/lang/java/ipc/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java @@ -74,7 +74,7 @@ public class TestSpecificCompiler { @Test public void testEsc() { - assertEquals("\\\"", SpecificCompiler.javaEscape("\"")); + assertEquals("\\\"", SpecificCompiler.escapeForJavaString("\"")); } @Test