diff --git a/.github/workflows/test-lang-py.yml b/.github/workflows/test-lang-py.yml index 00485b7103e..08e319a99fe 100644 --- a/.github/workflows/test-lang-py.yml +++ b/.github/workflows/test-lang-py.yml @@ -44,10 +44,7 @@ jobs: - '3.10' - '3.9' - '3.8' - - '3.7' - - '3.6' - - 'pypy-3.7' - - 'pypy-3.6' + - 'pypy-3.10' steps: - uses: actions/checkout@v4 @@ -91,10 +88,17 @@ jobs: - '3.10' - '3.9' - '3.8' +<<<<<<< HEAD - '3.7' +<<<<<<< HEAD - '3.6' - 'pypy-3.7' - 'pypy-3.6' +======= +======= +>>>>>>> 6e5e37876b (Drop apparently unsupported pypy version) + - 'pypy-3.10' +>>>>>>> 982ad792ce (ci: Remove deprecated pypy-3.9 test (#3211)) steps: - uses: actions/checkout@v4 diff --git a/build.sh b/build.sh index a6327b28d5c..d30c6b142c7 100755 --- a/build.sh +++ b/build.sh @@ -182,7 +182,8 @@ do # build docs cp -r doc/ build/staging-web/ find build/staging-web/ -type f -print0 | xargs -0 sed -r -i "s#\+\+version\+\+#${VERSION,,}#g" - mv build/staging-web/content/en/docs/++version++ build/staging-web/content/en/docs/"${VERSION,,}" + mkdir -p build/staging-web/public/docs/ + mv build/staging-web/content/en/docs/++version++ build/staging-web/public/docs/"${VERSION,,}" read -n 1 -s -r -p "Build build/staging-web/ manually now. Press a key to continue..." # If it was a SNAPSHOT, it was lowercased during the build. cp -R build/staging-web/public/docs/"${VERSION,,}"/* "build/$DOC_DIR/" @@ -217,7 +218,6 @@ do else gpg --pinentry-mode loopback --local-user="$GPG_LOCAL_USER" --passphrase "$password" --armor --output "$f.asc" --detach-sig "$f" fi - done set -x diff --git a/doc/examples/java-example/pom.xml b/doc/examples/java-example/pom.xml index a71aab28c09..6fc2cfffc0e 100644 --- a/doc/examples/java-example/pom.xml +++ b/doc/examples/java-example/pom.xml @@ -38,7 +38,7 @@ org.apache.avro avro - 1.11.4 + 1.11.5 @@ -55,7 +55,7 @@ org.apache.avro avro-maven-plugin - 1.11.4 + 1.11.5 generate-sources @@ -72,7 +72,7 @@ org.apache.maven.plugins maven-plugin - 1.11.4 + 1.11.5 1.8 1.8 @@ -92,7 +92,7 @@ org.apache.avro avro-maven-plugin - [1.11.4,) + [1.11.5,) schema diff --git a/doc/examples/mr-example/pom.xml b/doc/examples/mr-example/pom.xml index aae8c723c1e..66fa36f2499 100644 --- a/doc/examples/mr-example/pom.xml +++ b/doc/examples/mr-example/pom.xml @@ -45,7 +45,7 @@ org.apache.avro avro-maven-plugin - 1.11.4 + 1.11.5 generate-sources @@ -73,7 +73,7 @@ org.apache.avro avro-maven-plugin - [1.11.4,) + [1.11.5,) schema @@ -94,12 +94,12 @@ org.apache.avro avro - 1.11.4 + 1.11.5 org.apache.avro avro-mapred - 1.11.4 + 1.11.5 org.apache.hadoop diff --git a/lang/java/android/pom.xml b/lang/java/android/pom.xml index 46904c52af6..49ff1832a1c 100644 --- a/lang/java/android/pom.xml +++ b/lang/java/android/pom.xml @@ -22,7 +22,7 @@ avro-parent org.apache.avro - 1.11.4 + 1.11.5 ../pom.xml diff --git a/lang/java/archetypes/avro-service-archetype/pom.xml b/lang/java/archetypes/avro-service-archetype/pom.xml index d293b0936fe..4204aad6b95 100644 --- a/lang/java/archetypes/avro-service-archetype/pom.xml +++ b/lang/java/archetypes/avro-service-archetype/pom.xml @@ -23,7 +23,7 @@ avro-archetypes-parent org.apache.avro - 1.11.4 + 1.11.5 ../pom.xml diff --git a/lang/java/archetypes/pom.xml b/lang/java/archetypes/pom.xml index a632ae7b5d5..aff41d24305 100644 --- a/lang/java/archetypes/pom.xml +++ b/lang/java/archetypes/pom.xml @@ -22,7 +22,7 @@ org.apache.avro avro-parent - 1.11.4 + 1.11.5 ../pom.xml diff --git a/lang/java/avro/pom.xml b/lang/java/avro/pom.xml index 7064f080f7f..3ef5d02ea8c 100644 --- a/lang/java/avro/pom.xml +++ b/lang/java/avro/pom.xml @@ -23,7 +23,7 @@ avro-parent org.apache.avro - 1.11.4 + 1.11.5 ../ @@ -72,6 +72,9 @@ maven-surefire-plugin none + + java.math.BigDecimal,java.math.BigInteger,java.net.URI,java.net.URL,java.io.File,java.lang.Integer,org.apache.avro.reflect.TestReflect$R10 + diff --git a/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java b/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java index 8950f165991..510c9fed84d 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java +++ b/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java @@ -23,10 +23,14 @@ import org.apache.avro.generic.GenericDatumReader; import org.apache.avro.io.ResolvingDecoder; import org.apache.avro.util.ClassUtils; + import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Stream; /** * {@link org.apache.avro.io.DatumReader DatumReader} for generated Java @@ -34,14 +38,48 @@ */ public class SpecificDatumReader extends GenericDatumReader { + /** + * @deprecated prefer to use {@link #SERIALIZABLE_CLASSES} instead. + */ + @Deprecated public static final String[] SERIALIZABLE_PACKAGES; + public static final String[] SERIALIZABLE_CLASSES; + static { - SERIALIZABLE_PACKAGES = System.getProperty("org.apache.avro.SERIALIZABLE_PACKAGES", - "java.lang,java.math,java.io,java.net,org.apache.avro.reflect").split(","); + // no serializable classes by default + SERIALIZABLE_CLASSES = streamPropertyEntries(System.getProperty("org.apache.avro.SERIALIZABLE_CLASSES")) + .toArray(String[]::new); + + // no serializable packages by default + SERIALIZABLE_PACKAGES = streamPropertyEntries(System.getProperty("org.apache.avro.SERIALIZABLE_PACKAGES")) + // Add a '.' suffix to ensure we'll be matching package names instead of + // arbitrary prefixes, except for the wildcard "*", which allows all + // packages (this is only safe in fully controlled environments!). + .map(entry -> "*".equals(entry) ? entry : entry + ".").toArray(String[]::new); + } + + /** + * Parse a comma separated list into non-empty entries. Leading and trailing + * whitespace is stripped. + * + * @param commaSeparatedEntries the comma separated list of entries + * @return a stream of the entries + */ + private static Stream streamPropertyEntries(String commaSeparatedEntries) { + if (commaSeparatedEntries == null) { + return Stream.empty(); + } + return Stream.of(commaSeparatedEntries.split(",")).map(String::trim).filter(s -> !s.isEmpty()); } + // The primitive "class names" based on Class.isPrimitive() + private static final Set PRIMITIVES = new HashSet<>(Arrays.asList(Boolean.TYPE.getName(), + Character.TYPE.getName(), Byte.TYPE.getName(), Short.TYPE.getName(), Integer.TYPE.getName(), Long.TYPE.getName(), + Float.TYPE.getName(), Double.TYPE.getName(), Void.TYPE.getName())); + private final List trustedPackages = new ArrayList<>(); + private final List trustedClasses = new ArrayList<>(); public SpecificDatumReader() { this(null, null, SpecificData.get()); @@ -69,11 +107,14 @@ public SpecificDatumReader(Schema writer, Schema reader) { public SpecificDatumReader(Schema writer, Schema reader, SpecificData data) { super(writer, reader, data); trustedPackages.addAll(Arrays.asList(SERIALIZABLE_PACKAGES)); + trustedClasses.addAll(Arrays.asList(SERIALIZABLE_CLASSES)); } /** Construct given a {@link SpecificData}. */ public SpecificDatumReader(SpecificData data) { super(data); + trustedPackages.addAll(Arrays.asList(SERIALIZABLE_PACKAGES)); + trustedClasses.addAll(Arrays.asList(SERIALIZABLE_CLASSES)); } /** Return the contained {@link SpecificData}. */ @@ -115,8 +156,8 @@ private Class getPropAsClass(Schema schema, String prop) { if (name == null) return null; try { + checkSecurity(name); Class clazz = ClassUtils.forName(getData().getClassLoader(), name); - checkSecurity(clazz); return clazz; } catch (ClassNotFoundException e) { throw new AvroRuntimeException(e); @@ -127,31 +168,41 @@ private boolean trustAllPackages() { return (trustedPackages.size() == 1 && "*".equals(trustedPackages.get(0))); } - private void checkSecurity(Class clazz) throws ClassNotFoundException { - if (trustAllPackages() || clazz.isPrimitive()) { + private void checkSecurity(String className) throws ClassNotFoundException { + if (trustAllPackages() || PRIMITIVES.contains(className)) { return; } - boolean found = false; - Package thePackage = clazz.getPackage(); - if (thePackage != null) { - for (String trustedPackage : getTrustedPackages()) { - if (thePackage.getName().equals(trustedPackage) || thePackage.getName().startsWith(trustedPackage + ".")) { - found = true; - break; - } + for (String trustedClass : getTrustedClasses()) { + if (className.equals(trustedClass)) { + return; } - if (!found) { - throw new SecurityException("Forbidden " + clazz - + "! This class is not trusted to be included in Avro schema using java-class. Please set org.apache.avro.SERIALIZABLE_PACKAGES system property with the packages you trust."); + } + + for (String trustedPackage : getTrustedPackages()) { + if (className.startsWith(trustedPackage)) { + return; } } + + throw new SecurityException("Forbidden " + className + "! This class is not trusted to be included in Avro " + + "schemas using java-class. Please set the system property org.apache.avro.SERIALIZABLE_CLASSES to the comma " + + "separated list of classes you trust. You can also set the system property " + + "org.apache.avro.SERIALIZABLE_PACKAGES to the comma separated list of the packages you trust."); } + /** + * @deprecated Use getTrustedClasses() instead + */ + @Deprecated public final List getTrustedPackages() { return trustedPackages; } + public final List getTrustedClasses() { + return trustedClasses; + } + @Override protected Object readRecord(Object old, Schema expected, ResolvingDecoder in) throws IOException { SpecificData data = getSpecificData(); diff --git a/lang/java/avro/src/test/java/org/apache/avro/reflect/TestReflect.java b/lang/java/avro/src/test/java/org/apache/avro/reflect/TestReflect.java index e3065d59b87..5eab3551ff0 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/reflect/TestReflect.java +++ b/lang/java/avro/src/test/java/org/apache/avro/reflect/TestReflect.java @@ -570,6 +570,7 @@ void checkReadWrite(Object object) throws Exception { } void checkReadWrite(Object object, Schema s) throws Exception { + ReflectDatumWriter writer = new ReflectDatumWriter<>(s); ByteArrayOutputStream out = new ByteArrayOutputStream(); writer.write(object, factory.directBinaryEncoder(out, null)); diff --git a/lang/java/compiler/pom.xml b/lang/java/compiler/pom.xml index 33718dba924..cbdb5468346 100644 --- a/lang/java/compiler/pom.xml +++ b/lang/java/compiler/pom.xml @@ -23,7 +23,7 @@ avro-parent org.apache.avro - 1.11.4 + 1.11.5 ../pom.xml diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java index 03e8761d8ce..746f629d08c 100644 --- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java @@ -36,6 +36,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.avro.Conversion; @@ -1012,27 +1013,43 @@ public String conversionInstance(Schema schema) { */ public String[] javaAnnotations(JsonProperties props) { final Object value = props.getObjectProp("javaAnnotation"); - if (value == null) - return new String[0]; - if (value instanceof String) + if (value instanceof String && isValidAsAnnotation((String) value)) return new String[] { value.toString() }; if (value instanceof List) { final List list = (List) value; final List annots = new ArrayList<>(list.size()); for (Object o : list) { - annots.add(o.toString()); + if (isValidAsAnnotation(o.toString())) + annots.add(o.toString()); } return annots.toArray(new String[0]); } return new String[0]; } + private static final String PATTERN_IDENTIFIER_PART = "\\p{javaJavaIdentifierStart}\\p{javaJavaIdentifierPart}*"; + private static final String PATTERN_IDENTIFIER = String.format("(?:%s(?:\\.%s)*)", PATTERN_IDENTIFIER_PART, + PATTERN_IDENTIFIER_PART); + private static final String PATTERN_STRING = "\"(?:\\\\[\\\\\"ntfb]|(?", ">"); } /** diff --git a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/enum.vm b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/enum.vm index c15b7ecd124..1e34e85610a 100644 --- a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/enum.vm +++ b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/enum.vm @@ -19,7 +19,7 @@ package $this.mangle($schema.getNamespace()); #end #if ($schema.getDoc()) -/** $schema.getDoc() */ +/** $this.escapeForJavadoc($schema.getDoc()) */ #end #foreach ($annotation in $this.javaAnnotations($schema)) @$annotation @@ -28,7 +28,7 @@ package $this.mangle($schema.getNamespace()); public enum ${this.mangleTypeIdentifier($schema.getName())} implements org.apache.avro.generic.GenericEnumSymbol<${this.mangleTypeIdentifier($schema.getName())}> { #foreach ($symbol in ${schema.getEnumSymbols()})${this.mangle($symbol)}#if ($foreach.hasNext), #end#end ; - public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("${this.javaEscape($schema.toString())}"); + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("${this.escapeForJavaString($schema.toString())}"); public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } @Override diff --git a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/fixed.vm b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/fixed.vm index dbbef6ffb12..0f305af3190 100644 --- a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/fixed.vm +++ b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/fixed.vm @@ -19,7 +19,7 @@ package $this.mangle($schema.getNamespace()); #end #if ($schema.getDoc()) -/** $schema.getDoc() */ +/** $this.escapeForJavadoc($schema.getDoc()) */ #end #foreach ($annotation in $this.javaAnnotations($schema)) @$annotation @@ -28,7 +28,7 @@ package $this.mangle($schema.getNamespace()); @org.apache.avro.specific.AvroGenerated public class ${this.mangleTypeIdentifier($schema.getName())} extends org.apache.avro.specific.SpecificFixed { private static final long serialVersionUID = ${this.fingerprint64($schema)}L; - public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("${this.javaEscape($schema.toString())}"); + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("${this.escapeForJavaString($schema.toString())}"); public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } public org.apache.avro.Schema getSchema() { return SCHEMA$; } diff --git a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/protocol.vm b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/protocol.vm index f9a4d1aeccd..46ac443ac8d 100644 --- a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/protocol.vm +++ b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/protocol.vm @@ -20,7 +20,7 @@ package $this.mangle($protocol.getNamespace()); #end #if ($protocol.getDoc()) -/** $protocol.getDoc() */ +/** $this.escapeForJavadoc($protocol.getDoc()) */ #end #foreach ($annotation in $this.javaAnnotations($protocol)) @$annotation @@ -37,7 +37,7 @@ public interface $this.mangleTypeIdentifier($protocol.getName()) { * $this.escapeForJavadoc($message.getDoc()) #end #foreach ($p in $message.getRequest().getFields())## -#if ($p.doc()) * @param ${this.mangle($p.name())} $p.doc() +#if ($p.doc()) * @param ${this.mangle($p.name())} $this.escapeForJavadoc($p.doc()) #end #end */ @@ -62,7 +62,7 @@ ${this.mangle($error.getFullName())}## ## Generate nested callback API #if ($protocol.getDoc()) - /** $protocol.getDoc() */ + /** $this.escapeForJavadoc($protocol.getDoc()) */ #end @org.apache.avro.specific.AvroGenerated public interface Callback extends $this.mangleTypeIdentifier($protocol.getName()) { @@ -78,7 +78,7 @@ ${this.mangle($error.getFullName())}## * $this.escapeForJavadoc($message.getDoc()) #end #foreach ($p in $message.getRequest().getFields())## -#if ($p.doc()) * @param ${this.mangle($p.name())} $p.doc() +#if ($p.doc()) * @param ${this.mangle($p.name())} $this.escapeForJavadoc($p.doc()) #end #end * @throws java.io.IOException The async call could not be completed. diff --git a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm index 971cfa67758..fa89ebb7845 100755 --- a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm +++ b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm @@ -30,7 +30,7 @@ import org.apache.avro.message.SchemaStore; #if (${this.gettersReturnOptional} || ${this.createOptionalGetters})import java.util.Optional;#end #if ($schema.getDoc()) -/** $schema.getDoc() */ +/** $this.escapeForJavadoc($schema.getDoc()) */ #end #foreach ($annotation in $this.javaAnnotations($schema)) @$annotation @@ -117,7 +117,7 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError #foreach ($field in $schema.getFields()) #if ($field.doc()) - /** $field.doc() */ + /** $this.escapeForJavadoc($field.doc()) */ #end #foreach ($annotation in $this.javaAnnotations($field)) @$annotation @@ -156,7 +156,7 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError /** * All-args constructor. #foreach ($field in $schema.getFields()) -#if ($field.doc()) * @param ${this.mangle($field.name())} $field.doc() +#if ($field.doc()) * @param ${this.mangle($field.name())} $this.escapeForJavadoc($field.doc()) #else * @param ${this.mangle($field.name())} The new value for ${field.name()} #end #end @@ -228,8 +228,8 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError #foreach ($field in $schema.getFields()) #if (${this.gettersReturnOptional} && (!${this.optionalGettersForNullableFieldsOnly} || ${field.schema().isNullable()})) /** - * Gets the value of the '${this.mangle($field.name(), $schema.isError())}' field as an Optional<${this.javaType($field.schema())}>. -#if ($field.doc()) * $field.doc() + * Gets the value of the '${this.mangle($field.name(), $schema.isError())}' field as an Optional<${this.escapeForJavadoc(${this.javaType($field.schema())})}>. +#if ($field.doc()) * $this.escapeForJavadoc($field.doc()) #end * @return The value wrapped in an Optional<${this.javaType($field.schema())}>. */ @@ -239,7 +239,7 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError #else /** * Gets the value of the '${this.mangle($field.name(), $schema.isError())}' field. -#if ($field.doc()) * @return $field.doc() +#if ($field.doc()) * @return $this.escapeForJavadoc($field.doc()) #else * @return The value of the '${this.mangle($field.name(), $schema.isError())}' field. #end */ @@ -250,8 +250,8 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError #if (${this.createOptionalGetters}) /** - * Gets the value of the '${this.mangle($field.name(), $schema.isError())}' field as an Optional<${this.javaType($field.schema())}>. -#if ($field.doc()) * $field.doc() + * Gets the value of the '${this.mangle($field.name(), $schema.isError())}' field as an Optional<${this.escapeForJavadoc(${this.javaType($field.schema())})}>. +#if ($field.doc()) * $this.escapeForJavadoc($field.doc()) #end * @return The value wrapped in an Optional<${this.javaType($field.schema())}>. */ @@ -263,7 +263,7 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError #if ($this.createSetters) /** * Sets the value of the '${this.mangle($field.name(), $schema.isError())}' field. -#if ($field.doc()) * $field.doc() +#if ($field.doc()) * $this.escapeForJavadoc($field.doc()) #end * @param value the value to set. */ @@ -317,7 +317,7 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError #foreach ($field in $schema.getFields()) #if ($field.doc()) - /** $field.doc() */ + /** $this.escapeForJavadoc($field.doc()) */ #end private ${this.javaUnbox($field.schema(), false)} ${this.mangle($field.name(), $schema.isError())}; #if (${this.hasBuilder($field.schema())}) @@ -396,7 +396,7 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError #foreach ($field in $schema.getFields()) /** * Gets the value of the '${this.mangle($field.name(), $schema.isError())}' field. -#if ($field.doc()) * $field.doc() +#if ($field.doc()) * $this.escapeForJavadoc($field.doc()) #end * @return The value. */ @@ -406,8 +406,8 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError #if (${this.createOptionalGetters}) /** - * Gets the value of the '${this.mangle($field.name(), $schema.isError())}' field as an Optional<${this.javaType($field.schema())}>. -#if ($field.doc()) * $field.doc() + * Gets the value of the '${this.mangle($field.name(), $schema.isError())}' field as an Optional<${this.escapeForJavadoc(${this.javaType($field.schema())})}>. +#if ($field.doc()) * $this.escapeForJavadoc($field.doc()) #end * @return The value wrapped in an Optional<${this.javaType($field.schema())}>. */ @@ -418,7 +418,7 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError /** * Sets the value of the '${this.mangle($field.name(), $schema.isError())}' field. -#if ($field.doc()) * $field.doc() +#if ($field.doc()) * $this.escapeForJavadoc($field.doc()) #end * @param value The value of '${this.mangle($field.name(), $schema.isError())}'. * @return This builder. @@ -435,7 +435,7 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError /** * Checks whether the '${this.mangle($field.name(), $schema.isError())}' field has been set. -#if ($field.doc()) * $field.doc() +#if ($field.doc()) * $this.escapeForJavadoc($field.doc()) #end * @return True if the '${this.mangle($field.name(), $schema.isError())}' field has been set, false otherwise. */ @@ -446,7 +446,7 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError #if (${this.hasBuilder($field.schema())}) /** * Gets the Builder instance for the '${this.mangle($field.name(), $schema.isError())}' field and creates one if it doesn't exist yet. -#if ($field.doc()) * $field.doc() +#if ($field.doc()) * $this.escapeForJavadoc($field.doc()) #end * @return This builder. */ @@ -463,7 +463,7 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError /** * Sets the Builder instance for the '${this.mangle($field.name(), $schema.isError())}' field -#if ($field.doc()) * $field.doc() +#if ($field.doc()) * $this.escapeForJavadoc($field.doc()) #end * @param value The builder instance that must be set. * @return This builder. @@ -477,7 +477,7 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError /** * Checks whether the '${this.mangle($field.name(), $schema.isError())}' field has an active Builder instance -#if ($field.doc()) * $field.doc() +#if ($field.doc()) * $this.escapeForJavadoc($field.doc()) #end * @return True if the '${this.mangle($field.name(), $schema.isError())}' field has an active Builder instance */ @@ -488,7 +488,7 @@ public class ${this.mangleTypeIdentifier($schema.getName())}#if ($schema.isError /** * Clears the value of the '${this.mangle($field.name(), $schema.isError())}' field. -#if ($field.doc()) * $field.doc() +#if ($field.doc()) * $this.escapeForJavadoc($field.doc()) #end * @return This builder. */ diff --git a/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java b/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java index 1d075bda7ce..6a102cfa06e 100644 --- a/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java +++ b/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java @@ -36,6 +36,8 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import javax.tools.Diagnostic; import javax.tools.DiagnosticListener; import javax.tools.JavaCompiler; @@ -50,6 +52,7 @@ import org.apache.avro.LogicalType; import org.apache.avro.LogicalTypes; +import org.apache.avro.Protocol; import org.apache.avro.Schema; import org.apache.avro.SchemaBuilder; import org.apache.avro.generic.GenericData.StringType; @@ -951,4 +954,49 @@ public LogicalType fromSchema(Schema schema) { } } + @Test + public void docsAreEscaped_avro4053() { + String jsonSchema = "{\n" + " \"protocol\": \"DummyProtocol\",\n" + + " \"doc\": \"*/\\nTest escaping \\n/*\",\n" + " \"types\" : [\n" + + " {\"type\": \"fixed\", \"name\": \"Hash\", \"size\": 16, \"doc\": \"*/\\nTest escaping \\n/*\"" + + "},\n" + + " {\"type\": \"enum\", \"name\": \"Status\", \"symbols\": [\"ON\", \"OFF\"], \"doc\": \"*/\\nTest escaping \\n/*\"},\n" + + " " + " {\"type\": \"record\", \"name\": \"Message\", \"fields\" : [\n" + + " {\"name\": \"content\", \"type\": \"string\", \"doc\": \"*/\\nTest escaping \\n/*\"},\n" + + " {\"name\": \"status\", \"type\": \"Status\", \"doc\": \"*/\\nTest escaping \\n/*\"},\n" + + " {\"name\": \"signature\", \"type\": \"Hash\", \"doc\": \"*/\\nTest escaping \\n/*\"}\n" + + " ]}\n" + " ],\n" + " \"messages\" : {\n" + " \"echo\": {\"request\": [" + + "{\"name\": \"msg\", \"type\": \"Message\"}" + + "], \"response\": \"Message\", \"doc\": \"*/\\nTest escaping \\n/*\"}\n" + " },\n" + + " \"javaAnnotation\": [\n" + " \"Deprecated(forRemoval = true, since = \\\"forever\\\")\",\n" + + " \"SuppressWarnings(\\\"ALL\\\")\",\n" + " \"SuppressWarnings(\\\"CodeInjection\\\")/*\",\n" + + " \" This is inside a comment as each line is prefixed with @\",\n" + + " \" and the next bit is really dangerous... */ static { System.exit(); }\"\n" + " ]\n" + "}"; + Collection outputs = new SpecificCompiler(Protocol.parse(jsonSchema)).compile(); + for (SpecificCompiler.OutputFile outputFile : outputs) { + assertFalse("Threats present?", outputFile.contents.contains("*/\\nTest escaping \\n/*")); + + int expectedEscapeCount = countOccurrences(Pattern.compile("Test escaping", Pattern.LITERAL), + outputFile.contents); + int escapedJavaDocCount = countOccurrences(Pattern.compile("\\*/\\s*Test escaping <threats>\\s*/\\*"), + outputFile.contents); + // noinspection RegExpRedundantEscape + int escapedDocStringCount = countOccurrences( + Pattern.compile("\\\"doc\\\":\\\"*/\\\\nTest escaping \\\\n/*\\\"", Pattern.LITERAL), + outputFile.contents); + assertEquals("Escaped threats in " + outputFile.path, expectedEscapeCount, + escapedJavaDocCount + escapedDocStringCount); + + assertFalse("Code injection present? " + outputFile.contents, + Pattern.compile("\\{ System.exit\\(\\); }(?!\\\\\")").matcher(outputFile.contents).find()); + } + } + + private int countOccurrences(Pattern pattern, String textToSearch) { + int count = 0; + for (Matcher matcher = pattern.matcher(textToSearch); matcher.find();) { + count++; + } + return count; + } } diff --git a/lang/java/compiler/src/test/resources/simple_record.avsc b/lang/java/compiler/src/test/resources/simple_record.avsc index d78fd17e64e..e6e7cb79564 100644 --- a/lang/java/compiler/src/test/resources/simple_record.avsc +++ b/lang/java/compiler/src/test/resources/simple_record.avsc @@ -1,8 +1,9 @@ { - "type": "record", + "type": "record", "name": "SimpleRecord", + "doc": ",*/\n hoping the compiler won't barf on strange comments\n/*", "fields" : [ {"name": "value", "type": "int"}, {"name": "nullableValue", "type": ["null","int"], "doc" : "doc"} ] -} \ No newline at end of file +} diff --git a/lang/java/grpc/pom.xml b/lang/java/grpc/pom.xml index 33ed8b87a96..02d0b56b91d 100644 --- a/lang/java/grpc/pom.xml +++ b/lang/java/grpc/pom.xml @@ -23,7 +23,7 @@ org.apache.avro avro-parent - 1.11.4 + 1.11.5 ../ diff --git a/lang/java/integration-test/codegen-test/pom.xml b/lang/java/integration-test/codegen-test/pom.xml index 09ce70bba71..7f8a5738bdf 100644 --- a/lang/java/integration-test/codegen-test/pom.xml +++ b/lang/java/integration-test/codegen-test/pom.xml @@ -23,7 +23,7 @@ avro-integration-test org.apache.avro - 1.11.4 + 1.11.5 ../ diff --git a/lang/java/integration-test/pom.xml b/lang/java/integration-test/pom.xml index f93feb766dd..de48110b3d8 100644 --- a/lang/java/integration-test/pom.xml +++ b/lang/java/integration-test/pom.xml @@ -23,7 +23,7 @@ avro-parent org.apache.avro - 1.11.4 + 1.11.5 ../ diff --git a/lang/java/integration-test/test-custom-conversions/pom.xml b/lang/java/integration-test/test-custom-conversions/pom.xml index 38a9f66aaf5..d99b7fd8d2c 100644 --- a/lang/java/integration-test/test-custom-conversions/pom.xml +++ b/lang/java/integration-test/test-custom-conversions/pom.xml @@ -23,7 +23,7 @@ avro-integration-test org.apache.avro - 1.11.4 + 1.11.5 ../ diff --git a/lang/java/ipc-jetty/pom.xml b/lang/java/ipc-jetty/pom.xml index cbd20f2314e..7ebe2561ba8 100644 --- a/lang/java/ipc-jetty/pom.xml +++ b/lang/java/ipc-jetty/pom.xml @@ -23,7 +23,7 @@ avro-parent org.apache.avro - 1.11.4 + 1.11.5 ../ diff --git a/lang/java/ipc-netty/pom.xml b/lang/java/ipc-netty/pom.xml index 6a5fe621a10..0f77bfcb32a 100644 --- a/lang/java/ipc-netty/pom.xml +++ b/lang/java/ipc-netty/pom.xml @@ -23,7 +23,7 @@ avro-parent org.apache.avro - 1.11.4 + 1.11.5 ../ diff --git a/lang/java/ipc/pom.xml b/lang/java/ipc/pom.xml index dc468f878ee..ef7275f88bf 100644 --- a/lang/java/ipc/pom.xml +++ b/lang/java/ipc/pom.xml @@ -23,7 +23,7 @@ avro-parent org.apache.avro - 1.11.4 + 1.11.5 ../ @@ -62,6 +62,9 @@ 1 false none + + java.math.BigDecimal,java.math.BigInteger + diff --git a/lang/java/ipc/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java b/lang/java/ipc/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java index 7c42ca9d9fc..c59d9e7a73f 100644 --- a/lang/java/ipc/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java +++ b/lang/java/ipc/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java @@ -74,7 +74,7 @@ public class TestSpecificCompiler { @Test public void testEsc() { - assertEquals("\\\"", SpecificCompiler.javaEscape("\"")); + assertEquals("\\\"", SpecificCompiler.escapeForJavaString("\"")); } @Test diff --git a/lang/java/mapred/pom.xml b/lang/java/mapred/pom.xml index e6fb35e7864..0a83cdfb31e 100644 --- a/lang/java/mapred/pom.xml +++ b/lang/java/mapred/pom.xml @@ -23,7 +23,7 @@ avro-parent org.apache.avro - 1.11.4 + 1.11.5 ../ diff --git a/lang/java/maven-plugin/pom.xml b/lang/java/maven-plugin/pom.xml index f1351a3fe63..9114dfd8f83 100644 --- a/lang/java/maven-plugin/pom.xml +++ b/lang/java/maven-plugin/pom.xml @@ -23,7 +23,7 @@ avro-parent org.apache.avro - 1.11.4 + 1.11.5 ../pom.xml diff --git a/lang/java/perf/pom.xml b/lang/java/perf/pom.xml index 73d86c702bd..b6dd4f1a069 100644 --- a/lang/java/perf/pom.xml +++ b/lang/java/perf/pom.xml @@ -23,7 +23,7 @@ avro-parent org.apache.avro - 1.11.4 + 1.11.5 ../ diff --git a/lang/java/pom.xml b/lang/java/pom.xml index 77bd5803e06..d2ac4440c20 100644 --- a/lang/java/pom.xml +++ b/lang/java/pom.xml @@ -22,7 +22,7 @@ org.apache.avro avro-toplevel - 1.11.4 + 1.11.5 ../../pom.xml diff --git a/lang/java/protobuf/pom.xml b/lang/java/protobuf/pom.xml index bc92e9042a3..22245ea6d87 100644 --- a/lang/java/protobuf/pom.xml +++ b/lang/java/protobuf/pom.xml @@ -23,7 +23,7 @@ avro-parent org.apache.avro - 1.11.4 + 1.11.5 ../ diff --git a/lang/java/thrift/pom.xml b/lang/java/thrift/pom.xml index a85f024b588..38989fb57bf 100644 --- a/lang/java/thrift/pom.xml +++ b/lang/java/thrift/pom.xml @@ -23,7 +23,7 @@ avro-parent org.apache.avro - 1.11.4 + 1.11.5 ../ diff --git a/lang/java/tools/pom.xml b/lang/java/tools/pom.xml index 436cf8e1c85..87f7895b4b0 100644 --- a/lang/java/tools/pom.xml +++ b/lang/java/tools/pom.xml @@ -23,7 +23,7 @@ avro-parent org.apache.avro - 1.11.4 + 1.11.5 ../ diff --git a/lang/java/trevni/avro/pom.xml b/lang/java/trevni/avro/pom.xml index a035f05a9f7..32c420f5e95 100644 --- a/lang/java/trevni/avro/pom.xml +++ b/lang/java/trevni/avro/pom.xml @@ -22,7 +22,7 @@ trevni-java org.apache.avro - 1.11.4 + 1.11.5 ../ diff --git a/lang/java/trevni/core/pom.xml b/lang/java/trevni/core/pom.xml index 08a5fd57fa8..089f95ab25e 100644 --- a/lang/java/trevni/core/pom.xml +++ b/lang/java/trevni/core/pom.xml @@ -22,7 +22,7 @@ trevni-java org.apache.avro - 1.11.4 + 1.11.5 ../ diff --git a/lang/java/trevni/doc/pom.xml b/lang/java/trevni/doc/pom.xml index fc6bae86a4d..28aa6ea82aa 100644 --- a/lang/java/trevni/doc/pom.xml +++ b/lang/java/trevni/doc/pom.xml @@ -22,7 +22,7 @@ trevni-java org.apache.avro - 1.11.4 + 1.11.5 .. diff --git a/lang/java/trevni/pom.xml b/lang/java/trevni/pom.xml index 064be245134..b005484aeb9 100644 --- a/lang/java/trevni/pom.xml +++ b/lang/java/trevni/pom.xml @@ -23,7 +23,7 @@ avro-parent org.apache.avro - 1.11.4 + 1.11.5 ../ diff --git a/lang/js/package-lock.json b/lang/js/package-lock.json index cb61de31f17..f2d667dff99 100644 --- a/lang/js/package-lock.json +++ b/lang/js/package-lock.json @@ -1,6 +1,6 @@ { "name": "avro-js", - "version": "1.11.4-SNAPSHOT", + "version": "1.11.5", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/lang/js/package.json b/lang/js/package.json index 84383ce0b99..4c8678bc29a 100644 --- a/lang/js/package.json +++ b/lang/js/package.json @@ -1,6 +1,6 @@ { "name": "avro-js", - "version": "1.11.4", + "version": "1.11.5", "author": "Avro Developers ", "description": "JavaScript Avro implementation", "contributors": [ diff --git a/lang/perl/Makefile.PL b/lang/perl/Makefile.PL index ffac5e9d11b..13508b67197 100644 --- a/lang/perl/Makefile.PL +++ b/lang/perl/Makefile.PL @@ -26,10 +26,13 @@ license 'apache'; version $version; readme_from 'lib/Avro.pm'; all_from 'lib/Avro.pm'; -build_requires 'Test::More', 0.88; +configure_requires 'Module::Install'; +configure_requires 'Module::Install::ReadmeFromPod'; test_requires 'Math::BigInt'; test_requires 'Perl::Critic'; test_requires 'Test::Exception'; +test_requires 'Test::More', 0.88; +test_requires 'Test::Pod'; requires 'Compress::Zlib'; requires 'Compress::Zstd'; requires 'Encode'; diff --git a/lang/py/avro/io.py b/lang/py/avro/io.py index 7b5576697eb..15427c48302 100644 --- a/lang/py/avro/io.py +++ b/lang/py/avro/io.py @@ -644,9 +644,10 @@ def readers_schema(self, readers_schema: avro.schema.Schema) -> None: def read(self, decoder: "BinaryDecoder") -> object: if self.writers_schema is None: raise avro.errors.IONotReadyException("Cannot read without a writer's schema.") - if self.readers_schema is None: - self.readers_schema = self.writers_schema - return self.read_data(self.writers_schema, self.readers_schema, decoder) + reader_schema = self.readers_schema + if reader_schema is None: + reader_schema = self.writers_schema + return self.read_data(self.writers_schema, reader_schema, decoder) def read_data(self, writers_schema: avro.schema.Schema, readers_schema: avro.schema.Schema, decoder: "BinaryDecoder") -> object: # schema matching diff --git a/lang/py/tox.ini b/lang/py/tox.ini index e7bba45baf1..6a3c5d2d752 100644 --- a/lang/py/tox.ini +++ b/lang/py/tox.ini @@ -26,8 +26,8 @@ envlist = py38 py39 py310 - pypy3.6 - pypy3.7 + py311 + pypy3.10 [coverage:run] diff --git a/pom.xml b/pom.xml index bf99f329c3e..7bf72f6e924 100644 --- a/pom.xml +++ b/pom.xml @@ -27,7 +27,7 @@ org.apache.avro avro-toplevel - 1.11.4 + 1.11.5 pom Apache Avro Toplevel @@ -64,7 +64,7 @@ 3.2.5 - 1723820671 + 1754831786 diff --git a/share/VERSION.txt b/share/VERSION.txt index 3d0e62313ce..e6dbb7c238f 100644 --- a/share/VERSION.txt +++ b/share/VERSION.txt @@ -1 +1 @@ -1.11.4 +1.11.5 diff --git a/share/docker/Dockerfile b/share/docker/Dockerfile index c1c2565e593..b976f9772fa 100644 --- a/share/docker/Dockerfile +++ b/share/docker/Dockerfile @@ -114,8 +114,11 @@ RUN set -eux; \ ENV PATH="/opt/maven/bin:${PATH}" # Install nodejs +# The node deprecation warnings cause a 20 second sleep. +# But mom, I'm not even tired! RUN curl -sSL https://deb.nodesource.com/setup_14.x \ - | bash - \ + | sed 's/sleep/echo "But mom!"/' \ + | bash \ && apt-get -qqy install nodejs \ && apt-get -qqy clean \ && npm install -g grunt-cli \ @@ -167,10 +170,24 @@ RUN curl -sSL https://cpanmin.us \ Module::Install::Repository \ && rm -rf .cpanm -# Install Python packages -ENV PIP_NO_CACHE_DIR=off +# Install Python3 +ENV PATH="${PATH}:/opt/pypy3.10/bin" \ + PIP_NO_CACHE_DIR=off + +# https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope +ARG BUILDARCH +RUN case "${BUILDARCH:?}" in \ + arm64) pypyarch=aarch64;; \ + *) pypyarch=linux64;; \ + esac \ + && cd /opt \ + && for url in https://downloads.python.org/pypy/pypy3.10-v7.3.12-"$pypyarch".tar.bz2; \ + do curl -fsSL "$url" | tar -xvjpf -; \ + done \ + && ln -s pypy3.10* pypy3.10 # Install Python3 packages +ENV PIP_NO_CACHE_DIR=off RUN python3 -m pip install --upgrade pip setuptools wheel \ && python3 -m pip install tox zstandard