diff --git a/.zenodo.json b/.zenodo.json new file mode 100644 index 00000000000000..7161180c51ae3e --- /dev/null +++ b/.zenodo.json @@ -0,0 +1,13 @@ +{ + "description": "TensorFlow is an end-to-end open source platform for machine learning. It has a comprehensive, flexible ecosystem of tools, libraries, and community resources that lets researchers push the state-of-the-art in ML and developers easily build and deploy ML-powered applications.", + "license": "Apache-2.0", + "title": "TensorFlow", + "upload_type": "software", + "creators": [ + { + "name": "TensorFlow Developers" + } + ], + "access_right": "open", + "notes": "Specific TensorFlow versions can be found in the \"Versions\" list on the right side of this page.
See the full list of authors on GitHub." +} diff --git a/RELEASE.md b/RELEASE.md index 5dc9456da9c8b1..39db9362064462 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,3 +1,208 @@ +# Release 2.3.4 + +This release introduces several vulnerability fixes: + +* Fixes a heap out of bounds access in sparse reduction operations ([CVE-2021-37635](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37635)) +* Fixes a floating point exception in `SparseDenseCwiseDiv` ([CVE-2021-37636](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37636)) +* Fixes a null pointer dereference in `CompressElement` ([CVE-2021-37637](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37637)) +* Fixes a null pointer dereference in `RaggedTensorToTensor` ([CVE-2021-37638](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37638)) +* Fixes a null pointer dereference and a heap OOB read arising from operations restoring tensors ([CVE-2021-37639](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37639)) +* Fixes an integer division by 0 in sparse reshaping ([CVE-2021-37640](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37640)) +* Fixes a division by 0 in `ResourceScatterDiv` ([CVE-2021-37642](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37642)) +* Fixes a heap OOB in `RaggedGather` ([CVE-2021-37641](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37641)) +* Fixes a `std::abort` raised from `TensorListReserve` ([CVE-2021-37644](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37644)) +* Fixes a null pointer dereference in `MatrixDiagPartOp` ([CVE-2021-37643](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37643)) +* Fixes an integer overflow due to conversion to unsigned ([CVE-2021-37645](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37645)) +* Fixes a bad allocation error in `StringNGrams` caused by integer conversion ([CVE-2021-37646](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37646)) +* Fixes a null pointer dereference in `SparseTensorSliceDataset` ([CVE-2021-37647](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37647)) +* Fixes an incorrect validation of `SaveV2` inputs ([CVE-2021-37648](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37648)) +* Fixes a null pointer dereference in `UncompressElement` ([CVE-2021-37649](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37649)) +* Fixes a segfault and a heap buffer overflow in `{Experimental,}DatasetToTFRecord` ([CVE-2021-37650](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37650)) +* Fixes a heap buffer overflow in `FractionalAvgPoolGrad` ([CVE-2021-37651](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37651)) +* Fixes a use after free in boosted trees creation ([CVE-2021-37652](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37652)) +* Fixes a division by 0 in `ResourceGather` ([CVE-2021-37653](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37653)) +* Fixes a heap OOB and a `CHECK` fail in `ResourceGather` ([CVE-2021-37654](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37654)) +* Fixes a heap OOB in `ResourceScatterUpdate` ([CVE-2021-37655](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37655)) +* Fixes an undefined behavior arising from reference binding to nullptr in `RaggedTensorToSparse` ([CVE-2021-37656](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37656)) +* Fixes an undefined behavior arising from reference binding to nullptr in `MatrixDiagV*` ops ([CVE-2021-37657](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37657)) +* Fixes an undefined behavior arising from reference binding to nullptr in `MatrixSetDiagV*` ops ([CVE-2021-37658](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37658)) +* Fixes an undefined behavior arising from reference binding to nullptr and heap OOB in binary cwise ops ([CVE-2021-37659](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37659)) +* Fixes a division by 0 in inplace operations ([CVE-2021-37660](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37660)) +* Fixes a crash caused by integer conversion to unsigned ([CVE-2021-37661](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37661)) +* Fixes an undefined behavior arising from reference binding to nullptr in boosted trees ([CVE-2021-37662](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37662)) +* Fixes a heap OOB in boosted trees ([CVE-2021-37664](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37664)) +* Fixes vulnerabilities arising from incomplete validation in `QuantizeV2` ([CVE-2021-37663](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37663)) +* Fixes vulnerabilities arising from incomplete validation in MKL requantization ([CVE-2021-37665](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37665)) +* Fixes an undefined behavior arising from reference binding to nullptr in `RaggedTensorToVariant` ([CVE-2021-37666](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37666)) +* Fixes an undefined behavior arising from reference binding to nullptr in unicode encoding ([CVE-2021-37667](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37667)) +* Fixes an FPE in `tf.raw_ops.UnravelIndex` ([CVE-2021-37668](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37668)) +* Fixes a crash in NMS ops caused by integer conversion to unsigned ([CVE-2021-37669](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37669)) +* Fixes a heap OOB in `UpperBound` and `LowerBound` ([CVE-2021-37670](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37670)) +* Fixes an undefined behavior arising from reference binding to nullptr in map operations ([CVE-2021-37671](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37671)) +* Fixes a heap OOB in `SdcaOptimizerV2` ([CVE-2021-37672](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37672)) +* Fixes a `CHECK`-fail in `MapStage` ([CVE-2021-37673](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37673)) +* Fixes a vulnerability arising from incomplete validation in `MaxPoolGrad` ([CVE-2021-37674](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37674)) +* Fixes an undefined behavior arising from reference binding to nullptr in shape inference ([CVE-2021-37676](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37676)) +* Fixes a division by 0 in most convolution operators ([CVE-2021-37675](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37675)) +* Fixes vulnerabilities arising from missing validation in shape inference for `Dequantize` ([CVE-2021-37677](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37677)) +* Fixes an arbitrary code execution due to YAML deserialization ([CVE-2021-37678](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37678)) +* Fixes a heap OOB in nested `tf.map_fn` with `RaggedTensor`s ([CVE-2021-37679](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37679)) +* Fixes a division by zero in TFLite ([CVE-2021-37680](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37680)) +* Fixes an NPE in TFLite ([CVE-2021-37681](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37681)) +* Fixes a vulnerability arising from use of unitialized value in TFLite ([CVE-2021-37682](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37682)) +* Fixes an FPE in TFLite division operations ([CVE-2021-37683](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37683)) +* Fixes an FPE in TFLite pooling operations ([CVE-2021-37684](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37684)) +* Fixes an infinite loop in TFLite ([CVE-2021-37686](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37686)) +* Fixes a heap OOB in TFLite ([CVE-2021-37685](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37685)) +* Fixes a heap OOB in TFLite's `Gather*` implementations ([CVE-2021-37687](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37687)) +* Fixes an undefined behavior arising from null pointer dereference in TFLite ([CVE-2021-37688](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37688)) +* Fixes an undefined behavior arising from null pointer dereference in TFLite MLIR optimizations ([CVE-2021-37689](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37689)) +* Fixes a FPE in LSH in TFLite ([CVE-2021-37691](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37691)) +* Fixes a segfault on strings tensors with mismatched dimensions, arising in Go code ([CVE-2021-37692](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37692)) +* Fixes a use after free and a potential segfault in shape inference functions ([CVE-2021-37690](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-37690)) +* Updates `curl` to `7.77.0` to handle [CVE-2021-22876](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-22876), [CVE-2021-22897](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-22897), [CVE-2021-22898](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-22898), and [CVE-2021-22901](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-22901). + +# Release 2.3.3 + +This release introduces several vulnerability fixes: + +* Fixes a heap buffer overflow in `RaggedBinCount` ([CVE-2021-29512](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29512)) +* Fixes a heap out of bounds write in `RaggedBinCount` ([CVE-2021-29514](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29514)) +* Fixes a type confusion during tensor casts which leads to dereferencing null pointers ([CVE-2021-29513](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29513)) +* Fixes a reference binding to null pointer in `MatrixDiag*` ops ([CVE-2021-29515](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29515)) +* Fixes a null pointer dereference via invalid Ragged Tensors ([CVE-2021-29516](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29516)) +* Fixes a division by zero in `Conv3D` ([CVE-2021-29517](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29517)) +* Fixes vulnerabilities where session operations in eager mode lead to null pointer dereferences ([CVE-2021-29518](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29518)) +* Fixes a `CHECK`-fail in `SparseCross` caused by type confusion ([CVE-2021-29519](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29519)) +* Fixes a segfault in `SparseCountSparseOutput` ([CVE-2021-29521](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29521)) +* Fixes a heap buffer overflow in `Conv3DBackprop*` ([CVE-2021-29520](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29520)) +* Fixes a division by 0 in `Conv3DBackprop*` ([CVE-2021-29522](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29522)) +* Fixes a `CHECK`-fail in `AddManySparseToTensorsMap` ([CVE-2021-29523](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29523)) +* Fixes a division by 0 in `Conv2DBackpropFilter` ([CVE-2021-29524](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29524)) +* Fixes a division by 0 in `Conv2DBackpropInput` ([CVE-2021-29525](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29525)) +* Fixes a division by 0 in `Conv2D` ([CVE-2021-29526](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29526)) +* Fixes a division by 0 in `QuantizedConv2D` ([CVE-2021-29527](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29527)) +* Fixes a division by 0 in `QuantizedMul` ([CVE-2021-29528](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29528)) +* Fixes vulnerabilities caused by invalid validation in `SparseMatrixSparseCholesky` ([CVE-2021-29530](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29530)) +* Fixes a heap buffer overflow caused by rounding ([CVE-2021-29529](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29529)) +* Fixes a `CHECK`-fail in `tf.raw_ops.EncodePng` ([CVE-2021-29531](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29531)) +* Fixes a heap out of bounds read in `RaggedCross` ([CVE-2021-29532](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29532)) +* Fixes a `CHECK`-fail in `DrawBoundingBoxes` ([CVE-2021-29533](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29533)) +* Fixes a heap buffer overflow in `QuantizedMul` ([CVE-2021-29535](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29535)) +* Fixes a `CHECK`-fail in `SparseConcat` ([CVE-2021-29534](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29534)) +* Fixes a heap buffer overflow in `QuantizedResizeBilinear` ([CVE-2021-29537](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29537)) +* Fixes a heap buffer overflow in `QuantizedReshape` ([CVE-2021-29536](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29536)) +* Fixes a division by zero in `Conv2DBackpropFilter` ([CVE-2021-29538](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29538)) +* Fixes a heap buffer overflow in `Conv2DBackpropFilter` ([CVE-2021-29540](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29540)) +* Fixes a heap buffer overflow in `StringNGrams` ([CVE-2021-29542](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29542)) +* Fixes a null pointer dereference in `StringNGrams` ([CVE-2021-29541](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29541)) +* Fixes a `CHECK`-fail in `QuantizeAndDequantizeV4Grad` ([CVE-2021-29544](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29544)) +* Fixes a `CHECK`-fail in `CTCGreedyDecoder` ([CVE-2021-29543](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29543)) +* Fixes a heap buffer overflow in `SparseTensorToCSRSparseMatrix` ([CVE-2021-29545](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29545)) +* Fixes a division by 0 in `QuantizedBiasAdd` ([CVE-2021-29546](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29546)) +* Fixes a heap out of bounds in `QuantizedBatchNormWithGlobalNormalization` ([CVE-2021-29547](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29547)) +* Fixes a division by 0 in `QuantizedBatchNormWithGlobalNormalization` ([CVE-2021-29548](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29548)) +* Fixes a division by 0 in `QuantizedAdd` ([CVE-2021-29549](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29549)) +* Fixes a division by 0 in `FractionalAvgPool` ([CVE-2021-29550](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29550)) +* Fixes an OOB read in `MatrixTriangularSolve` ([CVE-2021-29551](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29551)) +* Fixes a heap OOB in `QuantizeAndDequantizeV3` ([CVE-2021-29553](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29553)) +* Fixes a `CHECK`-failure in `UnsortedSegmentJoin` ([CVE-2021-29552](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29552)) +* Fixes a division by 0 in `DenseCountSparseOutput` ([CVE-2021-29554](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29554)) +* Fixes a division by 0 in `FusedBatchNorm` ([CVE-2021-29555](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29555)) +* Fixes a division by 0 in `SparseMatMul` ([CVE-2021-29557](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29557)) +* Fixes a division by 0 in `Reverse` ([CVE-2021-29556](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29556)) +* Fixes a heap buffer overflow in `SparseSplit` ([CVE-2021-29558](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29558)) +* Fixes a heap OOB access in unicode ops ([CVE-2021-29559](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29559)) +* Fixes a heap buffer overflow in `RaggedTensorToTensor` ([CVE-2021-29560](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29560)) +* Fixes a `CHECK`-fail in `LoadAndRemapMatrix` ([CVE-2021-29561](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29561)) +* Fixes a `CHECK`-fail in `tf.raw_ops.IRFFT` ([CVE-2021-29562](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29562)) +* Fixes a `CHECK`-fail in `tf.raw_ops.RFFT` ([CVE-2021-29563](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29563)) +* Fixes a null pointer dereference in `EditDistance` ([CVE-2021-29564](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29564)) +* Fixes a null pointer dereference in `SparseFillEmptyRows` ([CVE-2021-29565](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29565)) +* Fixes a heap OOB access in `Dilation2DBackpropInput` ([CVE-2021-29566](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29566)) +* Fixes a reference binding to null in `ParameterizedTruncatedNormal` ([CVE-2021-29568](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29568)) +* Fixes a set of vulnerabilities caused by lack of validation in `SparseDenseCwiseMul` ([CVE-2021-29567](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29567)) +* Fixes a heap out of bounds read in `MaxPoolGradWithArgmax` ([CVE-2021-29570](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29570)) +* Fixes a heap out of bounds read in `RequantizationRange` ([CVE-2021-29569](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29569)) +* Fixes a memory corruption in `DrawBoundingBoxesV2` ([CVE-2021-29571](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29571)) +* Fixes a reference binding to nullptr in `SdcaOptimizer` ([CVE-2021-29572](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29572)) +* Fixes an overflow and a denial of service in `tf.raw_ops.ReverseSequence` ([CVE-2021-29575](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29575)) +* Fixes a division by 0 in `MaxPoolGradWithArgmax` ([CVE-2021-29573](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29573)) +* Fixes an undefined behavior in `MaxPool3DGradGrad` ([CVE-2021-29574](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29574)) +* Fixes a heap buffer overflow in `MaxPool3DGradGrad` ([CVE-2021-29576](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29576)) +* Fixes a heap buffer overflow in `AvgPool3DGrad` ([CVE-2021-29577](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29577)) +* Fixes an undefined behavior and a `CHECK`-fail in `FractionalMaxPoolGrad` ([CVE-2021-29580](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29580)) +* Fixes a heap buffer overflow in `FractionalAvgPoolGrad` ([CVE-2021-29578](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29578)) +* Fixes a heap buffer overflow in `MaxPoolGrad` ([CVE-2021-29579](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29579)) +* Fixes a segfault in `CTCBeamSearchDecoder` ([CVE-2021-29581](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29581)) +* Fixes a heap OOB read in `tf.raw_ops.Dequantize` ([CVE-2021-29582](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29582)) +* Fixes a `CHECK`-fail due to integer overflow ([CVE-2021-29584](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29584)) +* Fixes a heap buffer overflow and undefined behavior in `FusedBatchNorm` ([CVE-2021-29583](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29583)) +* Fixes a division by zero in padding computation in TFLite ([CVE-2021-29585](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29585)) +* Fixes a division by zero in optimized pooling implementations in TFLite ([CVE-2021-29586](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29586)) +* Fixes a division by zero in TFLite's implementation of `SpaceToDepth` ([CVE-2021-29587](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29587)) +* Fixes a division by zero in TFLite's implementation of `GatherNd` ([CVE-2021-29589](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29589)) +* Fixes a division by zero in TFLite's implementation of `TransposeConv` ([CVE-2021-29588](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29588)) +* Fixes a heap OOB read in TFLite's implementation of `Minimum` or `Maximum` ([CVE-2021-29590](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29590)) +* Fixes a null pointer dereference in TFLite's `Reshape` operator ([CVE-2021-29592](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29592)) +* Fixes a stack overflow due to looping TFLite subgraph ([CVE-2021-29591](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29591)) +* Fixes a division by zero in TFLite's implementation of `DepthToSpace` ([CVE-2021-29595](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29595)) +* Fixes a division by zero in TFLite's convolution code ([CVE-2021-29594](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29594)) +* Fixes a division by zero in TFLite's implementation of `EmbeddingLookup` ([CVE-2021-29596](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29596)) +* Fixes a division by zero in TFLite's implementation of `BatchToSpaceNd` ([CVE-2021-29593](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29593)) +* Fixes a division by zero in TFLite's implementation of `SpaceToBatchNd` ([CVE-2021-29597](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29597)) +* Fixes a division by zero in TFLite's implementation of `SVDF` ([CVE-2021-29598](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29598)) +* Fixes a division by zero in TFLite's implementation of `Split` ([CVE-2021-29599](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29599)) +* Fixes a division by zero in TFLite's implementation of `OneHot` ([CVE-2021-29600](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29600)) +* Fixes a division by zero in TFLite's implementation of `DepthwiseConv` ([CVE-2021-29602](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29602)) +* Fixes a division by zero in TFLite's implementation of hashtable lookup ([CVE-2021-29604](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29604)) +* Fixes a integer overflow in TFLite concatentation ([CVE-2021-29601](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29601)) +* Fixes a integer overflow in TFLite memory allocation ([CVE-2021-29605](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29605)) +* Fixes a heap OOB write in TFLite ([CVE-2021-29603](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29603)) +* Fixes a heap OOB read in TFLite ([CVE-2021-29606](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29606)) +* Fixes a heap OOB and null pointer dereference in `RaggedTensorToTensor` ([CVE-2021-29608](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29608)) +* Fixes vulnerabilities caused by incomplete validation in `SparseAdd` ([CVE-2021-29609](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29609)) +* Fixes vulnerabilities caused by incomplete validation in `SparseSparseMinimum` ([CVE-2021-29607](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29607)) +* Fixes vulnerabilities caused by incomplete validation in `SparseReshape` ([CVE-2021-29611](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29611)) +* Fixes vulnerabilities caused by invalid validation in `QuantizeAndDequantizeV2` ([CVE-2021-29610](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29610)) +* Fixes a heap buffer overflow in `BandedTriangularSolve` ([CVE-2021-29612](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29612)) +* Fixes vulnerabilities caused by incomplete validation in `tf.raw_ops.CTCLoss` ([CVE-2021-29613](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29613)) +* Fixes an interpreter crash from vulnerabilities in `tf.io.decode_raw` ([CVE-2021-29614](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29614)) +* Fixes a stack overflow in `ParseAttrValue` with nested tensors ([CVE-2021-29615](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29615)) +* Fixes a null dereference in Grappler's `TrySimplify` ([CVE-2021-29616](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29616)) +* Fixes a crash in `tf.transpose` with complex inputs ([CVE-2021-29618](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29618)) +* Fixes a crash in `tf.strings.substr` due to `CHECK`-fail ([CVE-2021-29617](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29617)) +* Fixes a segfault in `tf.raw_ops.SparseCountSparseOutput` ([CVE-2021-29619](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29619)) +* Fixes a segfault in `tf.raw_ops.ImmutableConst` ([CVE-2021-29539](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29539)) +* Updates `curl` to `7.76.0` to handle [CVE-2020-8169](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-8169), [CVE-2020-8177](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-8177), [CVE-2020-8231](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-8231), [CVE-2020-8284](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-8284), [CVE-2020-8285](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-8285) and [CVE-2020-8286](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-8286). + +# Release 2.3.2 + +## Bug Fixes and Other Changes +* Fixes an access to unitialized memory in Eigen code + ([CVE-2020-26266](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-26266)) +* Fixes a security vulnerability caused by lack of validation in + `tf.raw_ops.DataFormatVecPermute` and `tf.raw_ops.DataFormatDimMap` + ([CVE-2020-26267](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-26267)) +* Fixes a vulnerability caused by attempting to write to immutable memory region in + `tf.raw_ops.ImmutableConst` + ([CVE-2020-26268](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-26268) +* Fixes a `CHECK`-fail in LSTM with zero-length input + ([CVE-2020-26270](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-26270)) +* Fixes a security vulnerability caused by accessing heap data outside of bounds + when loading a specially crafted `SavedModel` + ([CVE-2020-26271](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-26271)) +* Solves an OOM issue on TPUs when XLA contexts use fused average updates +* Updates `libjpeg-turbo` to `2.0.5` to handle + [CVE-2020-13790](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-13790). +* Updates `junit` to `4.13.1` to handle + [CVE-2020-15250](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-15250). +* Updates `PCRE` to `8.44` to handle + [CVE-2019-20838](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-20838) + and + [CVE-2020-14155](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-14155). +* Updates `sqlite3` to `3.44.0` to keep in sync with master branch. + # Release 2.3.1 ## Bug Fixes and Other Changes diff --git a/tensorflow/compiler/mlir/lite/transforms/optimize.cc b/tensorflow/compiler/mlir/lite/transforms/optimize.cc index 30ae4b81f4f324..0be9d2d109ff82 100644 --- a/tensorflow/compiler/mlir/lite/transforms/optimize.cc +++ b/tensorflow/compiler/mlir/lite/transforms/optimize.cc @@ -56,6 +56,9 @@ constexpr char kRelu6[] = "RELU6"; constexpr char kRelu1[] = "RELU_N1_TO_1"; bool L2NormalizeReduceAxis(Value sq_op, DenseElementsAttr axis) { + if (axis.getNumElements() == 0) { + return false; + } if (sq_op.getType().cast().getRank() - 1 == *axis.getValues().begin() || *axis.getValues().begin() == -1) { diff --git a/tensorflow/core/common_runtime/graph_constructor.cc b/tensorflow/core/common_runtime/graph_constructor.cc index ab5b086b25c55b..4456c28be9ea21 100644 --- a/tensorflow/core/common_runtime/graph_constructor.cc +++ b/tensorflow/core/common_runtime/graph_constructor.cc @@ -44,6 +44,7 @@ limitations under the License. #include "tensorflow/core/lib/gtl/inlined_vector.h" #include "tensorflow/core/lib/strings/scanner.h" #include "tensorflow/core/lib/strings/str_util.h" +#include "tensorflow/core/platform/errors.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/public/version.h" @@ -1425,6 +1426,17 @@ void GraphConstructor::Undo() { Status GraphConstructor::MakeEdge(Node* src, int output_index, Node* dst, int input_index) { + if (output_index >= src->num_outputs()) { + return errors::InvalidArgument( + "Output ", output_index, " of node ", src->name(), + " does not exist. Node only has ", src->num_outputs(), " outputs."); + } + if (input_index >= dst->num_inputs()) { + return errors::InvalidArgument( + "Input ", input_index, " of node ", dst->name(), + " does not exist. Node only has ", dst->num_inputs(), " inputs."); + } + DataType src_out = src->output_type(output_index); DataType dst_in = dst->input_type(input_index); if (!TypesCompatible(dst_in, src_out)) { diff --git a/tensorflow/core/common_runtime/shape_refiner.cc b/tensorflow/core/common_runtime/shape_refiner.cc index a968aaf09b6ad3..3c5421a9507076 100644 --- a/tensorflow/core/common_runtime/shape_refiner.cc +++ b/tensorflow/core/common_runtime/shape_refiner.cc @@ -117,9 +117,26 @@ Status InferShapesForFunctionSubNode(const Node* node, ShapeRefiner* refiner, TF_RETURN_IF_ERROR(outer_context->MakeShapeFromShapeProto(proto, &handle)); outer_context->set_output(index, handle); - auto* resource = node_context->input_handle_shapes_and_types(0); + const std::vector* resource = + node_context->input_handle_shapes_and_types(0); if (resource) { - outer_context->set_output_handle_shapes_and_types(index, *resource); + // `ShapesAndType`s contain `ShapeHandle`s. These `ShapeHandle`s point + // to `Shape`s that are owned by a different inference context too. We + // need to copy them to the outer context to prevent them from being + // destroyed before they are used. + std::vector copied_shapes_and_types; + for (auto& shape_and_type : *resource) { + ShapeHandle handle; + TensorShapeProto proto; + node_context->ShapeHandleToProto(shape_and_type.shape, &proto); + TF_RETURN_IF_ERROR( + outer_context->MakeShapeFromShapeProto(proto, &handle)); + copied_shapes_and_types.push_back( + ShapeAndType(handle, shape_and_type.dtype)); + } + + outer_context->set_output_handle_shapes_and_types( + index, copied_shapes_and_types); } } diff --git a/tensorflow/core/data/compression_utils.cc b/tensorflow/core/data/compression_utils.cc index d132bdca8dabfc..f550b150ce945d 100644 --- a/tensorflow/core/data/compression_utils.cc +++ b/tensorflow/core/data/compression_utils.cc @@ -29,9 +29,10 @@ Status CompressElement(const std::vector& element, int64 total_size = 0; for (auto& component : element) { if (DataTypeCanUseMemcpy(component.dtype())) { - // Some datatypes can be memcopied, allowing us to save two copies - // (AsProtoTensorContent and SerializeToArray). - total_size += DMAHelper::buffer(&component)->size(); + const TensorBuffer* buffer = DMAHelper::buffer(&component); + if (buffer) { + total_size += buffer->size(); + } } else { non_memcpy_components.emplace_back(); component.AsProtoTensorContent(&non_memcpy_components.back()); @@ -53,8 +54,10 @@ Status CompressElement(const std::vector& element, component.shape().AsProto(metadata->mutable_tensor_shape()); if (DataTypeCanUseMemcpy(component.dtype())) { const TensorBuffer* buffer = DMAHelper::buffer(&component); - memcpy(position, buffer->data(), buffer->size()); - metadata->set_tensor_size_bytes(buffer->size()); + if (buffer) { + memcpy(position, buffer->data(), buffer->size()); + metadata->set_tensor_size_bytes(buffer->size()); + } } else { TensorProto& proto = non_memcpy_components[non_memcpy_component_index++]; proto.SerializeToArray(position, proto.ByteSizeLong()); @@ -94,8 +97,13 @@ Status UncompressElement(const CompressedElement& compressed, if (DataTypeCanUseMemcpy(metadata.dtype())) { out->emplace_back(metadata.dtype(), metadata.tensor_shape()); TensorBuffer* buffer = DMAHelper::buffer(&out->back()); - iov[i].iov_base = buffer->data(); - iov[i].iov_len = buffer->size(); + if (buffer) { + iov[i].iov_base = buffer->data(); + iov[i].iov_len = buffer->size(); + } else { + iov[i].iov_base = nullptr; + iov[i].iov_len = 0; + } } else { // Allocate an empty Tensor. We will fill it out later after // uncompressing into the tensor_proto_str. diff --git a/tensorflow/core/framework/attr_value_util.cc b/tensorflow/core/framework/attr_value_util.cc index a307c8a18c1862..ca1f316409b39b 100644 --- a/tensorflow/core/framework/attr_value_util.cc +++ b/tensorflow/core/framework/attr_value_util.cc @@ -38,6 +38,9 @@ namespace { // Do not construct large tensors to compute their hash or compare for equality. constexpr int kMaxAttrValueTensorByteSize = 32 * 1024 * 1024; // 32mb +// Limit nesting of tensors to 100 deep to prevent memory overflow. +constexpr int kMaxTensorNestDepth = 100; + // Return the size of the tensor represented by this TensorProto. If shape is // not fully defined return -1. int64 TensorByteSize(const TensorProto& t) { @@ -224,6 +227,54 @@ string SummarizeFunc(const NameAttrList& func) { return strings::StrCat(func.name(), "[", absl::StrJoin(entries, ", "), "]"); } +bool ParseAttrValueHelper_TensorNestsUnderLimit(int limit, string to_parse) { + int nests = 0; + int maxed_out = to_parse.length(); + int open_curly = to_parse.find('{'); + int open_bracket = to_parse.find('<'); + int close_curly = to_parse.find('}'); + int close_bracket = to_parse.find('>'); + if (open_curly == -1) { + open_curly = maxed_out; + } + if (open_bracket == -1) { + open_bracket = maxed_out; + } + int min = std::min(open_curly, open_bracket); + do { + if (open_curly == maxed_out && open_bracket == maxed_out) { + return true; + } + if (min == open_curly) { + nests += 1; + open_curly = to_parse.find('{', open_curly + 1); + if (open_curly == -1) { + open_curly = maxed_out; + } + } else if (min == open_bracket) { + nests += 1; + open_bracket = to_parse.find('<', open_bracket + 1); + if (open_bracket == -1) { + open_bracket = maxed_out; + } + } else if (min == close_curly) { + nests -= 1; + close_curly = to_parse.find('}', close_curly + 1); + if (close_curly == -1) { + close_curly = maxed_out; + } + } else if (min == close_bracket) { + nests -= 1; + close_bracket = to_parse.find('>', close_bracket + 1); + if (close_bracket == -1) { + close_bracket = maxed_out; + } + } + min = std::min({open_curly, open_bracket, close_curly, close_bracket}); + } while (nests < 100); + return false; +} + } // namespace string SummarizeAttrValue(const AttrValue& attr_value) { @@ -448,7 +499,12 @@ bool ParseAttrValue(StringPiece type, StringPiece text, AttrValue* out) { } else { to_parse = strings::StrCat(field_name, ": ", text); } - + if (field_name == "tensor") { + if (!ParseAttrValueHelper_TensorNestsUnderLimit(kMaxTensorNestDepth, + to_parse)) { + return false; + } + } return ProtoParseFromString(to_parse, out); } diff --git a/tensorflow/core/framework/common_shape_fns.cc b/tensorflow/core/framework/common_shape_fns.cc index b9efddf4cdbc99..a81f7400389843 100644 --- a/tensorflow/core/framework/common_shape_fns.cc +++ b/tensorflow/core/framework/common_shape_fns.cc @@ -659,6 +659,8 @@ Status Conv2DShapeImpl(shape_inference::InferenceContext* c, if (c->ValueKnown(input_depth_dim) && c->ValueKnown(filter_input_depth_dim)) { int64 input_depth_value = c->Value(input_depth_dim), filter_input_depth_value = c->Value(filter_input_depth_dim); + if (filter_input_depth_value == 0) + return errors::InvalidArgument("Depth of filter must not be 0"); if (input_depth_value % filter_input_depth_value != 0) return errors::InvalidArgument( "Depth of input (", input_depth_value, @@ -668,6 +670,8 @@ Status Conv2DShapeImpl(shape_inference::InferenceContext* c, int64 num_groups = input_depth_value / filter_input_depth_value; if (c->ValueKnown(output_depth_dim)) { int64 output_depth_value = c->Value(output_depth_dim); + if (num_groups == 0) + return errors::InvalidArgument("Number of groups must not be 0"); if (output_depth_value % num_groups != 0) return errors::InvalidArgument( "Depth of output (", output_depth_value, @@ -798,6 +802,8 @@ Status Conv3DShape(shape_inference::InferenceContext* c) { if (c->ValueKnown(input_depth_dim) && c->ValueKnown(filter_input_depth_dim)) { int64 input_depth_value = c->Value(input_depth_dim), filter_input_depth_value = c->Value(filter_input_depth_dim); + if (filter_input_depth_value == 0) + return errors::InvalidArgument("Depth of filter must not be 0"); if (input_depth_value % filter_input_depth_value != 0) return errors::InvalidArgument( "Depth of input (", input_depth_value, @@ -807,6 +813,8 @@ Status Conv3DShape(shape_inference::InferenceContext* c) { int64 num_groups = input_depth_value / filter_input_depth_value; if (c->ValueKnown(output_depth_dim)) { int64 output_depth_value = c->Value(output_depth_dim); + if (num_groups == 0) + return errors::InvalidArgument("Number of groups must not be 0"); if (output_depth_value % num_groups != 0) return errors::InvalidArgument( "Depth of output (", output_depth_value, @@ -2364,6 +2372,9 @@ Status SparseReduceShapeFn(InferenceContext* c) { int64 ndims = shape_vec.size(); absl::flat_hash_set axes; + if (ndims == 0) + return errors::InvalidArgument( + "Number of dims in shape tensor must not be 0"); for (int i = 0; i < axes_vec.size(); i++) { axes.insert((axes_vec(i) + ndims) % ndims); } diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc index 520346b0166a33..2aeeed75ef0b22 100644 --- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc @@ -2000,6 +2000,12 @@ class ReorderCastLikeAndValuePreserving : public ArithmeticOptimizerStage { Status TrySimplify(NodeDef* consumer, string* simplified_node_name) override { NodeDef* producer; + + if (consumer->input_size() < 1) { + return errors::FailedPrecondition("Node ", simplified_node_name, + " lacks inputs"); + } + TF_RETURN_IF_ERROR(GetInputNode(consumer->input(0), &producer)); const bool producer_is_cast = IsCastLike(*producer); const bool can_optimize = @@ -2402,6 +2408,11 @@ class ReplaceMulWithSquare : public ArithmeticOptimizerStage { ~ReplaceMulWithSquare() override = default; bool IsSupported(const NodeDef* node) const override { + if (!node || node->input_size() < 2) { + // Invalid node + return false; + } + return IsAnyMul(*node) && node->input(0) == node->input(1); } diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer.cc b/tensorflow/core/grappler/optimizers/dependency_optimizer.cc index 58ef14e3d3d60f..1febfc01e2d741 100644 --- a/tensorflow/core/grappler/optimizers/dependency_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/dependency_optimizer.cc @@ -68,6 +68,12 @@ bool DependencyOptimizer::SafeToRemoveIdentity(const NodeDef& node) const { // The output values of this node may be needed. return false; } + + if (node.input_size() < 1) { + // Node lacks input, is invalid + return false; + } + const NodeDef* input = node_map_->GetNode(NodeName(node.input(0))); CHECK(input != nullptr) << "node = " << node.name() << " input = " << node.input(0); diff --git a/tensorflow/core/kernels/banded_triangular_solve_op.cc b/tensorflow/core/kernels/banded_triangular_solve_op.cc index 666282e52c8031..ad2467ebefaf1a 100644 --- a/tensorflow/core/kernels/banded_triangular_solve_op.cc +++ b/tensorflow/core/kernels/banded_triangular_solve_op.cc @@ -217,6 +217,7 @@ class BandedTriangularSolveOpCpu : public OpKernel { const Tensor& in1 = ctx->input(1); ValidateInputTensors(ctx, in0, in1); + if (!ctx->status().ok()) return; MatMulBCast bcast(in0.shape().dim_sizes(), in1.shape().dim_sizes()); OP_REQUIRES( @@ -275,6 +276,14 @@ class BandedTriangularSolveOpCpu : public OpKernel { OP_REQUIRES( ctx, in1.dims() >= 2, errors::InvalidArgument("In[1] ndims must be >= 2: ", in1.dims())); + + OP_REQUIRES(ctx, in0.NumElements() > 0, + errors::InvalidArgument("In[0] must not be an empty tensor: ", + in0.DebugString())); + + OP_REQUIRES(ctx, in1.NumElements() > 0, + errors::InvalidArgument("In[1] must not be an empty tensor: ", + in1.DebugString())); } bool lower_; bool adjoint_; diff --git a/tensorflow/core/kernels/bincount_op.cc b/tensorflow/core/kernels/bincount_op.cc index a84b25f2541013..c75f67bb56009a 100644 --- a/tensorflow/core/kernels/bincount_op.cc +++ b/tensorflow/core/kernels/bincount_op.cc @@ -414,6 +414,15 @@ class RaggedBincountOp : public OpKernel { int num_values = values.size(); int batch_idx = 0; + OP_REQUIRES(ctx, splits(0) == 0, + errors::InvalidArgument("Splits must start with 0, not with ", + splits(0))); + + OP_REQUIRES(ctx, splits(num_rows) == num_values, + errors::InvalidArgument( + "Splits must end with the number of values, got ", + splits(num_rows), " instead of ", num_values)); + Tensor* out_t; OP_REQUIRES_OK( ctx, ctx->allocate_output(0, TensorShape({num_rows, size}), &out_t)); diff --git a/tensorflow/core/kernels/boosted_trees/quantile_ops.cc b/tensorflow/core/kernels/boosted_trees/quantile_ops.cc index 0065bdd66aa708..916db1f436148b 100644 --- a/tensorflow/core/kernels/boosted_trees/quantile_ops.cc +++ b/tensorflow/core/kernels/boosted_trees/quantile_ops.cc @@ -116,6 +116,9 @@ class BoostedTreesCreateQuantileStreamResourceOp : public OpKernel { const Tensor* num_streams_t; OP_REQUIRES_OK(context, context->input(kNumStreamsName, &num_streams_t)); int64 num_streams = num_streams_t->scalar()(); + OP_REQUIRES(context, num_streams >= 0, + errors::InvalidArgument( + "Num_streams input cannot be a negative integer")); auto result = new QuantileStreamResource(epsilon, max_elements_, num_streams); diff --git a/tensorflow/core/kernels/boosted_trees/resource_ops.cc b/tensorflow/core/kernels/boosted_trees/resource_ops.cc index ac1fb5652da5f9..8036f2b20f36bb 100644 --- a/tensorflow/core/kernels/boosted_trees/resource_ops.cc +++ b/tensorflow/core/kernels/boosted_trees/resource_ops.cc @@ -53,6 +53,7 @@ class BoostedTreesCreateEnsembleOp : public OpKernel { if (!result->InitFromSerialized( tree_ensemble_serialized_t->scalar()(), stamp_token)) { result->Unref(); + result.release(); // Needed due to the `->Unref` above, to prevent UAF OP_REQUIRES( context, false, errors::InvalidArgument("Unable to parse tree ensemble proto.")); diff --git a/tensorflow/core/kernels/boosted_trees/stats_ops.cc b/tensorflow/core/kernels/boosted_trees/stats_ops.cc index 851e5b78e847b7..dc8c4110b47259 100644 --- a/tensorflow/core/kernels/boosted_trees/stats_ops.cc +++ b/tensorflow/core/kernels/boosted_trees/stats_ops.cc @@ -14,6 +14,7 @@ limitations under the License. ==============================================================================*/ #include +#include #include #include "third_party/eigen3/Eigen/Core" @@ -22,6 +23,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/kernels/boosted_trees/boosted_trees.pb.h" #include "tensorflow/core/kernels/boosted_trees/tree_helper.h" +#include "tensorflow/core/platform/errors.h" #include "tensorflow/core/platform/logging.h" namespace tensorflow { @@ -51,6 +53,16 @@ class BoostedTreesCalculateBestGainsPerFeatureOp : public OpKernel { // node_id_range const Tensor* node_id_range_t; OP_REQUIRES_OK(context, context->input("node_id_range", &node_id_range_t)); + OP_REQUIRES( + context, node_id_range_t->dims() == 1, + errors::InvalidArgument("node_id_range must be a rank 1 tensor, but " + "given node_id_range has dims of ", + node_id_range_t->dims())); + OP_REQUIRES(context, node_id_range_t->dim_size(0) == 2, + errors::InvalidArgument( + "node_id_range must be a rank 1 tensor with shape=[2], but " + "given node_id_range has shape ", + node_id_range_t->dim_size(0), " on its first dim")); const auto node_id_range = node_id_range_t->vec(); const int32 node_id_first = node_id_range(0); // inclusive const int32 node_id_last = node_id_range(1); // exclusive @@ -244,12 +256,18 @@ class BoostedTreesCalculateBestFeatureSplitOp : public OpKernel { // node_id_range const Tensor* node_id_range_t; OP_REQUIRES_OK(context, context->input("node_id_range", &node_id_range_t)); + OP_REQUIRES( + context, node_id_range_t->NumElements() == 2, + errors::InvalidArgument("node_id_range argument must have shape [2]")); const auto node_id_range = node_id_range_t->vec(); const int32 node_id_first = node_id_range(0); // inclusive const int32 node_id_last = node_id_range(1); // exclusive const Tensor* stats_summary_t; OP_REQUIRES_OK(context, context->input("stats_summary", &stats_summary_t)); + OP_REQUIRES( + context, stats_summary_t->shape().dims() == 4, + errors::InvalidArgument("stats_summary argument must have rank 4")); TTypes::ConstTensor stats_summary = stats_summary_t->tensor(); const int32 feature_dims = stats_summary_t->dim_size(1); @@ -262,6 +280,8 @@ class BoostedTreesCalculateBestFeatureSplitOp : public OpKernel { const Tensor* l1_t; OP_REQUIRES_OK(context, context->input("l1", &l1_t)); + OP_REQUIRES(context, l1_t->NumElements() == 1, + errors::InvalidArgument("l1 argument must be a scalar")); const auto l1 = l1_t->scalar()(); DCHECK_GE(l1, 0); if (logits_dim_ > 1) { @@ -271,17 +291,25 @@ class BoostedTreesCalculateBestFeatureSplitOp : public OpKernel { const Tensor* l2_t; OP_REQUIRES_OK(context, context->input("l2", &l2_t)); + OP_REQUIRES(context, l2_t->NumElements() == 1, + errors::InvalidArgument("l2 argument must be a scalar")); const auto l2 = l2_t->scalar()(); DCHECK_GE(l2, 0); const Tensor* tree_complexity_t; OP_REQUIRES_OK(context, context->input("tree_complexity", &tree_complexity_t)); + OP_REQUIRES( + context, tree_complexity_t->NumElements() == 1, + errors::InvalidArgument("tree_complexity argument must be a scalar")); const auto tree_complexity = tree_complexity_t->scalar()(); const Tensor* min_node_weight_t; OP_REQUIRES_OK(context, context->input("min_node_weight", &min_node_weight_t)); + OP_REQUIRES( + context, min_node_weight_t->NumElements() == 1, + errors::InvalidArgument("min_node_weight argument must be a scalar")); const auto min_node_weight = min_node_weight_t->scalar()(); std::vector output_node_ids; @@ -290,7 +318,7 @@ class BoostedTreesCalculateBestFeatureSplitOp : public OpKernel { std::vector output_thresholds; std::vector output_left_node_contribs; std::vector output_right_node_contribs; - std::vector output_split_types; + std::vector output_split_types; // TODO(tanzheny) parallelize the computation. // Iterate each node and find the best gain per node. @@ -567,6 +595,16 @@ class BoostedTreesCalculateBestFeatureSplitV2 : public OpKernel { // node_id_range const Tensor* node_id_range_t; OP_REQUIRES_OK(context, context->input("node_id_range", &node_id_range_t)); + OP_REQUIRES( + context, node_id_range_t->dims() == 1, + errors::InvalidArgument("node_id_range must be a rank 1 tensor, but " + "given node_id_range has dims of ", + node_id_range_t->dims())); + OP_REQUIRES(context, node_id_range_t->dim_size(0) == 2, + errors::InvalidArgument( + "node_id_range must be a rank 1 tensor with shape=[2], but " + "given node_id_range has shape ", + node_id_range_t->dim_size(0), " on its first dim")); const auto node_id_range = node_id_range_t->vec(); const int32 node_id_first = node_id_range(0); // Inclusive. const int32 node_id_last = node_id_range(1); // Exclusive. @@ -1025,6 +1063,13 @@ class BoostedTreesSparseCalculateBestFeatureSplitOp : public OpKernel { const int32 feature_dim = stats_summary_indices(idx, 1); const int32 bucket_id = stats_summary_indices(idx, 2); const int32 stat_dim = stats_summary_indices(idx, 3); + OP_REQUIRES(context, stat_dim < stats_dims, + errors::InvalidArgument( + "Stat dim, the sum of logits dim and hessian dim in " + "stats_summary_indices, cannot be greater than stats " + "dims, the last value in stats_summary_shape, which was ", + stats_dims, ". At index (", idx, + ", 4), stats_summary_indices contains value ", stat_dim)); std::pair const& f_insert_result = f_map.insert( FeatureMapIterator::value_type(feature_dim, BucketMap())); auto& b_map = f_insert_result.first->second; diff --git a/tensorflow/core/kernels/conv_grad_filter_ops.cc b/tensorflow/core/kernels/conv_grad_filter_ops.cc index b16d3c7270fde0..d37ac5af59470a 100644 --- a/tensorflow/core/kernels/conv_grad_filter_ops.cc +++ b/tensorflow/core/kernels/conv_grad_filter_ops.cc @@ -496,6 +496,14 @@ class Conv2DCustomBackpropFilterOp : public OpKernel { const int filter_total_size = dims.spatial_dims[0].filter_size * dims.spatial_dims[1].filter_size * dims.in_depth; + OP_REQUIRES( + context, + filter_total_size * dims.out_depth == filter_backprop->NumElements(), + errors::InvalidArgument( + "filter_size does not have enough elements, requested ", + filter_total_size * dims.out_depth, ", got ", + filter_backprop->NumElements())); + // The output image size is the spatial size of the output. const int output_image_size = dims.spatial_dims[0].output_size * dims.spatial_dims[1].output_size; @@ -519,6 +527,11 @@ class Conv2DCustomBackpropFilterOp : public OpKernel { const size_t work_unit_size = size_A + size_B + size_C; + OP_REQUIRES( + context, work_unit_size != 0, + errors::InvalidArgument( + "Work size for convolution would be 0, which is not acceptable")); + const size_t shard_size = (target_working_set_size + work_unit_size - 1) / work_unit_size; diff --git a/tensorflow/core/kernels/conv_grad_input_ops.cc b/tensorflow/core/kernels/conv_grad_input_ops.cc index 2dd63d1f4d05b7..a89e5c7185c0f6 100644 --- a/tensorflow/core/kernels/conv_grad_input_ops.cc +++ b/tensorflow/core/kernels/conv_grad_input_ops.cc @@ -668,6 +668,11 @@ class Conv2DCustomBackpropInputOp : public OpKernel { dims.batch_size == 1 || thread_work_unit_size >= min_thread_work_unit_size; + OP_REQUIRES( + context, work_unit_size > 0, + errors::InvalidArgument("input, filter_sizes and out_backprop tensors " + "must all have at least 1 element")); + const size_t shard_size = use_parallel_contraction ? 1 diff --git a/tensorflow/core/kernels/conv_grad_ops_3d.cc b/tensorflow/core/kernels/conv_grad_ops_3d.cc index 322da2537f0da5..1ef931a97d93da 100644 --- a/tensorflow/core/kernels/conv_grad_ops_3d.cc +++ b/tensorflow/core/kernels/conv_grad_ops_3d.cc @@ -239,6 +239,28 @@ class Conv3DBackpropInputOp : public OpKernel { input_shape = context->input(0).shape(); } + OP_REQUIRES(context, input_shape.dims() == 5, + errors::InvalidArgument("input tensor must have 5 dimensions")); + OP_REQUIRES( + context, filter_shape.dims() == 5, + errors::InvalidArgument("filter_sizes tensor must have 5 dimensions")); + OP_REQUIRES( + context, out_backprop_shape.dims() == 5, + errors::InvalidArgument("out_backprop tensor must have 5 dimensions")); + OP_REQUIRES( + context, input_shape.dim_size(4) == filter_shape.dim_size(3), + errors::InvalidArgument("input and filter_sizes must have the same " + "number of channels. Got ", + input_shape.dim_size(4), " for input and ", + filter_shape.dim_size(3), " for filter_sizes")); + OP_REQUIRES( + context, out_backprop_shape.dim_size(4) == filter_shape.dim_size(4), + errors::InvalidArgument("out_backprop and filter_sizes must have the " + "same number of channels. Got ", + out_backprop_shape.dim_size(4), + " for out_backprop and ", + filter_shape.dim_size(4), " for filter_sizes")); + ConvBackpropDimensions dims; OP_REQUIRES_OK(context, ConvBackpropComputeDimensions( "Conv3DBackpropInputOp", /*num_spatial_dims=*/3, @@ -346,6 +368,28 @@ class Conv3DCustomBackpropInputOp : public OpKernel { input_shape = context->input(0).shape(); } + OP_REQUIRES(context, input_shape.dims() == 5, + errors::InvalidArgument("input tensor must have 5 dimensions")); + OP_REQUIRES( + context, filter_shape.dims() == 5, + errors::InvalidArgument("filter_sizes tensor must have 5 dimensions")); + OP_REQUIRES( + context, out_backprop_shape.dims() == 5, + errors::InvalidArgument("out_backprop tensor must have 5 dimensions")); + OP_REQUIRES( + context, input_shape.dim_size(4) == filter_shape.dim_size(3), + errors::InvalidArgument("input and filter_sizes must have the same " + "number of channels. Got ", + input_shape.dim_size(4), " for input and ", + filter_shape.dim_size(3), " for filter_sizes")); + OP_REQUIRES( + context, out_backprop_shape.dim_size(4) == filter_shape.dim_size(4), + errors::InvalidArgument("out_backprop and filter_sizes must have the " + "same number of channels. Got ", + out_backprop_shape.dim_size(4), + " for out_backprop and ", + filter_shape.dim_size(4), " for filter_sizes")); + ConvBackpropDimensions dims; OP_REQUIRES_OK(context, ConvBackpropComputeDimensions( "Conv3DBackpropInputOp", /*num_spatial_dims=*/3, @@ -416,6 +460,11 @@ class Conv3DCustomBackpropInputOp : public OpKernel { // contraction compared to sharding and matmuls. const bool use_parallel_contraction = dims.batch_size == 1; + OP_REQUIRES( + context, work_unit_size > 0, + errors::InvalidArgument("input, filter_sizes and out_backprop tensors " + "must all have at least 1 element")); + const size_t shard_size = use_parallel_contraction ? 1 @@ -696,6 +745,28 @@ class Conv3DBackpropFilterOp : public OpKernel { filter_shape = context->input(1).shape(); } + OP_REQUIRES(context, input_shape.dims() == 5, + errors::InvalidArgument("input tensor must have 5 dimensions")); + OP_REQUIRES( + context, filter_shape.dims() == 5, + errors::InvalidArgument("filter_sizes tensor must have 5 dimensions")); + OP_REQUIRES( + context, out_backprop_shape.dims() == 5, + errors::InvalidArgument("out_backprop tensor must have 5 dimensions")); + OP_REQUIRES( + context, input_shape.dim_size(4) == filter_shape.dim_size(3), + errors::InvalidArgument("input and filter_sizes must have the same " + "number of channels. Got ", + input_shape.dim_size(4), " for input and ", + filter_shape.dim_size(3), " for filter_sizes")); + OP_REQUIRES( + context, out_backprop_shape.dim_size(4) == filter_shape.dim_size(4), + errors::InvalidArgument("out_backprop and filter_sizes must have the " + "same number of channels. Got ", + out_backprop_shape.dim_size(4), + " for out_backprop and ", + filter_shape.dim_size(4), " for filter_sizes")); + ConvBackpropDimensions dims; OP_REQUIRES_OK(context, ConvBackpropComputeDimensions( @@ -808,6 +879,28 @@ class Conv3DCustomBackpropFilterOp : public OpKernel { filter_shape = context->input(1).shape(); } + OP_REQUIRES(context, input_shape.dims() == 5, + errors::InvalidArgument("input tensor must have 5 dimensions")); + OP_REQUIRES( + context, filter_shape.dims() == 5, + errors::InvalidArgument("filter_sizes tensor must have 5 dimensions")); + OP_REQUIRES( + context, out_backprop_shape.dims() == 5, + errors::InvalidArgument("out_backprop tensor must have 5 dimensions")); + OP_REQUIRES( + context, input_shape.dim_size(4) == filter_shape.dim_size(3), + errors::InvalidArgument("input and filter_sizes must have the same " + "number of channels. Got ", + input_shape.dim_size(4), " for input and ", + filter_shape.dim_size(3), " for filter_sizes")); + OP_REQUIRES( + context, out_backprop_shape.dim_size(4) == filter_shape.dim_size(4), + errors::InvalidArgument("out_backprop and filter_sizes must have the " + "same number of channels. Got ", + out_backprop_shape.dim_size(4), + " for out_backprop and ", + filter_shape.dim_size(4), " for filter_sizes")); + ConvBackpropDimensions dims; OP_REQUIRES_OK(context, ConvBackpropComputeDimensions( @@ -880,6 +973,11 @@ class Conv3DCustomBackpropFilterOp : public OpKernel { const int64 work_unit_size = size_A + size_B + size_C; + OP_REQUIRES( + context, work_unit_size > 0, + errors::InvalidArgument("input, filter_sizes and out_backprop tensors " + "must all have at least 1 element")); + const size_t shard_size = (target_working_set_size + work_unit_size - 1) / work_unit_size; diff --git a/tensorflow/core/kernels/conv_grad_shape_utils.cc b/tensorflow/core/kernels/conv_grad_shape_utils.cc index acb052968e1708..942e085b8ac3b2 100644 --- a/tensorflow/core/kernels/conv_grad_shape_utils.cc +++ b/tensorflow/core/kernels/conv_grad_shape_utils.cc @@ -127,6 +127,10 @@ Status ConvBackpropComputeDimensionsV2( // dimensions of the filter Tensor. VLOG(2) << "input vs filter_in depth " << dims->in_depth << " " << filter_shape.dim_size(num_dims - 2); + if (filter_shape.dim_size(num_dims - 2) <= 0) { + return errors ::InvalidArgument( + label, ": filter depth must be strictly greated than zero"); + } if (dims->in_depth % filter_shape.dim_size(num_dims - 2)) { return errors::InvalidArgument( label, ": input depth must be evenly divisible by filter depth"); diff --git a/tensorflow/core/kernels/conv_ops.cc b/tensorflow/core/kernels/conv_ops.cc index ab8e24a311ff68..287cf4a923b31c 100644 --- a/tensorflow/core/kernels/conv_ops.cc +++ b/tensorflow/core/kernels/conv_ops.cc @@ -425,6 +425,9 @@ Status ComputeConv2DDimension(const Conv2DParameters& params, errors::InvalidArgument("Patch depth too large")); const int in_depth = static_cast(in_depth_raw); const int patch_depth = static_cast(patch_depth_raw); + TF_REQUIRES(patch_depth > 0, + errors::InvalidArgument( + "filter depth must be stricly positive, got ", patch_depth)); TF_REQUIRES(in_depth % patch_depth == 0, errors::InvalidArgument( "input depth must be evenly divisible by filter depth: ", diff --git a/tensorflow/core/kernels/conv_ops_3d.h b/tensorflow/core/kernels/conv_ops_3d.h index 9dcdea5b18f10b..8073ca5a9dfdce 100644 --- a/tensorflow/core/kernels/conv_ops_3d.h +++ b/tensorflow/core/kernels/conv_ops_3d.h @@ -56,6 +56,11 @@ struct LaunchConvOp { errors::InvalidArgument("CPU implementation of Conv3D " "currently only supports dilated rates " "of 1.")); + OP_REQUIRES(context, filter.dim_size(3) == input.dim_size(input.dims() - 1), + errors::InvalidArgument( + "Number of channels in filter (", filter.dim_size(3), + ") must match last dimension of input (", + input.dim_size(input.dims() - 1), ")")); functor::CuboidConvolution()( context->template eigen_device(), output->tensor(), input.tensor(), filter.tensor(), strides[2], strides[1], @@ -135,6 +140,8 @@ class Conv3DOp : public BinaryOpBase { const int64 filter_depth = filter.dim_size(3); const int64 out_depth = filter.dim_size(4); + OP_REQUIRES(context, filter_depth != 0, + errors::InvalidArgument("filter_depth must be non-zero")); OP_REQUIRES(context, in_depth % filter_depth == 0, errors::InvalidArgument( "Input depth must be evenly divisible by filter depth: ", diff --git a/tensorflow/core/kernels/count_ops.cc b/tensorflow/core/kernels/count_ops.cc index 087deef0812f00..40aa1fe458c1ee 100644 --- a/tensorflow/core/kernels/count_ops.cc +++ b/tensorflow/core/kernels/count_ops.cc @@ -122,6 +122,9 @@ class DenseCount : public OpKernel { int num_batch_elements = 1; for (int i = 0; i < num_batch_dimensions; ++i) { + OP_REQUIRES(context, data.shape().dim_size(i) != 0, + errors::InvalidArgument( + "Invalid input: Shapes dimension cannot be 0.")); num_batch_elements *= data.shape().dim_size(i); } int num_value_elements = data.shape().num_elements() / num_batch_elements; @@ -192,10 +195,22 @@ class SparseCount : public OpKernel { "; values shape: ", values.shape().DebugString())); } + OP_REQUIRES(context, shape.NumElements() != 0, + errors::InvalidArgument( + "The shape argument requires at least one element.")); + bool is_1d = shape.NumElements() == 1; - int num_batches = is_1d ? 1 : shape.flat()(0); + auto shape_vector = shape.flat(); + int num_batches = is_1d ? 1 : shape_vector(0); int num_values = values.NumElements(); + for (int b = 0; b < shape_vector.size(); b++) { + OP_REQUIRES(context, shape_vector(b) >= 0, + errors::InvalidArgument( + "Elements in dense_shape must be >= 0. Instead got:", + shape.DebugString())); + } + OP_REQUIRES(context, num_values == indices.shape().dim_size(0), errors::InvalidArgument( "Number of values must match first dimension of indices.", @@ -212,6 +227,14 @@ class SparseCount : public OpKernel { for (int idx = 0; idx < num_values; ++idx) { int batch = is_1d ? 0 : indices_values(idx, 0); + if (batch >= num_batches) { + OP_REQUIRES(context, batch < num_batches, + errors::InvalidArgument( + "Indices value along the first dimension must be ", + "lower than the first index of the shape.", "Got ", + batch, " as batch and ", num_batches, + " as the first dimension of the shape.")); + } const auto& value = values_values(idx); if (value >= 0 && (maxlength_ <= 0 || value < maxlength_)) { if (binary_output_) { diff --git a/tensorflow/core/kernels/ctc_decoder_ops.cc b/tensorflow/core/kernels/ctc_decoder_ops.cc index d62aef2d03b988..9efdac60e369c2 100644 --- a/tensorflow/core/kernels/ctc_decoder_ops.cc +++ b/tensorflow/core/kernels/ctc_decoder_ops.cc @@ -70,6 +70,9 @@ class CTCDecodeHelper { if (inputs_shape.dims() != 3) { return errors::InvalidArgument("inputs is not a 3-Tensor"); } + if (inputs_shape.num_elements() == 0) { + return errors::InvalidArgument("inputs must not be empty"); + } const int64 max_time = inputs_shape.dim_size(0); const int64 batch_size = inputs_shape.dim_size(1); @@ -232,6 +235,8 @@ class CTCGreedyDecoderOp : public OpKernel { int prev_indices = -1; for (int t = 0; t < seq_len_t(b); ++t) { int max_class_indices; + OP_REQUIRES(ctx, input_list_t[t].dimension(1) > 0, + errors::InvalidArgument("Invalid input dimensions.")); log_prob_t(b, 0) += -RowMax(input_list_t[t], b, &max_class_indices); if (max_class_indices != blank_index && diff --git a/tensorflow/core/kernels/ctc_loss_op.cc b/tensorflow/core/kernels/ctc_loss_op.cc index 6358e82fdda853..ca505e1db93145 100644 --- a/tensorflow/core/kernels/ctc_loss_op.cc +++ b/tensorflow/core/kernels/ctc_loss_op.cc @@ -100,11 +100,18 @@ class CTCLossOp : public OpKernel { errors::InvalidArgument("sequence_length is not a vector")); OP_REQUIRES(ctx, TensorShapeUtils::IsMatrix(labels_indices->shape()), errors::InvalidArgument("labels_indices is not a matrix")); + OP_REQUIRES(ctx, labels_indices->dim_size(1) > 1, + errors::InvalidArgument( + "labels_indices second dimension must be >= 1. Received ", + labels_indices->dim_size(1))); OP_REQUIRES(ctx, TensorShapeUtils::IsVector(labels_values->shape()), errors::InvalidArgument("labels_values is not a vector")); const TensorShape& inputs_shape = inputs->shape(); const int64 max_time = inputs_shape.dim_size(0); + OP_REQUIRES(ctx, max_time != 0, + errors::InvalidArgument( + "Max time or first dimension of input cannot be 0.")); const int64 batch_size = inputs_shape.dim_size(1); const int64 num_classes_raw = inputs_shape.dim_size(2); OP_REQUIRES( diff --git a/tensorflow/core/kernels/cwise_ops_common.h b/tensorflow/core/kernels/cwise_ops_common.h index c0aee43d26800a..45efaf34892135 100644 --- a/tensorflow/core/kernels/cwise_ops_common.h +++ b/tensorflow/core/kernels/cwise_ops_common.h @@ -271,6 +271,11 @@ class SimpleBinaryOp : public OpKernel { void Compute(OpKernelContext* ctx) override { const Tensor& in0 = ctx->input(0); const Tensor& in1 = ctx->input(1); + OP_REQUIRES( + ctx, in0.NumElements() == in1.NumElements(), + errors::InvalidArgument("The two arguments to a cwise op must have " + "same number of elements, got ", + in0.NumElements(), " and ", in1.NumElements())); auto in0_flat = in0.flat(); auto in1_flat = in1.flat(); const Device& eigen_device = ctx->eigen_device(); diff --git a/tensorflow/core/kernels/data/experimental/compression_ops.cc b/tensorflow/core/kernels/data/experimental/compression_ops.cc index efa7018acb6293..8cc214671bd742 100644 --- a/tensorflow/core/kernels/data/experimental/compression_ops.cc +++ b/tensorflow/core/kernels/data/experimental/compression_ops.cc @@ -48,6 +48,11 @@ void UncompressElementOp::Compute(OpKernelContext* ctx) { Tensor tensor = ctx->input(0); const Variant& variant = tensor.scalar()(); const CompressedElement* compressed = variant.get(); + OP_REQUIRES( + ctx, compressed != nullptr, + errors::InvalidArgument( + "Input does not contain a compressed element. Instead got tensor ", + tensor.DebugString())); std::vector components; OP_REQUIRES_OK(ctx, UncompressElement(*compressed, &components)); diff --git a/tensorflow/core/kernels/data/experimental/to_tf_record_op.cc b/tensorflow/core/kernels/data/experimental/to_tf_record_op.cc index bfa894cd473b40..56401bb91f5753 100644 --- a/tensorflow/core/kernels/data/experimental/to_tf_record_op.cc +++ b/tensorflow/core/kernels/data/experimental/to_tf_record_op.cc @@ -16,6 +16,7 @@ limitations under the License. #include "tensorflow/core/framework/function_handle_cache.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/resource_mgr.h" +#include "tensorflow/core/framework/types.h" #include "tensorflow/core/kernels/data/dataset_utils.h" #include "tensorflow/core/kernels/ops_util.h" #include "tensorflow/core/lib/core/threadpool.h" @@ -87,8 +88,20 @@ class ToTFRecordOp : public AsyncOpKernel { TF_RETURN_IF_ERROR(dataset->MakeIterator( &iter_ctx, /*parent=*/nullptr, "ToTFRecordOpIterator", &iterator)); + const int num_output_dtypes = dataset->output_dtypes().size(); + if (num_output_dtypes != 1) { + return errors::InvalidArgument( + "ToTFRecordOp currently only support datasets of 1 single column, ", + "but got ", num_output_dtypes); + } + const DataType dt = dataset->output_dtypes()[0]; + if (dt != DT_STRING) { + return errors::InvalidArgument( + "ToTFRecordOp currently only supports DT_STRING dataypes, but got ", + DataTypeString(dt)); + } std::vector components; - components.reserve(dataset->output_dtypes().size()); + components.reserve(num_output_dtypes); bool end_of_sequence; do { TF_RETURN_IF_ERROR( diff --git a/tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc b/tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc index 1e3ed53d6c6d6e..212c2b4e96715f 100644 --- a/tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc +++ b/tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc @@ -237,6 +237,17 @@ class SparseTensorSliceDatasetOp : public DatasetOpKernel { errors::InvalidArgument( "Input indices should be a matrix but received shape ", indices->shape().DebugString())); + + const auto num_indices = indices->NumElements(); + const auto num_values = values->NumElements(); + if (num_indices == 0 || num_values == 0) { + OP_REQUIRES(ctx, num_indices == num_values, + errors::InvalidArgument( + "If indices or values are empty, the other one must also " + "be. Got indices of shape ", + indices->shape().DebugString(), " and values of shape ", + values->shape().DebugString())); + } OP_REQUIRES(ctx, TensorShapeUtils::IsVector(values->shape()), errors::InvalidArgument( "Input values should be a vector but received shape ", diff --git a/tensorflow/core/kernels/data_format_ops.cc b/tensorflow/core/kernels/data_format_ops.cc index 181aa1b8a2cab2..771986f2ee84d4 100644 --- a/tensorflow/core/kernels/data_format_ops.cc +++ b/tensorflow/core/kernels/data_format_ops.cc @@ -18,16 +18,52 @@ limitations under the License. #define EIGEN_USE_THREADS #include "tensorflow/core/kernels/data_format_ops.h" + +#include + #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/platform/errors.h" namespace tensorflow { typedef Eigen::ThreadPoolDevice CPUDevice; typedef Eigen::GpuDevice GPUDevice; +// Ensure that `src` and `dst` define a valid permutation. +// Ops defined in this file assume that user specifies a permutation via two +// string attributes. This check validates that these attributes properly define +// it to prevent security vulnerabilities. +static bool IsValidPermutation(const std::string& src, const std::string& dst) { + if (src.size() != dst.size()) { + return false; + } + + std::map characters; + + // Every character in `src` must be present only once + for (const auto c : src) { + if (characters[c]) { + return false; + } + characters[c] = true; + } + + // Every character in `dst` must show up in `src` exactly once + for (const auto c : dst) { + if (!characters[c]) { + return false; + } + characters[c] = false; + } + + // At this point, characters[] has been switched to true and false exactly + // once for all character in `src` (and `dst`) so we have a valid permutation + return true; +} + template class DataFormatDimMapOp : public OpKernel { public: @@ -37,15 +73,20 @@ class DataFormatDimMapOp : public OpKernel { OP_REQUIRES_OK(context, context->GetAttr("src_format", &src_format)); string dst_format; OP_REQUIRES_OK(context, context->GetAttr("dst_format", &dst_format)); - OP_REQUIRES(context, src_format.size() == 4, - errors::InvalidArgument(strings::StrCat( - "Source format must of length 4, received src_format = ", - src_format))); + OP_REQUIRES(context, src_format.size() == 4 || src_format.size() == 5, + errors::InvalidArgument( + "Source format must be of length 4 or 5, received " + "src_format = ", + src_format)); + OP_REQUIRES(context, dst_format.size() == 4 || dst_format.size() == 5, + errors::InvalidArgument("Destination format must be of length " + "4 or 5, received dst_format = ", + dst_format)); OP_REQUIRES( - context, dst_format.size() == 4, - errors::InvalidArgument(strings::StrCat( - "Destination format must of length 4, received dst_format = ", - dst_format))); + context, IsValidPermutation(src_format, dst_format), + errors::InvalidArgument( + "Destination and source format must determine a permutation, got ", + src_format, " and ", dst_format)); dst_idx_ = Tensor(DT_INT32, {static_cast(src_format.size())}); for (int i = 0; i < src_format.size(); ++i) { for (int j = 0; j < dst_format.size(); ++j) { @@ -77,8 +118,22 @@ class DataFormatVecPermuteOp : public OpKernel { : OpKernel(context) { string src_format; OP_REQUIRES_OK(context, context->GetAttr("src_format", &src_format)); + OP_REQUIRES(context, src_format.size() == 4 || src_format.size() == 5, + errors::InvalidArgument( + "Source format must be of length 4 or 5, received " + "src_format = ", + src_format)); string dst_format; OP_REQUIRES_OK(context, context->GetAttr("dst_format", &dst_format)); + OP_REQUIRES(context, dst_format.size() == 4 || dst_format.size() == 5, + errors::InvalidArgument("Destination format must be of length " + "4 or 5, received dst_format = ", + dst_format)); + OP_REQUIRES( + context, IsValidPermutation(src_format, dst_format), + errors::InvalidArgument( + "Destination and source format must determine a permutation, got ", + src_format, " and ", dst_format)); src_format_ = src_format; dst_format_ = dst_format; } @@ -124,6 +179,10 @@ class DataFormatVecPermuteOp : public OpKernel { }; keep_only_spatial_dimensions(&src_format_str); keep_only_spatial_dimensions(&dst_format_str); + OP_REQUIRES(context, + src_format_str.size() == 2 && dst_format_str.size() == 2, + errors::InvalidArgument( + "Format specifier must contain H and W for 2D case")); } ComputeDstIndex(src_format_str, dst_format_str, input.dims(), &dst_idx); diff --git a/tensorflow/core/kernels/decode_padded_raw_op.cc b/tensorflow/core/kernels/decode_padded_raw_op.cc index 12e8ec6aff0d41..ca7c7104b442d2 100644 --- a/tensorflow/core/kernels/decode_padded_raw_op.cc +++ b/tensorflow/core/kernels/decode_padded_raw_op.cc @@ -83,14 +83,13 @@ class DecodePaddedRawOp : public OpKernel { // can copy the memory directly. if (!convert_data_endianness_ || sizeof(T) == 1) { for (int64 i = 0; i < flat_in.size(); ++i) { - const T* in_data = reinterpret_cast(flat_in(i).data()); - - if (flat_in(i).size() > fixed_length) { - memcpy(out_data, in_data, fixed_length); - } else { - memcpy(out_data, in_data, flat_in(i).size()); - } - out_data += fixed_length; + const auto to_copy = + std::min(flat_in(i).size(), static_cast(fixed_length)); + memcpy(out_data, flat_in(i).data(), to_copy); + // Note: increase out_data by width since it's already of type T* so + // each shift amount is implicitly multiplied by sizeof(T) according to + // pointer arithmetic rules. + out_data += width; } } else { // Otherwise, the data is not in the host's byte order, and rather than a @@ -105,7 +104,10 @@ class DecodePaddedRawOp : public OpKernel { p_in += sizeof(T), p_out += sizeof(T)) { std::reverse_copy(p_in, p_in + sizeof(T), p_out); } - out_data += fixed_length; + // Note: increase out_data by width since it's already of type T* so + // each shift amount is implicitly multiplied by sizeof(T) according to + // pointer arithmetic rules. + out_data += width; } } } diff --git a/tensorflow/core/kernels/dequantize_op.cc b/tensorflow/core/kernels/dequantize_op.cc index 3b38daf006768b..54c0f9de572f59 100644 --- a/tensorflow/core/kernels/dequantize_op.cc +++ b/tensorflow/core/kernels/dequantize_op.cc @@ -98,6 +98,18 @@ class DequantizeOp : public OpKernel { if (axis_ > -1) { num_slices = input.dim_size(axis_); } + OP_REQUIRES(ctx, input_min_tensor.NumElements() == num_slices, + errors::InvalidArgument( + "input_min_tensor must have as many elements as input on " + "the dequantization axis (", + axis_, "), got ", input_min_tensor.NumElements(), + ", expected ", num_slices)); + OP_REQUIRES(ctx, input_max_tensor.NumElements() == num_slices, + errors::InvalidArgument( + "input_max_tensor must have as many elements as input on " + "the dequantization axis (", + axis_, "), got ", input_max_tensor.NumElements(), + ", expected ", num_slices)); Tensor* output = nullptr; OP_REQUIRES_OK(ctx, ctx->allocate_output(0, input.shape(), &output)); diff --git a/tensorflow/core/kernels/dilation_ops.cc b/tensorflow/core/kernels/dilation_ops.cc index 738ea31d555d5f..996ddb62bfefeb 100644 --- a/tensorflow/core/kernels/dilation_ops.cc +++ b/tensorflow/core/kernels/dilation_ops.cc @@ -130,6 +130,7 @@ class DilationOp : public OpKernel { ParseSizes(context, strides_, rates_, padding_, &stride_rows, &stride_cols, &rate_rows, &rate_cols, &pad_top, &pad_left, &out_rows, &out_cols); + if (!context->status().ok()) return; // Output tensor is of the following dimensions: // [ batch, out_rows, out_cols, depth ] @@ -229,6 +230,7 @@ class DilationBackpropInputOp : public OpKernel { ParseSizes(context, strides_, rates_, padding_, &stride_rows, &stride_cols, &rate_rows, &rate_cols, &pad_top, &pad_left, &out_rows, &out_cols); + if (!context->status().ok()) return; // Verify that the incoming gradient tensor has the expected size // [ batch, out_rows, out_cols, depth ] @@ -318,8 +320,10 @@ struct DilationBackpropInput { } } } - in_backprop(b, h_in_max, w_in_max, d) += - out_backprop(b, h_out, w_out, d); + if (h_in_max < input_rows && w_in_max < input_cols) { + in_backprop(b, h_in_max, w_in_max, d) += + out_backprop(b, h_out, w_out, d); + } } } } @@ -349,6 +353,7 @@ class DilationBackpropFilterOp : public OpKernel { ParseSizes(context, strides_, rates_, padding_, &stride_rows, &stride_cols, &rate_rows, &rate_cols, &pad_top, &pad_left, &out_rows, &out_cols); + if (!context->status().ok()) return; // Verify that the incoming gradient tensor has the expected size // [ batch, out_rows, out_cols, depth ] @@ -438,8 +443,10 @@ struct DilationBackpropFilter { } } } - filter_backprop(h_max, w_max, d) += - out_backprop(b, h_out, w_out, d); + if (h_max < filter_rows && w_max < filter_cols) { + filter_backprop(h_max, w_max, d) += + out_backprop(b, h_out, w_out, d); + } } } } diff --git a/tensorflow/core/kernels/draw_bounding_box_op.cc b/tensorflow/core/kernels/draw_bounding_box_op.cc index 30de99b7d560a2..926ea368a58ba8 100644 --- a/tensorflow/core/kernels/draw_bounding_box_op.cc +++ b/tensorflow/core/kernels/draw_bounding_box_op.cc @@ -73,6 +73,12 @@ class DrawBoundingBoxesOp : public OpKernel { errors::InvalidArgument("Channel depth should be either 1 (GRY), " "3 (RGB), or 4 (RGBA)")); + OP_REQUIRES( + context, boxes.dim_size(2) == 4, + errors::InvalidArgument( + "The size of the third dimension of the box must be 4. Received: ", + boxes.dim_size(2))); + const int64 batch_size = images.dim_size(0); const int64 height = images.dim_size(1); const int64 width = images.dim_size(2); @@ -147,22 +153,46 @@ class DrawBoundingBoxesOp : public OpKernel { // At this point, {min,max}_box_{row,col}_clamp are inside the // image. - CHECK_GE(min_box_row_clamp, 0); - CHECK_GE(max_box_row_clamp, 0); - CHECK_LT(min_box_row_clamp, height); - CHECK_LT(max_box_row_clamp, height); - CHECK_GE(min_box_col_clamp, 0); - CHECK_GE(max_box_col_clamp, 0); - CHECK_LT(min_box_col_clamp, width); - CHECK_LT(max_box_col_clamp, width); + OP_REQUIRES( + context, min_box_row_clamp >= 0, + errors::InvalidArgument("Min box row clamp is less than 0.")); + OP_REQUIRES( + context, max_box_row_clamp >= 0, + errors::InvalidArgument("Max box row clamp is less than 0.")); + OP_REQUIRES(context, min_box_row_clamp <= height, + errors::InvalidArgument( + "Min box row clamp is greater than height.")); + OP_REQUIRES(context, max_box_row_clamp <= height, + errors::InvalidArgument( + "Max box row clamp is greater than height.")); + + OP_REQUIRES( + context, min_box_col_clamp >= 0, + errors::InvalidArgument("Min box col clamp is less than 0.")); + OP_REQUIRES( + context, max_box_col_clamp >= 0, + errors::InvalidArgument("Max box col clamp is less than 0.")); + OP_REQUIRES(context, min_box_col_clamp <= width, + errors::InvalidArgument( + "Min box col clamp is greater than width.")); + OP_REQUIRES(context, max_box_col_clamp <= width, + errors::InvalidArgument( + "Max box col clamp is greater than width.")); // At this point, the min_box_row and min_box_col are either // in the image or above/left of it, and max_box_row and // max_box_col are either in the image or below/right or it. - CHECK_LT(min_box_row, height); - CHECK_GE(max_box_row, 0); - CHECK_LT(min_box_col, width); - CHECK_GE(max_box_col, 0); + + OP_REQUIRES( + context, min_box_row <= height, + errors::InvalidArgument("Min box row is greater than height.")); + OP_REQUIRES(context, max_box_row >= 0, + errors::InvalidArgument("Max box row is less than 0.")); + OP_REQUIRES( + context, min_box_col <= width, + errors::InvalidArgument("Min box col is greater than width.")); + OP_REQUIRES(context, max_box_col >= 0, + errors::InvalidArgument("Max box col is less than 0.")); // Draw top line. if (min_box_row >= 0) { diff --git a/tensorflow/core/kernels/edit_distance_op.cc b/tensorflow/core/kernels/edit_distance_op.cc index 4aecdc9e414d36..386a1af08409f6 100644 --- a/tensorflow/core/kernels/edit_distance_op.cc +++ b/tensorflow/core/kernels/edit_distance_op.cc @@ -64,6 +64,12 @@ Status ValidateShapes(OpKernelContext* ctx, const Tensor& hypothesis_indices, return errors::InvalidArgument( "truth_shape should be a vector, but got shape: ", truth_shape.shape().DebugString()); + if (hypothesis_values.NumElements() != hypothesis_indices.dim_size(0)) + return errors::InvalidArgument( + "Expected hypothesis_values.NumElements == " + "#rows(hypothesis_indices), their shapes are: ", + hypothesis_values.shape().DebugString(), " and ", + hypothesis_indices.shape().DebugString()); if (hypothesis_shape.NumElements() != hypothesis_indices.dim_size(1)) return errors::InvalidArgument( "Expected hypothesis_shape.NumElements == " @@ -75,6 +81,12 @@ Status ValidateShapes(OpKernelContext* ctx, const Tensor& hypothesis_indices, "Input SparseTensors must have rank at least 2, but truth_shape " "rank is: ", truth_shape.NumElements()); + if (truth_values.NumElements() != truth_indices.dim_size(0)) + return errors::InvalidArgument( + "Expected truth_values.NumElements == " + "#rows(truth_indices), their shapes are: ", + truth_values.shape().DebugString(), " and ", + truth_indices.shape().DebugString()); if (truth_shape.NumElements() != truth_indices.dim_size(1)) return errors::InvalidArgument( "Expected truth_shape.NumElements == " @@ -153,6 +165,11 @@ class EditDistanceOp : public OpKernel { output_shape.AddDim(std::max(hypothesis_st_shape.dim_size(d), truth_st_shape.dim_size(d))); } + const auto output_elements = output_shape.num_elements(); + OP_REQUIRES( + ctx, output_elements > 0, + errors::InvalidArgument("Got output shape ", output_shape.DebugString(), + " which has 0 elements")); Tensor* output = nullptr; OP_REQUIRES_OK(ctx, ctx->allocate_output("output", output_shape, &output)); @@ -185,6 +202,12 @@ class EditDistanceOp : public OpKernel { if (g_truth == g_hypothesis) { auto loc = std::inner_product(g_truth.begin(), g_truth.end(), output_strides.begin(), int64{0}); + OP_REQUIRES( + ctx, loc < output_elements, + errors::Internal("Got an inner product ", loc, + " which would require in writing to outside of " + "the buffer for the output tensor (max elements ", + output_elements, ")")); output_t(loc) = gtl::LevenshteinDistance(truth_seq, hypothesis_seq, cmp); if (normalize_) output_t(loc) /= truth_seq.size(); @@ -194,6 +217,12 @@ class EditDistanceOp : public OpKernel { } else if (g_truth > g_hypothesis) { // zero-length truth auto loc = std::inner_product(g_hypothesis.begin(), g_hypothesis.end(), output_strides.begin(), int64{0}); + OP_REQUIRES( + ctx, loc < output_elements, + errors::Internal("Got an inner product ", loc, + " which would require in writing to outside of " + "the buffer for the output tensor (max elements ", + output_elements, ")")); output_t(loc) = hypothesis_seq.size(); if (normalize_ && output_t(loc) != 0.0f) { output_t(loc) = std::numeric_limits::infinity(); @@ -202,6 +231,12 @@ class EditDistanceOp : public OpKernel { } else { // zero-length hypothesis auto loc = std::inner_product(g_truth.begin(), g_truth.end(), output_strides.begin(), int64{0}); + OP_REQUIRES( + ctx, loc < output_elements, + errors::Internal("Got an inner product ", loc, + " which would require in writing to outside of " + "the buffer for the output tensor (max elements ", + output_elements, ")")); output_t(loc) = (normalize_) ? 1.0 : truth_seq.size(); ++truth_iter; } @@ -212,6 +247,12 @@ class EditDistanceOp : public OpKernel { auto hypothesis_seq = hypothesis_j.values(); auto loc = std::inner_product(g_hypothesis.begin(), g_hypothesis.end(), output_strides.begin(), int64{0}); + OP_REQUIRES( + ctx, loc < output_elements, + errors::Internal("Got an inner product ", loc, + " which would require in writing to outside of the " + "buffer for the output tensor (max elements ", + output_elements, ")")); output_t(loc) = hypothesis_seq.size(); if (normalize_ && output_t(loc) != 0.0f) { output_t(loc) = std::numeric_limits::infinity(); @@ -224,6 +265,12 @@ class EditDistanceOp : public OpKernel { auto truth_seq = truth_i.values(); auto loc = std::inner_product(g_truth.begin(), g_truth.end(), output_strides.begin(), int64{0}); + OP_REQUIRES( + ctx, loc < output_elements, + errors::Internal("Got an inner product ", loc, + " which would require in writing to outside of the " + "buffer for the output tensor (max elements ", + output_elements, ")")); output_t(loc) = (normalize_) ? 1.0 : truth_seq.size(); ++truth_iter; } diff --git a/tensorflow/core/kernels/encode_png_op.cc b/tensorflow/core/kernels/encode_png_op.cc index 8dbe1d377df5c6..09bcdbe5e3db0b 100644 --- a/tensorflow/core/kernels/encode_png_op.cc +++ b/tensorflow/core/kernels/encode_png_op.cc @@ -54,6 +54,8 @@ class EncodePngOp : public OpKernel { OP_REQUIRES(context, image.dims() == 3, errors::InvalidArgument("image must be 3-dimensional", image.shape().DebugString())); + OP_REQUIRES(context, image.NumElements() > 0, + errors::Internal("Invalid image provided.")); OP_REQUIRES( context, FastBoundsCheck(image.NumElements(), std::numeric_limits::max()), diff --git a/tensorflow/core/kernels/fft_ops.cc b/tensorflow/core/kernels/fft_ops.cc index 058435948394c5..603f9383616399 100644 --- a/tensorflow/core/kernels/fft_ops.cc +++ b/tensorflow/core/kernels/fft_ops.cc @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#include "tensorflow/core/platform/errors.h" #define EIGEN_USE_THREADS // See docs in ../ops/fft_ops.cc. @@ -221,6 +222,9 @@ class FFTCPU : public FFTBase { input_slice_sizes[i] = fft_shape[i - 1]; temp_shape.AddDim(fft_shape[i - 1]); } + OP_REQUIRES(ctx, temp_shape.num_elements() > 0, + errors::InvalidArgument("Obtained a FFT shape of 0 elements: ", + temp_shape.DebugString())); auto output = out->flat_inner_dims(); const Eigen::DSizes zero_start_indices; @@ -261,6 +265,9 @@ class FFTCPU : public FFTBase { i == FFTRank ? fft_shape[i - 1] / 2 + 1 : fft_shape[i - 1]; full_fft_shape.AddDim(fft_shape[i - 1]); } + OP_REQUIRES(ctx, full_fft_shape.num_elements() > 0, + errors::InvalidArgument("Obtained a FFT shape of 0 elements: ", + full_fft_shape.DebugString())); Tensor temp; OP_REQUIRES_OK(ctx, ctx->allocate_temp(DataTypeToEnum::v(), diff --git a/tensorflow/core/kernels/fractional_avg_pool_op.cc b/tensorflow/core/kernels/fractional_avg_pool_op.cc index dfc2382624e3fa..7c396126427473 100644 --- a/tensorflow/core/kernels/fractional_avg_pool_op.cc +++ b/tensorflow/core/kernels/fractional_avg_pool_op.cc @@ -80,6 +80,10 @@ class FractionalAvgPoolOp : public OpKernel { std::vector output_size(tensor_in_and_out_dims); for (int i = 0; i < tensor_in_and_out_dims; ++i) { input_size[i] = tensor_in.dim_size(i); + OP_REQUIRES( + context, pooling_ratio_[i] <= input_size[i], + errors::InvalidArgument( + "Pooling ratio cannot be bigger than input tensor dim size.")); } // Output size. for (int i = 0; i < tensor_in_and_out_dims; ++i) { @@ -246,6 +250,19 @@ class FractionalAvgPoolGradOp : public OpKernel { const int64 out_cols = out_backprop.dim_size(2); const int64 out_depth = out_backprop.dim_size(3); + OP_REQUIRES(context, row_seq_tensor.NumElements() > out_rows, + errors::InvalidArgument("Given out_backprop shape ", + out_backprop.shape().DebugString(), + ", row_seq_tensor must have at least ", + out_rows + 1, " elements, but got ", + row_seq_tensor.NumElements())); + OP_REQUIRES(context, col_seq_tensor.NumElements() > out_cols, + errors::InvalidArgument("Given out_backprop shape ", + out_backprop.shape().DebugString(), + ", col_seq_tensor must have at least ", + out_cols + 1, " elements, but got ", + col_seq_tensor.NumElements())); + auto row_seq_tensor_flat = row_seq_tensor.flat(); auto col_seq_tensor_flat = col_seq_tensor.flat(); auto orig_input_tensor_shape_flat = orig_input_tensor_shape.flat(); @@ -254,6 +271,18 @@ class FractionalAvgPoolGradOp : public OpKernel { const int64 in_rows = orig_input_tensor_shape_flat(1); const int64 in_cols = orig_input_tensor_shape_flat(2); const int64 in_depth = orig_input_tensor_shape_flat(3); + OP_REQUIRES( + context, in_batch != 0, + errors::InvalidArgument("Batch dimension of input must not be 0")); + OP_REQUIRES( + context, in_rows != 0, + errors::InvalidArgument("Rows dimension of input must not be 0")); + OP_REQUIRES( + context, in_cols != 0, + errors::InvalidArgument("Columns dimension of input must not be 0")); + OP_REQUIRES( + context, in_depth != 0, + errors::InvalidArgument("Depth dimension of input must not be 0")); constexpr int tensor_in_and_out_dims = 4; // Transform orig_input_tensor_shape into TensorShape diff --git a/tensorflow/core/kernels/fractional_max_pool_op.cc b/tensorflow/core/kernels/fractional_max_pool_op.cc index 619a3507ce415f..1a2a783d135c54 100644 --- a/tensorflow/core/kernels/fractional_max_pool_op.cc +++ b/tensorflow/core/kernels/fractional_max_pool_op.cc @@ -235,6 +235,20 @@ class FractionalMaxPoolGradOp : public OpKernel { // Just to make it similar to FractionalMaxPoolOp. constexpr int tensor_in_and_out_dims = 4; + OP_REQUIRES( + context, tensor_in.dims() == tensor_in_and_out_dims, + errors::InvalidArgument("orig_input should be a tensor of rank 4, got ", + tensor_in.DebugString())); + OP_REQUIRES(context, tensor_in.NumElements() > 0, + errors::InvalidArgument("orig_input must not be empty, got ", + tensor_in.DebugString())); + OP_REQUIRES(context, tensor_out.dims() == tensor_in_and_out_dims, + errors::InvalidArgument( + "orig_output should be a tensor of rank 4, got ", + tensor_out.DebugString())); + OP_REQUIRES(context, tensor_out.NumElements() > 0, + errors::InvalidArgument("orig_output must not be empty, got ", + tensor_out.DebugString())); std::vector input_size(tensor_in_and_out_dims); std::vector output_size(tensor_in_and_out_dims); for (int i = 0; i < tensor_in_and_out_dims; ++i) { diff --git a/tensorflow/core/kernels/fused_batch_norm_op.cc b/tensorflow/core/kernels/fused_batch_norm_op.cc index 00ac9be6dcd17f..bd1b94d34b96fc 100644 --- a/tensorflow/core/kernels/fused_batch_norm_op.cc +++ b/tensorflow/core/kernels/fused_batch_norm_op.cc @@ -293,6 +293,9 @@ struct FusedBatchNorm { const CPUDevice& d = context->eigen_device(); const int depth = x.dimension(3); + OP_REQUIRES( + context, depth != 0, + errors::Internal("The 4th element in the input shape cannot be 0.")); const int size = x.size(); const int rest_size = size / depth; Eigen::DSizes rest_by_depth(rest_size, depth); @@ -1264,6 +1267,33 @@ class FusedBatchNormOpBase : public OpKernel { context, estimated_variance.dims() == 1, errors::InvalidArgument("estimated_variance must be 1-dimensional", estimated_variance.shape().DebugString())); + + const auto num_channels = GetTensorDim(x, tensor_format_, 'C'); + OP_REQUIRES( + context, scale.NumElements() == num_channels, + errors::InvalidArgument("scale must have the same number of elements " + "as the channels of x, got ", + scale.NumElements(), " and ", num_channels)); + OP_REQUIRES( + context, offset.NumElements() == num_channels, + errors::InvalidArgument("offset must have the same number of elements " + "as the channels of x, got ", + offset.NumElements(), " and ", num_channels)); + if (estimated_mean.NumElements() != 0) { + OP_REQUIRES(context, estimated_mean.NumElements() == num_channels, + errors::InvalidArgument( + "mean must be empty or have the same number of " + "elements as the channels of x, got ", + estimated_mean.NumElements(), " and ", num_channels)); + } + if (estimated_variance.NumElements() != 0) { + OP_REQUIRES(context, estimated_variance.NumElements() == num_channels, + errors::InvalidArgument( + "variance must be empty or have the same number of " + "elements as the channels of x, got ", + estimated_variance.NumElements(), " and ", num_channels)); + } + if (has_side_input_) { OP_REQUIRES(context, side_input->shape() == x.shape(), errors::InvalidArgument( @@ -1276,7 +1306,7 @@ class FusedBatchNormOpBase : public OpKernel { // NOTE(ezhulenev): This requirement is coming from implementation // details of cudnnBatchNormalizationForwardTrainingEx. OP_REQUIRES( - context, !is_training_ || x.dim_size(3) % 4 == 0, + context, !is_training_ || num_channels % 4 == 0, errors::InvalidArgument("FusedBatchNorm with activation requires " "channel dimension to be a multiple of 4.")); } diff --git a/tensorflow/core/kernels/immutable_constant_op.cc b/tensorflow/core/kernels/immutable_constant_op.cc index 0dd08c694eb6c5..19aa865c1fbe4d 100644 --- a/tensorflow/core/kernels/immutable_constant_op.cc +++ b/tensorflow/core/kernels/immutable_constant_op.cc @@ -17,6 +17,8 @@ limitations under the License. #include +#include "tensorflow/core/framework/types.pb.h" + namespace tensorflow { namespace { @@ -62,6 +64,12 @@ class MemmappedTensorAllocator : public Allocator { void set_delete_on_deallocate() { delete_on_deallocate_ = true; } + // Make sure tensors or complex types (strings, variants, resources) don't get + // their constructor called via a placement new since that would require + // writing to immutable data. + // See also: tensorflow/core/framework/typed_allocator.h + bool AllocatesOpaqueHandle() const override { return true; } + private: std::unique_ptr memory_region_; // If there is an error during allocation we keep it in this status. @@ -80,6 +88,9 @@ ImmutableConstantOp::ImmutableConstantOp(OpKernelConstruction* context) OP_REQUIRES_OK(context, context->GetAttr(kMemoryRegionNameAttr, ®ion_name_)); OP_REQUIRES_OK(context, context->GetAttr(kDTypeAttr, &dtype_)); + OP_REQUIRES(context, dtype_ != DT_RESOURCE && dtype_ != DT_VARIANT, + errors::InvalidArgument( + "Resource and variant dtypes are invalid for this op.")); OP_REQUIRES_OK(context, context->GetAttr(kShapeAttr, &shape_)); } diff --git a/tensorflow/core/kernels/inplace_ops.cc b/tensorflow/core/kernels/inplace_ops.cc index b5191b9989f328..008e9732b55768 100644 --- a/tensorflow/core/kernels/inplace_ops.cc +++ b/tensorflow/core/kernels/inplace_ops.cc @@ -280,7 +280,7 @@ class InplaceOpBase : public OpKernel { Tensor y = x; // This creates an alias intentionally. // Skip processing if tensors are empty. - if (x.NumElements() > 0 || v.NumElements() > 0) { + if (x.NumElements() > 0 && v.NumElements() > 0) { OP_REQUIRES_OK(ctx, DoCompute(ctx, i, v, &y)); } ctx->set_output(0, y); diff --git a/tensorflow/core/kernels/list_kernels.cc b/tensorflow/core/kernels/list_kernels.cc index 9a2f373f5ce0cf..488e02337f707b 100644 --- a/tensorflow/core/kernels/list_kernels.cc +++ b/tensorflow/core/kernels/list_kernels.cc @@ -302,6 +302,10 @@ class TensorListReserve : public OpKernel { PartialTensorShape element_shape; OP_REQUIRES_OK(c, TensorShapeFromTensor(c->input(0), &element_shape)); int32 num_elements = c->input(1).scalar()(); + OP_REQUIRES(c, num_elements >= 0, + errors::InvalidArgument("The num_elements to reserve must be a " + "non negative number, but got ", + num_elements)); TensorList output; output.element_shape = element_shape; output.element_dtype = element_dtype_; diff --git a/tensorflow/core/kernels/load_and_remap_matrix_op.cc b/tensorflow/core/kernels/load_and_remap_matrix_op.cc index cb0245a9b61261..5ec28c70358132 100644 --- a/tensorflow/core/kernels/load_and_remap_matrix_op.cc +++ b/tensorflow/core/kernels/load_and_remap_matrix_op.cc @@ -123,6 +123,11 @@ class LoadAndRemapMatrixOp : public OpKernel { // Processes the checkpoint source and the provided Tensor name. const Tensor* ckpt_path_t; OP_REQUIRES_OK(context, context->input("ckpt_path", &ckpt_path_t)); + OP_REQUIRES( + context, ckpt_path_t->NumElements() == 1, + errors::InvalidArgument("The `ckpt_path` tensor must have exactly one " + "element, got tensor of shape ", + ckpt_path_t->shape().DebugString())); const string& ckpt_path = ckpt_path_t->scalar()(); const Tensor* old_tensor_name_t; OP_REQUIRES_OK(context, diff --git a/tensorflow/core/kernels/map_stage_op.cc b/tensorflow/core/kernels/map_stage_op.cc index 6c01e42ff8c9fd..ed5f363eb66305 100644 --- a/tensorflow/core/kernels/map_stage_op.cc +++ b/tensorflow/core/kernels/map_stage_op.cc @@ -210,9 +210,9 @@ class StagingMap : public ResourceBase { const OptionalTuple& tuple) TF_EXCLUSIVE_LOCKS_REQUIRED(mu_) { if (tuple[index].has_value()) { - return Status(errors::InvalidArgument( + return errors::InvalidArgument( "The tensor for index '", index, "' for key '", key.scalar()(), - "' was already initialized '", dtypes_.size(), "'.")); + "' was already initialized '", dtypes_.size(), "'."); } return Status::OK(); @@ -220,6 +220,10 @@ class StagingMap : public ResourceBase { // Check that the indices are strictly ordered Status check_index_ordering(const Tensor& indices) { + if (indices.NumElements() == 0) { + return errors::InvalidArgument("Indices are empty"); + } + auto findices = indices.flat(); for (std::size_t i = 0; i < findices.dimension(0) - 1; ++i) { @@ -227,8 +231,7 @@ class StagingMap : public ResourceBase { continue; } - return Status( - errors::InvalidArgument("Indices are not strictly ordered")); + return errors::InvalidArgument("Indices are not strictly ordered"); } return Status::OK(); @@ -238,10 +241,10 @@ class StagingMap : public ResourceBase { Status check_memory_limit(std::size_t bytes) TF_EXCLUSIVE_LOCKS_REQUIRED(mu_) { if (has_memory_limit() && bytes > memory_limit_) { - return Status(errors::ResourceExhausted( + return errors::ResourceExhausted( "Attempted to insert tensors with combined size of '", bytes, "' bytes into Staging Area with a memory limit of '", memory_limit_, - "'.")); + "'."); } return Status::OK(); @@ -527,6 +530,8 @@ class MapStageOp : public OpKernel { OP_REQUIRES_OK(ctx, ctx->input("key", &key_tensor)); OP_REQUIRES_OK(ctx, ctx->input("indices", &indices_tensor)); OP_REQUIRES_OK(ctx, ctx->input_list("values", &values_tensor)); + OP_REQUIRES(ctx, key_tensor->NumElements() > 0, + errors::InvalidArgument("key must not be empty")); // Create copy for insertion into Staging Area Tensor key(*key_tensor); diff --git a/tensorflow/core/kernels/matrix_diag_op.cc b/tensorflow/core/kernels/matrix_diag_op.cc index 05d7e4e6f86752..bb5b27c2cbe928 100644 --- a/tensorflow/core/kernels/matrix_diag_op.cc +++ b/tensorflow/core/kernels/matrix_diag_op.cc @@ -73,6 +73,9 @@ class MatrixDiagPartOp : public OpKernel { errors::InvalidArgument( "diag_index must be a scalar or vector, received shape: ", diag_index.shape().DebugString())); + OP_REQUIRES(context, diag_index.NumElements() > 0, + errors::InvalidArgument( + "Expected diag_index to have at least 1 element")); lower_diag_index = diag_index.flat()(0); upper_diag_index = lower_diag_index; if (TensorShapeUtils::IsVector(diag_index.shape())) { @@ -86,7 +89,10 @@ class MatrixDiagPartOp : public OpKernel { upper_diag_index = diag_index.flat()(1); } } - padding_value = context->input(2).flat()(0); + const Tensor& padding_in = context->input(2); + OP_REQUIRES(context, padding_in.NumElements() == 1, + errors::InvalidArgument("Padding must be scalar.")); + padding_value = padding_in.flat()(0); } const TensorShape& input_shape = input.shape(); @@ -179,6 +185,9 @@ class MatrixDiagOp : public OpKernel { errors::InvalidArgument( "diag_index must be a scalar or vector, received shape: ", diag_index.shape().DebugString())); + OP_REQUIRES(context, diag_index.NumElements() > 0, + errors::InvalidArgument( + "Expected diag_index to have at least 1 element")); lower_diag_index = diag_index.flat()(0); upper_diag_index = lower_diag_index; if (TensorShapeUtils::IsVector(diag_index.shape())) { @@ -192,9 +201,22 @@ class MatrixDiagOp : public OpKernel { upper_diag_index = diag_index.flat()(1); } } - num_rows = context->input(2).flat()(0); - num_cols = context->input(3).flat()(0); - padding_value = context->input(4).flat()(0); + + auto& num_rows_tensor = context->input(2); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(num_rows_tensor.shape()), + errors::InvalidArgument("num_rows must be a scalar")); + num_rows = num_rows_tensor.flat()(0); + + auto& num_cols_tensor = context->input(3); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(num_cols_tensor.shape()), + errors::InvalidArgument("num_cols must be a scalar")); + num_cols = num_cols_tensor.flat()(0); + + auto& padding_value_tensor = context->input(4); + OP_REQUIRES(context, + TensorShapeUtils::IsScalar(padding_value_tensor.shape()), + errors::InvalidArgument("padding_value must be a scalar")); + padding_value = padding_value_tensor.flat()(0); } // Size validations. diff --git a/tensorflow/core/kernels/matrix_set_diag_op.cc b/tensorflow/core/kernels/matrix_set_diag_op.cc index bf98fd0d47d65b..e1c47d814050e1 100644 --- a/tensorflow/core/kernels/matrix_set_diag_op.cc +++ b/tensorflow/core/kernels/matrix_set_diag_op.cc @@ -70,6 +70,9 @@ class MatrixSetDiagOp : public OpKernel { errors::InvalidArgument( "diag_index must be a scalar or vector, received shape: ", diag_index.shape().DebugString())); + OP_REQUIRES( + context, diag_index.NumElements() > 0, + errors::InvalidArgument("diag_index must have at least one element")); lower_diag_index = diag_index.flat()(0); upper_diag_index = lower_diag_index; if (TensorShapeUtils::IsVector(diag_index.shape())) { diff --git a/tensorflow/core/kernels/matrix_triangular_solve_op_impl.h b/tensorflow/core/kernels/matrix_triangular_solve_op_impl.h index fb7e6f0f5ffe1d..d4ffcf18a52290 100644 --- a/tensorflow/core/kernels/matrix_triangular_solve_op_impl.h +++ b/tensorflow/core/kernels/matrix_triangular_solve_op_impl.h @@ -162,6 +162,9 @@ class BaseMatrixTriangularSolveOp : public OpKernel { const Tensor& in1 = ctx->input(1); ValidateInputTensors(ctx, in0, in1); + if (!ctx->status().ok()) { + return; + } MatMulBCast bcast(in0.shape().dim_sizes(), in1.shape().dim_sizes()); OP_REQUIRES( @@ -230,13 +233,22 @@ class MatrixTriangularSolveOp private: void ValidateInputTensors(OpKernelContext* ctx, const Tensor& in0, const Tensor& in1) override { + const auto in0_num_dims = in0.dims(); OP_REQUIRES( - ctx, in0.dims() >= 2, - errors::InvalidArgument("In[0] ndims must be >= 2: ", in0.dims())); + ctx, in0_num_dims >= 2, + errors::InvalidArgument("In[0] ndims must be >= 2: ", in0_num_dims)); + const auto in1_num_dims = in1.dims(); OP_REQUIRES( - ctx, in1.dims() >= 2, - errors::InvalidArgument("In[0] ndims must be >= 2: ", in1.dims())); + ctx, in1_num_dims >= 2, + errors::InvalidArgument("In[1] ndims must be >= 2: ", in1_num_dims)); + + const auto in0_last_dim = in0.dim_size(in0_num_dims - 1); + const auto in0_prev_dim = in0.dim_size(in0_num_dims - 2); + OP_REQUIRES(ctx, in0_last_dim == in0_prev_dim, + errors::InvalidArgument( + "In[0] matrices in the last dimensions must be square (", + in0_last_dim, " =/= ", in0_prev_dim, ")")); } }; diff --git a/tensorflow/core/kernels/maxpooling_op.cc b/tensorflow/core/kernels/maxpooling_op.cc index 5652addd00a957..adfde788097deb 100644 --- a/tensorflow/core/kernels/maxpooling_op.cc +++ b/tensorflow/core/kernels/maxpooling_op.cc @@ -68,6 +68,7 @@ static void SpatialMaxPoolWithArgMaxHelper( "SpatialMaxPoolWithArgMaxHelper requires include_batch_in_index " "to be True when when input_backprop != nullptr")); } + if (tensor_in.NumElements() == 0 || output->NumElements() == 0) return; typedef Eigen::Map> ConstEigenMatrixMap; @@ -192,7 +193,9 @@ static void SpatialMaxPoolWithArgMaxHelper( // CHECK(input_backprop_index >= in_start && input_backprop_index < // in_end) FastBoundsCheck(input_backprop_index - in_start, in_end - in_start); - input_backprop_flat(input_backprop_index) += out_backprop_flat(index); + if (index < out_backprop.NumElements()) { + input_backprop_flat(input_backprop_index) += out_backprop_flat(index); + } } } }; @@ -922,6 +925,10 @@ class MaxPoolingWithArgmaxOp : public OpKernel { void Compute(OpKernelContext* context) override { const Tensor& tensor_in = context->input(0); + OP_REQUIRES(context, tensor_in.dims() == 4, + errors::InvalidArgument("tensor_in must be 4-dimensional (2)")); + OP_REQUIRES(context, tensor_in.NumElements() > 0, + errors::InvalidArgument("tensor_in must not be empty (2)")); PoolParameters params{context, ksize_, stride_, padding_, FORMAT_NHWC, tensor_in.shape()}; @@ -984,6 +991,9 @@ struct LaunchMaxPoolingGradWithArgmax { const int input_start = start * input_size_per_batch; const int input_end = limit * input_size_per_batch; for (int64 index = input_start; index < input_end; index++) { + if (index >= argmax.NumElements()) { + break; + } int64 grad_out_index = argmax_flat(index); if (!include_batch_in_index) { const int64 cur_batch = index / input_size_per_batch; @@ -1049,6 +1059,8 @@ class MaxPoolingGradWithArgmaxOp : public OpKernel { OP_REQUIRES_OK(context, context->forward_input_or_allocate_output( {0}, 0, out_shape, &grad_out)); + if (out_shape.num_elements() == 0) return; // nothing to be done + LaunchMaxPoolingGradWithArgmax::launch( context, params, grad_in, argmax, grad_out, include_batch_in_index_); } diff --git a/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc b/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc index 0cd4843c0d8659..bb1045c7f513d5 100644 --- a/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc +++ b/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc @@ -57,6 +57,20 @@ class MklRequantizationRangePerChannelOp : public OpKernel { ctx, input_max.dim_size(0) == depth, errors::InvalidArgument("input_max has incorrect size, expected ", depth, " was ", input_max.dim_size(0))); + OP_REQUIRES( + ctx, input_min.NumElements() == depth, + errors::InvalidArgument("input_min must have the same number of " + "elements as input_max, got ", + input_min.NumElements(), " and ", depth)); + OP_REQUIRES(ctx, input.NumElements() > 0, + errors::InvalidArgument("input must not be empty")); + OP_REQUIRES(ctx, input.dims() == 4, + errors::InvalidArgument("input must be in NHWC format")); + OP_REQUIRES( + ctx, input.dim_size(3) == depth, + errors::InvalidArgument( + "input must have same number of channels as length of input_min: ", + input.dim_size(3), " vs ", depth)); const float* input_min_data = input_min.flat().data(); const float* input_max_data = input_max.flat().data(); diff --git a/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc b/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc index 0a0464f648b95b..4cac0e0ecdc76e 100644 --- a/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc +++ b/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc @@ -49,35 +49,45 @@ class MklRequantizePerChannelOp : public OpKernel { void Compute(OpKernelContext* ctx) override { try { const Tensor& input = ctx->input(kInputTensorIndex); + OP_REQUIRES( + ctx, input.dims() == 4, + errors::InvalidArgument("Current RequantizePerChannel operator" + "supports 4D tensors only.")); + const Tensor& input_min_vec = ctx->input(kInputMinVecIndex); + size_t depth = input_min_vec.NumElements(); float* input_min_vec_data = (float*)const_cast( static_cast(input_min_vec.flat().data())); + const Tensor& input_max_vec = ctx->input(kInputMaxVecIndex); + OP_REQUIRES( + ctx, input_max_vec.NumElements() == depth, + errors::InvalidArgument("input_max has incorrect size, expected ", + depth, " was ", input_max_vec.NumElements())); float* input_max_vec_data = (float*)const_cast( static_cast(input_max_vec.flat().data())); const Tensor& input_requested_min = ctx->input(this->kRequestMinIndex); + OP_REQUIRES( + ctx, input_requested_min.NumElements() == 1, + errors::InvalidArgument("requested_output_min must be a scalar")); const float input_requested_min_float = input_requested_min.flat()(0); + const Tensor& input_requested_max = ctx->input(this->kRequestMaxIndex); + OP_REQUIRES( + ctx, input_requested_min.NumElements() == 1, + errors::InvalidArgument("requested_output_max must be a scalar")); const float input_requested_max_float = input_requested_max.flat()(0); - size_t depth = input_min_vec.NumElements(); - OP_REQUIRES( - ctx, input.dims() == 4, - errors::InvalidArgument("Current RequantizePerChannel operator" - "supports 4D tensors only.")); - OP_REQUIRES( - ctx, input_min_vec.dim_size(0) == depth, - errors::InvalidArgument("input_min has incorrect size, expected ", - depth, " was ", input_min_vec.dim_size(0))); - OP_REQUIRES( - ctx, input_max_vec.dim_size(0) == depth, - errors::InvalidArgument("input_max has incorrect size, expected ", - depth, " was ", input_max_vec.dim_size(0))); - - if (out_type_ == DT_QINT8) DCHECK(input_requested_min_float < 0.0f); + if (out_type_ == DT_QINT8) { + OP_REQUIRES(ctx, input_requested_min_float < 0.0f, + errors::InvalidArgument( + "If out_type is QINT8, requested_output_max must be " + "non negative, got ", + input_requested_min_float)); + } const float factor = (out_type_ == DT_QINT8) ? 127.0f : 255.0f; const float requested_min_max = diff --git a/tensorflow/core/kernels/non_max_suppression_op.cc b/tensorflow/core/kernels/non_max_suppression_op.cc index 20ae3a2e0d07f6..04ea7b5c8000a6 100644 --- a/tensorflow/core/kernels/non_max_suppression_op.cc +++ b/tensorflow/core/kernels/non_max_suppression_op.cc @@ -161,6 +161,8 @@ void DoNonMaxSuppressionOp(OpKernelContext* context, const Tensor& scores, bool pad_to_max_output_size = false, int* ptr_num_valid_outputs = nullptr) { const int output_size = max_output_size.scalar()(); + OP_REQUIRES(context, output_size >= 0, + errors::InvalidArgument("output size must be non-negative")); std::vector scores_data(num_boxes); std::copy_n(scores.flat().data(), num_boxes, scores_data.begin()); @@ -759,6 +761,9 @@ class NonMaxSuppressionV4Op : public OpKernel { context, scores, num_boxes, max_output_size, iou_threshold_val, score_threshold_val, dummy_soft_nms_sigma, similarity_fn, return_scores_tensor_, pad_to_max_output_size_, &num_valid_outputs); + if (!context->status().ok()) { + return; + } // Allocate scalar output tensor for number of indices computed. Tensor* num_outputs_t = nullptr; @@ -836,6 +841,9 @@ class NonMaxSuppressionV5Op : public OpKernel { context, scores, num_boxes, max_output_size, iou_threshold_val, score_threshold_val, soft_nms_sigma_val, similarity_fn, return_scores_tensor_, pad_to_max_output_size_, &num_valid_outputs); + if (!context->status().ok()) { + return; + } // Allocate scalar output tensor for number of indices computed. Tensor* num_outputs_t = nullptr; @@ -921,6 +929,8 @@ class CombinedNonMaxSuppressionOp : public OpKernel { errors::InvalidArgument("max_size_per_class must be 0-D, got shape ", max_output_size.shape().DebugString())); const int max_size_per_class = max_output_size.scalar()(); + OP_REQUIRES(context, max_size_per_class > 0, + errors::InvalidArgument("max_size_per_class must be positive")); // max_total_size: scalar const Tensor& max_total_size = context->input(3); OP_REQUIRES( diff --git a/tensorflow/core/kernels/parameterized_truncated_normal_op.cc b/tensorflow/core/kernels/parameterized_truncated_normal_op.cc index a63457551ac29b..116df3541d7cf6 100644 --- a/tensorflow/core/kernels/parameterized_truncated_normal_op.cc +++ b/tensorflow/core/kernels/parameterized_truncated_normal_op.cc @@ -627,6 +627,9 @@ class ParameterizedTruncatedNormalOp : public OpKernel { ctx, TensorShapeUtils::IsVector(shape_tensor.shape()), errors::InvalidArgument("Input shape should be a vector, got shape: ", shape_tensor.shape().DebugString())); + OP_REQUIRES(ctx, shape_tensor.NumElements() > 0, + errors::InvalidArgument("Shape tensor must not be empty, got ", + shape_tensor.DebugString())); int32 num_batches = shape_tensor.flat()(0); int32 samples_per_batch = 1; diff --git a/tensorflow/core/kernels/pooling_ops_3d.cc b/tensorflow/core/kernels/pooling_ops_3d.cc index 532d861e6158e5..fc857fad5886e0 100644 --- a/tensorflow/core/kernels/pooling_ops_3d.cc +++ b/tensorflow/core/kernels/pooling_ops_3d.cc @@ -389,6 +389,19 @@ struct LaunchAvgPooling3dGradOp { const std::array& output_shape, const std::array& padding, TensorFormat data_format, Tensor* output) { + OP_REQUIRES( + context, tensor_in_shape.dim_size(0) == out_backprop.dim_size(0), + errors::InvalidArgument( + "Expected first dimension of tensor_in_shape and " + "out_backprop to match, got ", + tensor_in_shape.dim_size(0), " and ", out_backprop.dim_size(0))); + OP_REQUIRES( + context, tensor_in_shape.dim_size(4) == out_backprop.dim_size(4), + errors::InvalidArgument( + "Expected last dimension of tensor_in_shape and " + "out_backprop to match, got ", + tensor_in_shape.dim_size(4), " and ", out_backprop.dim_size(4))); + output->flat().setZero(); std::array input_size = {{tensor_in_shape.dim_size(3), tensor_in_shape.dim_size(2), @@ -699,11 +712,36 @@ class MaxPooling3dGradGradOp : public OpKernel { Pool3dParameters params{context, ksize_, stride_, padding_, data_format_, tensor_in.shape()}; + if (!context->status().ok()) return; // params is invalid Tensor* output = nullptr; OP_REQUIRES_OK(context, context->forward_input_or_allocate_output( {2}, 0, tensor_out.shape(), &output)); + // Given access patterns in LaunchMaxPooling3dGradGradOp, these tensors must + // have elements. + OP_REQUIRES(context, tensor_in.NumElements() > 0, + errors::InvalidArgument("received empty tensor tensor_in: ", + tensor_in.DebugString())); + OP_REQUIRES(context, tensor_out.NumElements() > 0, + errors::InvalidArgument("received empty tensor tensor_out: ", + tensor_out.DebugString())); + OP_REQUIRES( + context, out_grad_backprop.NumElements() > 0, + errors::InvalidArgument("received empty tensor out_grad_backprop: ", + out_grad_backprop.DebugString())); + OP_REQUIRES(context, + tensor_in.NumElements() == out_grad_backprop.NumElements(), + errors::InvalidArgument("tensor_in and out_grad_backprop must " + "have same number of elements, got <", + tensor_in.DebugString(), "> and <", + out_grad_backprop.DebugString(), ">")); + OP_REQUIRES( + context, tensor_out.NumElements() == output->NumElements(), + errors::InvalidArgument( + "tensor_out and output must have same number of elements, got <", + tensor_out.DebugString(), "> and <", output->DebugString(), ">")); + LaunchMaxPooling3dGradGradOp::launch( context, params, tensor_in, tensor_out, out_grad_backprop, output); } diff --git a/tensorflow/core/kernels/pooling_ops_common.cc b/tensorflow/core/kernels/pooling_ops_common.cc index 4bd710546fec26..59fbe883642a1a 100644 --- a/tensorflow/core/kernels/pooling_ops_common.cc +++ b/tensorflow/core/kernels/pooling_ops_common.cc @@ -96,6 +96,8 @@ PoolParameters::PoolParameters(OpKernelContext* context, pad_depth = 0; out_depth = depth; } else { + OP_REQUIRES(context, depth_window > 0, + errors::InvalidArgument("depth_window must not be 0")); // Our current version of depthwise max pooling does not support // any padding, and expects the depth_window to equal the // depth_stride (no overlapping). diff --git a/tensorflow/core/kernels/quantize_and_dequantize_op.cc b/tensorflow/core/kernels/quantize_and_dequantize_op.cc index 8f71d09c0832e7..28be8a9fa11822 100644 --- a/tensorflow/core/kernels/quantize_and_dequantize_op.cc +++ b/tensorflow/core/kernels/quantize_and_dequantize_op.cc @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#include "tensorflow/core/framework/op_requires.h" #define EIGEN_USE_THREADS #if (defined(GOOGLE_CUDA) && GOOGLE_CUDA) || \ @@ -71,6 +72,13 @@ class QuantizeAndDequantizeV2Op : public OpKernel { void Compute(OpKernelContext* ctx) override { const Tensor& input = ctx->input(0); + OP_REQUIRES( + ctx, axis_ >= -1, + errors::InvalidArgument("Axis must be at least -1. Found ", axis_)); + OP_REQUIRES( + ctx, (axis_ == -1 || axis_ < input.shape().dims()), + errors::InvalidArgument("Shape must be at least rank ", axis_ + 1, + " but is rank ", input.shape().dims())); const int depth = (axis_ == -1) ? 1 : input.dim_size(axis_); Tensor input_min_tensor; Tensor input_max_tensor; @@ -151,6 +159,10 @@ class QuantizeAndDequantizeV3Op : public OpKernel { void Compute(OpKernelContext* ctx) override { const Tensor& input = ctx->input(0); + OP_REQUIRES(ctx, axis_ < input.dims(), + errors::InvalidArgument( + "Axis requested is larger than input dimensions. Axis: ", + axis_, " Input Dimensions: ", input.dims())); const int depth = (axis_ == -1) ? 1 : input.dim_size(axis_); Tensor* output = nullptr; OP_REQUIRES_OK(ctx, ctx->allocate_output(0, input.shape(), &output)); diff --git a/tensorflow/core/kernels/quantize_op.cc b/tensorflow/core/kernels/quantize_op.cc index a523c4b9cd0249..098991e4f436d8 100644 --- a/tensorflow/core/kernels/quantize_op.cc +++ b/tensorflow/core/kernels/quantize_op.cc @@ -113,7 +113,50 @@ class QuantizeV2Op : public OpKernel { int num_slices = 1; if (axis_ > -1) { + OP_REQUIRES( + ctx, input.dims() > axis_, + errors::InvalidArgument( + "Axis is on a zero-based index, so its value must always be less " + "than number of input's dims, but given axis value was ", + axis_, " and input's dims was ", input.dims())); num_slices = input.dim_size(axis_); + OP_REQUIRES(ctx, input_min_range.dims() == 1, + errors::InvalidArgument( + "If axis is specified, min_range must be a 1-D tensor " + "whose size matches the axis dimension of the input and " + "output tensors, but min_range dims are ", + input_min_range.dims())); + OP_REQUIRES(ctx, input_min_range.dim_size(0) == num_slices, + errors::InvalidArgument( + "If axis is specified, min_range must be a 1-D tensor " + "whose size matches the axis dimension of the input and " + "output tensors, but min_range is a 1-D tensor of size ", + input_min_range.dim_size(0), + " and input's axis dimension is of size ", num_slices)); + OP_REQUIRES(ctx, input_max_range.dims() == 1, + errors::InvalidArgument( + "If axis is specified, max_range must be a 1-D tensor " + "whose size matches the axis dimension of the input and " + "output tensors, but max_range dims are ", + input_max_range.dims())); + OP_REQUIRES(ctx, input_max_range.dim_size(0) == num_slices, + errors::InvalidArgument( + "If axis is specified, max_range must be a 1-D tensor " + "whose size matches the axis dimension of the input and " + "output tensors, but max_range is a 1-D tensor of size ", + input_max_range.dim_size(0), + " and input's axis dimension is of size ", num_slices)); + } else { + OP_REQUIRES(ctx, input_min_range.NumElements() == 1, + errors::InvalidArgument( + "If axis is not specified, min_range must contain a " + "single float element, but it contains ", + input_min_range.NumElements(), " elements")); + OP_REQUIRES(ctx, input_max_range.NumElements() == 1, + errors::InvalidArgument( + "If axis is not specified, max_range must contain a " + "single float element, but it contains ", + input_max_range.NumElements(), " elements")); } const TensorShape& minmax_shape = ctx->input(1).shape(); diff --git a/tensorflow/core/kernels/quantized_add_op.cc b/tensorflow/core/kernels/quantized_add_op.cc index 55c69de7d3ea6c..b186f00f15c061 100644 --- a/tensorflow/core/kernels/quantized_add_op.cc +++ b/tensorflow/core/kernels/quantized_add_op.cc @@ -538,6 +538,8 @@ class QuantizedAddOp : public OpKernel { tensor_min = min_x; tensor_max = max_x; } + OP_REQUIRES(context, vector_num_elements > 0, + errors::InvalidArgument("Must have some elements to add")); VectorTensorAddition( vector_data, vector_min, vector_max, vector_num_elements, tensor_data, tensor_min, tensor_max, tensor_num_elements, min_z_value, max_z_value, diff --git a/tensorflow/core/kernels/quantized_batch_norm_op.cc b/tensorflow/core/kernels/quantized_batch_norm_op.cc index b03da7ad17fab4..6dfe07f97a4007 100644 --- a/tensorflow/core/kernels/quantized_batch_norm_op.cc +++ b/tensorflow/core/kernels/quantized_batch_norm_op.cc @@ -173,20 +173,50 @@ class QuantizedBatchNormOp : public OpKernel { void Compute(OpKernelContext* context) override { const Tensor& input = context->input(0); - const float input_min = context->input(1).flat()(0); - const float input_max = context->input(2).flat()(0); + const auto& input_min_tensor = context->input(1); + OP_REQUIRES(context, input_min_tensor.NumElements() == 1, + errors::InvalidArgument("input_min must have 1 element")); + const float input_min = input_min_tensor.flat()(0); + const auto& input_max_tensor = context->input(2); + OP_REQUIRES(context, input_max_tensor.NumElements() == 1, + errors::InvalidArgument("input_max must have 1 element")); + const float input_max = input_max_tensor.flat()(0); const Tensor& mean = context->input(3); - const float mean_min = context->input(4).flat()(0); - const float mean_max = context->input(5).flat()(0); + const auto& mean_min_tensor = context->input(4); + OP_REQUIRES(context, mean_min_tensor.NumElements() == 1, + errors::InvalidArgument("mean_min must have 1 element")); + const float mean_min = mean_min_tensor.flat()(0); + const auto& mean_max_tensor = context->input(5); + OP_REQUIRES(context, mean_max_tensor.NumElements() == 1, + errors::InvalidArgument("mean_max must have 1 element")); + const float mean_max = mean_max_tensor.flat()(0); const Tensor& var = context->input(6); - const float var_min = context->input(7).flat()(0); - const float var_max = context->input(8).flat()(0); + const auto& var_min_tensor = context->input(7); + OP_REQUIRES(context, var_min_tensor.NumElements() == 1, + errors::InvalidArgument("var_min must have 1 element")); + const float var_min = var_min_tensor.flat()(0); + const auto& var_max_tensor = context->input(8); + OP_REQUIRES(context, var_max_tensor.NumElements() == 1, + errors::InvalidArgument("var_max must have 1 element")); + const float var_max = var_max_tensor.flat()(0); const Tensor& beta = context->input(9); - const float beta_min = context->input(10).flat()(0); - const float beta_max = context->input(11).flat()(0); + const auto& beta_min_tensor = context->input(10); + OP_REQUIRES(context, beta_min_tensor.NumElements() == 1, + errors::InvalidArgument("beta_min must have 1 element")); + const float beta_min = beta_min_tensor.flat()(0); + const auto& beta_max_tensor = context->input(11); + OP_REQUIRES(context, beta_max_tensor.NumElements() == 1, + errors::InvalidArgument("beta_max must have 1 element")); + const float beta_max = beta_max_tensor.flat()(0); const Tensor& gamma = context->input(12); - const float gamma_min = context->input(13).flat()(0); - const float gamma_max = context->input(14).flat()(0); + const auto& gamma_min_tensor = context->input(13); + OP_REQUIRES(context, gamma_min_tensor.NumElements() == 1, + errors::InvalidArgument("gamma_min must have 1 element")); + const float gamma_min = gamma_min_tensor.flat()(0); + const auto& gamma_max_tensor = context->input(14); + OP_REQUIRES(context, gamma_max_tensor.NumElements() == 1, + errors::InvalidArgument("gamma_max must have 1 element")); + const float gamma_max = gamma_max_tensor.flat()(0); OP_REQUIRES(context, input.dims() == 4, errors::InvalidArgument("input must be 4-dimensional", @@ -203,6 +233,33 @@ class QuantizedBatchNormOp : public OpKernel { OP_REQUIRES(context, gamma.dims() == 1, errors::InvalidArgument("gamma must be 1-dimensional", gamma.shape().DebugString())); + OP_REQUIRES(context, mean.NumElements() > 1, + errors::InvalidArgument("Must have at least a mean value", + gamma.shape().DebugString())); + OP_REQUIRES(context, mean.NumElements() > 1, + errors::InvalidArgument("Must have at least a mean value")); + const auto last_dim = input.shape().dims() - 1; + OP_REQUIRES(context, + mean.shape().dim_size(0) == input.shape().dim_size(last_dim), + errors::InvalidArgument("Must provide as many means as the " + "last dimension of the input tensor: ", + mean.shape().DebugString(), " vs. ", + input.shape().DebugString())); + OP_REQUIRES( + context, mean.shape().dim_size(0) == var.shape().dim_size(0), + errors::InvalidArgument( + "Mean and variance tensors must have the same shape: ", + mean.shape().DebugString(), " vs. ", var.shape().DebugString())); + OP_REQUIRES( + context, mean.shape().dim_size(0) == beta.shape().dim_size(0), + errors::InvalidArgument( + "Mean and beta tensors must have the same shape: ", + mean.shape().DebugString(), " vs. ", beta.shape().DebugString())); + OP_REQUIRES( + context, mean.shape().dim_size(0) == gamma.shape().dim_size(0), + errors::InvalidArgument( + "Mean and gamma tensors must have the same shape: ", + mean.shape().DebugString(), " vs. ", gamma.shape().DebugString())); Tensor* output = nullptr; OP_REQUIRES_OK(context, diff --git a/tensorflow/core/kernels/quantized_bias_add_op.cc b/tensorflow/core/kernels/quantized_bias_add_op.cc index 5457d290c2559c..db0e21a498011d 100644 --- a/tensorflow/core/kernels/quantized_bias_add_op.cc +++ b/tensorflow/core/kernels/quantized_bias_add_op.cc @@ -56,6 +56,8 @@ class QuantizedBiasAddOp : public OpKernel { "Must provide as many biases as the last dimension " "of the input tensor: ", bias.shape().DebugString(), " vs. ", input.shape().DebugString())); + OP_REQUIRES(context, bias.NumElements() > 0, + errors::InvalidArgument("Must provide at least 1 bias")); Tensor* output = nullptr; OP_REQUIRES_OK(context, diff --git a/tensorflow/core/kernels/quantized_conv_ops.cc b/tensorflow/core/kernels/quantized_conv_ops.cc index a4d36cca3e4088..a339de8cfc8fa3 100644 --- a/tensorflow/core/kernels/quantized_conv_ops.cc +++ b/tensorflow/core/kernels/quantized_conv_ops.cc @@ -18,6 +18,8 @@ limitations under the License. #include #include +#include "tensorflow/core/platform/errors.h" + #define EIGEN_USE_THREADS #define GEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK @@ -227,8 +229,12 @@ class Im2ColConvFunctor { return; } - CHECK_GT(output_width, 0); - CHECK_GT(output_height, 0); + OP_REQUIRES( + context, output_width > 0, + errors::InvalidArgument("output_width must be strictly positive")); + OP_REQUIRES( + context, output_height > 0, + errors::InvalidArgument("output_height must be strictly positive")); int filter_left_offset; int filter_top_offset; if (padding == VALID) { @@ -255,6 +261,9 @@ class Im2ColConvFunctor { // by the width, then the height. This is the standard memory order in the // image world if it helps to visualize it. const int filter_value_count = filter_width * filter_height * input_depth; + OP_REQUIRES(context, filter_value_count > 0, + errors::InvalidArgument( + "filter patch must contain at least one element")); const int64 patches_per_chunk = kMaxChunkSize / (filter_value_count * sizeof(T1)); const int64 chunk_value_count = diff --git a/tensorflow/core/kernels/quantized_mul_op.cc b/tensorflow/core/kernels/quantized_mul_op.cc index 4e191f162662bb..22cff8939449a6 100644 --- a/tensorflow/core/kernels/quantized_mul_op.cc +++ b/tensorflow/core/kernels/quantized_mul_op.cc @@ -284,10 +284,22 @@ class QuantizedMulOp : public OpKernel { void Compute(OpKernelContext* context) override { const Tensor& x = context->input(0); const Tensor& y = context->input(1); - const float min_x = context->input(2).flat()(0); - const float max_x = context->input(3).flat()(0); - const float min_y = context->input(4).flat()(0); - const float max_y = context->input(5).flat()(0); + auto& min_x_tensor = context->input(2); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(min_x_tensor.shape()), + errors::InvalidArgument("min_x must be a scalar")); + const float min_x = min_x_tensor.flat()(0); + auto& max_x_tensor = context->input(3); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(max_x_tensor.shape()), + errors::InvalidArgument("max_x must be a scalar")); + const float max_x = max_x_tensor.flat()(0); + auto& min_y_tensor = context->input(4); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(min_y_tensor.shape()), + errors::InvalidArgument("min_y must be a scalar")); + const float min_y = min_y_tensor.flat()(0); + auto& max_y_tensor = context->input(5); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(max_y_tensor.shape()), + errors::InvalidArgument("max_y must be a scalar")); + const float max_y = max_y_tensor.flat()(0); BCast bcast(BCast::FromShape(x.shape()), BCast::FromShape(y.shape())); if (!bcast.IsValid()) { @@ -347,6 +359,11 @@ class QuantizedMulOp : public OpKernel { tensor_num_elements = x.NumElements(); tensor_offset = offset_x; } + if (vector_num_elements == 0) { + context->SetStatus( + errors::InvalidArgument("vector must have at least 1 element")); + return; + } VectorTensorMultiply( vector_data, vector_offset, vector_num_elements, tensor_data, tensor_offset, tensor_num_elements, z_data); diff --git a/tensorflow/core/kernels/quantized_reshape_op.cc b/tensorflow/core/kernels/quantized_reshape_op.cc index bd76c94edeea7a..682f4aaa1f79e7 100644 --- a/tensorflow/core/kernels/quantized_reshape_op.cc +++ b/tensorflow/core/kernels/quantized_reshape_op.cc @@ -17,6 +17,7 @@ limitations under the License. #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/tensor_types.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/kernels/reshape_op.h" @@ -30,9 +31,29 @@ class QuantizedReshapeOp : public ReshapeOp { void Compute(OpKernelContext* ctx) override { // This call processes inputs 1 and 2 to write output 0. ReshapeOp::Compute(ctx); + if (!ctx->status().ok()) { + return; + } + + const auto& input_min_float_tensor = ctx->input(2); + const auto& input_min_float_shape = input_min_float_tensor.shape(); + OP_REQUIRES(ctx, + TensorShapeUtils::IsScalar(input_min_float_shape) || + (TensorShapeUtils::IsVector(input_min_float_shape) && + (input_min_float_shape.dim_size(0) == 1)), + errors::InvalidArgument( + "input_min must be a scalar or a vector of 1 element")); + const float input_min_float = input_min_float_tensor.flat()(0); + const auto& input_max_float_tensor = ctx->input(3); + const auto& input_max_float_shape = input_max_float_tensor.shape(); + OP_REQUIRES(ctx, + TensorShapeUtils::IsScalar(input_max_float_shape) || + (TensorShapeUtils::IsVector(input_max_float_shape) && + (input_max_float_shape.dim_size(0) == 1)), + errors::InvalidArgument( + "input_max must be a scalar or a vector of 1 element")); + const float input_max_float = input_max_float_tensor.flat()(0); - const float input_min_float = ctx->input(2).flat()(0); - const float input_max_float = ctx->input(3).flat()(0); Tensor* output_min = nullptr; OP_REQUIRES_OK(ctx, ctx->allocate_output(1, TensorShape({}), &output_min)); output_min->flat()(0) = input_min_float; diff --git a/tensorflow/core/kernels/quantized_resize_bilinear_op.cc b/tensorflow/core/kernels/quantized_resize_bilinear_op.cc index 4da56cde5478d5..7569cbc11bdd4e 100644 --- a/tensorflow/core/kernels/quantized_resize_bilinear_op.cc +++ b/tensorflow/core/kernels/quantized_resize_bilinear_op.cc @@ -64,6 +64,8 @@ inline void ComputeInterpolationWeights( std::max(static_cast(in_f), static_cast(0)); interpolation->upper[i] = std::min(static_cast(std::ceil(in)), in_size - 1); + interpolation->lower[i] = + std::min(interpolation->lower[i], interpolation->upper[i]); interpolation->lerp[i] = in - in_f; interpolation->ilerp[i] = static_cast((in - in_f) * (1 << resolution)); @@ -701,8 +703,14 @@ class QuantizedResizeBilinearOp : public OpKernel { void Compute(OpKernelContext* context) override { const Tensor& input = context->input(0); - const float in_min = context->input(2).flat()(0); - const float in_max = context->input(3).flat()(0); + const auto& in_min_tensor = context->input(2); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(in_min_tensor.shape()), + errors::InvalidArgument("min must be a scalar")); + const float in_min = in_min_tensor.flat()(0); + const auto& in_max_tensor = context->input(3); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(in_max_tensor.shape()), + errors::InvalidArgument("max must be a scalar")); + const float in_max = in_max_tensor.flat()(0); ImageResizerState st(align_corners_, false); st.ValidateAndCreateOutput(context, input); diff --git a/tensorflow/core/kernels/ragged_cross_op.cc b/tensorflow/core/kernels/ragged_cross_op.cc index ea65c0ee2b5b21..5dfe93f4166592 100644 --- a/tensorflow/core/kernels/ragged_cross_op.cc +++ b/tensorflow/core/kernels/ragged_cross_op.cc @@ -21,6 +21,7 @@ limitations under the License. #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/platform/errors.h" #include "tensorflow/core/platform/fingerprint.h" #include "tensorflow/core/util/util.h" #include "tensorflow/core/util/work_sharder.h" @@ -466,16 +467,45 @@ class RaggedCrossOp : public OpKernel { int next_dense = 0; for (char c : input_order_) { if (c == 'R') { + if (next_ragged >= ragged_values_list.size()) + return errors::InvalidArgument( + "input_order \"", input_order_, + "\" specifies reading a ragged tensor value at index ", + next_ragged, " from a list of ", ragged_values_list.size(), + " values."); + if (next_ragged >= ragged_splits_list.size()) + return errors::InvalidArgument( + "input_order \"", input_order_, + "\" specifies reading a ragged tensor split at index ", + next_ragged, " from a list of ", ragged_splits_list.size(), + " splits."); TF_RETURN_IF_ERROR(BuildRaggedFeatureReader( ragged_values_list[next_ragged], ragged_splits_list[next_ragged], features)); next_ragged++; } else if (c == 'S') { + if (next_sparse >= sparse_values_list.size()) + return errors::InvalidArgument( + "input_order \"", input_order_, + "\" specifies reading a sparse tensor value at index ", + next_sparse, " from a list of ", sparse_values_list.size(), + " values."); + if (next_sparse >= sparse_indices_list.size()) + return errors::InvalidArgument( + "input_order \"", input_order_, + "\" specifies reading a sparse tensor index at index ", + next_sparse, " from a list of ", sparse_indices_list.size(), + " indices."); TF_RETURN_IF_ERROR(BuildSparseFeatureReader( sparse_indices_list[next_sparse], sparse_values_list[next_sparse], batch_size, features)); next_sparse++; } else if (c == 'D') { + if (next_dense >= dense_list.size()) + return errors::InvalidArgument( + "input_order \"", input_order_, + "\" specifies reading a dense tensor at index ", next_dense, + " from a list of ", dense_list.size(), " tensors."); TF_RETURN_IF_ERROR( BuildDenseFeatureReader(dense_list[next_dense++], features)); } else { diff --git a/tensorflow/core/kernels/ragged_gather_op.cc b/tensorflow/core/kernels/ragged_gather_op.cc index 88c0d1ebd6959b..5939b4120beeca 100644 --- a/tensorflow/core/kernels/ragged_gather_op.cc +++ b/tensorflow/core/kernels/ragged_gather_op.cc @@ -58,15 +58,21 @@ class RaggedGatherOpBase : public OpKernel { void Compute(OpKernelContext* context) override { // Get the input Tensors. + OpInputList params_nested_splits_in; OP_REQUIRES_OK(context, context->input_list("params_nested_splits", ¶ms_nested_splits_in)); + OP_REQUIRES( + context, params_nested_splits_in.size() > 0, + errors::InvalidArgument("params_nested_splits must be non empty")); + const Tensor& params_dense_values_in = context->input(params_nested_splits_in.size()); const Tensor& indices_in = context->input(params_nested_splits_in.size() + 1); - DCHECK_GT(params_nested_splits_in.size(), 0); // Enforced by REGISTER_OP. + OP_REQUIRES(context, params_nested_splits_in[0].dims() > 0, + errors::InvalidArgument("Split tensors must not be scalars")); SPLITS_TYPE num_params = params_nested_splits_in[0].dim_size(0) - 1; OP_REQUIRES_OK(context, ValidateIndices(indices_in, num_params)); diff --git a/tensorflow/core/kernels/ragged_tensor_from_variant_op.cc b/tensorflow/core/kernels/ragged_tensor_from_variant_op.cc index f83bcb38c6c336..1d9ddfefdae6e5 100644 --- a/tensorflow/core/kernels/ragged_tensor_from_variant_op.cc +++ b/tensorflow/core/kernels/ragged_tensor_from_variant_op.cc @@ -194,7 +194,23 @@ Status NestedStackRaggedTensors( auto output_values_flat = output_ragged->values.flat_outer_dims(); int values_index = 0; + + TensorShape expected_value_shape = component_values_shape; + expected_value_shape.RemoveDim(0); + for (int i = 0; i < ragged_components.size(); i++) { + // Check that the flat_values tensor shape is compatible. + TensorShape value_shape = ragged_components[i].values.shape(); + value_shape.RemoveDim(0); + if (value_shape != expected_value_shape) { + return errors::InvalidArgument( + "All flat_values must have compatible shapes. Shape at index 0: ", + expected_value_shape, ". Shape at index ", i, ": ", value_shape, + ". If you are using tf.map_fn, then you may need to specify an " + "explicit fn_output_signature with appropriate ragged_rank, and/or " + "convert output tensors to RaggedTensors."); + } + auto component_values_flat = ragged_components[i].values.flat_outer_dims(); int num_inner_elements = ragged_components[i].values.NumElements(); diff --git a/tensorflow/core/kernels/ragged_tensor_to_sparse_kernel.cc b/tensorflow/core/kernels/ragged_tensor_to_sparse_kernel.cc index 39b530f4a15ead..336a38fa58fc8b 100644 --- a/tensorflow/core/kernels/ragged_tensor_to_sparse_kernel.cc +++ b/tensorflow/core/kernels/ragged_tensor_to_sparse_kernel.cc @@ -21,6 +21,7 @@ limitations under the License. #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/platform/errors.h" namespace tensorflow { @@ -38,7 +39,8 @@ class RaggedTensorToSparseOp : public OpKernel { OP_REQUIRES_OK( context, context->input_list("rt_nested_splits", &rt_nested_splits_in)); const int rt_nested_splits_len = rt_nested_splits_in.size(); - DCHECK_GT(rt_nested_splits_len, 0); // Enforced by REGISTER_OP. + OP_REQUIRES(context, rt_nested_splits_len > 0, + errors::InvalidArgument("rt_nested_splits must be non empty")); std::vector rt_nested_splits; rt_nested_splits.reserve(rt_nested_splits_len); for (int i = 0; i < rt_nested_splits_len; ++i) { @@ -161,6 +163,14 @@ class RaggedTensorToSparseOp : public OpKernel { if (rt_nested_splits[i](0) != 0) { return InvalidArgument("First value of ragged splits must be 0."); } + for (int j = 1; j < rt_nested_splits[i].size(); ++j) { + if (rt_nested_splits[i](j) < rt_nested_splits[i](j - 1)) { + return InvalidArgument( + "Ragged splits should be non decreasing, but we got ", + rt_nested_splits[i](j - 1), " followed by ", + rt_nested_splits[i](j)); + } + } if (i > 0) { SPLITS_TYPE last_split = rt_nested_splits[i - 1](rt_nested_splits[i - 1].size() - 1); diff --git a/tensorflow/core/kernels/ragged_tensor_to_tensor_op.cc b/tensorflow/core/kernels/ragged_tensor_to_tensor_op.cc index d729c43f25a4b8..bffa35875e553d 100644 --- a/tensorflow/core/kernels/ragged_tensor_to_tensor_op.cc +++ b/tensorflow/core/kernels/ragged_tensor_to_tensor_op.cc @@ -207,7 +207,7 @@ class RaggedTensorToTensorBaseOp : public OpKernel { DCHECK_EQ(result->size(), first_dimension); } - void CalculateOutputIndexRowSplit( + Status CalculateOutputIndexRowSplit( const RowPartitionTensor& row_split, const vector& parent_output_index, INDEX_TYPE output_index_multiplier, INDEX_TYPE output_size, @@ -232,9 +232,11 @@ class RaggedTensorToTensorBaseOp : public OpKernel { result->push_back(-1); } } - if (row_split_size > 0) { - DCHECK_EQ(result->size(), row_split(row_split_size - 1)); + if (row_split_size > 0 && result->size() != row_split(row_split_size - 1)) { + return errors::InvalidArgument("Invalid row split size."); } + + return Status::OK(); } // Calculate the output index of the first element of a list. @@ -258,7 +260,7 @@ class RaggedTensorToTensorBaseOp : public OpKernel { // result[6] = -1 because parent_output_index[value_rowids[6]] == -1 // result[7] = -1 because parent_output_index[value_rowids[6]] == -1 // result[8] = parent_output_index[value_rowids[7]] - void CalculateOutputIndexValueRowID( + Status CalculateOutputIndexValueRowID( const RowPartitionTensor& value_rowids, const vector& parent_output_index, INDEX_TYPE output_index_multiplier, INDEX_TYPE output_size, @@ -266,12 +268,18 @@ class RaggedTensorToTensorBaseOp : public OpKernel { const INDEX_TYPE index_size = value_rowids.size(); result->reserve(index_size); if (index_size == 0) { - return; + return Status::OK(); } INDEX_TYPE current_output_column = 0; INDEX_TYPE current_value_rowid = value_rowids(0); - DCHECK_LT(current_value_rowid, parent_output_index.size()); + + if (current_value_rowid >= parent_output_index.size()) { + return errors::InvalidArgument( + "Got current_value_rowid=", current_value_rowid, + " which is not less than ", parent_output_index.size()); + } + INDEX_TYPE current_output_index = parent_output_index[current_value_rowid]; result->push_back(current_output_index); for (INDEX_TYPE i = 1; i < index_size; ++i) { @@ -288,12 +296,23 @@ class RaggedTensorToTensorBaseOp : public OpKernel { } else { current_output_column = 0; current_value_rowid = next_value_rowid; - DCHECK_LT(next_value_rowid, parent_output_index.size()); + + if (next_value_rowid >= parent_output_index.size()) { + return errors::InvalidArgument( + "Got next_value_rowid=", next_value_rowid, + " which is not less than ", parent_output_index.size()); + } + current_output_index = parent_output_index[next_value_rowid]; } result->push_back(current_output_index); } - DCHECK_EQ(result->size(), value_rowids.size()); + + if (result->size() != value_rowids.size()) { + return errors::InvalidArgument("Invalid row ids."); + } + + return Status::OK(); } Status CalculateOutputIndex(OpKernelContext* context, int dimension, @@ -306,15 +325,19 @@ class RaggedTensorToTensorBaseOp : public OpKernel { auto partition_type = GetRowPartitionTypeByDimension(dimension); switch (partition_type) { case RowPartitionType::VALUE_ROWIDS: - CalculateOutputIndexValueRowID( + return CalculateOutputIndexValueRowID( row_partition_tensor, parent_output_index, output_index_multiplier, output_size, result); - return tensorflow::Status::OK(); case RowPartitionType::ROW_SPLITS: - CalculateOutputIndexRowSplit(row_partition_tensor, parent_output_index, - output_index_multiplier, output_size, - result); - return tensorflow::Status::OK(); + if (row_partition_tensor.size() - 1 > parent_output_index.size()) { + return errors::InvalidArgument( + "Row partition size is greater than output size: ", + row_partition_tensor.size() - 1, " > ", + parent_output_index.size()); + } + return CalculateOutputIndexRowSplit( + row_partition_tensor, parent_output_index, output_index_multiplier, + output_size, result); default: return errors::InvalidArgument( "Unsupported partition type:", @@ -325,6 +348,9 @@ class RaggedTensorToTensorBaseOp : public OpKernel { Status GetFirstDimensionSize(OpKernelContext* context, INDEX_TYPE* result) { const Tensor first_partition_tensor = context->input(kFirstPartitionInputIndex); + if (row_partition_types_.empty()) { + return errors::InvalidArgument("No row_partition_types given."); + } const RowPartitionType first_partition_type = row_partition_types_[0]; switch (first_partition_type) { case RowPartitionType::FIRST_DIM_SIZE: @@ -345,6 +371,11 @@ class RaggedTensorToTensorBaseOp : public OpKernel { void Compute(OpKernelContext* context) override { INDEX_TYPE first_dimension; + const Tensor first_partition_tensor = + context->input(kFirstPartitionInputIndex); + OP_REQUIRES(context, first_partition_tensor.NumElements() > 0, + errors::InvalidArgument("Invalid first partition input. Tensor " + "requires at least one element.")); OP_REQUIRES_OK(context, GetFirstDimensionSize(context, &first_dimension)); vector output_size; OP_REQUIRES_OK(context, diff --git a/tensorflow/core/kernels/ragged_tensor_to_variant_op.cc b/tensorflow/core/kernels/ragged_tensor_to_variant_op.cc index 7a5ae1c6240b55..a3ecc80ce41f9a 100644 --- a/tensorflow/core/kernels/ragged_tensor_to_variant_op.cc +++ b/tensorflow/core/kernels/ragged_tensor_to_variant_op.cc @@ -173,8 +173,19 @@ class RaggedTensorToVariantOp : public OpKernel { return; } + // Checked here instead of at input in case batched_input_ is false + OP_REQUIRES(context, ragged_nested_splits_len > 0, + errors::InvalidArgument( + "rt_nested_splits must be a list of one or more, but " + "received rt_nested_splits of length 0.")); + // Unbatch the Ragged Tensor and encode the components. std::vector ragged_components; + auto batched_splits_top_vec = + batched_ragged_input.nested_splits[0].vec(); + int num_components = batched_splits_top_vec.size() - 1; + OP_REQUIRES(context, num_components >= 0, + errors::Internal("Invalid split argument.")); OP_REQUIRES_OK(context, UnbatchRaggedZerothDim( batched_ragged_input, &ragged_components)); std::vector encoded_components(ragged_components.size()); diff --git a/tensorflow/core/kernels/requantization_range_op.cc b/tensorflow/core/kernels/requantization_range_op.cc index cc6e891a6b352b..f6e217499d1983 100644 --- a/tensorflow/core/kernels/requantization_range_op.cc +++ b/tensorflow/core/kernels/requantization_range_op.cc @@ -46,6 +46,10 @@ class RequantizationRangeOp : public OpKernel { void Compute(OpKernelContext* ctx) override { const Tensor& input = ctx->input(0); + OP_REQUIRES(ctx, ctx->input(1).NumElements() > 0, + errors::InvalidArgument("Input min must not be empty.")); + OP_REQUIRES(ctx, ctx->input(2).NumElements() > 0, + errors::InvalidArgument("Input max must not be empty.")); const float input_min_float = ctx->input(1).flat()(0); const float input_max_float = ctx->input(2).flat()(0); Tensor* output_min = nullptr; diff --git a/tensorflow/core/kernels/resource_variable_ops.cc b/tensorflow/core/kernels/resource_variable_ops.cc index b9c883c7e2ff30..f75cc50a8df752 100644 --- a/tensorflow/core/kernels/resource_variable_ops.cc +++ b/tensorflow/core/kernels/resource_variable_ops.cc @@ -643,6 +643,11 @@ class ResourceGatherOp : public OpKernel { OP_REQUIRES( c, TensorShapeUtils::IsVectorOrHigher(params.shape()), errors::InvalidArgument("params must be at least 1 dimensional")); + OP_REQUIRES( + c, params.shape().dims() >= batch_dims_, + errors::InvalidArgument("params must have at least ", batch_dims_, + " (batch_dims) dimensions but it has shape ", + params.shape().DebugString())); // Check that we have enough index space const int64 N = indices.NumElements(); @@ -688,7 +693,8 @@ class ResourceGatherOp : public OpKernel { copy_functor(c->eigen_device(), tmp_indices.flat(), indices.flat()); - AddBatchOffsets(&tmp_indices, params); + AddBatchOffsets(c, &tmp_indices, params); + if (!c->status().ok()) return; op_indices = &tmp_indices; } @@ -720,11 +726,17 @@ class ResourceGatherOp : public OpKernel { // Example: batch_dims = 1, indices = [[0, 1, 2], [0, 1, 2]] // If indexing into a params dimension of size 4, then the indices will become // [0, 1, 2, 4, 5, 6] - void AddBatchOffsets(Tensor* indices, const Tensor& params) { + void AddBatchOffsets(OpKernelContext* ctx, Tensor* indices, + const Tensor& params) { int64 batch_size = 1; // The size of all batch dimensions. for (int idx = 0; idx < batch_dims_; ++idx) { batch_size *= params.dim_size(idx); } + OP_REQUIRES( + ctx, batch_size != 0, + errors::InvalidArgument( + "Inner size of indices would result in batch_size of 0 and a ", + "division by 0 in the implementation. This is illegal")); auto indices_flat = indices->flat(); int64 const index_inner_size = indices->NumElements() / batch_size; @@ -844,6 +856,35 @@ TF_CALL_GPU_NUMBER_TYPES(REGISTER_GATHER_ND_GPU); #undef REGISTER_GATHER_ND_ALL_INDICES #undef REGISTER_GATHER_ND_FULL +namespace { + +template +bool isCPUDevice() { + return false; +} + +template <> +bool isCPUDevice() { + return true; +} + +template +bool ValidateInput(const Tensor& updates) { + const auto updates_flat = updates.flat(); + const T zero(0); + for (int i = 0; i < updates.NumElements(); i++) { + if (updates_flat(i) == zero) return false; + } + return true; +} + +template <> +bool ValidateInput(const Tensor& updates) { + return true; +} + +} // namespace + template class ResourceScatterUpdateOp : public OpKernel { public: @@ -910,6 +951,12 @@ class ResourceScatterUpdateOp : public OpKernel { " indexing: ", params->dim_size(0), " > ", std::numeric_limits::max())); + // Prevent division by 0 + if (isCPUDevice() && op == tensorflow::scatter_op::UpdateOp::DIV) { + OP_REQUIRES(c, ValidateInput(updates), + errors::InvalidArgument("updates must not contain 0")); + } + if (N > 0) { auto indices_flat = indices.flat(); auto params_flat = params->flat_outer_dims(); @@ -926,11 +973,12 @@ class ResourceScatterUpdateOp : public OpKernel { params->dim_size(0), ")")); } else { int64 num_updates = updates.NumElements(); - OP_REQUIRES(c, num_updates % N == 0, - errors::InvalidArgument( - "shape of indices (", indices.shape().DebugString(), - ") is not compatible with the shape of updates (", - updates.shape().DebugString(), ")")); + OP_REQUIRES( + c, TensorShapeUtils::StartsWith(updates.shape(), indices.shape()), + errors::InvalidArgument( + "The shape of indices (", indices.shape().DebugString(), + ") must be a prefix of the shape of updates (", + updates.shape().DebugString(), ")")); auto updates_flat = updates.shaped({N, num_updates / N}); functor::ScatterFunctor functor; diff --git a/tensorflow/core/kernels/reverse_op.cc b/tensorflow/core/kernels/reverse_op.cc index d551d1ee4bc66c..a7605be4660357 100644 --- a/tensorflow/core/kernels/reverse_op.cc +++ b/tensorflow/core/kernels/reverse_op.cc @@ -158,6 +158,12 @@ class ReverseOp : public OpKernel { void Compute(OpKernelContext* context) override { const Tensor& input = context->input(0); + // If input is provided, check to make sure the first dimension is valid. + if (input.dims() > 0) { + OP_REQUIRES( + context, input.dim_size(0) != 0, + errors::InvalidArgument("Invalid input first dimension. Found 0.")); + } const Tensor& dims = context->input(1); if (TensorShapeUtils::IsScalar(input.shape())) { diff --git a/tensorflow/core/kernels/reverse_sequence_op.cc b/tensorflow/core/kernels/reverse_sequence_op.cc index b5b62bc76ca524..1282deb26e8cd6 100644 --- a/tensorflow/core/kernels/reverse_sequence_op.cc +++ b/tensorflow/core/kernels/reverse_sequence_op.cc @@ -115,6 +115,10 @@ class ReverseSequenceOp : public OpKernel { : OpKernel(context) { OP_REQUIRES_OK(context, context->GetAttr("batch_dim", &batch_dim_)); OP_REQUIRES_OK(context, context->GetAttr("seq_dim", &seq_dim_)); + OP_REQUIRES(context, batch_dim_ >= 0, + errors::InvalidArgument("Invalid batch_dim ", batch_dim_)); + OP_REQUIRES(context, seq_dim_ >= 0, + errors::InvalidArgument("Invalid seq_dim ", seq_dim_)); } void Compute(OpKernelContext* context) override { diff --git a/tensorflow/core/kernels/save_restore_tensor.cc b/tensorflow/core/kernels/save_restore_tensor.cc index 1a5b6b92bd6a0a..6b91ccf26c9320 100644 --- a/tensorflow/core/kernels/save_restore_tensor.cc +++ b/tensorflow/core/kernels/save_restore_tensor.cc @@ -151,11 +151,18 @@ void RestoreTensor(OpKernelContext* context, context, size == 1, errors::InvalidArgument( "Input 0 (file_pattern) must be a string scalar; got a tensor of ", - size, "elements")); + size, " elements")); } const string& file_pattern = file_pattern_t.flat()(0); const Tensor& tensor_name_t = context->input(1); + { + const int64_t size = tensor_name_t.NumElements(); + OP_REQUIRES(context, size > restore_index, + errors::InvalidArgument( + "Input 1 (file_pattern) must be a have at least ", + restore_index + 1, " elements")); + } const string& tensor_name = tensor_name_t.flat()(restore_index); // If we cannot find a cached reader we will allocate our own. diff --git a/tensorflow/core/kernels/save_restore_v2_ops.cc b/tensorflow/core/kernels/save_restore_v2_ops.cc index 07e120e042cf3f..97e94cbd02ac03 100644 --- a/tensorflow/core/kernels/save_restore_v2_ops.cc +++ b/tensorflow/core/kernels/save_restore_v2_ops.cc @@ -98,6 +98,7 @@ class SaveV2 : public OpKernel { const Tensor& shape_and_slices = context->input(2); ValidateInputs(true /* is save op */, context, prefix, tensor_names, shape_and_slices); + if (!context->status().ok()) return; const int kFixedInputs = 3; // Prefix, tensor names, shape_and_slices. const int num_tensors = static_cast(tensor_names.NumElements()); @@ -156,6 +157,7 @@ class RestoreV2 : public OpKernel { " expected dtypes.")); ValidateInputs(false /* not save op */, context, prefix, tensor_names, shape_and_slices); + if (!context->status().ok()) return; const string& prefix_string = prefix.scalar()(); diff --git a/tensorflow/core/kernels/sdca_internal.cc b/tensorflow/core/kernels/sdca_internal.cc index cbc754af0e9bb1..ed7149bf8365d8 100644 --- a/tensorflow/core/kernels/sdca_internal.cc +++ b/tensorflow/core/kernels/sdca_internal.cc @@ -99,6 +99,10 @@ Status ModelWeights::Initialize(OpKernelContext* const context) { OpInputList sparse_weights_inputs; TF_RETURN_IF_ERROR( context->input_list("sparse_weights", &sparse_weights_inputs)); + if (sparse_indices_inputs.size() != sparse_weights_inputs.size()) + return errors::InvalidArgument( + "sparse_indices and sparse_weights must have the same length, got ", + sparse_indices_inputs.size(), " and ", sparse_weights_inputs.size()); OpInputList dense_weights_inputs; TF_RETURN_IF_ERROR( context->input_list("dense_weights", &dense_weights_inputs)); @@ -106,10 +110,20 @@ Status ModelWeights::Initialize(OpKernelContext* const context) { OpOutputList sparse_weights_outputs; TF_RETURN_IF_ERROR(context->output_list("out_delta_sparse_weights", &sparse_weights_outputs)); + if (sparse_weights_outputs.size() != sparse_weights_inputs.size()) + return errors::InvalidArgument( + "out_delta_sparse_weights and sparse_weights must have the same " + "length, got ", + sparse_weights_outputs.size(), " and ", sparse_weights_inputs.size()); OpOutputList dense_weights_outputs; TF_RETURN_IF_ERROR( context->output_list("out_delta_dense_weights", &dense_weights_outputs)); + if (dense_weights_outputs.size() != dense_weights_inputs.size()) + return errors::InvalidArgument( + "out_delta_dense_weights and dense_weights must have the same length, " + "got ", + dense_weights_outputs.size(), " and ", dense_weights_inputs.size()); for (int i = 0; i < sparse_weights_inputs.size(); ++i) { Tensor* delta_t; @@ -327,13 +341,28 @@ Status Examples::Initialize(OpKernelContext* const context, OpInputList sparse_example_indices_inputs; TF_RETURN_IF_ERROR(context->input_list("sparse_example_indices", &sparse_example_indices_inputs)); + if (sparse_example_indices_inputs.size() != num_sparse_features) + return errors::InvalidArgument( + "Expected ", num_sparse_features, + " tensors in sparse_example_indices but got ", + sparse_example_indices_inputs.size()); OpInputList sparse_feature_indices_inputs; TF_RETURN_IF_ERROR(context->input_list("sparse_feature_indices", &sparse_feature_indices_inputs)); + if (sparse_feature_indices_inputs.size() != num_sparse_features) + return errors::InvalidArgument( + "Expected ", num_sparse_features, + " tensors in sparse_feature_indices but got ", + sparse_feature_indices_inputs.size()); OpInputList sparse_feature_values_inputs; if (num_sparse_features_with_values > 0) { TF_RETURN_IF_ERROR(context->input_list("sparse_feature_values", &sparse_feature_values_inputs)); + if (sparse_feature_values_inputs.size() != num_sparse_features_with_values) + return errors::InvalidArgument( + "Expected ", num_sparse_features_with_values, + " tensors in sparse_feature_values but got ", + sparse_feature_values_inputs.size()); } const Tensor* example_weights_t; @@ -351,6 +380,11 @@ Status Examples::Initialize(OpKernelContext* const context, const Tensor* example_labels_t; TF_RETURN_IF_ERROR(context->input("example_labels", &example_labels_t)); auto example_labels = example_labels_t->flat(); + if (example_labels.size() != num_examples) { + return errors::InvalidArgument("Expected ", num_examples, + " example labels but got ", + example_labels.size()); + } OpInputList dense_features_inputs; TF_RETURN_IF_ERROR( @@ -400,6 +434,13 @@ Status Examples::CreateSparseFeatureRepresentation( sparse_example_indices_inputs[i].template flat(); auto feature_indices = sparse_feature_indices_inputs[i].template flat(); + if (example_indices.size() != feature_indices.size()) { + mutex_lock l(mu); + result = errors::InvalidArgument( + "Found mismatched example_indices and feature_indices [", + example_indices, "] vs [", feature_indices, "]"); + return; + } // Parse features for each example. Features for a particular example // are at the offsets (start_id, end_id] diff --git a/tensorflow/core/kernels/searchsorted_op.cc b/tensorflow/core/kernels/searchsorted_op.cc index 01e221dc471c4d..5f075a6a540e9f 100644 --- a/tensorflow/core/kernels/searchsorted_op.cc +++ b/tensorflow/core/kernels/searchsorted_op.cc @@ -86,6 +86,10 @@ class UpperBoundOp : public OpKernel { const Tensor& sorted_inputs_t = ctx->input(0); const Tensor& values_t = ctx->input(1); + // inputs must be at least a matrix + OP_REQUIRES( + ctx, sorted_inputs_t.shape().dims() >= 2, + errors::InvalidArgument("sorted input argument must be a matrix")); // must have same batch dim_size for both OP_REQUIRES(ctx, sorted_inputs_t.dim_size(0) == values_t.dim_size(0), Status(error::INVALID_ARGUMENT, @@ -127,6 +131,10 @@ class LowerBoundOp : public OpKernel { const Tensor& sorted_inputs_t = ctx->input(0); const Tensor& values_t = ctx->input(1); + // inputs must be at least a matrix + OP_REQUIRES( + ctx, sorted_inputs_t.shape().dims() >= 2, + errors::InvalidArgument("sorted input argument must be a matrix")); // must have same batch dim_size for both OP_REQUIRES(ctx, sorted_inputs_t.dim_size(0) == values_t.dim_size(0), Status(error::INVALID_ARGUMENT, diff --git a/tensorflow/core/kernels/session_ops.cc b/tensorflow/core/kernels/session_ops.cc index e7e73549bc32f3..2403fad55a43ed 100644 --- a/tensorflow/core/kernels/session_ops.cc +++ b/tensorflow/core/kernels/session_ops.cc @@ -118,7 +118,11 @@ class GetSessionTensorOp : public OpKernel { const Tensor& handle = ctx->input(0); const string& name = handle.scalar()(); Tensor val; - OP_REQUIRES_OK(ctx, ctx->session_state()->GetTensor(name, &val)); + auto session_state = ctx->session_state(); + OP_REQUIRES(ctx, session_state != nullptr, + errors::FailedPrecondition( + "GetSessionTensor called on null session state")); + OP_REQUIRES_OK(ctx, session_state->GetTensor(name, &val)); ctx->set_output(0, val); } @@ -160,7 +164,11 @@ class DeleteSessionTensorOp : public OpKernel { void Compute(OpKernelContext* ctx) override { const Tensor& handle = ctx->input(0); const string& name = handle.scalar()(); - OP_REQUIRES_OK(ctx, ctx->session_state()->DeleteTensor(name)); + auto session_state = ctx->session_state(); + OP_REQUIRES(ctx, session_state != nullptr, + errors::FailedPrecondition( + "DeleteSessionTensor called on null session state")); + OP_REQUIRES_OK(ctx, session_state->DeleteTensor(name)); } TF_DISALLOW_COPY_AND_ASSIGN(DeleteSessionTensorOp); diff --git a/tensorflow/core/kernels/sparse/kernels.cc b/tensorflow/core/kernels/sparse/kernels.cc index 0eea9f1feed5c3..dff9aeb83ccfec 100644 --- a/tensorflow/core/kernels/sparse/kernels.cc +++ b/tensorflow/core/kernels/sparse/kernels.cc @@ -22,6 +22,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor_types.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/errors.h" namespace tensorflow { namespace functor { @@ -63,6 +64,11 @@ Status SparseTensorToCSRSparseMatrixCPUFunctor::operator()( for (int64 i = 0; i < total_nnz; ++i) { // For now, the rows pointers store the corresponding row counts. + int64 ix = indices(i, 0) + 1; + if (ix >= csr_row_ptr.size()) { + return errors::InvalidArgument("Got an index ", ix, + " that is outside of csr_row_ptr"); + } csr_row_ptr(indices(i, 0) + 1) += 1; csr_col_ind(i) = indices(i, 1); } diff --git a/tensorflow/core/kernels/sparse/sparse_cholesky_op.cc b/tensorflow/core/kernels/sparse/sparse_cholesky_op.cc index 9a939276f0b6cb..47ab252317de5e 100644 --- a/tensorflow/core/kernels/sparse/sparse_cholesky_op.cc +++ b/tensorflow/core/kernels/sparse/sparse_cholesky_op.cc @@ -17,6 +17,8 @@ limitations under the License. #include #include +#include "tensorflow/core/framework/op_requires.h" + #define EIGEN_USE_THREADS #include "third_party/eigen3/Eigen/Core" @@ -82,8 +84,8 @@ class CSRSparseCholeskyCPUOp : public OpKernel { int64 num_rows; int batch_size; - ValidateInputs(ctx, *input_matrix, input_permutation_indices, &batch_size, - &num_rows); + OP_REQUIRES_OK(ctx, ValidateInputs(*input_matrix, input_permutation_indices, + &batch_size, &num_rows)); // Allocate batch pointers. Tensor batch_ptr(cpu_allocator(), DT_INT32, TensorShape({batch_size + 1})); @@ -226,49 +228,48 @@ class CSRSparseCholeskyCPUOp : public OpKernel { } private: - void ValidateInputs(OpKernelContext* ctx, - const CSRSparseMatrix& sparse_matrix, - const Tensor& permutation_indices, int* batch_size, - int64* num_rows) { - OP_REQUIRES(ctx, sparse_matrix.dtype() == DataTypeToEnum::value, - errors::InvalidArgument( - "Asked for a CSRSparseMatrix of type ", - DataTypeString(DataTypeToEnum::value), - " but saw dtype: ", DataTypeString(sparse_matrix.dtype()))); + Status ValidateInputs(const CSRSparseMatrix& sparse_matrix, + const Tensor& permutation_indices, int* batch_size, + int64* num_rows) { + if (sparse_matrix.dtype() != DataTypeToEnum::value) + return errors::InvalidArgument( + "Asked for a CSRSparseMatrix of type ", + DataTypeString(DataTypeToEnum::value), + " but saw dtype: ", DataTypeString(sparse_matrix.dtype())); const Tensor& dense_shape = sparse_matrix.dense_shape(); const int rank = dense_shape.dim_size(0); - OP_REQUIRES(ctx, rank == 2 || rank == 3, - errors::InvalidArgument("sparse matrix must have rank 2 or 3; ", - "but dense_shape has size ", rank)); + if (rank < 2 || rank > 3) + return errors::InvalidArgument("sparse matrix must have rank 2 or 3; ", + "but dense_shape has size ", rank); const int row_dim = (rank == 2) ? 0 : 1; auto dense_shape_vec = dense_shape.vec(); *num_rows = dense_shape_vec(row_dim); const int64 num_cols = dense_shape_vec(row_dim + 1); - OP_REQUIRES(ctx, *num_rows == num_cols, - errors::InvalidArgument("sparse matrix must be square; got: ", - *num_rows, " != ", num_cols)); + if (*num_rows != num_cols) + return errors::InvalidArgument( + "sparse matrix must be square; got: ", *num_rows, " != ", num_cols); const TensorShape& perm_shape = permutation_indices.shape(); - OP_REQUIRES( - ctx, perm_shape.dims() + 1 == rank, - errors::InvalidArgument( - "sparse matrix must have the same rank as permutation; got: ", rank, - " != ", perm_shape.dims(), " + 1.")); - OP_REQUIRES( - ctx, perm_shape.dim_size(rank - 2) == *num_rows, - errors::InvalidArgument( - "permutation must have the same number of elements in each batch " - "as the number of rows in sparse matrix; got: ", - perm_shape.dim_size(rank - 2), " != ", *num_rows)); + if (perm_shape.dims() + 1 != rank) + return errors::InvalidArgument( + "sparse matrix must have the same rank as permutation; got: ", rank, + " != ", perm_shape.dims(), " + 1."); + if (perm_shape.dim_size(rank - 2) != *num_rows) + return errors::InvalidArgument( + "permutation must have the same number of elements in each batch " + "as the number of rows in sparse matrix; got: ", + perm_shape.dim_size(rank - 2), " != ", *num_rows); *batch_size = sparse_matrix.batch_size(); if (*batch_size > 1) { - OP_REQUIRES( - ctx, perm_shape.dim_size(0) == *batch_size, - errors::InvalidArgument("permutation must have the same batch size " - "as sparse matrix; got: ", - perm_shape.dim_size(0), " != ", *batch_size)); + if (perm_shape.dim_size(0) != *batch_size) + return errors::InvalidArgument( + "permutation must have the same batch size " + "as sparse matrix; got: ", + perm_shape.dim_size(0), " != ", *batch_size); } + + return Status::OK(); } }; diff --git a/tensorflow/core/kernels/sparse_add_op.cc b/tensorflow/core/kernels/sparse_add_op.cc index 0cf40a709a39a7..2bd05fa41adc26 100644 --- a/tensorflow/core/kernels/sparse_add_op.cc +++ b/tensorflow/core/kernels/sparse_add_op.cc @@ -14,6 +14,7 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/op_requires.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_util.h" @@ -43,6 +44,11 @@ class SparseAddOp : public OpKernel { b_indices->shape().DebugString())); const int64 a_nnz = a_indices->dim_size(0); const int64 b_nnz = b_indices->dim_size(0); + const int num_dims = a_indices->dim_size(1); + OP_REQUIRES(ctx, b_indices->dim_size(1) == num_dims, + errors::InvalidArgument( + "Input indices must have the same dimension, got ", + num_dims, " and ", b_indices->dim_size(1))); OP_REQUIRES_OK(ctx, ctx->input("a_values", &a_values_t)); OP_REQUIRES_OK(ctx, ctx->input("b_values", &b_values_t)); @@ -71,6 +77,13 @@ class SparseAddOp : public OpKernel { "Input shapes should be a vector but received shapes ", a_shape->shape().DebugString(), " and ", b_shape->shape().DebugString())); + OP_REQUIRES( + ctx, a_shape->NumElements() == num_dims, + errors::InvalidArgument("Second dimension of a_indices and length of " + "a_shape must match, got ", + num_dims, " and ", a_shape->NumElements())); + OP_REQUIRES(ctx, num_dims > 0, + errors::InvalidArgument("Tesors must not be empty")); OP_REQUIRES( ctx, a_shape->IsSameSize(*b_shape), errors::InvalidArgument( @@ -99,7 +112,6 @@ class SparseAddOp : public OpKernel { std::vector> entries_to_copy; // from_a?, idx entries_to_copy.reserve(a_nnz + b_nnz); std::vector out_values; - const int num_dims = a_shape->dim_size(0); // The input and output sparse tensors are assumed to be ordered along // increasing dimension number. diff --git a/tensorflow/core/kernels/sparse_concat_op.cc b/tensorflow/core/kernels/sparse_concat_op.cc index 3b2a0cb0f34ed3..d49f92ea556eb2 100644 --- a/tensorflow/core/kernels/sparse_concat_op.cc +++ b/tensorflow/core/kernels/sparse_concat_op.cc @@ -27,6 +27,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor_util.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/lib/gtl/inlined_vector.h" +#include "tensorflow/core/util/overflow.h" #include "tensorflow/core/util/sparse/sparse_tensor.h" namespace tensorflow { @@ -66,13 +67,32 @@ class SparseConcatOp : public OpKernel { OP_REQUIRES(context, shapes.size() == N, errors::InvalidArgument("Expected ", N, " input shapes, got ", shapes.size())); + bool overflow_ocurred = false; for (int i = 0; i < N; i++) { + int64 new_num_elements = 1; OP_REQUIRES(context, TensorShapeUtils::IsVector(shapes[i].shape()), errors::InvalidArgument( "Input shapes should be a vector but received shape ", shapes[i].shape().DebugString(), " at position ", i)); + auto input_shape_vector = shapes[i].vec(); + for (int j = 0; j < input_shape_vector.size(); j++) { + new_num_elements = + MultiplyWithoutOverflow(new_num_elements, input_shape_vector(j)); + if (new_num_elements < 0) { + overflow_ocurred = true; + break; + } + } + + if (overflow_ocurred) { + break; + } } + OP_REQUIRES( + context, !overflow_ocurred, + errors::Internal("Encountered overflow from large input shape.")); + const TensorShape input_shape(shapes[0].vec()); const int input_rank = input_shape.dims(); const int concat_dim = (concat_dim_attr_ < 0) diff --git a/tensorflow/core/kernels/sparse_cross_op.cc b/tensorflow/core/kernels/sparse_cross_op.cc index 9a80aad5d04fae..78e7561825a8a1 100644 --- a/tensorflow/core/kernels/sparse_cross_op.cc +++ b/tensorflow/core/kernels/sparse_cross_op.cc @@ -27,6 +27,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/types.h" +#include "tensorflow/core/framework/types.pb.h" #include "tensorflow/core/lib/core/stringpiece.h" #include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/platform/fingerprint.h" @@ -460,10 +461,19 @@ int64 CalculateBatchSize(const OpInputList& shapes_list_in, Status ValidateInput(const OpInputList& indices_list_in, const OpInputList& values_list_in, const OpInputList& shapes_list_in, - const OpInputList& dense_list_in) { + const OpInputList& dense_list_in, + const DataType& internal_type) { const auto size = indices_list_in.size(); + // Only perform internal_type check for SparseCrossOp. + // Check if the internal_type is not invalid before doing so. + bool check_type = internal_type != DT_INVALID; // Validates indices_list_in OpInputList. for (int i = 0; i < size; i++) { + if (check_type && indices_list_in[i].dtype() != DT_INT64) { + return errors::InvalidArgument("Input indices should be of type ", + DT_INT64, " but received ", + indices_list_in[i].dtype()); + } if (!TensorShapeUtils::IsMatrix(indices_list_in[i].shape())) { return errors::InvalidArgument( "Input indices should be a matrix but received shape ", @@ -482,6 +492,14 @@ Status ValidateInput(const OpInputList& indices_list_in, values_list_in.size()); } for (int i = 0; i < size; i++) { + // Make sure to avoid the expected type to be string, but input values to be + // int64. + if (check_type && internal_type == DT_STRING && + values_list_in[i].dtype() == DT_INT64) { + return errors::InvalidArgument("Input values should be of internal type ", + internal_type, " but received ", + values_list_in[i].dtype()); + } if (!TensorShapeUtils::IsVector(values_list_in[i].shape())) { return errors::InvalidArgument( "Input values should be a vector but received shape ", @@ -502,6 +520,11 @@ Status ValidateInput(const OpInputList& indices_list_in, shapes_list_in.size()); } for (int i = 0; i < size; i++) { + if (check_type && shapes_list_in[i].dtype() != DT_INT64) { + return errors::InvalidArgument("Input shape should be of type ", DT_INT64, + " but received ", + shapes_list_in[i].dtype()); + } if (!TensorShapeUtils::IsVector(shapes_list_in[i].shape())) { return errors::InvalidArgument( "Input shapes should be a vector but received shape ", @@ -517,6 +540,14 @@ Status ValidateInput(const OpInputList& indices_list_in, // Validates dense_list_in OpInputList for (int i = 0; i < dense_list_in.size(); ++i) { + // Make sure to avoid the expected type to be string, but input values to be + // int64. + if (check_type && internal_type == DT_STRING && + dense_list_in[i].dtype() == DT_INT64) { + return errors::InvalidArgument("Dense inputs should be of internal type ", + internal_type, " but received ", + dense_list_in[i].dtype()); + } if (!TensorShapeUtils::IsMatrix(dense_list_in[i].shape())) { return errors::InvalidArgument( "Dense inputs should be a matrix but received shape ", @@ -698,6 +729,7 @@ class SparseCrossOp : public OpKernel { int64 signed_hash_key_; OP_REQUIRES_OK(context, context->GetAttr("hash_key", &signed_hash_key_)); hash_key_ = static_cast(signed_hash_key_); + OP_REQUIRES_OK(context, context->GetAttr("internal_type", &internal_type_)); } void Compute(OpKernelContext* context) override { @@ -711,8 +743,10 @@ class SparseCrossOp : public OpKernel { OP_REQUIRES_OK(context, context->input_list("dense_inputs", &dense_list_in)); - OP_REQUIRES_OK(context, ValidateInput(indices_list_in, values_list_in, - shapes_list_in, dense_list_in)); + DataType internal_type = internal_type_; + OP_REQUIRES_OK( + context, ValidateInput(indices_list_in, values_list_in, shapes_list_in, + dense_list_in, internal_type)); std::vector>> columns = GenerateColumnsFromInput(indices_list_in, values_list_in, @@ -756,6 +790,7 @@ class SparseCrossOp : public OpKernel { private: int64 num_buckets_; uint64 hash_key_; + DataType internal_type_; }; class SparseCrossV2Op : public OpKernel { @@ -773,8 +808,11 @@ class SparseCrossV2Op : public OpKernel { OP_REQUIRES_OK(context, context->input_list("dense_inputs", &dense_list_in)); - OP_REQUIRES_OK(context, ValidateInput(indices_list_in, values_list_in, - shapes_list_in, dense_list_in)); + // Set internal_type to invalid_type so that the check will be ignored. + DataType internal_type = DT_INVALID; + OP_REQUIRES_OK( + context, ValidateInput(indices_list_in, values_list_in, shapes_list_in, + dense_list_in, internal_type)); const Tensor* sep_t; OP_REQUIRES_OK(context, context->input("sep", &sep_t)); @@ -832,8 +870,11 @@ class SparseCrossHashedOp : public OpKernel { OP_REQUIRES_OK(context, context->input_list("dense_inputs", &dense_list_in)); - OP_REQUIRES_OK(context, ValidateInput(indices_list_in, values_list_in, - shapes_list_in, dense_list_in)); + // Set internal_type to invalid_type so that the check will be ignored. + DataType internal_type = DT_INVALID; + OP_REQUIRES_OK( + context, ValidateInput(indices_list_in, values_list_in, shapes_list_in, + dense_list_in, internal_type)); const Tensor* num_buckets_t; OP_REQUIRES_OK(context, context->input("num_buckets", &num_buckets_t)); diff --git a/tensorflow/core/kernels/sparse_dense_binary_op_shared.cc b/tensorflow/core/kernels/sparse_dense_binary_op_shared.cc index 3a5e66a0e73ea6..edc238faa98db9 100644 --- a/tensorflow/core/kernels/sparse_dense_binary_op_shared.cc +++ b/tensorflow/core/kernels/sparse_dense_binary_op_shared.cc @@ -78,6 +78,11 @@ class SparseDenseBinaryOpShared : public OpKernel { "but received shapes: ", values_t->shape().DebugString(), " and ", shape_t->shape().DebugString())); + OP_REQUIRES( + ctx, values_t->dim_size(0) == indices_t->dim_size(0), + errors::InvalidArgument( + "The first dimension of values and indices should match. (", + values_t->dim_size(0), " vs. ", indices_t->dim_size(0), ")")); const auto indices_mat = indices_t->matrix(); const auto shape_vec = shape_t->vec(); @@ -109,7 +114,10 @@ class SparseDenseBinaryOpShared : public OpKernel { OP_REQUIRES_OK( ctx, ctx->allocate_temp(DataTypeToEnum::value, TensorShape({nnz}), &dense_gathered)); - + bool op_is_div = false; + if (absl::StrContains(ctx->op_kernel().type_string_view(), "Div")) { + op_is_div = true; + } // Pulls relevant entries from the dense side, with reshape and broadcasting // *of the dense side* taken into account. Use a TensorRef to avoid blowing // up memory. @@ -138,6 +146,12 @@ class SparseDenseBinaryOpShared : public OpKernel { errors::InvalidArgument("Provided indices are out-of-bounds w.r.t. " \ "dense side with broadcasted shape")); \ dense_gathered_flat(i) = rhs_ref.coeff(idx); \ + if (op_is_div) { \ + OP_REQUIRES(ctx, dense_gathered_flat(i) != T(0), \ + errors::InvalidArgument( \ + "SparseDenseCwiseDiv cannot divide by zero," \ + "but input dense tensor contains zero ")); \ + } \ } \ break; \ } diff --git a/tensorflow/core/kernels/sparse_fill_empty_rows_op.cc b/tensorflow/core/kernels/sparse_fill_empty_rows_op.cc index 542069ccd88e18..2b80903a9210b1 100644 --- a/tensorflow/core/kernels/sparse_fill_empty_rows_op.cc +++ b/tensorflow/core/kernels/sparse_fill_empty_rows_op.cc @@ -69,7 +69,9 @@ class SparseFillEmptyRowsOp : public OpKernel { default_value_t.shape().DebugString())); // TODO(ebrevdo): add shape checks between values, indices, // dense_shape. Also add check that dense rank > 0. - + // Also add check that dense rank > 0. + OP_REQUIRES(context, dense_shape_t.NumElements() != 0, + errors::InvalidArgument("Dense shape cannot be empty.")); const T& default_value = default_value_t.scalar()(); const auto indices = indices_t.matrix(); const auto values = values_t.vec(); diff --git a/tensorflow/core/kernels/sparse_matmul_op.cc b/tensorflow/core/kernels/sparse_matmul_op.cc index eb460147d71f39..1f66f8058a613a 100644 --- a/tensorflow/core/kernels/sparse_matmul_op.cc +++ b/tensorflow/core/kernels/sparse_matmul_op.cc @@ -1039,6 +1039,10 @@ class SparseMatMulOp : public OpKernel { if (transpose_b) { // TODO(agarwal): avoid transposing the matrix here and directly handle // transpose in CreateDenseSlices. + OP_REQUIRES(ctx, right->dim_size(0) != 0, + errors::InvalidArgument("b has an entry 0 in it's shape.")); + OP_REQUIRES(ctx, right->dim_size(1) != 0, + errors::InvalidArgument("b has an entry 0 in it's shape.")); right_tr.reset( new Tensor(right->dtype(), TensorShape({right->dim_size(1), right->dim_size(0)}))); diff --git a/tensorflow/core/kernels/sparse_reduce_op.cc b/tensorflow/core/kernels/sparse_reduce_op.cc index b65f31e5eb174e..eb56b7390b0f9a 100644 --- a/tensorflow/core/kernels/sparse_reduce_op.cc +++ b/tensorflow/core/kernels/sparse_reduce_op.cc @@ -219,7 +219,20 @@ class SparseReduceOp : public OpKernel { sp.Reorder(reduction.reorder_dims); for (const auto &g : sp.group(reduction.group_by_dims)) { Op::template Run(ctx, reduced_val, g.template values()); + OP_REQUIRES(ctx, + output_strides.empty() || + (g.group().size() == output_strides.size()), + errors::Internal( + "Expected group size and output_strides size to match", + ", but got ", g.group().size(), " and ", + output_strides.size())); const int64 idx = CoordinatesToFlatIndex(g.group(), output_strides); + OP_REQUIRES(ctx, + idx >= 0 && idx < out_flat.size(), + errors::Internal( + "Obtained a write index of ", idx, + " which is outside of bounds of [0, ", + out_flat.size(), ")")); out_flat(idx) = reduced_val(); VLOG(2) << "coords: " << absl::StrJoin(g.group(), ",") << "; idx: " << idx << "; group " << Op::Name() << ": " diff --git a/tensorflow/core/kernels/sparse_reshape_op.cc b/tensorflow/core/kernels/sparse_reshape_op.cc index 6eb5f0af635c28..c3b1932a1735b0 100644 --- a/tensorflow/core/kernels/sparse_reshape_op.cc +++ b/tensorflow/core/kernels/sparse_reshape_op.cc @@ -26,6 +26,7 @@ limitations under the License. #include "tensorflow/core/framework/types.h" #include "tensorflow/core/kernels/reshape_util.h" #include "tensorflow/core/lib/gtl/inlined_vector.h" +#include "tensorflow/core/platform/errors.h" namespace tensorflow { @@ -34,6 +35,17 @@ class SparseReshapeOp : public OpKernel { explicit SparseReshapeOp(OpKernelConstruction* context) : OpKernel(context) {} void Compute(OpKernelContext* context) override { + const Tensor& input_indices_in = context->input(0); + const Tensor& input_shape_in = context->input(1); + + OP_REQUIRES(context, TensorShapeUtils::IsMatrix(input_indices_in.shape()), + errors::InvalidArgument("Input must be a matrix.")); + OP_REQUIRES(context, TensorShapeUtils::IsVector(input_shape_in.shape()), + errors::InvalidArgument("Input shape must be a vector.")); + OP_REQUIRES(context, + input_indices_in.dim_size(1) == input_shape_in.dim_size(0), + errors::InvalidArgument( + "Input tensor rank must match input shape length.")); ReshapeSparseTensor(context, context->input(0), context->input(1), context->input(2), 0 /* output indices index */, 1 /* output shape index */); diff --git a/tensorflow/core/kernels/sparse_sparse_binary_op_shared.cc b/tensorflow/core/kernels/sparse_sparse_binary_op_shared.cc index 43dc9ae70cd627..eb993a5965043b 100644 --- a/tensorflow/core/kernels/sparse_sparse_binary_op_shared.cc +++ b/tensorflow/core/kernels/sparse_sparse_binary_op_shared.cc @@ -150,6 +150,7 @@ class SparseSparseBinaryOpShared : public OpKernel { const int64 a_nnz = a_indices_t->dim_size(0); const int64 b_nnz = b_indices_t->dim_size(0); + const auto a_values = a_values_t->vec(); const auto b_values = b_values_t->vec(); @@ -166,6 +167,14 @@ class SparseSparseBinaryOpShared : public OpKernel { "Input shapes should be a vector but received shapes ", a_shape_t->shape().DebugString(), " and ", b_shape_t->shape().DebugString())); + const int num_dims = a_indices_t->dim_size(1); + OP_REQUIRES( + ctx, a_shape_t->NumElements() == num_dims, + errors::InvalidArgument("Second dimension of a_indices and length of " + "a_shape must match, got ", + num_dims, " and ", a_shape_t->NumElements())); + OP_REQUIRES(ctx, num_dims > 0, + errors::InvalidArgument("Tensors must not be empty")); OP_REQUIRES(ctx, a_shape_t->IsSameSize(*b_shape_t), errors::InvalidArgument( "Operands do not have the same ranks; got shapes: ", @@ -180,7 +189,6 @@ class SparseSparseBinaryOpShared : public OpKernel { " for dimension ", i)); } - const int num_dims = a_indices_t->dim_size(1); const auto a_indices_mat = a_indices_t->matrix(); const auto b_indices_mat = b_indices_t->matrix(); std::vector a_augmented_values, b_augmented_values; diff --git a/tensorflow/core/kernels/sparse_split_op.cc b/tensorflow/core/kernels/sparse_split_op.cc index 3d02be47cbbef5..b0c147da8a8344 100644 --- a/tensorflow/core/kernels/sparse_split_op.cc +++ b/tensorflow/core/kernels/sparse_split_op.cc @@ -18,6 +18,7 @@ limitations under the License. #include #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/util/overflow.h" #include "tensorflow/core/util/sparse/sparse_tensor.h" namespace tensorflow { @@ -63,6 +64,16 @@ class SparseSplitOp : public OpKernel { input_shape.vec()(split_dim), "), got ", num_split_)); + // Prevent overflow by constructing the dense shape separately + int64 total_elements = 1; + const auto input_shape_flat = input_shape.flat(); + for (int i = 0; i < input_shape.NumElements(); i++) { + total_elements = + MultiplyWithoutOverflow(total_elements, input_shape_flat(i)); + OP_REQUIRES(context, total_elements >= 0, + errors::Internal("Encountered overflow in dense shape")); + } + sparse::SparseTensor sparse_tensor; OP_REQUIRES_OK(context, sparse::SparseTensor::Create( diff --git a/tensorflow/core/kernels/sparse_tensors_map_ops.cc b/tensorflow/core/kernels/sparse_tensors_map_ops.cc index c2c0e43ca2ba8d..5ea5fca544d3e9 100644 --- a/tensorflow/core/kernels/sparse_tensors_map_ops.cc +++ b/tensorflow/core/kernels/sparse_tensors_map_ops.cc @@ -21,9 +21,6 @@ limitations under the License. #include #include -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/register_types.h" - #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/resource_mgr.h" @@ -31,6 +28,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor_util.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/lib/gtl/inlined_vector.h" +#include "tensorflow/core/util/overflow.h" #include "tensorflow/core/util/sparse/sparse_tensor.h" namespace tensorflow { @@ -254,7 +252,22 @@ class AddManySparseToTensorsMapOp : public SparseTensorAccessingOp { errors::InvalidArgument( "Rank of input SparseTensor should be > 1, but saw rank: ", rank)); - TensorShape tensor_input_shape(input_shape->vec()); + auto input_shape_vec = input_shape->vec(); + int new_num_elements = 1; + bool overflow_ocurred = false; + for (int i = 0; i < input_shape_vec.size(); i++) { + new_num_elements = + MultiplyWithoutOverflow(new_num_elements, input_shape_vec(i)); + if (new_num_elements < 0) { + overflow_ocurred = true; + } + } + + OP_REQUIRES( + context, !overflow_ocurred, + errors::Internal("Encountered overflow from large input shape.")); + + TensorShape tensor_input_shape(input_shape_vec); gtl::InlinedVector std_order(rank); std::iota(std_order.begin(), std_order.end(), 0); SparseTensor input_st; @@ -262,8 +275,7 @@ class AddManySparseToTensorsMapOp : public SparseTensorAccessingOp { tensor_input_shape, std_order, &input_st)); - auto input_shape_t = input_shape->vec(); - const int64 N = input_shape_t(0); + const int64 N = input_shape_vec(0); Tensor sparse_handles(DT_INT64, TensorShape({N})); auto sparse_handles_t = sparse_handles.vec(); @@ -274,7 +286,7 @@ class AddManySparseToTensorsMapOp : public SparseTensorAccessingOp { // minibatch entries. TensorShape output_shape; OP_REQUIRES_OK(context, TensorShapeUtils::MakeShape( - input_shape_t.data() + 1, + input_shape_vec.data() + 1, input_shape->NumElements() - 1, &output_shape)); // Get groups by minibatch dimension diff --git a/tensorflow/core/kernels/string_ngrams_op.cc b/tensorflow/core/kernels/string_ngrams_op.cc index 8aed2b3831a2f4..97af9abc4454ac 100644 --- a/tensorflow/core/kernels/string_ngrams_op.cc +++ b/tensorflow/core/kernels/string_ngrams_op.cc @@ -53,6 +53,12 @@ class StringNGramsOp : public tensorflow::OpKernel { } void Compute(tensorflow::OpKernelContext* context) override { + for (int ngram_width : ngram_widths_) { + OP_REQUIRES( + context, ngram_width > 0, + errors::InvalidArgument("ngram_widths must contain positive values")); + } + const tensorflow::Tensor* data; OP_REQUIRES_OK(context, context->input("data", &data)); const auto& input_data = data->flat().data(); @@ -61,16 +67,28 @@ class StringNGramsOp : public tensorflow::OpKernel { OP_REQUIRES_OK(context, context->input("data_splits", &splits)); const auto& splits_vec = splits->flat(); - // Validate that the splits are valid indices into data + // Validate that the splits are valid indices into data, only if there are + // splits specified. const int input_data_size = data->flat().size(); const int splits_vec_size = splits_vec.size(); - for (int i = 0; i < splits_vec_size; ++i) { - bool valid_splits = splits_vec(i) >= 0; - valid_splits = valid_splits && (splits_vec(i) <= input_data_size); - OP_REQUIRES( - context, valid_splits, - errors::InvalidArgument("Invalid split value ", splits_vec(i), - ", must be in [0,", input_data_size, "]")); + if (splits_vec_size > 0) { + int prev_split = splits_vec(0); + OP_REQUIRES(context, prev_split == 0, + errors::InvalidArgument("First split value must be 0, got ", + prev_split)); + for (int i = 1; i < splits_vec_size; ++i) { + bool valid_splits = splits_vec(i) >= prev_split; + valid_splits = valid_splits && (splits_vec(i) <= input_data_size); + OP_REQUIRES(context, valid_splits, + errors::InvalidArgument( + "Invalid split value ", splits_vec(i), ", must be in [", + prev_split, ", ", input_data_size, "]")); + prev_split = splits_vec(i); + } + OP_REQUIRES(context, prev_split == input_data_size, + errors::InvalidArgument( + "Last split value must be data size. Expected ", + input_data_size, ", got ", prev_split)); } int num_batch_items = splits_vec.size() - 1; @@ -174,13 +192,31 @@ class StringNGramsOp : public tensorflow::OpKernel { ngram->append(left_pad_); ngram->append(separator_); } + // Only output first num_tokens - 1 pairs of data and separator for (int n = 0; n < num_tokens - 1; ++n) { ngram->append(data[data_start_index + n]); ngram->append(separator_); } - ngram->append(data[data_start_index + num_tokens - 1]); - for (int n = 0; n < right_padding; ++n) { - ngram->append(separator_); + // Handle case when there are no tokens or no right padding as these can + // result in consecutive separators. + if (num_tokens > 0) { + // If we have tokens, then output last and then pair each separator with + // the right padding that follows, to ensure ngram ends either with the + // token or with the right pad. + ngram->append(data[data_start_index + num_tokens - 1]); + for (int n = 0; n < right_padding; ++n) { + ngram->append(separator_); + ngram->append(right_pad_); + } + } else { + // If we don't have tokens, then the last item inserted into the ngram + // has been the separator from the left padding loop above. Hence, + // output right pad and separator and make sure to finish with a + // padding, not a separator. + for (int n = 0; n < right_padding - 1; ++n) { + ngram->append(right_pad_); + ngram->append(separator_); + } ngram->append(right_pad_); } diff --git a/tensorflow/core/kernels/string_ngrams_op_test.cc b/tensorflow/core/kernels/string_ngrams_op_test.cc index b89de9ad16dab8..0d52283bd8fb9d 100644 --- a/tensorflow/core/kernels/string_ngrams_op_test.cc +++ b/tensorflow/core/kernels/string_ngrams_op_test.cc @@ -542,6 +542,40 @@ TEST_F(NgramKernelTest, TestEmptyInput) { assert_int64_equal(expected_splits, *GetOutput(1)); } +TEST_F(NgramKernelTest, TestNoTokens) { + MakeOp("|", {3}, "L", "R", -1, false); + // Batch items are: + // 0: + // 1: "a" + AddInputFromArray(TensorShape({1}), {"a"}); + AddInputFromArray(TensorShape({3}), {0, 0, 1}); + TF_ASSERT_OK(RunOpKernel()); + + std::vector expected_values( + {"L|L|R", "L|R|R", // no input in first split + "L|L|a", "L|a|R", "a|R|R"}); // second split + std::vector expected_splits({0, 2, 5}); + + assert_string_equal(expected_values, *GetOutput(0)); + assert_int64_equal(expected_splits, *GetOutput(1)); +} + +TEST_F(NgramKernelTest, TestNoTokensNoPad) { + MakeOp("|", {3}, "", "", 0, false); + // Batch items are: + // 0: + // 1: "a" + AddInputFromArray(TensorShape({1}), {"a"}); + AddInputFromArray(TensorShape({3}), {0, 0, 1}); + TF_ASSERT_OK(RunOpKernel()); + + std::vector expected_values({}); + std::vector expected_splits({0, 0, 0}); + + assert_string_equal(expected_values, *GetOutput(0)); + assert_int64_equal(expected_splits, *GetOutput(1)); +} + TEST_F(NgramKernelTest, ShapeFn) { ShapeInferenceTestOp op("StringNGrams"); INFER_OK(op, "?;?", "[?];[?]"); diff --git a/tensorflow/core/kernels/substr_op.cc b/tensorflow/core/kernels/substr_op.cc index e382381e122324..ab83efda2a2e17 100644 --- a/tensorflow/core/kernels/substr_op.cc +++ b/tensorflow/core/kernels/substr_op.cc @@ -51,6 +51,11 @@ class SubstrOp : public OpKernel { const Tensor& len_tensor = context->input(2); const TensorShape& input_shape = input_tensor.shape(); const TensorShape& pos_shape = pos_tensor.shape(); + const TensorShape& len_shape = len_tensor.shape(); + OP_REQUIRES(context, (pos_shape == len_shape), + errors::InvalidArgument( + "pos and len should have the same shape, got: ", + pos_shape.DebugString(), " vs. ", len_shape.DebugString())); bool is_scalar = TensorShapeUtils::IsScalar(pos_shape); diff --git a/tensorflow/core/kernels/transpose_functor.h b/tensorflow/core/kernels/transpose_functor.h index 0c22b11b7c6813..44193ab40273b1 100644 --- a/tensorflow/core/kernels/transpose_functor.h +++ b/tensorflow/core/kernels/transpose_functor.h @@ -19,6 +19,7 @@ limitations under the License. #include #include #include + #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_types.h" #include "tensorflow/core/platform/logging.h" @@ -166,7 +167,6 @@ template Status DoTransposeImpl(const Device& d, const Tensor& in, const gtl::ArraySlice perm, bool conjugate, Tensor* out) { - CHECK_GE(in.dims(), 2); CHECK_EQ(in.dims(), out->dims()); CHECK_EQ(in.dims(), perm.size()); CHECK_EQ(in.dtype(), out->dtype()); diff --git a/tensorflow/core/kernels/unicode_ops.cc b/tensorflow/core/kernels/unicode_ops.cc index d3a7ad7b2866f7..ab09dbe1d54293 100644 --- a/tensorflow/core/kernels/unicode_ops.cc +++ b/tensorflow/core/kernels/unicode_ops.cc @@ -533,6 +533,21 @@ class UnicodeEncodeOp : public OpKernel { const Tensor& input_splits = context->input(1); const auto input_splits_flat = input_splits.flat(); + OP_REQUIRES( + context, input_splits.NumElements() > 0, + errors::InvalidArgument("Input_splits should contain elements, but " + "given input_values has 0 elements")); + // Operation will treat first argument in input_splits as if it were zero + // regardless of its actual value since splits should begin with zero and + // end with the length of the input values vector. + OP_REQUIRES( + context, input_splits_flat(0) == 0, + errors::InvalidArgument("First value in input_splits must be zero.")); + OP_REQUIRES(context, + input_splits_flat(input_splits_flat.size() - 1) == + input_tensor_flat.size(), + errors::InvalidArgument("Last value in input_splits must be " + "equal to length of input_tensor.")); // Since we limit to a 2-D input (flat_values of rank 1 and a single splits // tensor), our output dimension will be 1 with it's size equal to the // number of splits (outer dimension or ragged tensor). @@ -548,6 +563,14 @@ class UnicodeEncodeOp : public OpKernel { for (int i = 1; i < input_splits_flat.size(); ++i) { icu::UnicodeString unicode_string; icu::UnicodeStringAppendable appendable_unicode_string(unicode_string); + OP_REQUIRES( + context, input_splits_flat(i - 1) <= input_splits_flat(i), + errors::InvalidArgument( + "Values in input_splits must be equal or in ascending order.")); + OP_REQUIRES( + context, input_splits_flat(i) <= input_tensor_flat.size(), + errors::InvalidArgument("Values in input_splits must be less than or " + "equal to input_tensor length.")); for (; idx < input_splits_flat(i); ++idx) { int32 code_point = input_tensor_flat(idx); // Check for invalid code point diff --git a/tensorflow/core/kernels/unravel_index_op.cc b/tensorflow/core/kernels/unravel_index_op.cc index b45ff5e5b85f04..5b895799bbf4c5 100644 --- a/tensorflow/core/kernels/unravel_index_op.cc +++ b/tensorflow/core/kernels/unravel_index_op.cc @@ -53,6 +53,14 @@ class UnravelIndexOp : public OpKernel { dims_tensor.shape().DebugString(), "\"")); auto dims = dims_tensor.vec(); + // Make sure dims does not contain a zero + for (int i = 0; i < dims.size(); i++) { + OP_REQUIRES( + ctx, dims(i) != 0, + errors::InvalidArgument("Input dims cannot contain a dim of zero, " + "but dims contains zero at index ", + i)); + } // Chek to make sure indices is not out of boundary Eigen::Tensor dims_prod_eigen = dims.prod(); diff --git a/tensorflow/core/kernels/unsorted_segment_join_op.cc b/tensorflow/core/kernels/unsorted_segment_join_op.cc index 7464e165e46c8b..9acfe7fb1e4952 100644 --- a/tensorflow/core/kernels/unsorted_segment_join_op.cc +++ b/tensorflow/core/kernels/unsorted_segment_join_op.cc @@ -90,6 +90,8 @@ class UnsortedSegmentJoinOp : public OpKernel { const int32 segment_dims = segment_id_shape.dims(); const Tensor& num_segments_tensor = context->input(2); + OP_REQUIRES(context, num_segments_tensor.NumElements() != 0, + errors::InvalidArgument("Number of segments cannot be empty.")); auto num_segments = num_segments_tensor.scalar()(); OP_REQUIRES(context, segment_dims != 0, diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc index 11bfb9a3346d04..ad11e0b7d64639 100644 --- a/tensorflow/core/ops/array_ops.cc +++ b/tensorflow/core/ops/array_ops.cc @@ -2887,6 +2887,10 @@ REGISTER_OP("Dequantize") if (!s.ok() && s.code() != error::NOT_FOUND) { return s; } + if (axis < -1) { + return errors::InvalidArgument("axis should be at least -1, got ", + axis); + } const int minmax_rank = (axis == -1) ? 0 : 1; TF_RETURN_IF_ERROR(shape_inference::UnchangedShape(c)); ShapeHandle minmax; diff --git a/tensorflow/core/ops/sparse_ops.cc b/tensorflow/core/ops/sparse_ops.cc index 906cef1f5ecafe..b1e40e66af8929 100644 --- a/tensorflow/core/ops/sparse_ops.cc +++ b/tensorflow/core/ops/sparse_ops.cc @@ -16,6 +16,7 @@ limitations under the License. #include "tensorflow/core/framework/common_shape_fns.h" #include "tensorflow/core/framework/op.h" #include "tensorflow/core/framework/shape_inference.h" +#include "tensorflow/core/platform/errors.h" namespace tensorflow { @@ -619,6 +620,8 @@ REGISTER_OP("SparseFillEmptyRows") DimensionHandle unused_dim; TF_RETURN_IF_ERROR(c->Merge(c->Dim(input_indices, 1), c->Dim(input_shape, 0), &unused_dim)); + if (c->Value(c->NumElements(input_shape)) == 0) + return errors::InvalidArgument("dense_shape must not be empty"); ShapeHandle output_indices = c->Matrix(InferenceContext::kUnknownDim, c->NumElements(input_shape)); ShapeHandle output_values = c->Vector(InferenceContext::kUnknownDim); diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index 2e3bf62f6089a9..efd1256f3f1d79 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -22,7 +22,7 @@ limitations under the License. // tensorflow/tools/pip_package/setup.py #define TF_MAJOR_VERSION 2 #define TF_MINOR_VERSION 3 -#define TF_PATCH_VERSION 1 +#define TF_PATCH_VERSION 4 // TF_VERSION_SUFFIX is non-empty for pre-releases (e.g. "-alpha", "-alpha.1", // "-beta", "-rc", "-rc.1") diff --git a/tensorflow/core/util/sparse/sparse_tensor.h b/tensorflow/core/util/sparse/sparse_tensor.h index bc4e2c88f1c99e..dac51aac08b7a3 100644 --- a/tensorflow/core/util/sparse/sparse_tensor.h +++ b/tensorflow/core/util/sparse/sparse_tensor.h @@ -527,6 +527,10 @@ inline Status SparseTensor::Split(const SparseTensor& input_tensor, for (int i = 0; i < input_tensor.indices().dim_size(0); ++i) { const int dim = input_tensor.indices().matrix()(i, split_dim); int slice_index = GetSliceIndex(dim, split_size, residual); + if (slice_index >= num_values.size()) { + return errors::InvalidArgument("Slice index ", slice_index, + " is larger than num_split."); + } num_values[slice_index]++; } diff --git a/tensorflow/java/maven/spark-tensorflow-connector/pom.xml b/tensorflow/java/maven/spark-tensorflow-connector/pom.xml index f40090ac45d6d9..19f5e29da2bf38 100644 --- a/tensorflow/java/maven/spark-tensorflow-connector/pom.xml +++ b/tensorflow/java/maven/spark-tensorflow-connector/pom.xml @@ -35,7 +35,7 @@ 1.8 2.4.5 2.7.3 - 4.11 + 4.13.1 diff --git a/tensorflow/java/maven/tensorflow-hadoop/pom.xml b/tensorflow/java/maven/tensorflow-hadoop/pom.xml index e900d81e5dab50..675a3369cf1ff3 100644 --- a/tensorflow/java/maven/tensorflow-hadoop/pom.xml +++ b/tensorflow/java/maven/tensorflow-hadoop/pom.xml @@ -16,7 +16,7 @@ 1.6 2.6.0 3.5.1 - 4.11 + 4.13.1 diff --git a/tensorflow/lite/BUILD b/tensorflow/lite/BUILD index 0eae6ad17c0e85..83e57ef4c4848a 100644 --- a/tensorflow/lite/BUILD +++ b/tensorflow/lite/BUILD @@ -478,6 +478,7 @@ cc_test( "testdata/test_min_runtime.bin", "testdata/test_model.bin", "testdata/test_model_broken.bin", + "testdata/unsupported_recursion.bin", ], tags = [ "tflite_not_portable", diff --git a/tensorflow/lite/c/common.c b/tensorflow/lite/c/common.c index e6b47896528a63..9af6d5151b50fa 100644 --- a/tensorflow/lite/c/common.c +++ b/tensorflow/lite/c/common.c @@ -43,8 +43,10 @@ int TfLiteIntArrayEqualsArray(const TfLiteIntArray* a, int b_size, #ifndef TF_LITE_STATIC_MEMORY TfLiteIntArray* TfLiteIntArrayCreate(int size) { - TfLiteIntArray* ret = - (TfLiteIntArray*)malloc(TfLiteIntArrayGetSizeInBytes(size)); + int alloc_size = TfLiteIntArrayGetSizeInBytes(size); + if (alloc_size <= 0) return NULL; + TfLiteIntArray* ret = (TfLiteIntArray*)malloc(alloc_size); + if (!ret) return ret; ret->size = size; return ret; } diff --git a/tensorflow/lite/core/subgraph.cc b/tensorflow/lite/core/subgraph.cc index 00a37815d21a4a..e1ed653ae1b6db 100644 --- a/tensorflow/lite/core/subgraph.cc +++ b/tensorflow/lite/core/subgraph.cc @@ -141,6 +141,42 @@ const char* GetTFLiteOpName(const TfLiteRegistration& op_reg) { return tflite::EnumNamesBuiltinOperator()[op_reg.builtin_code]; } +// An utility test to detect if the subgraph is abused: +// 1. Detects if recursion exists in the graph (recursion is not currently +// supported. +// 2. Detects if the interpreter / subgraph is used in multiple subgraphs. +// Note: It's clearly documented that the interpreter / subgraph are not +// thread-safe. This serves as a check with possible false negatives +// unless we switch to atomic boolean flags. +class SubgraphGuard { + public: + SubgraphGuard(TfLiteContext* context, bool* is_subgraph_in_use) + : is_subgraph_in_use_(is_subgraph_in_use) { + if (*is_subgraph_in_use_) { + TF_LITE_KERNEL_LOG( + context, + "Subgraph is already in use. Using an interpreter or a subgraph in " + "multiple threads is not supported. Recursion in the graph is not " + "supported."); + status_ = kTfLiteError; + } else { + *is_subgraph_in_use_ = true; + } + } + ~SubgraphGuard() { + // If tht original status was OK, recover the boolean flag. + if (status_ == kTfLiteOk) { + *is_subgraph_in_use_ = false; + } + } + + TfLiteStatus status() const { return status_; } + + private: + TfLiteStatus status_ = kTfLiteOk; + bool* is_subgraph_in_use_; +}; + } // namespace // A trivial implementation of GraphInfo around the Interpreter. @@ -637,6 +673,7 @@ TfLiteStatus Subgraph::BytesRequired(TfLiteType type, const int* dims, TfLiteStatus Subgraph::AllocateTensors() { TFLITE_SCOPED_TAGGED_DEFAULT_PROFILE(profiler_.get(), "AllocateTensors"); + if (!consistent_) { ReportError("AllocateTensors() called on inconsistent model."); return kTfLiteError; @@ -660,6 +697,12 @@ TfLiteStatus Subgraph::AllocateTensors() { return kTfLiteOk; } + // Note `AllocateTensors` sometimes calls itself recursively above + // for delegates. Therefore only the logic below need to be guarded + // by `SubgraphGuard`. + SubgraphGuard guard(&context_, &is_subgraph_in_use_); + TF_LITE_ENSURE_OK(&context_, guard.status()); + next_execution_plan_index_to_prepare_ = 0; next_execution_plan_index_to_plan_allocation_ = 0; if (memory_planner_) { @@ -917,6 +960,9 @@ TfLiteStatus Subgraph::PrepareOpsAndTensors() { } TfLiteStatus Subgraph::Invoke() { + SubgraphGuard guard(&context_, &is_subgraph_in_use_); + TF_LITE_ENSURE_OK(&context_, guard.status()); + if (!consistent_) { ReportError("Invoke called on model that is not consistent."); return kTfLiteError; @@ -973,10 +1019,17 @@ TfLiteStatus Subgraph::Invoke() { TF_LITE_ENSURE_STATUS(EnsureTensorDataIsReadable(tensor_index)); } if (tensor->data.raw == nullptr && tensor->bytes > 0) { - if (registration.builtin_code == kTfLiteBuiltinReshape && i == 1) { + if (registration.builtin_code == kTfLiteBuiltinReshape && i == 1 && + tensor->dims->size != 1) { // In general, having a tensor here with no buffer will be an error. - // However, for the reshape operator, the second input tensor is only - // used for the shape, not for the data. Thus, null buffer is ok. + // However, for the reshape operator, the second input tensor is + // sometimes only used for the shape, not for the data. Thus, null + // buffer is ok in this situation. + // The situation where null buffer is not ok for reshape operator is + // only when there are 2 inputs given to the node and the one + // corresponding to the shape (i == 1) is a vector that contains all + // dimensions. See `GetOutputShape()` function in + // `tensorflow/lite/kernels/reshape.cc` continue; } else { // In all other cases, we need to return an error as otherwise we will diff --git a/tensorflow/lite/core/subgraph.h b/tensorflow/lite/core/subgraph.h index 979c709614c90a..d66d3205cf101d 100644 --- a/tensorflow/lite/core/subgraph.h +++ b/tensorflow/lite/core/subgraph.h @@ -711,6 +711,10 @@ class Subgraph { // A map of resources. Owned by interpreter and shared by multiple subgraphs. resource::ResourceMap* resources_ = nullptr; + + // Whether the subgraph is currently in use (e.g. running the `Invoke` + // or `AllocateTensors` functions). + bool is_subgraph_in_use_ = false; }; } // namespace impl diff --git a/tensorflow/lite/kernels/arg_min_max.cc b/tensorflow/lite/kernels/arg_min_max.cc index 4a3902ac57c59c..e8b7201cc4b125 100644 --- a/tensorflow/lite/kernels/arg_min_max.cc +++ b/tensorflow/lite/kernels/arg_min_max.cc @@ -42,6 +42,9 @@ TfLiteStatus ResizeOutput(TfLiteContext* context, const TfLiteTensor* input, axis_value += NumDimensions(input); } + TF_LITE_ENSURE(context, axis_value >= 0); + TF_LITE_ENSURE(context, axis_value < NumDimensions(input)); + // Copy the input dimensions to output except the axis dimension. TfLiteIntArray* output_dims = TfLiteIntArrayCreate(NumDimensions(input) - 1); int j = 0; diff --git a/tensorflow/lite/kernels/batch_to_space_nd.cc b/tensorflow/lite/kernels/batch_to_space_nd.cc index 9d6492e0fcbf06..044ac1b3a5ee5d 100644 --- a/tensorflow/lite/kernels/batch_to_space_nd.cc +++ b/tensorflow/lite/kernels/batch_to_space_nd.cc @@ -78,6 +78,7 @@ TfLiteStatus ResizeOutputTensor(TfLiteContext* context, int output_batch_size = input_size->data[0]; for (int dim = 0; dim < spatial_dims_num; ++dim) { // Number of batch must be multiple of (block_shape[dim]). + TF_LITE_ENSURE(context, block_shape[dim] != 0); TF_LITE_ENSURE_EQ(context, output_batch_size % block_shape[dim], 0); output_batch_size = output_batch_size / block_shape[dim]; output_size->data[dim + 1] = input_size->data[dim + 1] * block_shape[dim] - diff --git a/tensorflow/lite/kernels/concatenation.cc b/tensorflow/lite/kernels/concatenation.cc index 5d5f06ba013645..df01792c182897 100644 --- a/tensorflow/lite/kernels/concatenation.cc +++ b/tensorflow/lite/kernels/concatenation.cc @@ -16,6 +16,8 @@ limitations under the License. #include +#include + #include "tensorflow/lite/c/builtin_op_data.h" #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/kernels/internal/compatibility.h" @@ -68,6 +70,10 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE_EQ(context, t->type, input_type); for (int d = 0; d < t0->dims->size; ++d) { if (d == axis) { + // Avoid integer overflow in sum_axis below + TF_LITE_ENSURE(context, t->dims->data[axis] >= 0); + TF_LITE_ENSURE(context, t->dims->data[axis] <= + std::numeric_limits::max() - sum_axis); sum_axis += t->dims->data[axis]; } else { TF_LITE_ENSURE_EQ(context, t->dims->data[d], t0->dims->data[d]); diff --git a/tensorflow/lite/kernels/conv.cc b/tensorflow/lite/kernels/conv.cc index 81069de1abe890..390565f6e79b7b 100644 --- a/tensorflow/lite/kernels/conv.cc +++ b/tensorflow/lite/kernels/conv.cc @@ -501,6 +501,7 @@ TfLiteStatus Prepare(KernelType kernel_type, TfLiteContext* context, // Only one scale factor per batch is typically necessary. See optimized // implementation for why we need to allocate for the height of the inputs // flattened to 2D. + TF_LITE_ENSURE(context, channels_in != 0); const int height = NumElements(input) / channels_in; int scaling_dims[1] = {height}; if (!TfLiteIntArrayEqualsArray(scaling_factors->dims, 1, scaling_dims)) { @@ -539,6 +540,7 @@ TfLiteStatus Prepare(KernelType kernel_type, TfLiteContext* context, input_offsets->type = kTfLiteInt32; input_offsets->allocation_type = kTfLiteArenaRw; // See above comment for the need to allocate for height of inputs. + TF_LITE_ENSURE(context, channels_in != 0); const int height = NumElements(input) / channels_in; const int input_offset_dims[1] = {height}; if (!TfLiteIntArrayEqualsArray(input_offsets->dims, 1, @@ -791,17 +793,19 @@ void EvalFloat(TfLiteContext* context, TfLiteNode* node, } template -void EvalHybridPerChannel(TfLiteContext* context, TfLiteNode* node, - TfLiteConvParams* params, OpData* data, - const TfLiteTensor* input, const TfLiteTensor* filter, - const TfLiteTensor* bias, TfLiteTensor* im2col, - TfLiteTensor* output) { +TfLiteStatus EvalHybridPerChannel(TfLiteContext* context, TfLiteNode* node, + TfLiteConvParams* params, OpData* data, + const TfLiteTensor* input, + const TfLiteTensor* filter, + const TfLiteTensor* bias, + TfLiteTensor* im2col, TfLiteTensor* output) { float output_activation_min, output_activation_max; CalculateActivationRange(params->activation, &output_activation_min, &output_activation_max); - const int input_size = NumElements(input) / SizeOfDimension(input, 0); const int batch_size = SizeOfDimension(input, 0); + TF_LITE_ENSURE(context, batch_size != 0); + const int input_size = NumElements(input) / batch_size; int8_t* quantized_input_ptr_batch = GetTensorData( GetTemporary(context, node, data->input_quantized_index)); float* scaling_factors_ptr = GetTensorData( @@ -869,17 +873,18 @@ void EvalHybridPerChannel(TfLiteContext* context, TfLiteNode* node, } template -void EvalHybrid(TfLiteContext* context, TfLiteNode* node, - TfLiteConvParams* params, OpData* data, - const TfLiteTensor* input, const TfLiteTensor* filter, - const TfLiteTensor* bias, TfLiteTensor* im2col, - TfLiteTensor* accum_scratch, TfLiteTensor* output) { +TfLiteStatus EvalHybrid(TfLiteContext* context, TfLiteNode* node, + TfLiteConvParams* params, OpData* data, + const TfLiteTensor* input, const TfLiteTensor* filter, + const TfLiteTensor* bias, TfLiteTensor* im2col, + TfLiteTensor* accum_scratch, TfLiteTensor* output) { float output_activation_min, output_activation_max; CalculateActivationRange(params->activation, &output_activation_min, &output_activation_max); - const int input_size = NumElements(input) / SizeOfDimension(input, 0); const int batch_size = SizeOfDimension(input, 0); + TF_LITE_ENSURE(context, batch_size != 0); + const int input_size = NumElements(input) / batch_size; const float* input_ptr = GetTensorData(input); int8_t* quantized_input_ptr_batch = GetTensorData( @@ -957,14 +962,17 @@ TfLiteStatus EvalImpl(TfLiteContext* context, TfLiteNode* node) { case kTfLiteFloat32: if (filter->type == kTfLiteUInt8 || filter->type == kTfLiteInt8) { if (data->is_hybrid_per_channel) { - EvalHybridPerChannel(context, node, params, data, input, - filter, bias, im2col, output); + TF_LITE_ENSURE_OK(context, EvalHybridPerChannel( + context, node, params, data, input, + filter, bias, im2col, output)); } else { TfLiteTensor* accum_scratch = &context->tensors[node->temporaries ->data[data->accum_scratch_index]]; - EvalHybrid(context, node, params, data, input, filter, - bias, im2col, accum_scratch, output); + TF_LITE_ENSURE_OK(context, + EvalHybrid(context, node, params, data, + input, filter, bias, im2col, + accum_scratch, output)); } } else { EvalFloat(context, node, params, data, input, filter, bias, diff --git a/tensorflow/lite/kernels/depth_to_space.cc b/tensorflow/lite/kernels/depth_to_space.cc index 1637ad4350f889..c2047f1062f493 100644 --- a/tensorflow/lite/kernels/depth_to_space.cc +++ b/tensorflow/lite/kernels/depth_to_space.cc @@ -58,6 +58,7 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE_TYPES_EQ(context, input->type, output->type); const int block_size = params->block_size; + TF_LITE_ENSURE(context, block_size > 0); const int input_height = input->dims->data[1]; const int input_width = input->dims->data[2]; const int input_channels = input->dims->data[3]; diff --git a/tensorflow/lite/kernels/depth_to_space_test.cc b/tensorflow/lite/kernels/depth_to_space_test.cc index 4429faf9909178..c03512dd710ad7 100644 --- a/tensorflow/lite/kernels/depth_to_space_test.cc +++ b/tensorflow/lite/kernels/depth_to_space_test.cc @@ -60,6 +60,11 @@ TEST(DepthToSpaceOpModel, BadBlockSize) { EXPECT_DEATH(DepthToSpaceOpModel({TensorType_FLOAT32, {1, 1, 1, 4}}, 4), "Cannot allocate tensors"); } + +TEST(DepthToSpaceOpModel, NoBlockSize) { + EXPECT_DEATH(DepthToSpaceOpModel({TensorType_FLOAT32, {1, 1, 1, 4}}, 0), + "Cannot allocate tensors"); +} #endif TEST(DepthToSpaceOpModel, Float32) { diff --git a/tensorflow/lite/kernels/depthwise_conv.cc b/tensorflow/lite/kernels/depthwise_conv.cc index 961a987cf028a0..d37c2700755b1b 100644 --- a/tensorflow/lite/kernels/depthwise_conv.cc +++ b/tensorflow/lite/kernels/depthwise_conv.cc @@ -171,6 +171,7 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { if (data_type != kTfLiteFloat32) { TF_LITE_ENSURE_EQ(context, filter->quantization.type, kTfLiteAffineQuantization); + TF_LITE_ENSURE(context, filter->quantization.type != kTfLiteNoQuantization); const auto* affine_quantization = reinterpret_cast( filter->quantization.params); @@ -190,6 +191,7 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { } if (is_hybrid) { + TF_LITE_ENSURE(context, filter->quantization.type != kTfLiteNoQuantization); const auto* affine_quantization = reinterpret_cast( filter->quantization.params); @@ -274,8 +276,8 @@ TfLiteStatus ComputeDepthMultiplier(TfLiteContext* context, int16* depth_multiplier) { int num_filter_channels = SizeOfDimension(filter, 3); int num_input_channels = SizeOfDimension(input, 3); + TF_LITE_ENSURE(context, num_input_channels != 0); TF_LITE_ENSURE_EQ(context, num_filter_channels % num_input_channels, 0); - *depth_multiplier = num_filter_channels / num_input_channels; return kTfLiteOk; } @@ -444,8 +446,9 @@ TfLiteStatus EvalHybridPerChannel(TfLiteContext* context, TfLiteNode* node, float output_activation_min, output_activation_max; CalculateActivationRange(params->activation, &output_activation_min, &output_activation_max); - const int input_size = NumElements(input) / SizeOfDimension(input, 0); const int batch_size = SizeOfDimension(input, 0); + TF_LITE_ENSURE(context, batch_size != 0); + const int input_size = NumElements(input) / batch_size; const TfLiteTensor* input_quantized = GetTemporary(context, node, data->input_quantized_index); int8_t* quantized_input_ptr_batch = input_quantized->data.int8; @@ -475,6 +478,7 @@ TfLiteStatus EvalHybridPerChannel(TfLiteContext* context, TfLiteNode* node, op_params.weights_offset = 0; op_params.float_activation_min = output_activation_min; op_params.float_activation_max = output_activation_max; + TF_LITE_ENSURE(context, filter->quantization.type != kTfLiteNoQuantization); const auto* affine_quantization = reinterpret_cast(filter->quantization.params); if (kernel_type == kReference) { diff --git a/tensorflow/lite/kernels/div.cc b/tensorflow/lite/kernels/div.cc index c9eb1db531a647..aafe00f0d0cbe9 100644 --- a/tensorflow/lite/kernels/div.cc +++ b/tensorflow/lite/kernels/div.cc @@ -204,9 +204,23 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { const TfLiteTensor* input2 = GetInput(context, node, kInputTensor2); TfLiteTensor* output = GetOutput(context, node, kOutputTensor); - if (output->type == kTfLiteFloat32 || output->type == kTfLiteInt32) { + // TODO(b/193904910): This can written with C++ templates +#define TF_LITE_CHECK_DIV_NON_ZERO(data_type) \ + const auto* input2_data = GetTensorData(input2); \ + const size_t input2_elements = input2->bytes / sizeof(data_type); \ + for (size_t i = 0; i < input2_elements; i++) { \ + TF_LITE_ENSURE(context, input2_data[i] != 0); \ + } + + if (output->type == kTfLiteFloat32) { + // Div by zero seems ok in this case, just like in TF case infinities are + // returned. So we don't do a check at this point. + EvalDiv(context, node, params, data, input1, input2, output); + } else if (output->type == kTfLiteInt32) { + TF_LITE_CHECK_DIV_NON_ZERO(int32_t); EvalDiv(context, node, params, data, input1, input2, output); } else if (output->type == kTfLiteUInt8) { + TF_LITE_CHECK_DIV_NON_ZERO(uint8_t); TF_LITE_ENSURE_OK( context, EvalQuantized(context, node, params, data, input1, input2, output)); @@ -217,6 +231,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { output->type); return kTfLiteError; } +#undef TF_LITE_CHECK_DIV_NON_ZERO return kTfLiteOk; } diff --git a/tensorflow/lite/kernels/embedding_lookup.cc b/tensorflow/lite/kernels/embedding_lookup.cc index 36e0737c7e2830..ea8fd5431a99cf 100644 --- a/tensorflow/lite/kernels/embedding_lookup.cc +++ b/tensorflow/lite/kernels/embedding_lookup.cc @@ -68,6 +68,10 @@ TfLiteStatus EvalSimple(TfLiteContext* context, TfLiteNode* node, const TfLiteTensor* lookup, const TfLiteTensor* value, TfLiteTensor* output) { const int row_size = SizeOfDimension(value, 0); + if (row_size == 0) { + // Propagate empty tensor if input is empty + return kTfLiteOk; + } const int row_bytes = value->bytes / row_size; char* output_raw = GetTensorData(output); diff --git a/tensorflow/lite/kernels/embedding_lookup_sparse.cc b/tensorflow/lite/kernels/embedding_lookup_sparse.cc index 745b5090094687..e798c6a87391ea 100644 --- a/tensorflow/lite/kernels/embedding_lookup_sparse.cc +++ b/tensorflow/lite/kernels/embedding_lookup_sparse.cc @@ -161,6 +161,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { // Resize output tensor. TfLiteIntArray* output_shape = TfLiteIntArrayCreate(output_rank); + TF_LITE_ENSURE(context, output_shape != nullptr); int k = 0; int embedding_size = 1; int lookup_size = 1; diff --git a/tensorflow/lite/kernels/expand_dims.cc b/tensorflow/lite/kernels/expand_dims.cc index 721ab3d510a526..6db2830ea12e78 100644 --- a/tensorflow/lite/kernels/expand_dims.cc +++ b/tensorflow/lite/kernels/expand_dims.cc @@ -38,6 +38,7 @@ TfLiteStatus ExpandTensorDim(TfLiteContext* context, const TfLiteTensor& input, axis = input_dims.size + 1 + axis; } TF_LITE_ENSURE(context, axis <= input_dims.size); + TF_LITE_ENSURE(context, axis >= 0); TfLiteIntArray* output_dims = TfLiteIntArrayCreate(input_dims.size + 1); for (int i = 0; i < output_dims->size; ++i) { diff --git a/tensorflow/lite/kernels/fully_connected.cc b/tensorflow/lite/kernels/fully_connected.cc index 9cbbcae9c51291..7fbbf9983ac675 100644 --- a/tensorflow/lite/kernels/fully_connected.cc +++ b/tensorflow/lite/kernels/fully_connected.cc @@ -175,6 +175,7 @@ TfLiteStatus PrepareImpl(TfLiteContext* context, TfLiteNode* node) { } TF_LITE_ENSURE_EQ(context, NumDimensions(filter), 2); + TF_LITE_ENSURE(context, filter->dims->data[1] != 0); const int batch_size = input_size / filter->dims->data[1]; const int num_units = filter->dims->data[0]; diff --git a/tensorflow/lite/kernels/gather.cc b/tensorflow/lite/kernels/gather.cc index 1de49f7c486c44..63e7cf8e2161f3 100644 --- a/tensorflow/lite/kernels/gather.cc +++ b/tensorflow/lite/kernels/gather.cc @@ -98,8 +98,20 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { } template -TfLiteStatus Gather(const TfLiteGatherParams& params, const TfLiteTensor* input, - const TfLiteTensor* positions, TfLiteTensor* output) { +TfLiteStatus Gather(TfLiteContext* context, const TfLiteGatherParams& params, + const TfLiteTensor* input, const TfLiteTensor* positions, + TfLiteTensor* output) { + const PositionsT* indexes = GetTensorData(positions); + bool indices_has_only_positive_elements = true; + const size_t num_indices = positions->bytes / sizeof(PositionsT); + for (size_t i = 0; i < num_indices; i++) { + if (indexes[i] < 0) { + indices_has_only_positive_elements = false; + break; + } + } + TF_LITE_ENSURE(context, indices_has_only_positive_elements); + tflite::GatherParams op_params; op_params.axis = params.axis; optimized_ops::Gather(op_params, GetTensorShape(input), @@ -114,7 +126,18 @@ TfLiteStatus GatherStrings(TfLiteContext* context, const TfLiteTensor* input, const TfLiteTensor* positions, TfLiteTensor* output) { DynamicBuffer buffer; + const PositionT* indexes = GetTensorData(positions); + bool indices_has_only_positive_elements = true; + const size_t num_indices = positions->bytes / sizeof(PositionT); + for (size_t i = 0; i < num_indices; i++) { + if (indexes[i] < 0) { + indices_has_only_positive_elements = false; + break; + } + } + TF_LITE_ENSURE(context, indices_has_only_positive_elements); + const PositionT num_strings = GetStringCount(input); const int num_indexes = NumElements(positions); @@ -138,17 +161,23 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { if (positions->type == kTfLiteInt32) { switch (input->type) { case kTfLiteFloat32: - return Gather(*params, input, positions, output); + return Gather(context, *params, input, positions, + output); case kTfLiteUInt8: - return Gather(*params, input, positions, output); + return Gather(context, *params, input, positions, + output); case kTfLiteInt8: - return Gather(*params, input, positions, output); + return Gather(context, *params, input, positions, + output); case kTfLiteInt32: - return Gather(*params, input, positions, output); + return Gather(context, *params, input, positions, + output); case kTfLiteInt64: - return Gather(*params, input, positions, output); + return Gather(context, *params, input, positions, + output); case kTfLiteBool: - return Gather(*params, input, positions, output); + return Gather(context, *params, input, positions, + output); case kTfLiteString: return GatherStrings(context, input, positions, output); default: @@ -160,17 +189,23 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { if (positions->type == kTfLiteInt64) { switch (input->type) { case kTfLiteFloat32: - return Gather(*params, input, positions, output); + return Gather(context, *params, input, positions, + output); case kTfLiteUInt8: - return Gather(*params, input, positions, output); + return Gather(context, *params, input, positions, + output); case kTfLiteInt8: - return Gather(*params, input, positions, output); + return Gather(context, *params, input, positions, + output); case kTfLiteInt32: - return Gather(*params, input, positions, output); + return Gather(context, *params, input, positions, + output); case kTfLiteInt64: - return Gather(*params, input, positions, output); + return Gather(context, *params, input, positions, + output); case kTfLiteBool: - return Gather(*params, input, positions, output); + return Gather(context, *params, input, positions, + output); case kTfLiteString: return GatherStrings(context, input, positions, output); default: diff --git a/tensorflow/lite/kernels/gather_nd.cc b/tensorflow/lite/kernels/gather_nd.cc index fd31b8c4ddd709..18108b00cdb1e3 100644 --- a/tensorflow/lite/kernels/gather_nd.cc +++ b/tensorflow/lite/kernels/gather_nd.cc @@ -118,6 +118,17 @@ TfLiteStatus GatherNdString(const TfLiteTensor* params, template TfLiteStatus EvalGatherNd(TfLiteContext* context, const TfLiteTensor* params, const TfLiteTensor* indices, TfLiteTensor* output) { + bool indices_has_only_positive_elements = true; + const auto* indices_values = GetTensorData(indices); + const size_t num_indices = indices->bytes / sizeof(IndicesT); + for (size_t i = 0; i < num_indices; i++) { + if (indices_values[i] < 0) { + indices_has_only_positive_elements = false; + break; + } + } + TF_LITE_ENSURE(context, indices_has_only_positive_elements); + switch (params->type) { case kTfLiteFloat32: return GatherNd(params, indices, output); @@ -144,6 +155,9 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { const TfLiteTensor* indices = GetInput(context, node, kIndices); TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + // Prevent division by 0 in the helper + TF_LITE_ENSURE(context, NumElements(params) > 0); + switch (indices->type) { case kTfLiteInt32: return EvalGatherNd(context, params, indices, output); diff --git a/tensorflow/lite/kernels/hashtable_lookup.cc b/tensorflow/lite/kernels/hashtable_lookup.cc index 65e50fe41c2331..9d947107c1bc2c 100644 --- a/tensorflow/lite/kernels/hashtable_lookup.cc +++ b/tensorflow/lite/kernels/hashtable_lookup.cc @@ -101,6 +101,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { const TfLiteTensor* value = GetInput(context, node, 2); const int num_rows = SizeOfDimension(value, 0); + TF_LITE_ENSURE(context, num_rows != 0); const int row_bytes = value->bytes / num_rows; void* pointer = nullptr; DynamicBuffer buf; diff --git a/tensorflow/lite/kernels/internal/averagepool_quantized_test.cc b/tensorflow/lite/kernels/internal/averagepool_quantized_test.cc index cbc863645b74b9..fea343ae6b8824 100644 --- a/tensorflow/lite/kernels/internal/averagepool_quantized_test.cc +++ b/tensorflow/lite/kernels/internal/averagepool_quantized_test.cc @@ -40,12 +40,14 @@ void RunOneAveragePoolTest(const PoolParams& params, std::vector optimized_averagePool_output(buffer_size); std::vector reference_averagePool_output(buffer_size); - reference_integer_ops::AveragePool(params, input_shape, input_data, - output_shape, - reference_averagePool_output.data()); - optimized_integer_ops::AveragePool(params, input_shape, input_data, - output_shape, - optimized_averagePool_output.data()); + bool reference_success = reference_integer_ops::AveragePool( + params, input_shape, input_data, output_shape, + reference_averagePool_output.data()); + bool optimized_success = optimized_integer_ops::AveragePool( + params, input_shape, input_data, output_shape, + optimized_averagePool_output.data()); + EXPECT_TRUE(reference_success); + EXPECT_TRUE(optimized_success); for (int i = 0; i < buffer_size; i++) { EXPECT_TRUE(reference_averagePool_output[i] == diff --git a/tensorflow/lite/kernels/internal/optimized/integer_ops/pooling.h b/tensorflow/lite/kernels/internal/optimized/integer_ops/pooling.h index f2696500ab9874..dfe8bd9b545fc6 100644 --- a/tensorflow/lite/kernels/internal/optimized/integer_ops/pooling.h +++ b/tensorflow/lite/kernels/internal/optimized/integer_ops/pooling.h @@ -145,7 +145,7 @@ inline void MaxPool(const PoolParams& params, const RuntimeShape& input_shape, } } -inline void AveragePool16(const PoolParams& params, +inline bool AveragePool16(const PoolParams& params, const RuntimeShape& input_shape, const int8* input_data, const RuntimeShape& output_shape, int8* output_data) { @@ -194,6 +194,7 @@ inline void AveragePool16(const PoolParams& params, std::min(params.filter_height, input_height - in_y_origin); const int filter_count = (filter_x_end - filter_x_start) * (filter_y_end - filter_y_start); + if (filter_count == 0) return false; memset(acc, 0, tranche_depth * sizeof(acc[0])); const int8* input_ptr = input_data + depth_base + @@ -281,16 +282,18 @@ inline void AveragePool16(const PoolParams& params, } } } + return true; } -inline void AveragePool(const PoolParams& params, +inline bool AveragePool(const PoolParams& params, const RuntimeShape& input_shape, const int8* input_data, const RuntimeShape& output_shape, int8* output_data) { if (params.filter_height * params.filter_width > 16 * 16) { - reference_integer_ops::AveragePool(params, input_shape, input_data, - output_shape, output_data); + return reference_integer_ops::AveragePool(params, input_shape, input_data, + output_shape, output_data); } else { - AveragePool16(params, input_shape, input_data, output_shape, output_data); + return AveragePool16(params, input_shape, input_data, output_shape, + output_data); } } diff --git a/tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h b/tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h index f206dfa9235428..0f1c50329c733d 100644 --- a/tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h +++ b/tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h @@ -3763,7 +3763,7 @@ inline void BroadcastMul(const uint8* input1_data, const Dims<4>& input1_dims, output_data, output_dims); } -inline void AveragePool(const float* input_data, const Dims<4>& input_dims, +inline bool AveragePool(const float* input_data, const Dims<4>& input_dims, int stride_width, int stride_height, int pad_width, int pad_height, int kwidth, int kheight, float output_activation_min, @@ -3778,35 +3778,37 @@ inline void AveragePool(const float* input_data, const Dims<4>& input_dims, params.padding_values.width = pad_width; params.float_activation_min = output_activation_min; params.float_activation_max = output_activation_max; - AveragePool(params, DimsToShape(input_dims), input_data, - DimsToShape(output_dims), output_data); + return AveragePool(params, DimsToShape(input_dims), input_data, + DimsToShape(output_dims), output_data); } // legacy, for compatibility with old checked-in code template -void AveragePool(const float* input_data, const Dims<4>& input_dims, +bool AveragePool(const float* input_data, const Dims<4>& input_dims, int stride_width, int stride_height, int pad_width, int pad_height, int kwidth, int kheight, float* output_data, const Dims<4>& output_dims) { float output_activation_min, output_activation_max; GetActivationMinMax(Ac, &output_activation_min, &output_activation_max); - AveragePool(input_data, input_dims, stride_width, stride_height, pad_width, - pad_height, kwidth, kheight, output_activation_min, - output_activation_max, output_data, output_dims); + return AveragePool(input_data, input_dims, stride_width, stride_height, + pad_width, pad_height, kwidth, kheight, + output_activation_min, output_activation_max, output_data, + output_dims); } // legacy, for compatibility with old checked-in code template -void AveragePool(const float* input_data, const Dims<4>& input_dims, int stride, +bool AveragePool(const float* input_data, const Dims<4>& input_dims, int stride, int pad_width, int pad_height, int filter_width, int filter_height, float* output_data, const Dims<4>& output_dims) { - AveragePool(input_data, input_dims, stride, stride, pad_width, pad_height, - filter_width, filter_height, output_data, output_dims); + return AveragePool(input_data, input_dims, stride, stride, pad_width, + pad_height, filter_width, filter_height, output_data, + output_dims); } -inline void AveragePool(const uint8* input_data, const Dims<4>& input_dims, +inline bool AveragePool(const uint8* input_data, const Dims<4>& input_dims, int stride_width, int stride_height, int pad_width, int pad_height, int filter_width, int filter_height, int32 output_activation_min, @@ -3821,13 +3823,13 @@ inline void AveragePool(const uint8* input_data, const Dims<4>& input_dims, params.padding_values.width = pad_width; params.quantized_activation_min = output_activation_min; params.quantized_activation_max = output_activation_max; - AveragePool(params, DimsToShape(input_dims), input_data, - DimsToShape(output_dims), output_data); + return AveragePool(params, DimsToShape(input_dims), input_data, + DimsToShape(output_dims), output_data); } // legacy, for compatibility with old checked-in code template -void AveragePool(const uint8* input_data, const Dims<4>& input_dims, +bool AveragePool(const uint8* input_data, const Dims<4>& input_dims, int stride_width, int stride_height, int pad_width, int pad_height, int filter_width, int filter_height, int32 output_activation_min, int32 output_activation_max, @@ -3841,21 +3843,23 @@ void AveragePool(const uint8* input_data, const Dims<4>& input_dims, TFLITE_DCHECK_EQ(output_activation_min, 0); TFLITE_DCHECK_EQ(output_activation_max, 255); } - AveragePool(input_data, input_dims, stride_width, stride_height, pad_width, - pad_height, filter_width, filter_height, output_activation_min, - output_activation_max, output_data, output_dims); + return AveragePool(input_data, input_dims, stride_width, stride_height, + pad_width, pad_height, filter_width, filter_height, + output_activation_min, output_activation_max, output_data, + output_dims); } // legacy, for compatibility with old checked-in code template -void AveragePool(const uint8* input_data, const Dims<4>& input_dims, int stride, +bool AveragePool(const uint8* input_data, const Dims<4>& input_dims, int stride, int pad_width, int pad_height, int filter_width, int filter_height, int32 output_activation_min, int32 output_activation_max, uint8* output_data, const Dims<4>& output_dims) { - AveragePool(input_data, input_dims, stride, stride, pad_width, pad_height, - filter_width, filter_height, output_activation_min, - output_activation_max, output_data, output_dims); + return AveragePool(input_data, input_dims, stride, stride, pad_width, + pad_height, filter_width, filter_height, + output_activation_min, output_activation_max, + output_data, output_dims); } inline void MaxPool(const float* input_data, const Dims<4>& input_dims, diff --git a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h index 528eea3d698678..bfb98ab937262e 100644 --- a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h +++ b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h @@ -304,7 +304,7 @@ inline void BinaryBroadcastFiveFold(const ArithmeticParams& unswitched_params, // We have broadcast y2*y3*y4 of input2 data y1 times, and now move on. input2_data_reset = input2_data_ptr; } - } else { + } else if (input1_data_ptr != nullptr) { // Special case of y4 == 1, in which the innermost loop is a single // element and can be combined with the next (y3) as an inner broadcast. // @@ -3223,7 +3223,7 @@ inline int NodeOffset(int b, int h, int w, int height, int width) { return (b * height + h) * width + w; } -inline void AveragePool(const PoolParams& params, +inline bool AveragePool(const PoolParams& params, const RuntimeShape& input_shape, const float* input_data, const RuntimeShape& output_shape, float* output_data) { @@ -3238,6 +3238,9 @@ inline void AveragePool(const PoolParams& params, const int stride_height = params.stride_height; const int stride_width = params.stride_width; + if (stride_height == 0) return false; + if (stride_width == 0) return false; + // TODO(benoitjacob) make this a proper reference impl without Eigen! const auto in_mat = MapAsMatrixWithLastDimAsRows(input_data, input_shape); auto out_mat = MapAsMatrixWithLastDimAsRows(output_data, output_shape); @@ -3283,9 +3286,11 @@ inline void AveragePool(const PoolParams& params, params.float_activation_min, params.float_activation_max); } + + return true; } -inline void AveragePool16(const PoolParams& params, +inline bool AveragePool16(const PoolParams& params, const RuntimeShape& input_shape, const uint8* input_data, const RuntimeShape& output_shape, @@ -3335,6 +3340,7 @@ inline void AveragePool16(const PoolParams& params, std::min(params.filter_height, input_height - in_y_origin); const int filter_count = (filter_x_end - filter_x_start) * (filter_y_end - filter_y_start); + if (filter_count == 0) return false; memset(acc, 0, tranche_depth * sizeof(acc[0])); const uint8* input_ptr = input_data + depth_base + @@ -3417,7 +3423,7 @@ inline void AveragePool16(const PoolParams& params, } } -inline void AveragePool32(const PoolParams& params, +inline bool AveragePool32(const PoolParams& params, const RuntimeShape& input_shape, const uint8* input_data, const RuntimeShape& output_shape, @@ -3467,6 +3473,7 @@ inline void AveragePool32(const PoolParams& params, std::min(params.filter_height, input_height - in_y_origin); const int filter_count = (filter_x_end - filter_x_start) * (filter_y_end - filter_y_start); + if (filter_count == 0) return false; memset(acc, 0, tranche_depth * sizeof(acc[0])); const uint8* input_ptr = input_data + depth_base + @@ -3553,16 +3560,19 @@ inline void AveragePool32(const PoolParams& params, } } } + return true; } -inline void AveragePool(const PoolParams& params, +inline bool AveragePool(const PoolParams& params, const RuntimeShape& input_shape, const uint8* input_data, const RuntimeShape& output_shape, uint8* output_data) { if (params.filter_height * params.filter_width > 16 * 16) { - AveragePool32(params, input_shape, input_data, output_shape, output_data); + return AveragePool32(params, input_shape, input_data, output_shape, + output_data); } else { - AveragePool16(params, input_shape, input_data, output_shape, output_data); + return AveragePool16(params, input_shape, input_data, output_shape, + output_data); } } diff --git a/tensorflow/lite/kernels/internal/reference/integer_ops/pooling.h b/tensorflow/lite/kernels/internal/reference/integer_ops/pooling.h index 6b49d2b150bf46..f0ef31269a34f7 100644 --- a/tensorflow/lite/kernels/internal/reference/integer_ops/pooling.h +++ b/tensorflow/lite/kernels/internal/reference/integer_ops/pooling.h @@ -21,7 +21,7 @@ limitations under the License. namespace tflite { namespace reference_integer_ops { -inline void AveragePool(const PoolParams& params, +inline bool AveragePool(const PoolParams& params, const RuntimeShape& input_shape, const int8* input_data, const RuntimeShape& output_shape, int8* output_data) { TFLITE_DCHECK_LE(params.quantized_activation_min, @@ -65,6 +65,7 @@ inline void AveragePool(const PoolParams& params, filter_count++; } } + if (filter_count == 0) return false; // Round to the closest integer value. acc = acc > 0 ? (acc + filter_count / 2) / filter_count : (acc - filter_count / 2) / filter_count; @@ -76,6 +77,7 @@ inline void AveragePool(const PoolParams& params, } } } + return true; } inline void MaxPool(const PoolParams& params, const RuntimeShape& input_shape, @@ -135,7 +137,7 @@ inline void MaxPool(const PoolParams& params, const RuntimeShape& input_shape, } } -inline void AveragePool(const PoolParams& params, +inline bool AveragePool(const PoolParams& params, const RuntimeShape& input_shape, const int16* input_data, const RuntimeShape& output_shape, int16* output_data) { @@ -180,6 +182,7 @@ inline void AveragePool(const PoolParams& params, filter_count++; } } + if (filter_count == 0) return false; // Round to the closest integer value. acc = acc > 0 ? (acc + filter_count / 2) / filter_count : (acc - filter_count / 2) / filter_count; @@ -191,6 +194,7 @@ inline void AveragePool(const PoolParams& params, } } } + return true; } inline void MaxPool(const PoolParams& params, const RuntimeShape& input_shape, diff --git a/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h b/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h index f62c9bd197c876..c204b3946b522f 100644 --- a/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h +++ b/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h @@ -1528,7 +1528,7 @@ void Sub(const T* input1_data, const Dims<4>& input1_dims, const T* input2_data, output_data); } -inline void AveragePool(const float* input_data, const Dims<4>& input_dims, +inline bool AveragePool(const float* input_data, const Dims<4>& input_dims, int stride_width, int stride_height, int pad_width, int pad_height, int kwidth, int kheight, float output_activation_min, @@ -1543,8 +1543,8 @@ inline void AveragePool(const float* input_data, const Dims<4>& input_dims, params.padding_values.width = pad_width; params.float_activation_min = output_activation_min; params.float_activation_max = output_activation_max; - AveragePool(params, DimsToShape(input_dims), input_data, - DimsToShape(output_dims), output_data); + return AveragePool(params, DimsToShape(input_dims), input_data, + DimsToShape(output_dims), output_data); } // Transitional version that will be moved shortly to legacy_reference_ops, as @@ -1603,29 +1603,31 @@ inline void BroadcastMul(const uint8* input1_data, const Dims<4>& input1_dims, // legacy, for compatibility with old checked-in code template -void AveragePool(const float* input_data, const Dims<4>& input_dims, +bool AveragePool(const float* input_data, const Dims<4>& input_dims, int stride_width, int stride_height, int pad_width, int pad_height, int kwidth, int kheight, float* output_data, const Dims<4>& output_dims) { float output_activation_min, output_activation_max; GetActivationMinMax(Ac, &output_activation_min, &output_activation_max); - AveragePool(input_data, input_dims, stride_width, stride_height, pad_width, - pad_height, kwidth, kheight, output_activation_min, - output_activation_max, output_data, output_dims); + return AveragePool(input_data, input_dims, stride_width, stride_height, + pad_width, pad_height, kwidth, kheight, + output_activation_min, output_activation_max, output_data, + output_dims); } // legacy, for compatibility with old checked-in code template -void AveragePool(const float* input_data, const Dims<4>& input_dims, int stride, +bool AveragePool(const float* input_data, const Dims<4>& input_dims, int stride, int pad_width, int pad_height, int filter_width, int filter_height, float* output_data, const Dims<4>& output_dims) { - AveragePool(input_data, input_dims, stride, stride, pad_width, pad_height, - filter_width, filter_height, output_data, output_dims); + return AveragePool(input_data, input_dims, stride, stride, pad_width, + pad_height, filter_width, filter_height, output_data, + output_dims); } -inline void AveragePool(const uint8* input_data, const Dims<4>& input_dims, +inline bool AveragePool(const uint8* input_data, const Dims<4>& input_dims, int stride_width, int stride_height, int pad_width, int pad_height, int filter_width, int filter_height, int32 output_activation_min, @@ -1640,13 +1642,13 @@ inline void AveragePool(const uint8* input_data, const Dims<4>& input_dims, params.padding_values.width = pad_width; params.quantized_activation_min = output_activation_min; params.quantized_activation_max = output_activation_max; - AveragePool(params, DimsToShape(input_dims), input_data, - DimsToShape(output_dims), output_data); + return AveragePool(params, DimsToShape(input_dims), input_data, + DimsToShape(output_dims), output_data); } // legacy, for compatibility with old checked-in code template -void AveragePool(const uint8* input_data, const Dims<4>& input_dims, +bool AveragePool(const uint8* input_data, const Dims<4>& input_dims, int stride_width, int stride_height, int pad_width, int pad_height, int filter_width, int filter_height, int32 output_activation_min, int32 output_activation_max, @@ -1660,21 +1662,23 @@ void AveragePool(const uint8* input_data, const Dims<4>& input_dims, TFLITE_DCHECK_EQ(output_activation_min, 0); TFLITE_DCHECK_EQ(output_activation_max, 255); } - AveragePool(input_data, input_dims, stride_width, stride_height, pad_width, - pad_height, filter_width, filter_height, output_activation_min, - output_activation_max, output_data, output_dims); + return AveragePool(input_data, input_dims, stride_width, stride_height, + pad_width, pad_height, filter_width, filter_height, + output_activation_min, output_activation_max, output_data, + output_dims); } // legacy, for compatibility with old checked-in code template -void AveragePool(const uint8* input_data, const Dims<4>& input_dims, int stride, +bool AveragePool(const uint8* input_data, const Dims<4>& input_dims, int stride, int pad_width, int pad_height, int filter_width, int filter_height, int32 output_activation_min, int32 output_activation_max, uint8* output_data, const Dims<4>& output_dims) { - AveragePool(input_data, input_dims, stride, stride, pad_width, pad_height, - filter_width, filter_height, output_activation_min, - output_activation_max, output_data, output_dims); + return AveragePool(input_data, input_dims, stride, stride, pad_width, + pad_height, filter_width, filter_height, + output_activation_min, output_activation_max, + output_data, output_dims); } inline void MaxPool(const float* input_data, const Dims<4>& input_dims, diff --git a/tensorflow/lite/kernels/internal/reference/pooling.h b/tensorflow/lite/kernels/internal/reference/pooling.h index a03359cda8217b..685e1f50a550f0 100644 --- a/tensorflow/lite/kernels/internal/reference/pooling.h +++ b/tensorflow/lite/kernels/internal/reference/pooling.h @@ -23,7 +23,7 @@ limitations under the License. namespace tflite { namespace reference_ops { -inline void AveragePool(const PoolParams& params, +inline bool AveragePool(const PoolParams& params, const RuntimeShape& input_shape, const float* input_data, const RuntimeShape& output_shape, float* output_data) { @@ -66,6 +66,7 @@ inline void AveragePool(const PoolParams& params, filter_count++; } } + if (filter_count == 0) return false; const float average = total / filter_count; output_data[Offset(output_shape, batch, out_y, out_x, channel)] = ActivationFunctionWithMinMax(average, params.float_activation_min, @@ -74,9 +75,10 @@ inline void AveragePool(const PoolParams& params, } } } + return true; } -inline void AveragePool(const PoolParams& params, +inline bool AveragePool(const PoolParams& params, const RuntimeShape& input_shape, const uint8* input_data, const RuntimeShape& output_shape, uint8* output_data) { @@ -121,6 +123,7 @@ inline void AveragePool(const PoolParams& params, filter_count++; } } + if (filter_count == 0) return false; acc = (acc + filter_count / 2) / filter_count; acc = std::max(acc, params.quantized_activation_min); acc = std::min(acc, params.quantized_activation_max); @@ -130,6 +133,7 @@ inline void AveragePool(const PoolParams& params, } } } + return true; } inline void L2Pool(const PoolParams& params, const RuntimeShape& input_shape, diff --git a/tensorflow/lite/kernels/lsh_projection.cc b/tensorflow/lite/kernels/lsh_projection.cc index b809748c59ca82..34cbf6e5c3aa78 100644 --- a/tensorflow/lite/kernels/lsh_projection.cc +++ b/tensorflow/lite/kernels/lsh_projection.cc @@ -28,7 +28,7 @@ limitations under the License. // // Input: // Tensor[0]: Hash functions. Dim.size == 2, DataType: Float. -// Tensor[0].Dim[0]: Num of hash functions. +// Tensor[0].Dim[0]: Num of hash functions. Must be at least 1. // Tensor[0].Dim[1]: Num of projected output bits generated by // each hash function. // In sparse case, Tensor[0].Dim[1] + ceil( log2(Tensor[0].Dim[0] )) <= 32. @@ -80,6 +80,7 @@ TfLiteStatus Resize(TfLiteContext* context, TfLiteNode* node) { const TfLiteTensor* input = GetInput(context, node, 1); TF_LITE_ENSURE(context, NumDimensions(input) >= 1); + TF_LITE_ENSURE(context, SizeOfDimension(input, 0) >= 1); if (NumInputs(node) == 3) { const TfLiteTensor* weight = GetInput(context, node, 2); diff --git a/tensorflow/lite/kernels/maximum_minimum.cc b/tensorflow/lite/kernels/maximum_minimum.cc index 777e51442f120e..176e020a5a8e55 100644 --- a/tensorflow/lite/kernels/maximum_minimum.cc +++ b/tensorflow/lite/kernels/maximum_minimum.cc @@ -157,35 +157,37 @@ template TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { OpContext op_context(context, node); - switch (op_context.output->type) { - case kTfLiteFloat32: - TFLiteOperation(context, node, op_context); - break; - case kTfLiteUInt8: - TFLiteOperation(context, node, - op_context); - break; - case kTfLiteInt8: - TFLiteOperation(context, node, op_context); - break; - case kTfLiteInt32: - TFLiteOperation(context, node, - op_context); - break; - case kTfLiteInt64: - TFLiteOperation(context, node, - op_context); - break; - case kTfLiteInt16: - TFLiteOperation(context, node, - op_context); - break; - default: - context->ReportError(context, - "Type %d is currently not supported by Maximum.", - op_context.output->type); - return kTfLiteError; - } + // If inputs have no element, shortcircuit. + if (NumElements(op_context.input1) == 0 || + NumElements(op_context.input2) == 0) { + return kTfLiteOk; + } + + switch (op_context.output->type) { + case kTfLiteFloat32: + TFLiteOperation(context, node, op_context); + break; + case kTfLiteUInt8: + TFLiteOperation(context, node, op_context); + break; + case kTfLiteInt8: + TFLiteOperation(context, node, op_context); + break; + case kTfLiteInt32: + TFLiteOperation(context, node, op_context); + break; + case kTfLiteInt64: + TFLiteOperation(context, node, op_context); + break; + case kTfLiteInt16: + TFLiteOperation(context, node, op_context); + break; + default: + context->ReportError(context, + "Type %d is currently not supported by Maximum.", + op_context.output->type); + return kTfLiteError; + } return kTfLiteOk; } diff --git a/tensorflow/lite/kernels/one_hot.cc b/tensorflow/lite/kernels/one_hot.cc index f7b4e8e7e19d57..75bfb48d6b19c8 100644 --- a/tensorflow/lite/kernels/one_hot.cc +++ b/tensorflow/lite/kernels/one_hot.cc @@ -69,6 +69,11 @@ void OneHotComputeImpl(const OneHotContext& op_context) { for (int i = 0; i < op_context.axis; ++i) { prefix_dim_size *= op_context.indices->dims->data[i]; } + if (prefix_dim_size == 0) { + // If indices tensor is degenerate, return a degenerate tensor, just like + // TensorFlow does. + return; + } const int suffix_dim_size = NumElements(op_context.indices) / prefix_dim_size; const int depth = *op_context.depth->data.i32; diff --git a/tensorflow/lite/kernels/padding.h b/tensorflow/lite/kernels/padding.h index 1116b1da852cf6..6b4ab7fa58d1aa 100644 --- a/tensorflow/lite/kernels/padding.h +++ b/tensorflow/lite/kernels/padding.h @@ -44,6 +44,11 @@ inline int ComputePaddingWithOffset(int stride, int dilation_rate, int in_size, inline int ComputeOutSize(TfLitePadding padding, int image_size, int filter_size, int stride, int dilation_rate = 1) { int effective_filter_size = (filter_size - 1) * dilation_rate + 1; + + // TODO(b/186448822): This uses 0 since the function has no other way to + // report error case + if (stride == 0) return 0; + switch (padding) { case kTfLitePaddingSame: return (image_size + stride - 1) / stride; diff --git a/tensorflow/lite/kernels/pooling.cc b/tensorflow/lite/kernels/pooling.cc index a1380080a1eb03..afce4ad77d8ca9 100644 --- a/tensorflow/lite/kernels/pooling.cc +++ b/tensorflow/lite/kernels/pooling.cc @@ -85,6 +85,10 @@ TfLiteStatus GenericPrepare(TfLiteContext* context, TfLiteNode* node) { auto padding = params->padding; int out_width, out_height; + // Prevent division by 0 in optimized pooling implementations + TF_LITE_ENSURE(context, params->stride_height > 0); + TF_LITE_ENSURE(context, params->stride_width > 0); + data->padding = ComputePaddingHeightWidth( params->stride_height, params->stride_width, 1, 1, height, width, params->filter_height, params->filter_width, padding, &out_height, @@ -111,117 +115,126 @@ TfLiteStatus GenericPrepare(TfLiteContext* context, TfLiteNode* node) { } template -void AverageEvalFloat(TfLiteContext* context, TfLiteNode* node, - TfLitePoolParams* params, OpData* data, - const TfLiteTensor* input, TfLiteTensor* output) { +TfLiteStatus AverageEvalFloat(TfLiteContext* context, TfLiteNode* node, + TfLitePoolParams* params, OpData* data, + const TfLiteTensor* input, TfLiteTensor* output) { float activation_min, activation_max; CalculateActivationRange(params->activation, &activation_min, &activation_max); -#define TF_LITE_AVERAGE_POOL(type) \ - tflite::PoolParams op_params; \ - op_params.stride_height = params->stride_height; \ - op_params.stride_width = params->stride_width; \ - op_params.filter_height = params->filter_height; \ - op_params.filter_width = params->filter_width; \ - op_params.padding_values.height = data->padding.height; \ - op_params.padding_values.width = data->padding.width; \ - op_params.float_activation_min = activation_min; \ - op_params.float_activation_max = activation_max; \ - type::AveragePool(op_params, GetTensorShape(input), \ - GetTensorData(input), GetTensorShape(output), \ - GetTensorData(output)) +#define TF_LITE_AVERAGE_POOL(type) \ + tflite::PoolParams op_params; \ + op_params.stride_height = params->stride_height; \ + op_params.stride_width = params->stride_width; \ + op_params.filter_height = params->filter_height; \ + op_params.filter_width = params->filter_width; \ + op_params.padding_values.height = data->padding.height; \ + op_params.padding_values.width = data->padding.width; \ + op_params.float_activation_min = activation_min; \ + op_params.float_activation_max = activation_max; \ + TF_LITE_ENSURE(context, type::AveragePool(op_params, GetTensorShape(input), \ + GetTensorData(input), \ + GetTensorShape(output), \ + GetTensorData(output))) if (kernel_type == kReference) { TF_LITE_AVERAGE_POOL(reference_ops); } else { TF_LITE_AVERAGE_POOL(optimized_ops); } #undef TF_LITE_AVERAGE_POOL + return kTfLiteOk; } template -void AverageEvalQuantizedUint8(TfLiteContext* context, TfLiteNode* node, - TfLitePoolParams* params, OpData* data, - const TfLiteTensor* input, - TfLiteTensor* output) { +TfLiteStatus AverageEvalQuantizedUint8(TfLiteContext* context, TfLiteNode* node, + TfLitePoolParams* params, OpData* data, + const TfLiteTensor* input, + TfLiteTensor* output) { int32_t activation_min; int32_t activation_max; (void)CalculateActivationRangeQuantized(context, params->activation, output, &activation_min, &activation_max); -#define TF_LITE_AVERAGE_POOL(type) \ - tflite::PoolParams op_params; \ - op_params.stride_height = params->stride_height; \ - op_params.stride_width = params->stride_width; \ - op_params.filter_height = params->filter_height; \ - op_params.filter_width = params->filter_width; \ - op_params.padding_values.height = data->padding.height; \ - op_params.padding_values.width = data->padding.width; \ - op_params.quantized_activation_min = activation_min; \ - op_params.quantized_activation_max = activation_max; \ - type::AveragePool(op_params, GetTensorShape(input), \ - GetTensorData(input), GetTensorShape(output), \ - GetTensorData(output)) +#define TF_LITE_AVERAGE_POOL(type) \ + tflite::PoolParams op_params; \ + op_params.stride_height = params->stride_height; \ + op_params.stride_width = params->stride_width; \ + op_params.filter_height = params->filter_height; \ + op_params.filter_width = params->filter_width; \ + op_params.padding_values.height = data->padding.height; \ + op_params.padding_values.width = data->padding.width; \ + op_params.quantized_activation_min = activation_min; \ + op_params.quantized_activation_max = activation_max; \ + TF_LITE_ENSURE(context, type::AveragePool(op_params, GetTensorShape(input), \ + GetTensorData(input), \ + GetTensorShape(output), \ + GetTensorData(output))) if (kernel_type == kReference) { TF_LITE_AVERAGE_POOL(reference_ops); } else { TF_LITE_AVERAGE_POOL(optimized_ops); } #undef TF_LITE_AVERAGE_POOL + return kTfLiteOk; } template -void AverageEvalQuantizedInt8(TfLiteContext* context, TfLiteNode* node, - TfLitePoolParams* params, OpData* data, - const TfLiteTensor* input, TfLiteTensor* output) { +TfLiteStatus AverageEvalQuantizedInt8(TfLiteContext* context, TfLiteNode* node, + TfLitePoolParams* params, OpData* data, + const TfLiteTensor* input, + TfLiteTensor* output) { int32_t activation_min; int32_t activation_max; (void)CalculateActivationRangeQuantized(context, params->activation, output, &activation_min, &activation_max); -#define TF_LITE_AVERAGE_POOL(type) \ - tflite::PoolParams op_params; \ - op_params.stride_height = params->stride_height; \ - op_params.stride_width = params->stride_width; \ - op_params.filter_height = params->filter_height; \ - op_params.filter_width = params->filter_width; \ - op_params.padding_values.height = data->padding.height; \ - op_params.padding_values.width = data->padding.width; \ - op_params.quantized_activation_min = activation_min; \ - op_params.quantized_activation_max = activation_max; \ - type::AveragePool(op_params, GetTensorShape(input), \ - GetTensorData(input), GetTensorShape(output), \ - GetTensorData(output)) +#define TF_LITE_AVERAGE_POOL(type) \ + tflite::PoolParams op_params; \ + op_params.stride_height = params->stride_height; \ + op_params.stride_width = params->stride_width; \ + op_params.filter_height = params->filter_height; \ + op_params.filter_width = params->filter_width; \ + op_params.padding_values.height = data->padding.height; \ + op_params.padding_values.width = data->padding.width; \ + op_params.quantized_activation_min = activation_min; \ + op_params.quantized_activation_max = activation_max; \ + TF_LITE_ENSURE(context, type::AveragePool(op_params, GetTensorShape(input), \ + GetTensorData(input), \ + GetTensorShape(output), \ + GetTensorData(output))) if (kernel_type == kReference) { TF_LITE_AVERAGE_POOL(reference_integer_ops); } else { TF_LITE_AVERAGE_POOL(optimized_integer_ops); } #undef TF_LITE_AVERAGE_POOL + return kTfLiteOk; } template -void AverageEvalQuantizedInt16(TfLiteContext* context, TfLiteNode* node, - TfLitePoolParams* params, OpData* data, - const TfLiteTensor* input, - TfLiteTensor* output) { +TfLiteStatus AverageEvalQuantizedInt16(TfLiteContext* context, TfLiteNode* node, + TfLitePoolParams* params, OpData* data, + const TfLiteTensor* input, + TfLiteTensor* output) { int32_t activation_min; int32_t activation_max; CalculateActivationRangeQuantized(context, params->activation, output, &activation_min, &activation_max); -#define TF_LITE_AVERAGE_POOL(type) \ - tflite::PoolParams op_params; \ - op_params.stride_height = params->stride_height; \ - op_params.stride_width = params->stride_width; \ - op_params.filter_height = params->filter_height; \ - op_params.filter_width = params->filter_width; \ - op_params.padding_values.height = data->padding.height; \ - op_params.padding_values.width = data->padding.width; \ - op_params.quantized_activation_min = activation_min; \ - op_params.quantized_activation_max = activation_max; \ - type::AveragePool(op_params, GetTensorShape(input), \ - GetTensorData(input), GetTensorShape(output), \ - GetTensorData(output)) +#define TF_LITE_AVERAGE_POOL(type) \ + tflite::PoolParams op_params; \ + op_params.stride_height = params->stride_height; \ + op_params.stride_width = params->stride_width; \ + op_params.filter_height = params->filter_height; \ + op_params.filter_width = params->filter_width; \ + op_params.padding_values.height = data->padding.height; \ + op_params.padding_values.width = data->padding.width; \ + op_params.quantized_activation_min = activation_min; \ + op_params.quantized_activation_max = activation_max; \ + TF_LITE_ENSURE(context, type::AveragePool(op_params, GetTensorShape(input), \ + GetTensorData(input), \ + GetTensorShape(output), \ + GetTensorData(output))) TF_LITE_AVERAGE_POOL(reference_integer_ops); #undef TF_LITE_AVERAGE_POOL + return kTfLiteOk; } template @@ -372,20 +385,17 @@ TfLiteStatus AverageEval(TfLiteContext* context, TfLiteNode* node) { const TfLiteTensor* input = GetInput(context, node, 0); switch (input->type) { // Already know in/out types are same. case kTfLiteFloat32: - AverageEvalFloat(context, node, params, data, input, output); - break; + return AverageEvalFloat(context, node, params, data, input, + output); case kTfLiteUInt8: - AverageEvalQuantizedUint8(context, node, params, data, input, - output); - break; + return AverageEvalQuantizedUint8(context, node, params, data, + input, output); case kTfLiteInt8: - AverageEvalQuantizedInt8(context, node, params, data, input, - output); - break; + return AverageEvalQuantizedInt8(context, node, params, data, + input, output); case kTfLiteInt16: - AverageEvalQuantizedInt16(context, node, params, data, input, - output); - break; + return AverageEvalQuantizedInt16(context, node, params, data, + input, output); default: TF_LITE_KERNEL_LOG(context, "Type %s not currently supported.", TfLiteTypeGetName(input->type)); diff --git a/tensorflow/lite/kernels/pooling_test.cc b/tensorflow/lite/kernels/pooling_test.cc index e614fedccfd500..108195388141df 100644 --- a/tensorflow/lite/kernels/pooling_test.cc +++ b/tensorflow/lite/kernels/pooling_test.cc @@ -1151,5 +1151,18 @@ TEST(FloatPoolingOpTest, L2PoolPaddingValidSlide1) { EXPECT_THAT(m.GetOutput(), ElementsAreArray({3.5, 6.0, 6.5})); } +#ifdef GTEST_HAS_DEATH_TEST +TEST(FloatPoolingOpTest, MaxPoolWithZeroStride) { + EXPECT_DEATH( + FloatPoolingOpModel m(BuiltinOperator_MAX_POOL_2D, + /*input=*/{TensorType_FLOAT32, {1, 2, 4, 1}}, + /*filter_width=*/2, /*filter_height=*/2, + /*output=*/{TensorType_FLOAT32, {}}, + /*padding=*/Padding_VALID, + /*stride_w=*/0, /*stride_h=*/0), + "Cannot allocate tensors"); +} +#endif + } // namespace } // namespace tflite diff --git a/tensorflow/lite/kernels/space_to_batch_nd.cc b/tensorflow/lite/kernels/space_to_batch_nd.cc index 0d537e2d1892fe..af7b9d9e914a1e 100644 --- a/tensorflow/lite/kernels/space_to_batch_nd.cc +++ b/tensorflow/lite/kernels/space_to_batch_nd.cc @@ -79,6 +79,7 @@ TfLiteStatus ResizeOutputTensor(TfLiteContext* context, for (int dim = 0; dim < spatial_dims_num; ++dim) { int final_dim_size = (input_size->data[dim + 1] + paddings_data[dim * 2] + paddings_data[dim * 2 + 1]); + TF_LITE_ENSURE(context, block_shape[dim] != 0); TF_LITE_ENSURE_EQ(context, final_dim_size % block_shape[dim], 0); output_size->data[dim + 1] = final_dim_size / block_shape[dim]; output_batch_size *= block_shape[dim]; diff --git a/tensorflow/lite/kernels/space_to_depth.cc b/tensorflow/lite/kernels/space_to_depth.cc index ac001d903a466c..e01381466300b6 100644 --- a/tensorflow/lite/kernels/space_to_depth.cc +++ b/tensorflow/lite/kernels/space_to_depth.cc @@ -58,6 +58,7 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE_TYPES_EQ(context, input->type, output->type); const int block_size = params->block_size; + TF_LITE_ENSURE(context, block_size > 0); const int input_height = input->dims->data[1]; const int input_width = input->dims->data[2]; int output_height = input_height / block_size; diff --git a/tensorflow/lite/kernels/split.cc b/tensorflow/lite/kernels/split.cc index 3b7781f409e2fe..dbd7384b487c20 100644 --- a/tensorflow/lite/kernels/split.cc +++ b/tensorflow/lite/kernels/split.cc @@ -58,6 +58,7 @@ TfLiteStatus ResizeOutputTensors(TfLiteContext* context, TfLiteNode* node, TF_LITE_ENSURE(context, axis_value < NumDimensions(input)); const int input_size = SizeOfDimension(input, axis_value); + TF_LITE_ENSURE(context, num_splits != 0); TF_LITE_ENSURE_MSG(context, input_size % num_splits == 0, "Not an even split"); const int slice_size = input_size / num_splits; diff --git a/tensorflow/lite/kernels/split_v.cc b/tensorflow/lite/kernels/split_v.cc index 7d60086a91ddd2..26bbc0d37ecc06 100644 --- a/tensorflow/lite/kernels/split_v.cc +++ b/tensorflow/lite/kernels/split_v.cc @@ -94,6 +94,8 @@ TfLiteStatus ResizeOutputTensors(TfLiteContext* context, TfLiteNode* node, } } + TF_LITE_ENSURE(context, axis_value >= 0); + TF_LITE_ENSURE(context, axis_value < NumDimensions(input)); const int input_size = SizeOfDimension(input, axis_value); if (minus_one_index != -1) { diff --git a/tensorflow/lite/kernels/svdf.cc b/tensorflow/lite/kernels/svdf.cc index 1b8bf904b8ac31..863c18fd3de66d 100644 --- a/tensorflow/lite/kernels/svdf.cc +++ b/tensorflow/lite/kernels/svdf.cc @@ -96,6 +96,7 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { const int rank = params->rank; const int batch_size = input->dims->data[0]; const int num_filters = weights_feature->dims->data[0]; + TF_LITE_ENSURE(context, rank != 0); TF_LITE_ENSURE_EQ(context, num_filters % rank, 0); const int num_units = num_filters / rank; const int memory_size = weights_time->dims->data[1]; @@ -235,14 +236,21 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { output_temp_size_array)); // Calculate effective scales. + TF_LITE_ENSURE(context, input->quantization.type != kTfLiteNoQuantization); auto* input_params = reinterpret_cast(input->quantization.params); + TF_LITE_ENSURE(context, + weights_feature->quantization.type != kTfLiteNoQuantization); auto* weights_feature_params = reinterpret_cast( weights_feature->quantization.params); + TF_LITE_ENSURE(context, state->quantization.type != kTfLiteNoQuantization); auto* state_params = reinterpret_cast(state->quantization.params); + TF_LITE_ENSURE(context, + weights_time->quantization.type != kTfLiteNoQuantization); auto* weight_time_params = reinterpret_cast( weights_time->quantization.params); + TF_LITE_ENSURE(context, output->quantization.type != kTfLiteNoQuantization); auto* output_params = reinterpret_cast( output->quantization.params); const double effective_scale_1 = input_params->scale->data[0] * @@ -273,6 +281,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { TfLiteTensor* scratch = GetTemporary(context, node, /*index=*/0); TfLiteTensor* state = GetVariableInput(context, node, kStateTensor); + TF_LITE_ENSURE(context, state != nullptr); TfLiteTensor* output = GetOutput(context, node, kOutputTensor); switch (weights_feature->type) { diff --git a/tensorflow/lite/kernels/transpose_conv.cc b/tensorflow/lite/kernels/transpose_conv.cc index 07dc4bbac53452..079d3bd381221f 100644 --- a/tensorflow/lite/kernels/transpose_conv.cc +++ b/tensorflow/lite/kernels/transpose_conv.cc @@ -573,6 +573,10 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { const auto* params = reinterpret_cast(node->builtin_data); + // Prevent divisions by 0 + TF_LITE_ENSURE(context, params->stride_height > 0); + TF_LITE_ENSURE(context, params->stride_width > 0); + // Resize any deferred dynamic tensors if (IsDynamicTensor(output)) { TF_LITE_ENSURE_OK(context, ResizeTensor(context, output_shape, output)); diff --git a/tensorflow/lite/model_test.cc b/tensorflow/lite/model_test.cc index ba96494225ccc2..a51e030e849dd0 100644 --- a/tensorflow/lite/model_test.cc +++ b/tensorflow/lite/model_test.cc @@ -438,6 +438,24 @@ TEST(BasicFlatBufferModel, TestParseModelWithSparseTensor) { } // TODO(b/150072943): Add malformed model with sparse tensor tests. +// Recursion & reentrant are not supported in TFLite. +// The test ensures it fails gracefullly instead of crashing with +// a stack overflow. +TEST(BasicFlatBufferModel, TestUnsupportedRecursion) { + const auto model_path = + "tensorflow/lite/testdata/unsupported_recursion.bin"; + + std::unique_ptr model = + FlatBufferModel::BuildFromFile(model_path); + ASSERT_NE(model, nullptr); + + tflite::ops::builtin::BuiltinOpResolver resolver; + InterpreterBuilder builder(*model, resolver); + std::unique_ptr interpreter; + ASSERT_EQ(builder(&interpreter), kTfLiteOk); + ASSERT_NE(interpreter, nullptr); + ASSERT_NE(interpreter->AllocateTensors(), kTfLiteOk); +} // TODO(aselle): Add tests for serialization of builtin op data types. // These tests will occur with the evaluation tests of individual operators, diff --git a/tensorflow/lite/testdata/unsupported_recursion.bin b/tensorflow/lite/testdata/unsupported_recursion.bin new file mode 100644 index 00000000000000..525c5383ab4ef6 Binary files /dev/null and b/tensorflow/lite/testdata/unsupported_recursion.bin differ diff --git a/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py b/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py index d7a2c158de9a13..444b83cb72e5d4 100644 --- a/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py +++ b/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py @@ -84,6 +84,26 @@ def testFromSparseTensorSlices(self): with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) + @combinations.generate(combinations.combine(tf_api_version=1, mode=["graph"])) + def testEmptySparseTensorSlicesInvalid(self): + """Test a dataset based on invalid `tf.sparse.SparseTensor`.""" + st = array_ops.sparse_placeholder(dtypes.float64) + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.from_sparse_tensor_slices(st)) + init_op = iterator.initializer + + with self.cached_session() as sess: + # Test with an empty sparse tensor but with non empty values. + empty_indices = np.empty((0, 4), dtype=np.int64) + non_empty_values = [1, 2, 3, 4] + empty_dense_shape = [0, 4, 37, 9] + sparse_feed = sparse_tensor.SparseTensorValue(empty_indices, + non_empty_values, + empty_dense_shape) + # Here, we expect the test to fail when running the feed. + with self.assertRaises(errors.InvalidArgumentError): + sess.run(init_op, feed_dict={st: sparse_feed}) + @combinations.generate(combinations.combine(tf_api_version=2, mode=["eager"])) def testFromSparseTensorSlicesError(self): with self.assertRaises(AttributeError): diff --git a/tensorflow/python/feature_column/BUILD b/tensorflow/python/feature_column/BUILD index b3c6e061c22460..a2e7082219abd1 100644 --- a/tensorflow/python/feature_column/BUILD +++ b/tensorflow/python/feature_column/BUILD @@ -231,7 +231,10 @@ py_test( srcs = ["sequence_feature_column_integration_test.py"], python_version = "PY3", srcs_version = "PY2AND3", - tags = ["no_pip"], + tags = [ + "no_mac", + "no_pip", + ], deps = [ ":feature_column_v2", "//tensorflow/python:client_testlib", diff --git a/tensorflow/python/keras/engine/functional.py b/tensorflow/python/keras/engine/functional.py index fd80e7f8bb4ef5..b0cf778a895bed 100644 --- a/tensorflow/python/keras/engine/functional.py +++ b/tensorflow/python/keras/engine/functional.py @@ -58,7 +58,7 @@ class Functional(training_lib.Model): than with subclassed `Model`s, specifically: - Model cloning (`keras.models.clone`) - - Serialization (`model.get_config()/from_config`, `model.to_json()/to_yaml()` + - Serialization (`model.get_config()/from_config`, `model.to_json()` - Whole-model saving (`model.save()`) A `Functional` model can be instantiated by passing two arguments to diff --git a/tensorflow/python/keras/engine/functional_test.py b/tensorflow/python/keras/engine/functional_test.py index b60373e8c9bd71..c91026a6ee3c8e 100644 --- a/tensorflow/python/keras/engine/functional_test.py +++ b/tensorflow/python/keras/engine/functional_test.py @@ -52,11 +52,6 @@ from tensorflow.python.platform import test from tensorflow.python.training.tracking.util import Checkpoint -try: - import yaml # pylint:disable=g-import-not-at-top -except ImportError: - yaml = None - class NetworkConstructionTest(keras_parameterized.TestCase): @@ -620,10 +615,6 @@ def test_multi_input_multi_output_recursion(self): json_str = model.to_json() models.model_from_json(json_str) - if yaml is not None: - yaml_str = model.to_yaml() - models.model_from_yaml(yaml_str) - @combinations.generate(combinations.combine(mode=['graph', 'eager'])) def test_invalid_graphs(self): a = layers.Input(shape=(32,), name='input_a') @@ -1261,10 +1252,6 @@ def test_constant_initializer_with_numpy(self): json_str = model.to_json() models.model_from_json(json_str) - if yaml is not None: - yaml_str = model.to_yaml() - models.model_from_yaml(yaml_str) - def test_subclassed_error_if_init_not_called(self): class MyNetwork(training_lib.Model): diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py index a0ebec4f95e25c..e000e62f5da42f 100644 --- a/tensorflow/python/keras/engine/training.py +++ b/tensorflow/python/keras/engine/training.py @@ -88,11 +88,6 @@ import h5py except ImportError: h5py = None - -try: - import yaml -except ImportError: - yaml = None # pylint: enable=g-import-not-at-top @@ -2258,6 +2253,9 @@ def to_json(self, **kwargs): def to_yaml(self, **kwargs): """Returns a yaml string containing the network configuration. + Note: Since TF 2.6, this method is no longer supported and will raise a + RuntimeError. + To load a network from a yaml save file, use `keras.models.model_from_yaml(yaml_string, custom_objects={})`. @@ -2273,12 +2271,12 @@ def to_yaml(self, **kwargs): A YAML string. Raises: - ImportError: if yaml module is not found. + RuntimeError: announces that the method poses a security risk """ - if yaml is None: - raise ImportError( - 'Requires yaml module installed (`pip install pyyaml`).') - return yaml.dump(self._updated_config(), **kwargs) + raise RuntimeError( + 'Method `model.to_yaml()` has been removed due to security risk of ' + 'arbitrary code execution. Please use `model.to_json()` instead.' + ) def reset_states(self): for layer in self.layers: diff --git a/tensorflow/python/keras/layers/normalization.py b/tensorflow/python/keras/layers/normalization.py index e5723a3ef98353..9ab606d8038785 100644 --- a/tensorflow/python/keras/layers/normalization.py +++ b/tensorflow/python/keras/layers/normalization.py @@ -30,12 +30,12 @@ from tensorflow.python.keras.engine.input_spec import InputSpec from tensorflow.python.keras.utils import tf_utils from tensorflow.python.ops import array_ops +from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import init_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import nn from tensorflow.python.ops import state_ops from tensorflow.python.ops import variables as tf_variables -from tensorflow.python.platform import device_context from tensorflow.python.platform import tf_logging as logging from tensorflow.python.util.tf_export import keras_export @@ -514,7 +514,7 @@ def _fused_batch_norm(self, inputs, training): use_fused_avg_updates = ( ops.executing_eagerly_outside_functions() and isinstance(self.momentum, (float, int)) and - device_context.enclosing_tpu_context() is None) + enclosing_xla_context() is None) if use_fused_avg_updates: exponential_avg_factor = 1.0 - self.momentum else: @@ -930,6 +930,23 @@ def replace_in_base_docstring(replacements): return string +def enclosing_xla_context(): + """Recursively find and return the XLAControlFlowContext.""" + graph = ops.get_default_graph() + while graph is not None: + # pylint: disable=protected-access + context_ = graph._get_control_flow_context() + # pylint: enable=protected-access + while context_ is not None: + if isinstance(context_, control_flow_ops.XLAControlFlowContext): + return context_ + context_ = context_.outer_context + # This may be a FuncGraph due to defuns or v2 control flow. We need to + # find the original graph with the XLAControlFlowContext. + graph = getattr(graph, 'outer_graph', None) + return None + + @keras_export(v1=['keras.layers.BatchNormalization']) # pylint: disable=missing-docstring class BatchNormalization(BatchNormalizationBase): diff --git a/tensorflow/python/keras/saving/model_config.py b/tensorflow/python/keras/saving/model_config.py index 63f82b404a4c1c..344e543f9930a6 100644 --- a/tensorflow/python/keras/saving/model_config.py +++ b/tensorflow/python/keras/saving/model_config.py @@ -23,13 +23,6 @@ from tensorflow.python.util.tf_export import keras_export -# pylint: disable=g-import-not-at-top -try: - import yaml -except ImportError: - yaml = None -# pylint: enable=g-import-not-at-top - @keras_export('keras.models.model_from_config') def model_from_config(config, custom_objects=None): @@ -59,17 +52,8 @@ def model_from_config(config, custom_objects=None): def model_from_yaml(yaml_string, custom_objects=None): """Parses a yaml model configuration file and returns a model instance. - Usage: - - >>> model = tf.keras.Sequential([ - ... tf.keras.layers.Dense(5, input_shape=(3,)), - ... tf.keras.layers.Softmax()]) - >>> try: - ... import yaml - ... config = model.to_yaml() - ... loaded_model = tf.keras.models.model_from_yaml(config) - ... except ImportError: - ... pass + Note: Since TF 2.6, this method is no longer supported and will raise a + RuntimeError. Arguments: yaml_string: YAML string or open file encoding a model configuration. @@ -81,19 +65,13 @@ def model_from_yaml(yaml_string, custom_objects=None): A Keras model instance (uncompiled). Raises: - ImportError: if yaml module is not found. + RuntimeError: announces that the method poses a security risk """ - if yaml is None: - raise ImportError('Requires yaml module installed (`pip install pyyaml`).') - # The method unsafe_load only exists in PyYAML 5.x+, so which branch of the - # try block is covered by tests depends on the installed version of PyYAML. - try: - # PyYAML 5.x+ - config = yaml.unsafe_load(yaml_string) - except AttributeError: - config = yaml.load(yaml_string) - from tensorflow.python.keras.layers import deserialize # pylint: disable=g-import-not-at-top - return deserialize(config, custom_objects=custom_objects) + raise RuntimeError( + 'Method `model_from_yaml()` has been removed due to security risk of ' + 'arbitrary code execution. Please use `Model.to_json()` and ' + '`model_from_json()` instead.' + ) @keras_export('keras.models.model_from_json') diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index d37b928ad79211..efad3df7c1a3de 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -790,6 +790,7 @@ tf_py_test( name = "parsing_ops_test", size = "medium", srcs = ["parsing_ops_test.py"], + tags = ["no_mac"], deps = [ "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", diff --git a/tensorflow/python/kernel_tests/array_ops_test.py b/tensorflow/python/kernel_tests/array_ops_test.py index dbff3a1b2f728f..31c5164d922eb4 100644 --- a/tensorflow/python/kernel_tests/array_ops_test.py +++ b/tensorflow/python/kernel_tests/array_ops_test.py @@ -1441,7 +1441,7 @@ def testUnravelIndexZeroDim(self): with self.cached_session(): for dtype in [dtypes.int32, dtypes.int64]: with self.assertRaisesRegexp(errors.InvalidArgumentError, - "index is out of bound as with dims"): + "dims cannot contain a dim of zero"): indices = constant_op.constant([2, 5, 7], dtype=dtype) dims = constant_op.constant([3, 0], dtype=dtype) self.evaluate(array_ops.unravel_index(indices=indices, dims=dims)) diff --git a/tensorflow/python/kernel_tests/boosted_trees/BUILD b/tensorflow/python/kernel_tests/boosted_trees/BUILD index 5b318324d4cd2e..68b27849773bf7 100644 --- a/tensorflow/python/kernel_tests/boosted_trees/BUILD +++ b/tensorflow/python/kernel_tests/boosted_trees/BUILD @@ -24,6 +24,7 @@ tf_py_test( name = "resource_ops_test", size = "small", srcs = ["resource_ops_test.py"], + tags = ["no_mac"], deps = [ "//tensorflow/core/kernels/boosted_trees:boosted_trees_proto_py", "//tensorflow/python:boosted_trees_ops", @@ -39,6 +40,7 @@ tf_py_test( name = "prediction_ops_test", size = "small", srcs = ["prediction_ops_test.py"], + tags = ["no_mac"], deps = [ "//tensorflow/core/kernels/boosted_trees:boosted_trees_proto_py", "//tensorflow/python:array_ops", @@ -69,6 +71,7 @@ tf_py_test( name = "training_ops_test", size = "small", srcs = ["training_ops_test.py"], + tags = ["no_mac"], deps = [ "//tensorflow/core/kernels/boosted_trees:boosted_trees_proto_py", "//tensorflow/python:array_ops", diff --git a/tensorflow/python/kernel_tests/distributions/BUILD b/tensorflow/python/kernel_tests/distributions/BUILD index 549d7b4c98ece1..3325c853cf23fd 100644 --- a/tensorflow/python/kernel_tests/distributions/BUILD +++ b/tensorflow/python/kernel_tests/distributions/BUILD @@ -60,6 +60,9 @@ cuda_py_test( name = "beta_test", size = "small", srcs = ["beta_test.py"], + tags = [ + "no_oss", + ], deps = [ "//tensorflow/python:client", "//tensorflow/python:client_testlib", diff --git a/tensorflow/python/kernel_tests/substr_op_test.py b/tensorflow/python/kernel_tests/substr_op_test.py index 9302152e82bfa9..eae4e10f378567 100644 --- a/tensorflow/python/kernel_tests/substr_op_test.py +++ b/tensorflow/python/kernel_tests/substr_op_test.py @@ -492,6 +492,15 @@ def testInvalidUnit(self): with self.assertRaises(ValueError): string_ops.substr(b"test", 3, 1, unit="UTF8") + def testInvalidPos(self): + # Test case for GitHub issue 46900. + with self.assertRaises((ValueError, errors_impl.InvalidArgumentError)): + x = string_ops.substr(b"abc", len=1, pos=[1, -1]) + self.evaluate(x) + + with self.assertRaises((ValueError, errors_impl.InvalidArgumentError)): + x = string_ops.substr(b"abc", len=1, pos=[1, 2]) + self.evaluate(x) if __name__ == "__main__": test.main() diff --git a/tensorflow/python/kernel_tests/transpose_op_test.py b/tensorflow/python/kernel_tests/transpose_op_test.py index 87096211a01494..ed634ae7543b54 100644 --- a/tensorflow/python/kernel_tests/transpose_op_test.py +++ b/tensorflow/python/kernel_tests/transpose_op_test.py @@ -387,6 +387,8 @@ def testDouble(self): @test_util.run_v1_only("b/120545219") def testComplex64(self): + self._testBoth(np.array(np.complex(1, 2)).astype(np.complex64)) + self._testBoth(np.complex(1, 2) * np.arange(0, 21).astype(np.complex64)) self._testBoth( np.complex(1, 2) * np.arange(0, 21).reshape([3, 7]).astype(np.complex64)) @@ -399,6 +401,8 @@ def testComplex64(self): @test_util.run_v1_only("b/120545219") def testComplex128(self): + self._testBoth(np.array(np.complex(1, 2)).astype(np.complex128)) + self._testBoth(np.complex(1, 2) * np.arange(0, 21).astype(np.complex128)) self._testBoth( np.complex(1, 2) * np.arange(0, 21).reshape([3, 7]).astype(np.complex128)) diff --git a/tensorflow/python/lib/core/ndarray_tensor.cc b/tensorflow/python/lib/core/ndarray_tensor.cc index 2afd2888e8fd3e..5f1bfc8c7485bb 100644 --- a/tensorflow/python/lib/core/ndarray_tensor.cc +++ b/tensorflow/python/lib/core/ndarray_tensor.cc @@ -16,6 +16,7 @@ limitations under the License. #include "tensorflow/python/lib/core/ndarray_tensor.h" #include +#include #include "tensorflow/c/eager/tfe_context_internal.h" #include "tensorflow/c/tf_tensor_internal.h" @@ -74,6 +75,13 @@ Status PyArrayDescr_to_TF_DataType(PyArray_Descr* descr, PyObject* key; PyObject* value; Py_ssize_t pos = 0; + + // Return an error if the fields attribute is null. + // Occurs with an improper conversion attempt to resource. + if (descr->fields == nullptr) { + return errors::Internal("Unexpected numpy data type"); + } + if (PyDict_Next(descr->fields, &pos, &key, &value)) { // In Python 3, the keys of numpy custom struct types are unicode, unlike // Python 2, where the keys are bytes. diff --git a/tensorflow/python/ops/nn_test.py b/tensorflow/python/ops/nn_test.py index bfe11b63eea8d1..345abc53546a6d 100644 --- a/tensorflow/python/ops/nn_test.py +++ b/tensorflow/python/ops/nn_test.py @@ -27,6 +27,7 @@ from tensorflow.python.eager import def_function from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_spec from tensorflow.python.framework import test_util @@ -1216,6 +1217,46 @@ def testArbitraryASCII(self): y_val = self.evaluate(y) self.assertAllEqual(y_val, y_val_expected) + @test_util.disable_xla("XLA catches the error and rethrows as different one") + def testInvalidLength(self): + x = [-4, -3, -2, -1, 0, 1, 2, 3] + with self.assertRaisesRegex(errors.InvalidArgumentError, + "Source format must be of length 4 or 5"): + op = nn_ops.data_format_dim_map( + x, src_format="12345678", dst_format="87654321") + with test_util.use_gpu(): + self.evaluate(op) + + @test_util.disable_xla("XLA catches the error and rethrows as different one") + def testDuplicateSrc(self): + x = [-4, -3, -2, -1, 0, 1, 2, 3] + with self.assertRaisesRegex( + errors.InvalidArgumentError, + "Destination and source format must determine a permutation"): + op = nn_ops.data_format_dim_map(x, src_format="1233", dst_format="4321") + with test_util.use_gpu(): + self.evaluate(op) + + @test_util.disable_xla("XLA catches the error and rethrows as different one") + def testDuplicateDst(self): + x = [-4, -3, -2, -1, 0, 1, 2, 3] + with self.assertRaisesRegex( + errors.InvalidArgumentError, + "Destination and source format must determine a permutation"): + op = nn_ops.data_format_dim_map(x, src_format="1234", dst_format="3321") + with test_util.use_gpu(): + self.evaluate(op) + + @test_util.disable_xla("XLA catches the error and rethrows as different one") + def testExtraSpecifiers(self): + x = [-4, -3, -2, -1, 0, 1, 2, 3] + with self.assertRaisesRegex( + errors.InvalidArgumentError, + "Destination and source format must determine a permutation"): + op = nn_ops.data_format_dim_map(x, src_format="1234", dst_format="5321") + with test_util.use_gpu(): + self.evaluate(op) + class DataFormatVectorPermuteTest(test_lib.TestCase): @@ -1317,6 +1358,60 @@ def testNCHWToNHWC2D(self): y_val = self.evaluate(y) self.assertAllEqual(y_val, [[7, 4], [4, 5], [5, 1], [9, 3]]) + @test_util.disable_xla("XLA catches the error and rethrows as different one") + def testInvalidLength(self): + x = [0, 1, 2, 3] + with self.assertRaisesRegex(errors.InvalidArgumentError, + "Source format must be of length 4 or 5"): + op = nn_ops.data_format_vec_permute( + x, src_format="12345678", dst_format="87654321") + with test_util.use_gpu(): + self.evaluate(op) + + @test_util.disable_xla("XLA catches the error and rethrows as different one") + def testDuplicateSrc(self): + x = [0, 1, 2, 3] + with self.assertRaisesRegex( + errors.InvalidArgumentError, + "Destination and source format must determine a permutation"): + op = nn_ops.data_format_vec_permute( + x, src_format="1233", dst_format="4321") + with test_util.use_gpu(): + self.evaluate(op) + + @test_util.disable_xla("XLA catches the error and rethrows as different one") + def testDuplicateDst(self): + x = [0, 1, 2, 3] + with self.assertRaisesRegex( + errors.InvalidArgumentError, + "Destination and source format must determine a permutation"): + op = nn_ops.data_format_vec_permute( + x, src_format="1234", dst_format="3321") + with test_util.use_gpu(): + self.evaluate(op) + + @test_util.disable_xla("XLA catches the error and rethrows as different one") + def testExtraSpecifiers(self): + x = [0, 1, 2, 3] + with self.assertRaisesRegex( + errors.InvalidArgumentError, + "Destination and source format must determine a permutation"): + op = nn_ops.data_format_vec_permute( + x, src_format="1234", dst_format="5321") + with test_util.use_gpu(): + self.evaluate(op) + + @test_util.disable_xla("XLA catches the error and rethrows as different one") + def test2DNoWH(self): + x = [[0, 1], [2, 3]] + with self.assertRaisesRegex( + errors.InvalidArgumentError, + "Format specifier must contain H and W for 2D case"): + op = nn_ops.data_format_vec_permute( + x, src_format="1234", dst_format="4321") + with test_util.use_gpu(): + self.evaluate(op) + @test_util.run_all_in_graph_and_eager_modes class AvgPoolTest(test_lib.TestCase): diff --git a/tensorflow/python/ops/ragged/ragged_map_fn_op_test.py b/tensorflow/python/ops/ragged/ragged_map_fn_op_test.py index 9e74de4bc358dc..4808a10f86e8c1 100644 --- a/tensorflow/python/ops/ragged/ragged_map_fn_op_test.py +++ b/tensorflow/python/ops/ragged/ragged_map_fn_op_test.py @@ -21,9 +21,11 @@ import numpy as np from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors from tensorflow.python.framework import sparse_tensor from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops +from tensorflow.python.ops import map_fn as map_fn_lib from tensorflow.python.ops import math_ops as mo from tensorflow.python.ops import string_ops from tensorflow.python.ops.ragged import ragged_factory_ops @@ -294,6 +296,27 @@ def testMapOnSparseTensor(self): ) self.assertAllEqual(id_t2, [[0, 5], [0, 4]]) + def testRaggedMapWithIncorrectFnOutputSignature(self): + x = ragged_factory_ops.constant([[1, 2, 3, 4], [1]]) + with self.assertRaisesRegex(errors.InvalidArgumentError, + 'All flat_values must have compatible shapes'): + y = map_fn_lib.map_fn(lambda r: map_fn_lib.map_fn(lambda y: r, r), x) + self.evaluate(y) + + def testNestedRaggedMapWithFnOutputSignature(self): + ragged1d = ragged_tensor.RaggedTensorSpec([None], dtypes.int32) + ragged2d = ragged_tensor.RaggedTensorSpec([None, None], dtypes.int32) + + x = ragged_factory_ops.constant([[1, 2, 3, 4], [1]]) + # pylint: disable=g-long-lambda + y = map_fn_lib.map_fn( + lambda r: map_fn_lib.map_fn( + lambda y: r, r, fn_output_signature=ragged1d), + x, + fn_output_signature=ragged2d) + expected = [[[1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4]], [[1]]] + self.assertAllEqual(y, expected) + if __name__ == '__main__': googletest.main() diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc index a97850bd8d5348..5ae19f27ec6d53 100644 --- a/tensorflow/stream_executor/cuda/cuda_dnn.cc +++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc @@ -1474,7 +1474,9 @@ class CudnnRnnSequenceTensorDescriptor static port::StatusOr Create( GpuExecutor* parent, int max_seq_length, int batch_size, int data_size, cudnnDataType_t data_type) { - CHECK_GT(max_seq_length, 0); + if (max_seq_length <= 0) { + return port::Status(port::error::INVALID_ARGUMENT, "max_seq_length <= 0"); + } int dims[] = {batch_size, data_size, 1}; int strides[] = {dims[1] * dims[2], dims[2], 1}; TensorDescriptor tensor_desc = CreateTensorDescriptor(); @@ -1495,7 +1497,9 @@ class CudnnRnnSequenceTensorDescriptor const absl::Span& seq_lengths, bool time_major, cudnnDataType_t data_type) { #if CUDNN_VERSION >= 7201 - CHECK_GT(max_seq_length, 0); + if (max_seq_length <= 0) { + return port::Status(port::error::INVALID_ARGUMENT, "max_seq_length <= 0"); + } int dims[] = {batch_size, data_size, 1}; int strides[] = {dims[1] * dims[2], dims[2], 1}; TensorDescriptor tensor_desc = CreateTensorDescriptor(); diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl index 3ba217c49309ce..1e20a6713051d9 100644 --- a/tensorflow/tensorflow.bzl +++ b/tensorflow/tensorflow.bzl @@ -59,7 +59,7 @@ load( # not contain rc or alpha, only numbers. # Also update tensorflow/core/public/version.h # and tensorflow/tools/pip_package/setup.py -VERSION = "2.3.1" +VERSION = "2.3.4" VERSION_MAJOR = VERSION.split(".")[0] # Sanitize a dependency so that it works correctly from code that includes diff --git a/tensorflow/tools/ci_build/builds/libtensorflow.sh b/tensorflow/tools/ci_build/builds/libtensorflow.sh index a281afe7442d21..a6fa334a395345 100755 --- a/tensorflow/tools/ci_build/builds/libtensorflow.sh +++ b/tensorflow/tools/ci_build/builds/libtensorflow.sh @@ -56,6 +56,7 @@ function build_libtensorflow_tarball() { if [ "${TF_NEED_CUDA}" == "1" ]; then BAZEL_OPTS="${BAZEL_OPTS} --config=cuda --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain" export TF_NEED_ROCM=0 + export TF_CUDA_COMPUTE_CAPABILITIES="sm_35,sm_50,sm_60,sm_70,sm_75" fi bazel clean --expunge yes "" | ./configure diff --git a/tensorflow/tools/ci_build/linux/libtensorflow_docker.sh b/tensorflow/tools/ci_build/linux/libtensorflow_docker.sh index 1b255682671a78..fc8fad8eb76d5a 100755 --- a/tensorflow/tools/ci_build/linux/libtensorflow_docker.sh +++ b/tensorflow/tools/ci_build/linux/libtensorflow_docker.sh @@ -58,6 +58,7 @@ ${DOCKER_BINARY} run \ -e "TF_NEED_HDFS=0" \ -e "TF_NEED_CUDA=${TF_NEED_CUDA}" \ -e "TF_NEED_TENSORRT=${TF_NEED_CUDA}" \ + -e "TF_CUDA_COMPUTE_CAPABILITIES=${TF_CUDA_COMPUTE_CAPABILITIES}" \ -e "TF_NEED_ROCM=${TF_NEED_ROCM}" \ -e "TF_NEED_OPENCL_SYCL=0" \ "${DOCKER_IMAGE}" \ diff --git a/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile index 107d1b426c1722..deec0d21cb0c15 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile @@ -33,7 +33,7 @@ RUN apt-get update && apt-get install -y \ python3-pip RUN python3 -m pip --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary @@ -60,9 +60,7 @@ RUN jupyter serverextension enable --py jupyter_http_over_ws RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/ RUN mkdir /.local && chmod a+rwx /.local -RUN apt-get install -y --no-install-recommends wget -# some examples require git to fetch dependencies -RUN apt-get install -y --no-install-recommends git +RUN apt-get update && apt-get install -y --no-install-recommends wget git WORKDIR /tf/tensorflow-tutorials RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/classification.ipynb RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/overfit_and_underfit.ipynb diff --git a/tensorflow/tools/dockerfiles/dockerfiles/cpu.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/cpu.Dockerfile index e83592c5fd2408..e12571e34c1f22 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/cpu.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/cpu.Dockerfile @@ -33,7 +33,7 @@ RUN apt-get update && apt-get install -y \ python3-pip RUN python3 -m pip --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary diff --git a/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu-jupyter.Dockerfile index 78ec4416f47bc1..a496ad79df2cc4 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu-jupyter.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu-jupyter.Dockerfile @@ -62,7 +62,7 @@ RUN apt-get update && apt-get install -y \ python3-pip RUN python3 -m pip --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary @@ -111,9 +111,7 @@ RUN jupyter serverextension enable --py jupyter_http_over_ws RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/ RUN mkdir /.local && chmod a+rwx /.local -RUN apt-get install -y --no-install-recommends wget -# some examples require git to fetch dependencies -RUN apt-get install -y --no-install-recommends git +RUN apt-get update && apt-get install -y --no-install-recommends wget git WORKDIR /tf/tensorflow-tutorials RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/classification.ipynb RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/overfit_and_underfit.ipynb diff --git a/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu.Dockerfile index 018b7bb35bac12..4973ddd8026349 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu.Dockerfile @@ -62,7 +62,7 @@ RUN apt-get update && apt-get install -y \ python3-pip RUN python3 -m pip --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary diff --git a/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile index b99c384fe20e38..d7eca09e5a322e 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile @@ -104,7 +104,7 @@ RUN apt-get update && apt-get install -y \ python3-pip RUN python3 -m pip --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary @@ -153,9 +153,7 @@ RUN jupyter serverextension enable --py jupyter_http_over_ws RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/ RUN mkdir /.local && chmod a+rwx /.local -RUN apt-get install -y --no-install-recommends wget -# some examples require git to fetch dependencies -RUN apt-get install -y --no-install-recommends git +RUN apt-get update && apt-get install -y --no-install-recommends wget git WORKDIR /tf/tensorflow-tutorials RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/classification.ipynb RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/overfit_and_underfit.ipynb diff --git a/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile index 4493964cffc523..9602892bbf456e 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile @@ -104,7 +104,7 @@ RUN apt-get update && apt-get install -y \ python3-pip RUN python3 -m pip --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary diff --git a/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile index d4d913ce34a1c0..05f49d31fce62c 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile @@ -82,7 +82,7 @@ RUN apt-get update && apt-get install -y \ python3-pip RUN python3 -m pip --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary @@ -109,9 +109,7 @@ RUN jupyter serverextension enable --py jupyter_http_over_ws RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/ RUN mkdir /.local && chmod a+rwx /.local -RUN apt-get install -y --no-install-recommends wget -# some examples require git to fetch dependencies -RUN apt-get install -y --no-install-recommends git +RUN apt-get update && apt-get install -y --no-install-recommends wget git WORKDIR /tf/tensorflow-tutorials RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/classification.ipynb RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/overfit_and_underfit.ipynb diff --git a/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile index f563f2fc909804..4730a0dc145edf 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile @@ -82,7 +82,7 @@ RUN apt-get update && apt-get install -y \ python3-pip RUN python3 -m pip --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary diff --git a/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/devel-horovod-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/devel-horovod-jupyter.Dockerfile index 5ed856259a9170..baa5e7ce863562 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/devel-horovod-jupyter.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/devel-horovod-jupyter.Dockerfile @@ -62,7 +62,7 @@ RUN apt-get update && apt-get install -y \ python3-pip RUN python3 -m pip --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary @@ -163,9 +163,7 @@ RUN jupyter serverextension enable --py jupyter_http_over_ws RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/ RUN mkdir /.local && chmod a+rwx /.local -RUN apt-get install -y --no-install-recommends wget -# some examples require git to fetch dependencies -RUN apt-get install -y --no-install-recommends git +RUN apt-get update && apt-get install -y --no-install-recommends wget git WORKDIR /tf/tensorflow-tutorials RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/classification.ipynb RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/overfit_and_underfit.ipynb diff --git a/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/devel-horovod.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/devel-horovod.Dockerfile index a4a0bee0bc6b5c..0dfa5764537c02 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/devel-horovod.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/devel-horovod.Dockerfile @@ -62,7 +62,7 @@ RUN apt-get update && apt-get install -y \ python3-pip RUN python3 -m pip --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary diff --git a/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/horovod-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/horovod-jupyter.Dockerfile index 00c21e287f1393..68a1e3a432f31f 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/horovod-jupyter.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/horovod-jupyter.Dockerfile @@ -33,7 +33,7 @@ RUN apt-get update && apt-get install -y \ python3-pip RUN python3 -m pip --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary @@ -112,9 +112,7 @@ RUN jupyter serverextension enable --py jupyter_http_over_ws RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/ RUN mkdir /.local && chmod a+rwx /.local -RUN apt-get install -y --no-install-recommends wget -# some examples require git to fetch dependencies -RUN apt-get install -y --no-install-recommends git +RUN apt-get update && apt-get install -y --no-install-recommends wget git WORKDIR /tf/tensorflow-tutorials RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/classification.ipynb RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/overfit_and_underfit.ipynb diff --git a/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/horovod.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/horovod.Dockerfile index bef75f1e495432..c76e143b1850d0 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/horovod.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/mkl_horovod/horovod.Dockerfile @@ -33,7 +33,7 @@ RUN apt-get update && apt-get install -y \ python3-pip RUN python3 -m pip --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/cpu-ppc64le-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/cpu-ppc64le-jupyter.Dockerfile index 0a284f4dcb07e9..16163aeb1e538b 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/cpu-ppc64le-jupyter.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/cpu-ppc64le-jupyter.Dockerfile @@ -33,7 +33,7 @@ RUN apt-get update && apt-get install -y \ python3-pip RUN python3 -m pip --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary @@ -78,9 +78,7 @@ RUN jupyter serverextension enable --py jupyter_http_over_ws RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/ RUN mkdir /.local && chmod a+rwx /.local -RUN apt-get install -y --no-install-recommends wget -# some examples require git to fetch dependencies -RUN apt-get install -y --no-install-recommends git +RUN apt-get update && apt-get install -y --no-install-recommends wget git WORKDIR /tf/tensorflow-tutorials RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/classification.ipynb RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/overfit_and_underfit.ipynb diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/cpu-ppc64le.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/cpu-ppc64le.Dockerfile index 831e5aead0511d..cbcd2e0a8e00ee 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/cpu-ppc64le.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/cpu-ppc64le.Dockerfile @@ -33,7 +33,7 @@ RUN apt-get update && apt-get install -y \ python3-pip RUN python3 -m pip --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le-jupyter.Dockerfile index 14ae948c31a1a4..a222f8d51e5dff 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le-jupyter.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le-jupyter.Dockerfile @@ -62,7 +62,7 @@ RUN apt-get update && apt-get install -y \ python3-pip RUN python3 -m pip --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary @@ -112,9 +112,7 @@ RUN jupyter serverextension enable --py jupyter_http_over_ws RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/ RUN mkdir /.local && chmod a+rwx /.local -RUN apt-get install -y --no-install-recommends wget -# some examples require git to fetch dependencies -RUN apt-get install -y --no-install-recommends git +RUN apt-get update && apt-get install -y --no-install-recommends wget git WORKDIR /tf/tensorflow-tutorials RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/classification.ipynb RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/overfit_and_underfit.ipynb diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le.Dockerfile index c098b863eaa03c..81c67d90983de0 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le.Dockerfile @@ -62,7 +62,7 @@ RUN apt-get update && apt-get install -y \ python3-pip RUN python3 -m pip --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le-jupyter.Dockerfile index 1967c20419c03d..5dae92c82e3975 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le-jupyter.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le-jupyter.Dockerfile @@ -104,7 +104,7 @@ RUN apt-get update && apt-get install -y \ python3-pip RUN python3 -m pip --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary @@ -154,9 +154,7 @@ RUN jupyter serverextension enable --py jupyter_http_over_ws RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/ RUN mkdir /.local && chmod a+rwx /.local -RUN apt-get install -y --no-install-recommends wget -# some examples require git to fetch dependencies -RUN apt-get install -y --no-install-recommends git +RUN apt-get update && apt-get install -y --no-install-recommends wget git WORKDIR /tf/tensorflow-tutorials RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/classification.ipynb RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/overfit_and_underfit.ipynb diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le.Dockerfile index ffd74c52efa1c8..b8325567d4a413 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le.Dockerfile @@ -104,7 +104,7 @@ RUN apt-get update && apt-get install -y \ python3-pip RUN python3 -m pip --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le-jupyter.Dockerfile index 6ef081013047f5..7de36582fc3554 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le-jupyter.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le-jupyter.Dockerfile @@ -82,7 +82,7 @@ RUN apt-get update && apt-get install -y \ python3-pip RUN python3 -m pip --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary @@ -127,9 +127,7 @@ RUN jupyter serverextension enable --py jupyter_http_over_ws RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/ RUN mkdir /.local && chmod a+rwx /.local -RUN apt-get install -y --no-install-recommends wget -# some examples require git to fetch dependencies -RUN apt-get install -y --no-install-recommends git +RUN apt-get update && apt-get install -y --no-install-recommends wget git WORKDIR /tf/tensorflow-tutorials RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/classification.ipynb RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/overfit_and_underfit.ipynb diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le.Dockerfile index f10e9f95182224..4e43bdc638a4e8 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le.Dockerfile @@ -82,7 +82,7 @@ RUN apt-get update && apt-get install -y \ python3-pip RUN python3 -m pip --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary diff --git a/tensorflow/tools/dockerfiles/partials/jupyter.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/jupyter.partial.Dockerfile index cd84872a9864d7..49905e7289a7da 100644 --- a/tensorflow/tools/dockerfiles/partials/jupyter.partial.Dockerfile +++ b/tensorflow/tools/dockerfiles/partials/jupyter.partial.Dockerfile @@ -5,9 +5,7 @@ RUN jupyter serverextension enable --py jupyter_http_over_ws RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/ RUN mkdir /.local && chmod a+rwx /.local -RUN apt-get install -y --no-install-recommends wget -# some examples require git to fetch dependencies -RUN apt-get install -y --no-install-recommends git +RUN apt-get update && apt-get install -y --no-install-recommends wget git WORKDIR /tf/tensorflow-tutorials RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/classification.ipynb RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/overfit_and_underfit.ipynb diff --git a/tensorflow/tools/dockerfiles/partials/ubuntu/python.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/python.partial.Dockerfile index a3c07385cc892f..6318a5fb7ed11c 100644 --- a/tensorflow/tools/dockerfiles/partials/ubuntu/python.partial.Dockerfile +++ b/tensorflow/tools/dockerfiles/partials/ubuntu/python.partial.Dockerfile @@ -6,7 +6,7 @@ RUN apt-get update && apt-get install -y \ python3-pip RUN python3 -m pip --no-cache-dir install --upgrade \ - pip \ + "pip<20.3" \ setuptools # Some TF tools expect a "python" binary diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index 594e74f40c0340..d944993c48fa28 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -49,7 +49,7 @@ # result for pip. # Also update tensorflow/tensorflow.bzl and # tensorflow/core/public/version.h -_VERSION = '2.3.1' +_VERSION = '2.3.4' REQUIRED_PACKAGES = [ 'absl-py >= 0.7.0', diff --git a/tensorflow/tools/pip_package/setup.py.orig b/tensorflow/tools/pip_package/setup.py.orig new file mode 100644 index 00000000000000..6f158a8c84db91 --- /dev/null +++ b/tensorflow/tools/pip_package/setup.py.orig @@ -0,0 +1,313 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""TensorFlow is an open source machine learning framework for everyone. + +TensorFlow is an open source software library for high performance numerical +computation. Its flexible architecture allows easy deployment of computation +across a variety of platforms (CPUs, GPUs, TPUs), and from desktops to clusters +of servers to mobile and edge devices. + +Originally developed by researchers and engineers from the Google Brain team +within Google's AI organization, it comes with strong support for machine +learning and deep learning and the flexible numerical computation core is used +across many other scientific domains. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import fnmatch +import os +import re +import sys + +from setuptools import Command +from setuptools import find_packages +from setuptools import setup +from setuptools.command.install import install as InstallCommandBase +from setuptools.dist import Distribution + +DOCLINES = __doc__.split('\n') + +# This version string is semver compatible, but incompatible with pip. +# For pip, we will remove all '-' characters from this string, and use the +# result for pip. +# Also update tensorflow/tensorflow.bzl and +# tensorflow/core/public/version.h +_VERSION = '2.1.2' + +REQUIRED_PACKAGES = [ + 'absl-py >= 0.7.0', + 'astor >= 0.6.0', + 'backports.weakref >= 1.0rc1;python_version<"3.4"', + 'enum34 >= 1.1.6;python_version<"3.4"', + 'gast == 0.2.2', + 'google_pasta >= 0.1.6', + 'keras_applications >= 1.0.8', + 'keras_preprocessing == 1.1.0', + 'numpy >= 1.16.0, < 1.19.0', + 'opt_einsum >= 2.3.2', + 'protobuf >= 3.8.0', + 'tensorboard >= 2.1.0, < 2.2.0', + 'tensorflow_estimator >= 2.1.0rc0, < 2.2.0', + 'termcolor >= 1.1.0', + 'wrapt >= 1.11.1', + # python3 requires wheel 0.26 + 'wheel >= 0.26;python_version>="3"', + 'wheel;python_version<"3"', +<<<<<<< HEAD + # mock comes with unittest.mock for python3, need to install for python2 + 'mock >= 2.0.0;python_version<"3"', + # functools comes with python3, need to install the backport for python2 + 'functools32 >= 3.2.3;python_version<"3"', + 'six >= 1.12.0', +======= + 'wrapt >= 1.11.1', + # Pin h5py to at most 2.10.0 as newer versions break old keras tests + 'h5py <= 2.10.0', +>>>>>>> 03d7ca7871b (Add upper bound to `h5py`.) +] + +if sys.byteorder == 'little': + # grpcio does not build correctly on big-endian machines due to lack of + # BoringSSL support. + # See https://github.com/tensorflow/tensorflow/issues/17882. + REQUIRED_PACKAGES.append('grpcio >= 1.8.6') + +project_name = 'tensorflow' +if '--project_name' in sys.argv: + project_name_idx = sys.argv.index('--project_name') + project_name = sys.argv[project_name_idx + 1] + sys.argv.remove('--project_name') + sys.argv.pop(project_name_idx) + +# tf-nightly should depend on tb-nightly +if 'tf_nightly' in project_name: + for i, pkg in enumerate(REQUIRED_PACKAGES): + if 'tensorboard' in pkg: + REQUIRED_PACKAGES[i] = 'tb-nightly >= 2.1.0a0, < 2.2.0a0' + elif 'tensorflow_estimator' in pkg and '2.0' in project_name: + REQUIRED_PACKAGES[i] = 'tensorflow-estimator-2.0-preview' + elif 'tensorflow_estimator' in pkg: + REQUIRED_PACKAGES[i] = 'tf-estimator-nightly' + +# pylint: disable=line-too-long +CONSOLE_SCRIPTS = [ + 'toco_from_protos = tensorflow.lite.toco.python.toco_from_protos:main', + 'tflite_convert = tensorflow.lite.python.tflite_convert:main', + 'toco = tensorflow.lite.python.tflite_convert:main', + 'saved_model_cli = tensorflow.python.tools.saved_model_cli:main', + # We need to keep the TensorBoard command, even though the console script + # is now declared by the tensorboard pip package. If we remove the + # TensorBoard command, pip will inappropriately remove it during install, + # even though the command is not removed, just moved to a different wheel. + 'tensorboard = tensorboard.main:run_main', + 'tf_upgrade_v2 = tensorflow.tools.compatibility.tf_upgrade_v2_main:main', + 'estimator_ckpt_converter = tensorflow_estimator.python.estimator.tools.checkpoint_converter:main', +] +# pylint: enable=line-too-long + +# Only keep freeze_graph console script in 1.X. +if _VERSION.startswith('1.') and '_2.0' not in project_name: + CONSOLE_SCRIPTS.append( + 'freeze_graph = tensorflow.python.tools.freeze_graph:run_main') + +# remove the tensorboard console script if building tf_nightly +if 'tf_nightly' in project_name: + CONSOLE_SCRIPTS.remove('tensorboard = tensorboard.main:run_main') + +TEST_PACKAGES = [ + 'scipy >= 0.15.1', +] + + +class BinaryDistribution(Distribution): + + def has_ext_modules(self): + return True + + +class InstallCommand(InstallCommandBase): + """Override the dir where the headers go.""" + + def finalize_options(self): + ret = InstallCommandBase.finalize_options(self) + self.install_headers = os.path.join(self.install_purelib, 'tensorflow_core', + 'include') + self.install_lib = self.install_platlib + return ret + + +class InstallHeaders(Command): + """Override how headers are copied. + + The install_headers that comes with setuptools copies all files to + the same directory. But we need the files to be in a specific directory + hierarchy for -I to work correctly. + """ + description = 'install C/C++ header files' + + user_options = [('install-dir=', 'd', + 'directory to install header files to'), + ('force', 'f', + 'force installation (overwrite existing files)'), + ] + + boolean_options = ['force'] + + def initialize_options(self): + self.install_dir = None + self.force = 0 + self.outfiles = [] + + def finalize_options(self): + self.set_undefined_options('install', + ('install_headers', 'install_dir'), + ('force', 'force')) + + def mkdir_and_copy_file(self, header): + install_dir = os.path.join(self.install_dir, os.path.dirname(header)) + # Get rid of some extra intervening directories so we can have fewer + # directories for -I + install_dir = re.sub('/google/protobuf_archive/src', '', install_dir) + install_dir = re.sub('/include/tensorflow_core/', '/include/tensorflow/', + install_dir) + + # Copy external code headers into tensorflow_core/include. + # A symlink would do, but the wheel file that gets created ignores + # symlink within the directory hierarchy. + # NOTE(keveman): Figure out how to customize bdist_wheel package so + # we can do the symlink. + external_header_locations = [ + 'tensorflow_core/include/external/eigen_archive/', + 'tensorflow_core/include/external/com_google_absl/', + ] + for location in external_header_locations: + if location in install_dir: + extra_dir = install_dir.replace(location, '') + if not os.path.exists(extra_dir): + self.mkpath(extra_dir) + self.copy_file(header, extra_dir) + + if not os.path.exists(install_dir): + self.mkpath(install_dir) + return self.copy_file(header, install_dir) + + def run(self): + hdrs = self.distribution.headers + if not hdrs: + return + + self.mkpath(self.install_dir) + for header in hdrs: + (out, _) = self.mkdir_and_copy_file(header) + self.outfiles.append(out) + + def get_inputs(self): + return self.distribution.headers or [] + + def get_outputs(self): + return self.outfiles + + +def find_files(pattern, root): + """Return all the files matching pattern below root dir.""" + for dirpath, _, files in os.walk(root): + for filename in fnmatch.filter(files, pattern): + yield os.path.join(dirpath, filename) + + +so_lib_paths = [ + i for i in os.listdir('.') + if os.path.isdir(i) and fnmatch.fnmatch(i, '_solib_*') +] + +matches = [] +for path in so_lib_paths: + matches.extend( + ['../' + x for x in find_files('*', path) if '.py' not in x] + ) + +if os.name == 'nt': + EXTENSION_NAME = 'python/_pywrap_tensorflow_internal.pyd' +else: + EXTENSION_NAME = 'python/_pywrap_tensorflow_internal.so' + +headers = ( + list(find_files('*.h', 'tensorflow_core/core')) + + list(find_files('*.h', 'tensorflow_core/stream_executor')) + + list(find_files('*.h', 'google/com_google_protobuf/src')) + + list(find_files('*.inc', 'google/com_google_protobuf/src')) + + list(find_files('*', 'third_party/eigen3')) + list( + find_files('*.h', 'tensorflow_core/include/external/com_google_absl')) + + list( + find_files('*.inc', 'tensorflow_core/include/external/com_google_absl')) + + list(find_files('*', 'tensorflow_core/include/external/eigen_archive'))) + +setup( + name=project_name, + version=_VERSION.replace('-', ''), + description=DOCLINES[0], + long_description='\n'.join(DOCLINES[2:]), + url='https://www.tensorflow.org/', + download_url='https://github.com/tensorflow/tensorflow/tags', + author='Google Inc.', + author_email='packages@tensorflow.org', + # Contained modules and scripts. + packages=find_packages(), + entry_points={ + 'console_scripts': CONSOLE_SCRIPTS, + }, + headers=headers, + install_requires=REQUIRED_PACKAGES, + tests_require=REQUIRED_PACKAGES + TEST_PACKAGES, + # Add in any packaged data. + include_package_data=True, + package_data={ + 'tensorflow': [ + EXTENSION_NAME, + ] + matches, + }, + zip_safe=False, + distclass=BinaryDistribution, + cmdclass={ + 'install_headers': InstallHeaders, + 'install': InstallCommand, + }, + # PyPI package information. + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Intended Audience :: Education', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Topic :: Scientific/Engineering', + 'Topic :: Scientific/Engineering :: Mathematics', + 'Topic :: Scientific/Engineering :: Artificial Intelligence', + 'Topic :: Software Development', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], + license='Apache 2.0', + keywords='tensorflow tensor machine learning', +) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index c9615431456ac2..c38c80cc30c93f 100755 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -409,12 +409,12 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): tf_http_archive( name = "org_sqlite", build_file = clean_dep("//third_party:sqlite.BUILD"), - sha256 = "b34f4c0c0eefad9a7e515c030c18702e477f4ef7d8ade6142bdab8011b487ac6", - strip_prefix = "sqlite-amalgamation-3330000", + sha256 = "8ff0b79fd9118af7a760f1f6a98cac3e69daed325c8f9f0a581ecb62f797fd64", + strip_prefix = "sqlite-amalgamation-3340000", system_build_file = clean_dep("//third_party/systemlibs:sqlite.BUILD"), urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/www.sqlite.org/2020/sqlite-amalgamation-3330000.zip", - "https://www.sqlite.org/2020/sqlite-amalgamation-3330000.zip", + "https://storage.googleapis.com/mirror.tensorflow.org/www.sqlite.org/2020/sqlite-amalgamation-3340000.zip", + "https://www.sqlite.org/2020/sqlite-amalgamation-3340000.zip", ], ) @@ -647,12 +647,12 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): tf_http_archive( name = "pcre", build_file = clean_dep("//third_party:pcre.BUILD"), - sha256 = "69acbc2fbdefb955d42a4c606dfde800c2885711d2979e356c0636efde9ec3b5", - strip_prefix = "pcre-8.42", + sha256 = "aecafd4af3bd0f3935721af77b889d9024b2e01d96b58471bd91a3063fb47728", + strip_prefix = "pcre-8.44", system_build_file = clean_dep("//third_party/systemlibs:pcre.BUILD"), urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/ftp.exim.org/pub/pcre/pcre-8.42.tar.gz", - "https://ftp.exim.org/pub/pcre/pcre-8.42.tar.gz", + "https://storage.googleapis.com/mirror.tensorflow.org/ftp.exim.org/pub/pcre/pcre-8.44.tar.gz", + "https://ftp.exim.org/pub/pcre/pcre-8.44.tar.gz", ], ) @@ -672,12 +672,12 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): tf_http_archive( name = "curl", build_file = clean_dep("//third_party:curl.BUILD"), - sha256 = "01ae0c123dee45b01bbaef94c0bc00ed2aec89cb2ee0fd598e0d302a6b5e0a98", - strip_prefix = "curl-7.69.1", + sha256 = "b0a3428acb60fa59044c4d0baae4e4fc09ae9af1d8a3aa84b2e3fbcd99841f77", + strip_prefix = "curl-7.77.0", system_build_file = clean_dep("//third_party/systemlibs:curl.BUILD"), urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/curl.haxx.se/download/curl-7.69.1.tar.gz", - "https://curl.haxx.se/download/curl-7.69.1.tar.gz", + "https://storage.googleapis.com/mirror.tensorflow.org/curl.haxx.se/download/curl-7.77.0.tar.gz", + "https://curl.haxx.se/download/curl-7.77.0.tar.gz", ], ) @@ -743,12 +743,12 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): tf_http_archive( name = "jsoncpp_git", build_file = clean_dep("//third_party:jsoncpp.BUILD"), - sha256 = "77a402fb577b2e0e5d0bdc1cf9c65278915cdb25171e3452c68b6da8a561f8f0", - strip_prefix = "jsoncpp-1.9.2", + sha256 = "e34a628a8142643b976c7233ef381457efad79468c67cb1ae0b83a33d7493999", + strip_prefix = "jsoncpp-1.9.4", system_build_file = clean_dep("//third_party/systemlibs:jsoncpp.BUILD"), urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/github.com/open-source-parsers/jsoncpp/archive/1.9.2.tar.gz", - "https://github.com/open-source-parsers/jsoncpp/archive/1.9.2.tar.gz", + "https://storage.googleapis.com/mirror.tensorflow.org/github.com/open-source-parsers/jsoncpp/archive/1.9.4.tar.gz", + "https://github.com/open-source-parsers/jsoncpp/archive/1.9.4.tar.gz", ], ) diff --git a/third_party/curl.BUILD b/third_party/curl.BUILD index 62fc946956da85..85b09cf1bf4c72 100644 --- a/third_party/curl.BUILD +++ b/third_party/curl.BUILD @@ -25,22 +25,35 @@ CURL_WIN_SRCS = [ "lib/asyn-thread.c", "lib/inet_ntop.c", "lib/system_win32.c", - "lib/x509asn1.c", - "lib/vtls/schannel.c", - "lib/vtls/schannel_verify.c", - "lib/idn_win32.c", + "lib/setup-win32.h", ] cc_library( name = "curl", srcs = [ "include/curl_config.h", + "lib/altsvc.c", + "lib/altsvc.h", + "lib/amigaos.c", "lib/amigaos.h", "lib/arpa_telnet.h", - "lib/asyn.h", "lib/asyn-ares.c", + "lib/asyn.h", "lib/base64.c", + "lib/bufref.c", + "lib/bufref.h", + "lib/c-hyper.c", + "lib/c-hyper.h", + "lib/config-amigaos.h", + "lib/config-dos.h", + "lib/config-mac.h", + "lib/config-os400.h", + "lib/config-plan9.h", + "lib/config-riscos.h", + "lib/config-tpf.h", + "lib/config-vxworks.h", "lib/config-win32.h", + "lib/config-win32ce.h", "lib/conncache.c", "lib/conncache.h", "lib/connect.c", @@ -54,14 +67,20 @@ cc_library( "lib/curl_base64.h", "lib/curl_ctype.c", "lib/curl_ctype.h", + "lib/curl_des.c", "lib/curl_des.h", + "lib/curl_endian.c", "lib/curl_endian.h", "lib/curl_fnmatch.c", "lib/curl_fnmatch.h", + "lib/curl_get_line.c", + "lib/curl_get_line.h", "lib/curl_gethostname.c", "lib/curl_gethostname.h", + "lib/curl_gssapi.c", "lib/curl_gssapi.h", "lib/curl_hmac.h", + "lib/curl_krb5.h", "lib/curl_ldap.h", "lib/curl_md4.h", "lib/curl_md5.h", @@ -70,14 +89,19 @@ cc_library( "lib/curl_memrchr.h", "lib/curl_multibyte.c", "lib/curl_multibyte.h", + "lib/curl_ntlm_core.c", "lib/curl_ntlm_core.h", + "lib/curl_ntlm_wb.c", "lib/curl_ntlm_wb.h", + "lib/curl_path.c", + "lib/curl_path.h", "lib/curl_printf.h", + "lib/curl_range.c", + "lib/curl_range.h", "lib/curl_rtmp.c", "lib/curl_rtmp.h", "lib/curl_sasl.c", "lib/curl_sasl.h", - "lib/curl_sec.h", "lib/curl_setup.h", "lib/curl_setup_once.h", "lib/curl_sha256.h", @@ -86,23 +110,35 @@ cc_library( "lib/curl_threads.c", "lib/curl_threads.h", "lib/curlx.h", + "lib/dict.c", "lib/dict.h", + "lib/doh.c", + "lib/doh.h", "lib/dotdot.c", "lib/dotdot.h", + "lib/dynbuf.c", + "lib/dynbuf.h", "lib/easy.c", + "lib/easygetopt.c", "lib/easyif.h", + "lib/easyoptions.c", + "lib/easyoptions.h", "lib/escape.c", "lib/escape.h", + "lib/file.c", "lib/file.h", "lib/fileinfo.c", "lib/fileinfo.h", "lib/formdata.c", "lib/formdata.h", + "lib/ftp.c", "lib/ftp.h", + "lib/ftplistparser.c", "lib/ftplistparser.h", "lib/getenv.c", "lib/getinfo.c", "lib/getinfo.h", + "lib/gopher.c", "lib/gopher.h", "lib/hash.c", "lib/hash.h", @@ -115,6 +151,8 @@ cc_library( "lib/hostip4.c", "lib/hostip6.c", "lib/hostsyn.c", + "lib/hsts.c", + "lib/hsts.h", "lib/http.c", "lib/http.h", "lib/http2.c", @@ -123,17 +161,24 @@ cc_library( "lib/http_chunks.h", "lib/http_digest.c", "lib/http_digest.h", + "lib/http_negotiate.c", "lib/http_negotiate.h", + "lib/http_ntlm.c", "lib/http_ntlm.h", "lib/http_proxy.c", "lib/http_proxy.h", + "lib/http_aws_sigv4.c", + "lib/http_aws_sigv4.h", + "lib/idn_win32.c", "lib/if2ip.c", "lib/if2ip.h", + "lib/imap.c", "lib/imap.h", "lib/inet_ntop.h", "lib/inet_pton.c", "lib/inet_pton.h", "lib/krb5.c", + "lib/ldap.c", "lib/llist.c", "lib/llist.h", "lib/md4.c", @@ -143,38 +188,43 @@ cc_library( "lib/mime.c", "lib/mime.h", "lib/mprintf.c", + "lib/mqtt.c", + "lib/mqtt.h", "lib/multi.c", "lib/multihandle.h", "lib/multiif.h", "lib/netrc.c", "lib/netrc.h", + "lib/non-ascii.c", "lib/non-ascii.h", "lib/nonblock.c", "lib/nonblock.h", - "lib/nwlib.c", - "lib/nwos.c", + #"lib/nwlib.c", + #"lib/nwos.c", + "lib/openldap.c", "lib/parsedate.c", "lib/parsedate.h", - "lib/pingpong.h", "lib/pingpong.c", + "lib/pingpong.h", + "lib/pop3.c", "lib/pop3.h", "lib/progress.c", "lib/progress.h", + "lib/psl.c", + "lib/psl.h", "lib/quic.h", "lib/rand.c", "lib/rand.h", - "lib/rename.h", "lib/rename.c", + "lib/rename.h", "lib/rtsp.c", "lib/rtsp.h", - "lib/security.c", "lib/select.c", "lib/select.h", "lib/sendf.c", "lib/sendf.h", "lib/setopt.c", "lib/setopt.h", - "lib/setup-os400.h", "lib/setup-vms.h", "lib/sha256.c", "lib/share.c", @@ -182,13 +232,17 @@ cc_library( "lib/sigpipe.h", "lib/slist.c", "lib/slist.h", + "lib/smb.c", "lib/smb.h", + "lib/smtp.c", "lib/smtp.h", "lib/sockaddr.h", - "lib/socketpair.h", "lib/socketpair.c", + "lib/socketpair.h", "lib/socks.c", "lib/socks.h", + "lib/socks_gssapi.c", + "lib/socks_sspi.c", "lib/speedcheck.c", "lib/speedcheck.h", "lib/splay.c", @@ -204,7 +258,9 @@ cc_library( "lib/strtoofft.c", "lib/strtoofft.h", "lib/system_win32.h", + "lib/telnet.c", "lib/telnet.h", + "lib/tftp.c", "lib/tftp.h", "lib/timeval.c", "lib/timeval.h", @@ -213,44 +269,69 @@ cc_library( "lib/url.c", "lib/url.h", "lib/urldata.h", + "lib/urlapi-int.h", + "lib/urlapi.c", + "lib/version.c", + "lib/version_win32.c", + "lib/version_win32.h", + "lib/warnless.c", + "lib/warnless.h", + "lib/wildcard.c", + "lib/wildcard.h", + "lib/x509asn1.c", + "lib/x509asn1.h", "lib/vauth/cleartext.c", "lib/vauth/cram.c", "lib/vauth/digest.c", "lib/vauth/digest.h", + "lib/vauth/digest_sspi.c", + "lib/vauth/krb5_gssapi.c", + "lib/vauth/krb5_sspi.c", + "lib/vauth/ntlm.c", "lib/vauth/ntlm.h", + "lib/vauth/ntlm_sspi.c", "lib/vauth/oauth2.c", + "lib/vauth/spnego_sspi.c", "lib/vauth/vauth.c", "lib/vauth/vauth.h", - "lib/version.c", + "lib/vquic/ngtcp2.c", + "lib/vquic/ngtcp2.h", + "lib/vquic/quiche.c", + "lib/vquic/quiche.h", + "lib/vquic/vquic.c", + "lib/vquic/vquic.h", + "lib/vssh/libssh.c", + "lib/vssh/libssh2.c", "lib/vssh/ssh.h", + "lib/vssh/wolfssh.c", + "lib/vtls/bearssl.c", "lib/vtls/bearssl.h", + "lib/vtls/gskit.c", "lib/vtls/gskit.h", + "lib/vtls/gtls.c", "lib/vtls/gtls.h", + "lib/vtls/keylog.c", + "lib/vtls/keylog.h", + "lib/vtls/mbedtls.c", "lib/vtls/mbedtls.h", + "lib/vtls/mbedtls_threadlock.c", + "lib/vtls/mbedtls_threadlock.h", + "lib/vtls/mesalink.c", + "lib/vtls/mesalink.h", + "lib/vtls/nss.c", "lib/vtls/nssg.h", + "lib/vtls/openssl.c", "lib/vtls/openssl.h", + "lib/vtls/rustls.c", + "lib/vtls/rustls.h", + "lib/vtls/schannel.c", "lib/vtls/schannel.h", + "lib/vtls/schannel_verify.c", + "lib/vtls/sectransp.h", "lib/vtls/vtls.c", "lib/vtls/vtls.h", + "lib/vtls/wolfssl.c", "lib/vtls/wolfssl.h", - "lib/warnless.c", - "lib/warnless.h", - "lib/wildcard.c", - "lib/wildcard.h", - "lib/x509asn1.h", - "lib/psl.h", - "lib/psl.c", - "lib/vtls/sectransp.h", - "lib/vtls/mesalink.h", - "lib/vtls/mesalink.c", - "lib/curl_get_line.h", - "lib/curl_get_line.c", - "lib/urlapi-int.h", - "lib/urlapi.c", - "lib/altsvc.h", - "lib/altsvc.c", - "lib/doh.h", - "lib/doh.c", ] + select({ "@org_tensorflow//tensorflow:macos": [ "lib/vtls/sectransp.c", @@ -260,7 +341,6 @@ cc_library( ], "@org_tensorflow//tensorflow:windows": CURL_WIN_SRCS, "//conditions:default": [ - "lib/vtls/openssl.c", ], }), hdrs = [ @@ -269,6 +349,7 @@ cc_library( "include/curl/easy.h", "include/curl/mprintf.h", "include/curl/multi.h", + "include/curl/options.h", "include/curl/stdcheaders.h", "include/curl/system.h", "include/curl/typecheck-gcc.h", @@ -372,6 +453,8 @@ cc_binary( "src/tool_doswin.h", "src/tool_easysrc.c", "src/tool_easysrc.h", + "src/tool_filetime.c", + "src/tool_filetime.h", "src/tool_formparse.c", "src/tool_formparse.h", "src/tool_getparam.c", @@ -406,6 +489,8 @@ cc_binary( "src/tool_paramhlp.h", "src/tool_parsecfg.c", "src/tool_parsecfg.h", + "src/tool_progress.c", + "src/tool_progress.h", "src/tool_sdecls.h", "src/tool_setopt.c", "src/tool_setopt.h", @@ -425,6 +510,8 @@ cc_binary( "src/tool_writeenv.h", "src/tool_writeout.c", "src/tool_writeout.h", + "src/tool_writeout_json.c", + "src/tool_writeout_json.h", "src/tool_xattr.c", "src/tool_xattr.h", ], diff --git a/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/FixedPointTypes.h b/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/FixedPointTypes.h index ff359cedced961..fd35360da28208 100644 --- a/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/FixedPointTypes.h +++ b/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/FixedPointTypes.h @@ -49,7 +49,7 @@ struct scalar_product_traits { // the compiler from silently type cast the mantissa into a bigger or a smaller // representation. struct QInt8 { - QInt8() {} + QInt8() : value(0) {} QInt8(const int8_t v) : value(v) {} QInt8(const QInt32 v); @@ -59,7 +59,7 @@ struct QInt8 { }; struct QUInt8 { - QUInt8() {} + QUInt8() : value(0) {} QUInt8(const uint8_t v) : value(v) {} QUInt8(const QInt32 v); @@ -69,7 +69,7 @@ struct QUInt8 { }; struct QInt16 { - QInt16() {} + QInt16() : value(0) {} QInt16(const int16_t v) : value(v) {} QInt16(const QInt32 v); operator int() const { return static_cast(value); } @@ -78,7 +78,7 @@ struct QInt16 { }; struct QUInt16 { - QUInt16() {} + QUInt16() : value(0) {} QUInt16(const uint16_t v) : value(v) {} QUInt16(const QInt32 v); operator int() const { return static_cast(value); } @@ -87,7 +87,7 @@ struct QUInt16 { }; struct QInt32 { - QInt32() {} + QInt32() : value(0) {} QInt32(const int8_t v) : value(v) {} QInt32(const int32_t v) : value(v) {} QInt32(const uint32_t v) : value(static_cast(v)) {} diff --git a/third_party/jpeg/workspace.bzl b/third_party/jpeg/workspace.bzl index c458ff12ba8248..60f989df722152 100644 --- a/third_party/jpeg/workspace.bzl +++ b/third_party/jpeg/workspace.bzl @@ -6,11 +6,11 @@ def repo(): third_party_http_archive( name = "libjpeg_turbo", urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/github.com/libjpeg-turbo/libjpeg-turbo/archive/2.0.4.tar.gz", - "https://github.com/libjpeg-turbo/libjpeg-turbo/archive/2.0.4.tar.gz", + "https://storage.googleapis.com/mirror.tensorflow.org/github.com/libjpeg-turbo/libjpeg-turbo/archive/2.0.5.tar.gz", + "https://github.com/libjpeg-turbo/libjpeg-turbo/archive/2.0.5.tar.gz", ], - sha256 = "7777c3c19762940cff42b3ba4d7cd5c52d1671b39a79532050c85efb99079064", - strip_prefix = "libjpeg-turbo-2.0.4", + sha256 = "b3090cd37b5a8b3e4dbd30a1311b3989a894e5d3c668f14cbc6739d77c9402b7", + strip_prefix = "libjpeg-turbo-2.0.5", build_file = "//third_party/jpeg:BUILD.bazel", system_build_file = "//third_party/jpeg:BUILD.system", ) diff --git a/third_party/jsoncpp.BUILD b/third_party/jsoncpp.BUILD index 7bc466c664f71e..3b4642c81098c7 100644 --- a/third_party/jsoncpp.BUILD +++ b/third_party/jsoncpp.BUILD @@ -13,7 +13,6 @@ cc_library( ], hdrs = [ "include/json/allocator.h", - "include/json/autolink.h", "include/json/config.h", "include/json/forwards.h", "include/json/json.h",