diff --git a/RELEASE.md b/RELEASE.md index 257b822306443c..0e27725431867e 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,3 +1,47 @@ +# Release 2.5.2 + +This release introduces several vulnerability fixes: + +* Fixes a code injection issue in `saved_model_cli` ([CVE-2021-41228](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41228)) +* Fixes a vulnerability due to use of uninitialized value in Tensorflow ([CVE-2021-41225](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41225)) +* Fixes a heap OOB in `FusedBatchNorm` kernels ([CVE-2021-41223](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41223)) +* Fixes an arbitrary memory read in `ImmutableConst` ([CVE-2021-41227](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41227)) +* Fixes a heap OOB in `SparseBinCount` ([CVE-2021-41226](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41226)) +* Fixes a heap OOB in `SparseFillEmptyRows` ([CVE-2021-41224](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41224)) +* Fixes a segfault due to negative splits in `SplitV` ([CVE-2021-41222](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41222)) +* Fixes segfaults and vulnerabilities caused by accesses to invalid memory during shape inference in `Cudnn*` ops ([CVE-2021-41221](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41221)) +* Fixes a null pointer exception when `Exit` node is not preceded by `Enter` op ([CVE-2021-41217](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41217)) +* Fixes an integer division by 0 in `tf.raw_ops.AllToAll` ([CVE-2021-41218](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41218)) +* Fixes an undefined behavior via `nullptr` reference binding in sparse matrix multiplication ([CVE-2021-41219](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41219)) +* Fixes a heap buffer overflow in `Transpose` ([CVE-2021-41216](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41216)) +* Prevents deadlocks arising from mutually recursive `tf.function` objects ([CVE-2021-41213](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41213)) +* Fixes a null pointer exception in `DeserializeSparse` ([CVE-2021-41215](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41215)) +* Fixes an undefined behavior arising from reference binding to `nullptr` in `tf.ragged.cross` ([CVE-2021-41214](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41214)) +* Fixes a heap OOB read in `tf.ragged.cross` ([CVE-2021-41212](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41212)) +* Fixes a heap OOB read in all `tf.raw_ops.QuantizeAndDequantizeV*` ops ([CVE-2021-41205](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41205)) +* Fixes an FPE in `ParallelConcat` ([CVE-2021-41207](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41207)) +* Fixes FPE issues in convolutions with zero size filters ([CVE-2021-41209](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41209)) +* Fixes a heap OOB read in `tf.raw_ops.SparseCountSparseOutput` ([CVE-2021-41210](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41210)) +* Fixes vulnerabilities caused by incomplete validation in boosted trees code ([CVE-2021-41208](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41208)) +* Fixes vulnerabilities caused by incomplete validation of shapes in multiple TF ops ([CVE-2021-41206](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41206)) +* Fixes a segfault produced while copying constant resource tensor ([CVE-2021-41204](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41204)) +* Fixes a vulnerability caused by unitialized access in `EinsumHelper::ParseEquation` ([CVE-2021-41201](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41201)) +* Fixes several vulnerabilities and segfaults caused by missing validation during checkpoint loading ([CVE-2021-41203](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41203)) +* Fixes an overflow producing a crash in `tf.range` ([CVE-2021-41202](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41202)) +* Fixes an overflow producing a crash in `tf.image.resize` when size is large ([CVE-2021-41199](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41199)) +* Fixes an overflow producing a crash in `tf.tile` when tiling tensor is large ([CVE-2021-41198](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41198)) +* Fixes a vulnerability produced due to incomplete validation in `tf.summary.create_file_writer` ([CVE-2021-41200](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41200)) +* Fixes multiple crashes due to overflow and `CHECK`-fail in ops with large tensor shapes ([CVE-2021-41197](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41197)) +* Fixes a crash in `max_pool3d` when size argument is 0 or negative ([CVE-2021-41196](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41196)) +* Fixes a crash in `tf.math.segment_*` operations ([CVE-2021-41195](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41195)) +* Updates `curl` to `7.78.0` to handle + [CVE-2021-22922](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-22922), + [CVE-2021-22923](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-22923), + [CVE-2021-22924](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-22924), + [CVE-2021-22925](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-22925), + and + [CVE-2021-22926](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-22926). + # Release 2.5.1 This release introduces several vulnerability fixes: diff --git a/tensorflow/core/common_runtime/constant_folding.cc b/tensorflow/core/common_runtime/constant_folding.cc index 384ec836cdf9b4..64ed1c398ada30 100644 --- a/tensorflow/core/common_runtime/constant_folding.cc +++ b/tensorflow/core/common_runtime/constant_folding.cc @@ -30,6 +30,7 @@ limitations under the License. #include "tensorflow/core/framework/log_memory.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/types.h" +#include "tensorflow/core/framework/types.pb.h" #include "tensorflow/core/graph/algorithm.h" #include "tensorflow/core/graph/node_builder.h" #include "tensorflow/core/graph/subgraph.h" @@ -223,7 +224,8 @@ bool IsConstantFoldable( std::unordered_map>* shape_replacement_map) { if (n->IsConstant()) { - return true; + // Skip constant folding resources as they cannot be deep copied. + return n->output_type(0) != DT_RESOURCE; } if (MaybeReplaceShapeOp(n, shape_map, shape_replacement_map)) { return true; diff --git a/tensorflow/core/common_runtime/immutable_executor_state.cc b/tensorflow/core/common_runtime/immutable_executor_state.cc index 03d12a0e98abd5..3cde56bc85ab81 100644 --- a/tensorflow/core/common_runtime/immutable_executor_state.cc +++ b/tensorflow/core/common_runtime/immutable_executor_state.cc @@ -315,6 +315,10 @@ Status ImmutableExecutorState::BuildControlFlowInfo(const Graph* g, } else if (IsExit(curr_node)) { // Exit to the parent frame. parent = parent_nodes[curr_id]; + if (!parent) { + return errors::InvalidArgument( + "Invalid Exit op: Cannot find a corresponding Enter op."); + } frame_name = cf_info->frame_names[parent->id()]; parent = parent_nodes[parent->id()]; } else { diff --git a/tensorflow/core/framework/BUILD b/tensorflow/core/framework/BUILD index de44be07c292d2..653e7da49fcd1b 100644 --- a/tensorflow/core/framework/BUILD +++ b/tensorflow/core/framework/BUILD @@ -795,6 +795,7 @@ tf_cuda_library( "//tensorflow/core/lib/strings:str_util", "//tensorflow/core/lib/strings:strcat", "//tensorflow/core/platform:abi", + "//tensorflow/core/platform:errors", "//tensorflow/core/platform:logging", "//tensorflow/core/platform:macros", "//tensorflow/core/platform:platform_port", diff --git a/tensorflow/core/framework/tensor.cc b/tensorflow/core/framework/tensor.cc index e5eb512a6422a9..167ba5e6587be2 100644 --- a/tensorflow/core/framework/tensor.cc +++ b/tensorflow/core/framework/tensor.cc @@ -48,6 +48,7 @@ limitations under the License. #include "tensorflow/core/lib/gtl/inlined_vector.h" #include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/errors.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/platform/protobuf.h" @@ -717,11 +718,11 @@ bool Tensor::RefCountIsOne() const { // The macro CASES() expands to a switch statement conditioned on // TYPE_ENUM. Each case expands the STMTS after a typedef for T. #define SINGLE_ARG(...) __VA_ARGS__ -#define CASE(TYPE, STMTS) \ - case DataTypeToEnum::value: { \ - typedef TYPE T; \ - STMTS; \ - break; \ +#define CASE(TYPE, STMTS) \ + case DataTypeToEnum::value: { \ + typedef TF_ATTRIBUTE_UNUSED TYPE T; \ + STMTS; \ + break; \ } #define CASES_WITH_DEFAULT(TYPE_ENUM, STMTS, INVALID, DEFAULT) \ switch (TYPE_ENUM) { \ @@ -757,9 +758,8 @@ bool Tensor::RefCountIsOne() const { } #define CASES(TYPE_ENUM, STMTS) \ - CASES_WITH_DEFAULT(TYPE_ENUM, STMTS, \ - LOG(FATAL) << "Unexpected type: " << TYPE_ENUM; \ - , LOG(FATAL) << "Type not set";) + CASES_WITH_DEFAULT(TYPE_ENUM, STMTS, LOG(FATAL) << "Type not set"; \ + , LOG(FATAL) << "Unexpected type: " << TYPE_ENUM;) Tensor::Tensor(Allocator* a, DataType type, const TensorShape& shape) : shape_(shape), buf_(nullptr) { @@ -789,6 +789,16 @@ Tensor::Tensor(Allocator* a, DataType type, const TensorShape& shape, } } +Status Tensor::BuildTensor(DataType type, const TensorShape& shape, + Tensor* out_tensor) { + // Avoid crashes due to invalid or unsupported types. + CASES_WITH_DEFAULT( + type, {}, return errors::InvalidArgument("Type not set"), + return errors::InvalidArgument("Unexpected type: ", DataType_Name(type))); + *out_tensor = Tensor(type, shape); + return Status::OK(); +} + // NOTE(mrry): The default allocator for a Tensor (when none is specified) is // the default CPU allocator for NUMA zone 0. Accessing that currently involves // acquiring a lock, which guards initialization of the per-NUMA zone diff --git a/tensorflow/core/framework/tensor.h b/tensorflow/core/framework/tensor.h index 33a240d85dc5c9..dd01bfc272010a 100644 --- a/tensorflow/core/framework/tensor.h +++ b/tensorflow/core/framework/tensor.h @@ -170,6 +170,15 @@ class Tensor { /// for details. explicit Tensor(DataType type); + /// \brief Initializes a tensor with the input `type` and `shape`, or returns + /// an error and leaves `out_tensor` unmodified. This factory method should be + /// used instead of the corresponding constructor if calling code cannot + /// validate that the `DataType` is valid and supported. + /// + /// The underlying buffer is allocated using a `CPUAllocator`. + static Status BuildTensor(DataType type, const TensorShape& shape, + Tensor* out_tensor); + private: // A tag type for selecting the `Tensor` constructor overload that creates a // scalar tensor in host memory. diff --git a/tensorflow/core/framework/tensor_slice.cc b/tensorflow/core/framework/tensor_slice.cc index 975e1e2e24a439..7041b011157434 100644 --- a/tensorflow/core/framework/tensor_slice.cc +++ b/tensorflow/core/framework/tensor_slice.cc @@ -14,7 +14,10 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/core/framework/tensor_slice.h" + +#include #include + #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/strings/numbers.h" #include "tensorflow/core/lib/strings/str_util.h" @@ -44,6 +47,34 @@ TensorSlice::TensorSlice( } } +Status TensorSlice::BuildTensorSlice(const TensorSliceProto& proto, + TensorSlice* output) { + output->Clear(); + output->starts_.reserve(proto.extent_size()); + output->lengths_.reserve(proto.extent_size()); + for (const auto& e : proto.extent()) { + int64_t l = GetExtentLength(e); + if (e.start() != 0 || l != kFullExtent) { + if (e.start() < 0 || l <= 0) { + return errors::InvalidArgument( + "Expected non-negative start and positive length but got start = ", + e.start(), ", length = ", l, ": extent = ", e.ShortDebugString()); + } + // Calculating the extent end must not cause signed integer overflow. + if (static_cast(e.start()) + static_cast(e.length()) > + std::numeric_limits::max()) { + return errors::InvalidArgument( + "Extent end exceeds the maximum possible size: extent = ", + e.ShortDebugString()); + } + } + output->starts_.push_back(e.start()); + output->lengths_.push_back(l); + } + + return Status::OK(); +} + Status TensorSlice::Parse(const string& str, TensorSlice* slice) { std::vector items = str_util::Split(str, ':', str_util::SkipEmpty()); slice->starts_.reserve(items.size()); diff --git a/tensorflow/core/framework/tensor_slice.h b/tensorflow/core/framework/tensor_slice.h index 82f21fb17eec78..4c2795694564da 100644 --- a/tensorflow/core/framework/tensor_slice.h +++ b/tensorflow/core/framework/tensor_slice.h @@ -47,6 +47,12 @@ class TensorSlice { explicit TensorSlice(const TensorSliceProto& proto); explicit TensorSlice(std::initializer_list> extents); + // This factory methods should be used instead of the constructor that takes a + // `TensorSliceProto` if calling code cannot validate that the sizes specify a + // valid `TensorSlice`. + static Status BuildTensorSlice(const TensorSliceProto& proto, + TensorSlice* output); + static Status Parse(const string& str, TensorSlice* output); static TensorSlice ParseOrDie(const string& str) { TensorSlice ret; diff --git a/tensorflow/core/framework/tensor_slice_test.cc b/tensorflow/core/framework/tensor_slice_test.cc index 54e680484e228b..69b7c7cd084e33 100644 --- a/tensorflow/core/framework/tensor_slice_test.cc +++ b/tensorflow/core/framework/tensor_slice_test.cc @@ -15,6 +15,8 @@ limitations under the License. #include "tensorflow/core/framework/tensor_slice.h" +#include + #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/protobuf.h" @@ -123,6 +125,48 @@ TEST(TensorSliceTest, Serialization) { } } +// Testing `BuildTensorSlice` with valid and invalid input protos. +TEST(TensorSliceTest, BuildTensorSlice) { + TensorSliceProto proto; + TensorSlice({{0, -1}, {0, 10}, {14, 1}}).AsProto(&proto); + TensorSlice s; + + // Successful building. + { + TF_ASSERT_OK(TensorSlice::BuildTensorSlice(proto, &s)); + EXPECT_EQ("-:0,10:14,1", s.DebugString()); + } + + // Failed building due to negative extent start. + { + TensorSliceProto invalid_proto = proto; + invalid_proto.mutable_extent(0)->set_start(-1); + EXPECT_FALSE(TensorSlice::BuildTensorSlice(invalid_proto, &s).ok()); + } + + // Failed building due to negative extent length. + { + TensorSliceProto invalid_proto = proto; + invalid_proto.mutable_extent(2)->set_length(-1); + EXPECT_FALSE(TensorSlice::BuildTensorSlice(invalid_proto, &s).ok()); + } + + // Failed building due to missing extent length. + { + TensorSliceProto invalid_proto = proto; + invalid_proto.mutable_extent(2)->clear_length(); + EXPECT_FALSE(TensorSlice::BuildTensorSlice(invalid_proto, &s).ok()); + } + + // Failed building due to extent end overflowing. + { + TensorSliceProto invalid_proto = proto; + invalid_proto.mutable_extent(2)->set_length( + std::numeric_limits::max()); + EXPECT_FALSE(TensorSlice::BuildTensorSlice(invalid_proto, &s).ok()); + } +} + // Testing the slice intersection TEST(TensorSliceTest, Intersection) { // "EVERYTHING" intersects with everything diff --git a/tensorflow/core/grappler/optimizers/auto_parallel.cc b/tensorflow/core/grappler/optimizers/auto_parallel.cc index a537fa256babba..e4e8009f9ccb20 100644 --- a/tensorflow/core/grappler/optimizers/auto_parallel.cc +++ b/tensorflow/core/grappler/optimizers/auto_parallel.cc @@ -152,7 +152,7 @@ Status AutoParallel::Initialize(const GrapplerItem& item) { TF_RETURN_IF_ERROR(ComputeTransitiveFanin(graph_, item.fetch, &train_nodes)); LOG(INFO) << "Number of training nodes: " << train_nodes.size(); - const NodeDef* dequeue_node; + const NodeDef* dequeue_node = nullptr; for (const auto& train_node : train_nodes) { if (IsDequeueOp(*train_node)) { dequeue_node = train_node; diff --git a/tensorflow/core/grappler/optimizers/auto_parallel_test.cc b/tensorflow/core/grappler/optimizers/auto_parallel_test.cc index 1c3186f1ee6e68..3af03a09613883 100644 --- a/tensorflow/core/grappler/optimizers/auto_parallel_test.cc +++ b/tensorflow/core/grappler/optimizers/auto_parallel_test.cc @@ -126,6 +126,30 @@ TEST_F(AutoParallelTest, SimpleParallel) { EXPECT_EQ("^AutoParallel-Control-Fetch", node_gradient.input(0)); } +TEST_F(AutoParallelTest, SimpleParallelNoDequeue) { + tensorflow::Scope s = tensorflow::Scope::DisabledShapeInferenceScope(); + Output constant_a = ops::Const(s.WithOpName("constant_a"), 1.0f, {1}); + Output constant_c = ops::Const(s.WithOpName("constant_c"), 1.0f, {1}); + Output constant_b = ops::Const(s.WithOpName("constant_b"), 1, {1}); + Output var = ops::Variable(s.WithOpName("var"), {1}, DT_FLOAT); + Output assign = ops::Assign(s.WithOpName("assign"), {var}, {constant_a}); + Output add = ops::AddN(s.WithOpName("add"), {constant_a, constant_c}); + Output learning_rate = ops::Const(s.WithOpName("learning_rate"), 0.01f, {1}); + Output apply_gradient = ops::ApplyGradientDescent( + s.WithOpName("apply_gradient"), {var}, {learning_rate}, {add}); + + GrapplerItem item; + item.init_ops.push_back("assign"); + item.fetch.push_back("apply_gradient"); + item.init_ops.push_back("assign"); + TF_CHECK_OK(s.ToGraphDef(&item.graph)); + + AutoParallel parallel(2); + GraphDef output; + Status status = parallel.Optimize(nullptr, item, &output); + TF_EXPECT_OK(status); +} + } // namespace } // namespace grappler } // namespace tensorflow diff --git a/tensorflow/core/kernels/bincount_op.cc b/tensorflow/core/kernels/bincount_op.cc index 258266ab29d33f..6299ca3e3b1d90 100644 --- a/tensorflow/core/kernels/bincount_op.cc +++ b/tensorflow/core/kernels/bincount_op.cc @@ -364,6 +364,16 @@ class SparseBincountOp : public OpKernel { for (int64 i = 0; i < indices_mat.dimension(0); ++i) { const int64 batch = indices_mat(i, 0); const Tidx bin = values(i); + OP_REQUIRES( + ctx, batch < out.dimension(0), + errors::InvalidArgument("Index out of bound. `batch` (", batch, + ") must be less than the dimension size (", + out.dimension(0), ").")); + OP_REQUIRES( + ctx, bin < out.dimension(1), + errors::InvalidArgument("Index out ouf bound. `bin` (", bin, + ") must be less then the dimension size (", + out.dimension(1), ").")); if (bin < size) { if (binary_output_) { out(batch, bin) = T(1); diff --git a/tensorflow/core/kernels/boosted_trees/stats_ops.cc b/tensorflow/core/kernels/boosted_trees/stats_ops.cc index dc8c4110b47259..bb6709d32d5e2f 100644 --- a/tensorflow/core/kernels/boosted_trees/stats_ops.cc +++ b/tensorflow/core/kernels/boosted_trees/stats_ops.cc @@ -72,7 +72,10 @@ class BoostedTreesCalculateBestGainsPerFeatureOp : public OpKernel { &stats_summary_list)); const int64 num_buckets = stats_summary_list[0].dim_size(1); // Check for single logit: 1 gradient + 1 hessian value. - DCHECK_EQ(stats_summary_list[0].dim_size(2), 2); + OP_REQUIRES(context, stats_summary_list[0].dim_size(2) == 2, + errors::InvalidArgument("stats_summary_list[0] must have " + "exactly 2 dimensions, obtained: ", + stats_summary_list[0].dim_size(2))); std::vector::ConstTensor> stats_summary; stats_summary.reserve(stats_summary_list.size()); for (const auto& tensor : stats_summary_list) { @@ -275,8 +278,13 @@ class BoostedTreesCalculateBestFeatureSplitOp : public OpKernel { const int32 num_buckets = stats_summary_t->dim_size(2) - 1; const int32 logits_dim = logits_dim_; const int32 hessian_dim = stats_summary_t->dim_size(3) - logits_dim; - DCHECK_GT(hessian_dim, 0); - DCHECK_LE(hessian_dim, logits_dim * logits_dim); + OP_REQUIRES(context, hessian_dim > 0, + errors::InvalidArgument("hessian dim should be < 0, got ", + hessian_dim)); + OP_REQUIRES(context, hessian_dim <= logits_dim * logits_dim, + errors::InvalidArgument( + "hessian dim should be <= ", logits_dim * logits_dim, + " but got: ", hessian_dim)); const Tensor* l1_t; OP_REQUIRES_OK(context, context->input("l1", &l1_t)); @@ -621,8 +629,13 @@ class BoostedTreesCalculateBestFeatureSplitV2 : public OpKernel { const int32 num_buckets = stats_summaries_list[0].dim_size(2) - 1; const int32 logits_dim = logits_dim_; const int32 hessian_dim = stats_summaries_list[0].dim_size(3) - logits_dim; - DCHECK_GT(hessian_dim, 0); - DCHECK_LE(hessian_dim, logits_dim * logits_dim); + OP_REQUIRES(context, hessian_dim > 0, + errors::InvalidArgument("hessian dim should be < 0, got ", + hessian_dim)); + OP_REQUIRES(context, hessian_dim <= logits_dim * logits_dim, + errors::InvalidArgument( + "hessian dim should be <= ", logits_dim * logits_dim, + " but got: ", hessian_dim)); // Vector of stats_summaries; each element is stats for feature of shape // [max_splits, feature_dim, num_buckets, logits_dim + hessian_dim]. @@ -997,6 +1010,10 @@ class BoostedTreesSparseCalculateBestFeatureSplitOp : public OpKernel { const Tensor* node_id_range_t; OP_REQUIRES_OK(context, context->input("node_id_range", &node_id_range_t)); const auto node_id_range = node_id_range_t->vec(); + OP_REQUIRES( + context, node_id_range.size() == 2, + errors::InvalidArgument("node_id_range should have 2 entries, got: ", + node_id_range.size())); const int32 node_id_first = node_id_range(0); // inclusive const int32 node_id_last = node_id_range(1); // exclusive @@ -1070,6 +1087,11 @@ class BoostedTreesSparseCalculateBestFeatureSplitOp : public OpKernel { "dims, the last value in stats_summary_shape, which was ", stats_dims, ". At index (", idx, ", 4), stats_summary_indices contains value ", stat_dim)); + OP_REQUIRES(context, stat_dim >= 0, + errors::InvalidArgument( + "Stat dim, the sum of logits dim and hessian dim in " + "stats_summary_indices, should be >= 0, which was ", + stat_dim, " at index ", idx)); std::pair const& f_insert_result = f_map.insert( FeatureMapIterator::value_type(feature_dim, BucketMap())); auto& b_map = f_insert_result.first->second; @@ -1302,6 +1324,12 @@ class BoostedTreesMakeStatsSummaryOp : public OpKernel { const Tensor* gradients_t; OP_REQUIRES_OK(context, context->input("gradients", &gradients_t)); const auto gradients = gradients_t->matrix(); + OP_REQUIRES( + context, node_ids.size() == gradients.dimension(0), + errors::InvalidArgument( + "node_ids size should match 0th dim of gradients. node ids " + "size: ", + node_ids.size(), ", gradients dim0: ", gradients.dimension(0))); // hessians const Tensor* hessians_t; OP_REQUIRES_OK(context, context->input("hessians", &hessians_t)); @@ -1371,6 +1399,13 @@ class BoostedTreesAggregateStatsOp : public OpKernel { OP_REQUIRES_OK(context, context->input("gradients", &gradients_t)); const auto gradients = gradients_t->matrix(); + OP_REQUIRES( + context, node_ids.size() == gradients.dimension(0), + errors::InvalidArgument( + "node_ids size should match 0th dim of gradients. node ids " + "size: ", + node_ids.size(), ", gradients dim0: ", gradients.dimension(0))); + // hessians. const Tensor* hessians_t; OP_REQUIRES_OK(context, context->input("hessians", &hessians_t)); @@ -1401,6 +1436,9 @@ class BoostedTreesAggregateStatsOp : public OpKernel { for (int i = 0; i < batch_size; ++i) { const int32 node = node_ids(i); + OP_REQUIRES(context, node >= 0, + errors::InvalidArgument( + "node_ids ", i, "th entry should be >=0, got: ", node)); for (int feature_dim = 0; feature_dim < feature_dims; ++feature_dim) { const int32 feature_value = feature(i, feature_dim); const int32 bucket = @@ -1608,7 +1646,12 @@ class BoostedTreesSparseAggregateStatsOp : public OpKernel { const int64 stats_dims = logits_dims + hessians_dims; const int64 num_sparse_entries = feature_indices_t->dim_size(0); const int32 feature_dims = feature_shape(1); - DCHECK_LE(num_sparse_entries, batch_size * feature_dims); + OP_REQUIRES(context, num_sparse_entries <= batch_size * feature_dims, + errors::InvalidArgument( + "feature_indices dim0 should be <= gradients dim0 * " + "feature_shape[1]. features_indices dim0: ", + num_sparse_entries, " gradients dim0: ", batch_size, + ", feature_shape[1]: ", feature_dims)); // Aggregate statistics info to map. StatsPartitionMap stats_map; diff --git a/tensorflow/core/kernels/conv_ops.cc b/tensorflow/core/kernels/conv_ops.cc index 9bacebe7d265dc..3c4860ba45f059 100644 --- a/tensorflow/core/kernels/conv_ops.cc +++ b/tensorflow/core/kernels/conv_ops.cc @@ -271,6 +271,12 @@ struct LaunchConv2DOp { " vs ", patch_depth)); return; } + if (filter.NumElements() <= 0) { + ctx->SetStatus( + errors::InvalidArgument("filter must not have zero elements " + "(i.e. all dimensions must be non-zero)")); + return; + } const int64 num_groups = in_depth / patch_depth; if (num_groups <= 0) { @@ -322,6 +328,10 @@ struct LaunchConv2DOp { "attempted to be run because the input depth of ", in_depth, " does not match the filter input depth of ", filter.dim_size(2))); + OP_REQUIRES( + ctx, filter.NumElements() > 0, + errors::InvalidArgument("filter must not have zero elements " + "(i.e. all dimensions must be non-zero)")); for (int64 explicit_padding : explicit_paddings) { if (!FastBoundsCheck(explicit_padding, std::numeric_limits::max())) { @@ -796,6 +806,11 @@ void LaunchConv2DOp::operator()( const int64 patch_cols = filter.dim_size(1); const int64 patch_depths = filter.dim_size(2); + OP_REQUIRES( + ctx, filter.NumElements() > 0, + errors::InvalidArgument("filter must not have zero elements " + "(i.e. all dimensions must be non-zero)")); + // If the filter in-depth (patch_depths) is 1 and smaller than the input // depth, it's a depthwise convolution. More generally, if the filter in-depth // divides but is smaller than the input depth, it is a grouped convolution. diff --git a/tensorflow/core/kernels/conv_ops_3d.cc b/tensorflow/core/kernels/conv_ops_3d.cc index 505c55c7e6feaa..dc275bfb8281ef 100644 --- a/tensorflow/core/kernels/conv_ops_3d.cc +++ b/tensorflow/core/kernels/conv_ops_3d.cc @@ -153,6 +153,10 @@ class Conv3DOp : public BinaryOp { errors::InvalidArgument( "Input depth must be evenly divisible by filter depth: ", in_depth, " vs ", filter_depth)); + OP_REQUIRES( + context, filter.NumElements() > 0, + errors::InvalidArgument("filter must not have zero elements " + "(i.e. all dimensions must be non-zero)")); // Dimension order for these arrays is: z, y, x. std::array input_size = { diff --git a/tensorflow/core/kernels/fused_batch_norm_op.cc b/tensorflow/core/kernels/fused_batch_norm_op.cc index 7b0932d953261c..8d99d25ccba6dd 100644 --- a/tensorflow/core/kernels/fused_batch_norm_op.cc +++ b/tensorflow/core/kernels/fused_batch_norm_op.cc @@ -1293,18 +1293,20 @@ class FusedBatchNormOpBase : public OpKernel { errors::InvalidArgument("offset must have the same number of elements " "as the channels of x, got ", offset.NumElements(), " and ", num_channels)); - if (estimated_mean.NumElements() != 0) { + if (!is_training_ || exponential_avg_factor_ != 1.) { + std::string prefix_msg = is_training_ ? "When exponential_avg_factor != 1" + : "When is_training=false"; OP_REQUIRES(context, estimated_mean.NumElements() == num_channels, errors::InvalidArgument( - "mean must be empty or have the same number of " - "elements as the channels of x, got ", + prefix_msg, + ", mean must have the same number " + "of elements as the channels of x, got ", estimated_mean.NumElements(), " and ", num_channels)); - } - if (estimated_variance.NumElements() != 0) { OP_REQUIRES(context, estimated_variance.NumElements() == num_channels, errors::InvalidArgument( - "variance must be empty or have the same number of " - "elements as the channels of x, got ", + prefix_msg, + ", variance must have the same " + "number of elements as the channels of x, got ", estimated_variance.NumElements(), " and ", num_channels)); } @@ -1454,6 +1456,11 @@ class FusedBatchNormGradOpBase : public OpKernel { errors::InvalidArgument( "saved variance must be 1-dimensional", saved_maybe_inv_var_or_pop_var.shape().DebugString())); + OP_REQUIRES( + context, x.shape() == y_backprop.shape(), + errors::InvalidArgument( + "x and y_backprop must have same shape, but x has shape ", + x.shape(), " and y_backprop has shape ", y_backprop.shape())); bool use_reshape = (x.dims() == 5); auto x_shape = x.shape(); TensorShape dest_shape; @@ -1471,6 +1478,23 @@ class FusedBatchNormGradOpBase : public OpKernel { errors::InvalidArgument("Error during tensor copy.")); } + const auto num_channels = GetTensorDim(x, tensor_format_, 'C'); + OP_REQUIRES( + context, scale.NumElements() == num_channels, + errors::InvalidArgument("scale must have the same number of elements " + "as the channels of x, got ", + scale.NumElements(), " and ", num_channels)); + OP_REQUIRES( + context, saved_mean_or_pop_mean.NumElements() == num_channels, + errors::InvalidArgument("reserve_space_1 must have the same number of " + "elements as the channels of x, got ", + scale.NumElements(), " and ", num_channels)); + OP_REQUIRES( + context, saved_maybe_inv_var_or_pop_var.NumElements() == num_channels, + errors::InvalidArgument("reserve_space_2 must have the same number of " + "elements as the channels of x, got ", + scale.NumElements(), " and ", num_channels)); + Tensor* x_backprop = nullptr; auto alloc_shape = use_reshape ? dest_shape : x_shape; OP_REQUIRES_OK(context, diff --git a/tensorflow/core/kernels/image/attention_ops.cc b/tensorflow/core/kernels/image/attention_ops.cc index 6e5e07a9fb1b3c..100be63d98e44d 100644 --- a/tensorflow/core/kernels/image/attention_ops.cc +++ b/tensorflow/core/kernels/image/attention_ops.cc @@ -85,11 +85,12 @@ class ExtractGlimpseOp : public OpKernel { "input must be a vector of size 2 (height, width)", window_size.shape().DebugString())); - const int64 output_height = window_size.tensor()(0); - const int64 output_width = window_size.tensor()(1); + const int64_t output_height = window_size.tensor()(0); + const int64_t output_width = window_size.tensor()(1); + TensorShape output_shape = input_shape; - output_shape.set_dim(1, output_height); - output_shape.set_dim(2, output_width); + OP_REQUIRES_OK(context, output_shape.SetDimWithStatus(1, output_height)); + OP_REQUIRES_OK(context, output_shape.SetDimWithStatus(2, output_width)); const Tensor& offsets = context->input(2); OP_REQUIRES(context, offsets.shape().dims() == 2, diff --git a/tensorflow/core/kernels/image/crop_and_resize_op.cc b/tensorflow/core/kernels/image/crop_and_resize_op.cc index 4efc4ae8846d17..65f972e1730752 100644 --- a/tensorflow/core/kernels/image/crop_and_resize_op.cc +++ b/tensorflow/core/kernels/image/crop_and_resize_op.cc @@ -169,14 +169,15 @@ class CropAndResizeOp : public AsyncOpKernel { context, crop_height > 0 && crop_width > 0, errors::InvalidArgument("crop dimensions must be positive"), done); + TensorShape shape; + OP_REQUIRES_OK_ASYNC(context, shape.AddDimWithStatus(num_boxes), done); + OP_REQUIRES_OK_ASYNC(context, shape.AddDimWithStatus(crop_height), done); + OP_REQUIRES_OK_ASYNC(context, shape.AddDimWithStatus(crop_width), done); + OP_REQUIRES_OK_ASYNC(context, shape.AddDimWithStatus(depth), done); // Allocate output tensor. Tensor* output = nullptr; - OP_REQUIRES_OK_ASYNC( - context, - context->allocate_output( - 0, TensorShape({num_boxes, crop_height, crop_width, depth}), - &output), - done); + OP_REQUIRES_OK_ASYNC(context, context->allocate_output(0, shape, &output), + done); auto compute_callback = [this, context, output]() { const Tensor& image = context->input(0); @@ -407,14 +408,15 @@ class CropAndResizeGradImageOp : public AsyncOpKernel { context, grads.dim_size(3) == depth, errors::InvalidArgument("image_size and grads are incompatible"), done); + TensorShape shape; + OP_REQUIRES_OK_ASYNC(context, shape.AddDimWithStatus(batch_size), done); + OP_REQUIRES_OK_ASYNC(context, shape.AddDimWithStatus(image_height), done); + OP_REQUIRES_OK_ASYNC(context, shape.AddDimWithStatus(image_width), done); + OP_REQUIRES_OK_ASYNC(context, shape.AddDimWithStatus(depth), done); // Allocate output tensor. Tensor* output = nullptr; - OP_REQUIRES_OK_ASYNC( - context, - context->allocate_output( - 0, TensorShape({batch_size, image_height, image_width, depth}), - &output), - done); + OP_REQUIRES_OK_ASYNC(context, context->allocate_output(0, shape, &output), + done); auto compute_callback = [this, context, output]() { const Tensor& grads = context->input(0); diff --git a/tensorflow/core/kernels/immutable_constant_op.cc b/tensorflow/core/kernels/immutable_constant_op.cc index 19aa865c1fbe4d..df0d76ce633e9b 100644 --- a/tensorflow/core/kernels/immutable_constant_op.cc +++ b/tensorflow/core/kernels/immutable_constant_op.cc @@ -100,6 +100,9 @@ void ImmutableConstantOp::Compute(OpKernelContext* ctx) { OP_REQUIRES_OK(ctx, allocator->InitializeFromRegion(region_name_, ctx->env())); + OP_REQUIRES(ctx, dtype_ != DT_STRING, + errors::Unimplemented("Sorry, DT_STRING is not currently " + "supported for ImmutableConstOp.")); ctx->set_output(0, Tensor(allocator.get(), dtype_, shape_)); OP_REQUIRES_OK(ctx, allocator->allocation_status()); // Allocator is owned by the tensor from this point. diff --git a/tensorflow/core/kernels/immutable_constant_op_test.cc b/tensorflow/core/kernels/immutable_constant_op_test.cc index d52a8b55a35d79..40ce8918a39ade 100644 --- a/tensorflow/core/kernels/immutable_constant_op_test.cc +++ b/tensorflow/core/kernels/immutable_constant_op_test.cc @@ -146,7 +146,8 @@ TEST(ImmutableConstantOpTest, ExecutionError) { error::INTERNAL); } -Status CreateTempFile(Env* env, float value, uint64 size, string* filename) { +Status CreateTempFileFloat(Env* env, float value, uint64 size, + string* filename) { const string dir = testing::TmpDir(); *filename = io::JoinPath(dir, strings::StrCat("file_", value)); std::unique_ptr file; @@ -166,8 +167,8 @@ TEST(ImmutableConstantOpTest, FromFile) { auto root = Scope::NewRootScope().ExitOnError(); string two_file, three_file; - TF_ASSERT_OK(CreateTempFile(env, 2.0f, 1000, &two_file)); - TF_ASSERT_OK(CreateTempFile(env, 3.0f, 1000, &three_file)); + TF_ASSERT_OK(CreateTempFileFloat(env, 2.0f, 1000, &two_file)); + TF_ASSERT_OK(CreateTempFileFloat(env, 3.0f, 1000, &three_file)); auto node1 = ops::ImmutableConst(root, DT_FLOAT, kFileTensorShape, two_file); auto node2 = ops::ImmutableConst(root, DT_FLOAT, kFileTensorShape, three_file); @@ -190,5 +191,39 @@ TEST(ImmutableConstantOpTest, FromFile) { EXPECT_EQ(outputs.front().flat()(2), 2.0f * 3.0f); } +Status CreateTempFileBadString(Env* env, char value, uint64 size, + const string suffix, string* filename) { + const string dir = testing::TmpDir(); + *filename = io::JoinPath(dir, strings::StrCat("file_", suffix)); + std::unique_ptr file; + TF_RETURN_IF_ERROR(env->NewWritableFile(*filename, &file)); + TF_RETURN_IF_ERROR(file->Append(std::string(size, value))); + TF_RETURN_IF_ERROR(file->Close()); + return Status::OK(); +} + +TEST(ImmutableConstantOpTest, FromFileStringUnimplmented) { + const TensorShape kFileTensorShape({1}); + Env* env = Env::Default(); + auto root = Scope::NewRootScope().ExitOnError(); + + string bad_file; + TF_ASSERT_OK(CreateTempFileBadString(env, '\xe2', 128, "bad_e2", &bad_file)); + auto result = + ops::ImmutableConst(root, DT_STRING, kFileTensorShape, bad_file); + GraphDef graph_def; + TF_ASSERT_OK(root.ToGraphDef(&graph_def)); + SessionOptions session_options; + session_options.env = Env::Default(); + std::unique_ptr session(NewSession(session_options)); + ASSERT_TRUE(session != nullptr) << "Failed to create session"; + TF_ASSERT_OK(session->Create(graph_def)) << "Can't create test graph"; + std::vector outputs; + // Check that the run returned error. + EXPECT_EQ( + session->Run({}, {result.node()->name() + ":0"}, {}, &outputs).code(), + error::UNIMPLEMENTED); +} + } // namespace } // namespace tensorflow diff --git a/tensorflow/core/kernels/inplace_ops.cc b/tensorflow/core/kernels/inplace_ops.cc index 8e5174cb17d085..e72732e99b24f2 100644 --- a/tensorflow/core/kernels/inplace_ops.cc +++ b/tensorflow/core/kernels/inplace_ops.cc @@ -71,6 +71,15 @@ class ParallelConcatUpdate : public OpKernel { void Compute(OpKernelContext* ctx) override { auto value = ctx->input(0); + // Value should be at least rank 1. Also the 0th dimension should be + // at least loc_. + OP_REQUIRES(ctx, value.dims() >= 1, + errors::InvalidArgument("value should be at least rank 1.")); + OP_REQUIRES( + ctx, value.dim_size(0) > loc_, + errors::InvalidArgument("0th dimension of value = ", value.dim_size(0), + " is less than loc_=", loc_)); + auto update = ctx->input(1); OP_REQUIRES( diff --git a/tensorflow/core/kernels/linalg/einsum_op_impl.h b/tensorflow/core/kernels/linalg/einsum_op_impl.h index ba01d108453b7c..ed7839457d2d49 100644 --- a/tensorflow/core/kernels/linalg/einsum_op_impl.h +++ b/tensorflow/core/kernels/linalg/einsum_op_impl.h @@ -153,6 +153,7 @@ struct EinsumHelper { input_has_ellipsis->resize(num_inputs); for (int i = 0; i < num_inputs; ++i) { input_label_counts->at(i).resize(num_labels); + input_has_ellipsis->at(i) = false; for (const int label : input_labels->at(i)) { if (label != kEllipsisLabel) input_label_counts->at(i)[label] += 1; @@ -161,6 +162,7 @@ struct EinsumHelper { } } output_label_counts->resize(num_labels); + *output_has_ellipsis = false; for (const int label : *output_labels) { if (label != kEllipsisLabel) output_label_counts->at(label) += 1; diff --git a/tensorflow/core/kernels/linalg/matrix_solve_op.cc b/tensorflow/core/kernels/linalg/matrix_solve_op.cc index 70f02bddf9b785..aeb0203b4a337d 100644 --- a/tensorflow/core/kernels/linalg/matrix_solve_op.cc +++ b/tensorflow/core/kernels/linalg/matrix_solve_op.cc @@ -143,15 +143,22 @@ class MatrixSolveOpGpu : public AsyncOpKernel { done); OP_REQUIRES_ASYNC( context, input.dim_size(ndims - 2) == n, - errors::InvalidArgument("Input matrices must be squares, got", + errors::InvalidArgument("Input matrices must be squares, got ", input.dim_size(ndims - 2), " != ", n), done); OP_REQUIRES_ASYNC(context, rhs.dim_size(ndims - 2) == n, errors::InvalidArgument( "Input matrix and right-hand side must have the " - "same number of rows, got", + "same number of rows, got ", n, " != ", rhs.dim_size(ndims - 2)), done); + for (int dim = 0; dim < ndims - 2; dim++) { + OP_REQUIRES_ASYNC( + context, input.dim_size(dim) == rhs.dim_size(dim), + errors::InvalidArgument( + "All input tensors must have the same outer dimensions."), + done); + } // Allocate output. Tensor* output; diff --git a/tensorflow/core/kernels/linalg/tridiagonal_matmul_op_gpu.cu.cc b/tensorflow/core/kernels/linalg/tridiagonal_matmul_op_gpu.cu.cc index a65db40d822abc..1f59d4311c3ab4 100644 --- a/tensorflow/core/kernels/linalg/tridiagonal_matmul_op_gpu.cu.cc +++ b/tensorflow/core/kernels/linalg/tridiagonal_matmul_op_gpu.cu.cc @@ -66,6 +66,12 @@ class TridiagonalMatMulOpGpu : public OpKernel { const Tensor& rhs = context->input(3); const int ndims = rhs.dims(); + OP_REQUIRES( + context, ndims >= 2, + errors::InvalidArgument("Input must have rank >= 2, but got ", ndims)); + OP_REQUIRES_OK(context, ValidateInputTensor(superdiag, "superdiag", rhs)); + OP_REQUIRES_OK(context, ValidateInputTensor(maindiag, "maindiag", rhs)); + OP_REQUIRES_OK(context, ValidateInputTensor(subdiag, "subdiag", rhs)); int64 batch_size = 1; for (int i = 0; i < ndims - 2; i++) { batch_size *= rhs.dim_size(i); @@ -85,6 +91,39 @@ class TridiagonalMatMulOpGpu : public OpKernel { maindiag.flat().data(), subdiag.flat().data(), rhs.flat().data(), output->flat().data())); } + + private: + Status ValidateInputTensor(const Tensor& tensor, + const std::string& tensor_name, + const Tensor& rhs) { + const int ndims = rhs.dims(); + if (tensor.dims() != ndims) { + return errors::InvalidArgument(tensor_name, + " must have same rank as rhs, but got ", + tensor.dims(), " and ", ndims); + } + for (int i = 0; i < ndims - 2; i++) { + if (tensor.dim_size(i) != rhs.dim_size(i)) { + return errors::InvalidArgument( + tensor_name, + " must have same outer dimensions as rhs, but for index ", i, + ", got ", tensor.dim_size(i), " and ", rhs.dim_size(i)); + } + } + if (tensor.dim_size(ndims - 2) != 1) { + return errors::InvalidArgument( + tensor_name, "'s second-to-last dimension must be 1, but got ", + tensor.dim_size(ndims - 2)); + } + if (tensor.dim_size(ndims - 1) != rhs.dim_size(ndims - 2)) { + return errors::InvalidArgument(tensor_name, + "'s last dimension size must be rhs's " + "second-to-last dimension size, but got ", + tensor.dim_size(ndims - 1), " and ", + rhs.dim_size(ndims - 2)); + } + return Status::OK(); + } }; REGISTER_LINALG_OP_GPU("TridiagonalMatMul", (TridiagonalMatMulOpGpu), diff --git a/tensorflow/core/kernels/pad_op.cc b/tensorflow/core/kernels/pad_op.cc index 4a1d0cfc3e2374..b59db5a59a7bd3 100644 --- a/tensorflow/core/kernels/pad_op.cc +++ b/tensorflow/core/kernels/pad_op.cc @@ -84,8 +84,10 @@ class PadOp : public OpKernel { OP_REQUIRES(context, before_d >= 0 && after_d >= 0, errors::InvalidArgument("Paddings must be non-negative: ", before_d, " ", after_d)); - const int64 size_d = in0.dim_size(d); - output_shape.AddDim(before_d + size_d + after_d); + const int64_t size_d = in0.dim_size(d); + OP_REQUIRES_OK( + context, output_shape.AddDimWithStatus(before_d + size_d + after_d)); + } // If there is no padding to be done, forward the input to output. diff --git a/tensorflow/core/kernels/pooling_ops_3d.cc b/tensorflow/core/kernels/pooling_ops_3d.cc index 56a55bc2ec87bf..2faf90bc072cad 100644 --- a/tensorflow/core/kernels/pooling_ops_3d.cc +++ b/tensorflow/core/kernels/pooling_ops_3d.cc @@ -141,6 +141,11 @@ class Pooling3DOp : public UnaryOp { OP_REQUIRES(context, ksize_.size() == 5, errors::InvalidArgument("Sliding window ksize field must " "specify 5 dimensions")); + bool non_negative = + std::all_of(ksize_.begin(), ksize_.end(), [](int k) { return k > 0; }); + OP_REQUIRES(context, non_negative, + errors::InvalidArgument("Sliding window ksize field must " + "have non-negative dimensions")); OP_REQUIRES_OK(context, context->GetAttr("strides", &stride_)); OP_REQUIRES(context, stride_.size() == 5, errors::InvalidArgument("Sliding window stride field must " diff --git a/tensorflow/core/kernels/pooling_ops_common.h b/tensorflow/core/kernels/pooling_ops_common.h index dacbb872cf04a7..642fb4b413a0ef 100644 --- a/tensorflow/core/kernels/pooling_ops_common.h +++ b/tensorflow/core/kernels/pooling_ops_common.h @@ -194,6 +194,9 @@ class MaxPoolingOp : public OpKernel { void SpatialMaxPool(OpKernelContext* context, Tensor* output, const Tensor& tensor_in, const PoolParameters& params, const Padding& padding) { + if (output->NumElements() == 0) { + return; + } // On GPU, use Eigen's Spatial Max Pooling. On CPU, use an // EigenMatrix version that is currently faster than Eigen's // Spatial MaxPooling implementation. @@ -447,6 +450,9 @@ class MaxPoolingV2Op : public OpKernel { void SpatialMaxPool(OpKernelContext* context, Tensor* output, const Tensor& tensor_in, const PoolParameters& params, const Padding& padding) { + if (output->NumElements() == 0) { + return; + } // On GPU, use Eigen's Spatial Max Pooling. On CPU, use an // EigenMatrix version that is currently faster than Eigen's // Spatial MaxPooling implementation. @@ -564,6 +570,9 @@ template void SpatialAvgPool(OpKernelContext* context, Tensor* output, const Tensor& input, const PoolParameters& params, const Padding& padding) { + if (output->NumElements() == 0) { + return; + } typedef Eigen::Map> ConstEigenMatrixMap; typedef Eigen::Map> diff --git a/tensorflow/core/kernels/segment_reduction_ops_impl.h b/tensorflow/core/kernels/segment_reduction_ops_impl.h index 81c9ba869ab50c..6a9ec8934deffc 100644 --- a/tensorflow/core/kernels/segment_reduction_ops_impl.h +++ b/tensorflow/core/kernels/segment_reduction_ops_impl.h @@ -111,7 +111,9 @@ class SegmentReductionOp : public OpKernel { errors::InvalidArgument("segment ids must be >= 0")); TensorShape output_shape = input.shape(); - output_shape.set_dim(0, output_rows); + // Since we're changing the first dimension of the shape, we need to make + // sure the new shape won't overflow. + OP_REQUIRES_OK(context, output_shape.SetDimWithStatus(0, output_rows)); // Note that we do not initialize the output buffer with a default value, so // we need to explicitly set missing indices to the default value. @@ -287,7 +289,10 @@ class SegmentReductionGPUOp : public AsyncOpKernel { done); TensorShape output_shape = input.shape(); - output_shape.set_dim(0, output_rows); + // Since we're changing the first dimension of the shape, we need to make + // sure the new shape won't overflow. + OP_REQUIRES_OK_ASYNC(context, + output_shape.SetDimWithStatus(0, output_rows), done); Tensor* output = nullptr; OP_REQUIRES_OK_ASYNC( diff --git a/tensorflow/core/kernels/sequence_ops.cc b/tensorflow/core/kernels/sequence_ops.cc index d15f95125e0ac3..ebf31d61e5f85b 100644 --- a/tensorflow/core/kernels/sequence_ops.cc +++ b/tensorflow/core/kernels/sequence_ops.cc @@ -71,13 +71,17 @@ class RangeOp : public OpKernel { errors::InvalidArgument( "Requires start >= limit when delta < 0: ", start, "/", limit)); } - int64 size = (std::is_integral::value - ? ((std::abs(limit - start) + std::abs(delta) - 1) / - std::abs(delta)) - : std::ceil(std::abs((limit - start) / delta))); + int64 size = 0; + if (std::is_integral::value) { + size = static_cast( + (std::abs(limit - start) + std::abs(delta) - 1) / std::abs(delta)); + } else { + size = static_cast(std::ceil(std::abs((limit - start) / delta))); + } + TensorShape shape; + OP_REQUIRES_OK(context, shape.AddDimWithStatus(size)); Tensor* out = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output(0, TensorShape({size}), &out)); + OP_REQUIRES_OK(context, context->allocate_output(0, shape, &out)); auto flat = out->flat(); T val = start; for (int64 i = 0; i < size; ++i) { diff --git a/tensorflow/core/kernels/sparse_fill_empty_rows_op.cc b/tensorflow/core/kernels/sparse_fill_empty_rows_op.cc index af3201463344ad..1dfb0bad2969eb 100644 --- a/tensorflow/core/kernels/sparse_fill_empty_rows_op.cc +++ b/tensorflow/core/kernels/sparse_fill_empty_rows_op.cc @@ -24,11 +24,13 @@ limitations under the License. #include #include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/op_requires.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_util.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/lib/gtl/inlined_vector.h" +#include "tensorflow/core/platform/errors.h" #include "tensorflow/core/util/sparse/sparse_tensor.h" namespace tensorflow { @@ -211,6 +213,10 @@ class SparseFillEmptyRowsOp : public OpKernel { OP_REQUIRES(context, TensorShapeUtils::IsVector(values_t.shape()), errors::InvalidArgument("values must be a vector, saw: ", values_t.shape().DebugString())); + OP_REQUIRES(context, indices_t.dim_size(0) == values_t.dim_size(0), + errors::InvalidArgument("The length of `values` (", values_t.dim_size(0), + ") must match the first dimension of `indices` (", + indices_t.dim_size(0), ").")); OP_REQUIRES(context, TensorShapeUtils::IsScalar(default_value_t.shape()), errors::InvalidArgument("default_value must be a scalar, saw: ", default_value_t.shape().DebugString())); diff --git a/tensorflow/core/kernels/sparse_matmul_op.cc b/tensorflow/core/kernels/sparse_matmul_op.cc index a02afafa33e3ad..6bf9dfa3d8bb75 100644 --- a/tensorflow/core/kernels/sparse_matmul_op.cc +++ b/tensorflow/core/kernels/sparse_matmul_op.cc @@ -32,6 +32,7 @@ limitations under the License. #include "tensorflow/core/kernels/fill_functor.h" #include "tensorflow/core/lib/core/blocking_counter.h" #include "tensorflow/core/lib/core/threadpool.h" +#include "tensorflow/core/platform/errors.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/platform/mutex.h" @@ -980,9 +981,18 @@ class SparseMatMulOp : public OpKernel { errors::InvalidArgument( "Matrix size incompatible: a: ", a.shape().DebugString(), ", b: ", b.shape().DebugString())); + OP_REQUIRES(ctx, m >= 0 && n >= 0 && k >= 0, + errors::InvalidArgument( + "Matrix dimensions cannot be negative: a: ", + a.shape().DebugString(), ", b: ", b.shape().DebugString())); Tensor* output = nullptr; OP_REQUIRES_OK(ctx, ctx->allocate_output(0, TensorShape({m, n}), &output)); + // Return early if at least one of the output dimension size is 0. + if (m == 0 || n == 0) { + return; + } + if (k == 0) { // If the inner dimension k in the matrix multiplication is zero, we fill // the output with zeros. diff --git a/tensorflow/core/kernels/split_v_op.cc b/tensorflow/core/kernels/split_v_op.cc index fc070610877d5f..3497f95f13d306 100644 --- a/tensorflow/core/kernels/split_v_op.cc +++ b/tensorflow/core/kernels/split_v_op.cc @@ -138,8 +138,17 @@ class SplitVOpBase : public OpKernel { (*split_sizes_vec)[neg_one_dim] = input_size_split_dim - determined_size; } - // Special case 2: split along the 1st dimension. We can share the - // underlying buffer. + for (int i = 0; i < split_sizes_vec->size(); ++i) { + const Tlen& split_size = (*split_sizes_vec)[i]; + OP_REQUIRES(context, split_size >= Tlen(0), + errors::InvalidArgument("Split size at index ", i, + " must be >= 0. Got: ", split_size)); + } + + // Special case 2: split along the 1st dimension. The requirements are that + // either we are splitting the outer dimension of two or more such that + // every outer subpart is aligned or that the split sizes mean that they are + // always aligned. In these cases, we can share the underlying buffer. // // Apply this optimization conservatively: if input is aligned, // the resulting tensors must be aligned. It's conservative diff --git a/tensorflow/core/kernels/summary_kernels.cc b/tensorflow/core/kernels/summary_kernels.cc index 7f888da69d6c7f..6764deee21d286 100644 --- a/tensorflow/core/kernels/summary_kernels.cc +++ b/tensorflow/core/kernels/summary_kernels.cc @@ -38,12 +38,20 @@ class CreateSummaryFileWriterOp : public OpKernel { void Compute(OpKernelContext* ctx) override { const Tensor* tmp; OP_REQUIRES_OK(ctx, ctx->input("logdir", &tmp)); + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(tmp->shape()), + errors::InvalidArgument("logdir must be a scalar")); const string logdir = tmp->scalar()(); OP_REQUIRES_OK(ctx, ctx->input("max_queue", &tmp)); - const int32 max_queue = tmp->scalar()(); + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(tmp->shape()), + errors::InvalidArgument("max_queue must be a scalar")); + const int32_t max_queue = tmp->scalar()(); OP_REQUIRES_OK(ctx, ctx->input("flush_millis", &tmp)); - const int32 flush_millis = tmp->scalar()(); + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(tmp->shape()), + errors::InvalidArgument("flush_millis must be a scalar")); + const int32_t flush_millis = tmp->scalar()(); OP_REQUIRES_OK(ctx, ctx->input("filename_suffix", &tmp)); + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(tmp->shape()), + errors::InvalidArgument("filename_suffix must be a scalar")); const string filename_suffix = tmp->scalar()(); core::RefCountPtr s; diff --git a/tensorflow/core/kernels/tile_ops.cc b/tensorflow/core/kernels/tile_ops.cc index 7d967d8761077f..4cc6557247d272 100644 --- a/tensorflow/core/kernels/tile_ops.cc +++ b/tensorflow/core/kernels/tile_ops.cc @@ -188,7 +188,8 @@ class TileOp : public OpKernel { context, multiples_array[i] >= 0, errors::InvalidArgument("Expected multiples[", i, "] >= 0, but got ", multiples_array[i])); - output_shape.AddDim(input.dim_size(i) * multiples_array[i]); + OP_REQUIRES_OK(context, output_shape.AddDimWithStatus( + input.dim_size(i) * multiples_array[i])); } if (output_shape == input.shape()) { context->set_output(0, input); diff --git a/tensorflow/core/kernels/xent_op.cc b/tensorflow/core/kernels/xent_op.cc index 0e826274f2ebd3..56c3fd9881bea8 100644 --- a/tensorflow/core/kernels/xent_op.cc +++ b/tensorflow/core/kernels/xent_op.cc @@ -44,7 +44,8 @@ class SoftmaxXentWithLogitsOp : public OpKernel { TensorShape shape_in = logits_in.shape(); BCast bcast(BCast::FromShape(logits_in.shape()), - BCast::FromShape(labels_in.shape())); + BCast::FromShape(labels_in.shape()), + /*fewer_dims_optimization=*/false); if (!logits_in.IsSameSize(labels_in)) { OP_REQUIRES(context, bcast.IsValid(), errors::InvalidArgument( @@ -76,20 +77,12 @@ class SoftmaxXentWithLogitsOp : public OpKernel { {0}, 1, shape_in, &back_out)); if (shape_in.dim_size(0) > 0) { functor::XentFunctor functor; - if (logits_in.IsSameSize(labels_in)) { - functor(context->eigen_device(), shape_in.AsEigenDSizes<2>(), - Eigen::array{1, 1}, - Eigen::array{1, 1}, logits_in.matrix(), - labels_in.matrix(), scratch.matrix(), loss_out->vec(), - back_out->matrix()); - } else { - functor(context->eigen_device(), shape_in.AsEigenDSizes<2>(), - BCast::ToIndexArray<2>(bcast.x_bcast()), - BCast::ToIndexArray<2>(bcast.y_bcast()), - logits_in.template shaped(bcast.x_reshape()), - labels_in.template shaped(bcast.y_reshape()), - scratch.matrix(), loss_out->vec(), back_out->matrix()); - } + functor(context->eigen_device(), shape_in.AsEigenDSizes<2>(), + BCast::ToIndexArray<2>(bcast.x_bcast()), + BCast::ToIndexArray<2>(bcast.y_bcast()), + logits_in.template shaped(bcast.x_reshape()), + labels_in.template shaped(bcast.y_reshape()), + scratch.matrix(), loss_out->vec(), back_out->matrix()); } } }; diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc index 572cf824fcfc8f..90bc2cecd0b869 100644 --- a/tensorflow/core/ops/array_ops.cc +++ b/tensorflow/core/ops/array_ops.cc @@ -168,7 +168,7 @@ Status TransposeShapeFn(InferenceContext* c) { for (int32 i = 0; i < rank; ++i) { int64 in_idx = data[i]; - if (in_idx >= rank) { + if (in_idx >= rank || in_idx <= -rank) { return errors::InvalidArgument("perm dim ", in_idx, " is out of range of input rank ", rank); } @@ -680,6 +680,12 @@ REGISTER_OP("SplitV") if (data[i] == -1 && c->ValueKnown(split_dim_size)) { size = split_dim_size - total_size; } + // If we have a negative known size (either explicit, or computed + // via -1), then the split sizes are invalid. + if (size < -1 || (size == -1 && c->ValueKnown(split_dim_size))) { + return errors::InvalidArgument("Split size at index ", i, + " must be >= 0. Got: ", size); + } TF_RETURN_IF_ERROR( c->ReplaceDim(input, split_dim, c->MakeDim(size), &output_shape)); c->set_output(i, output_shape); @@ -2841,7 +2847,10 @@ REGISTER_OP("QuantizeAndDequantizeV2") ShapeHandle minmax; TF_RETURN_IF_ERROR(c->WithRank(c->input(1), minmax_rank, &minmax)); TF_RETURN_IF_ERROR(c->Merge(c->input(2), minmax, &minmax)); - if (axis != -1) { + if (axis < -1) { + return errors::InvalidArgument("axis should be at least -1, got ", + axis); + } else if (axis != -1) { ShapeHandle input; TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), axis + 1, &input)); DimensionHandle depth; @@ -2873,7 +2882,10 @@ REGISTER_OP("QuantizeAndDequantizeV4") ShapeHandle minmax; TF_RETURN_IF_ERROR(c->WithRank(c->input(1), minmax_rank, &minmax)); TF_RETURN_IF_ERROR(c->Merge(c->input(2), minmax, &minmax)); - if (axis != -1) { + if (axis < -1) { + return errors::InvalidArgument("axis should be at least -1, got ", + axis); + } else if (axis != -1) { ShapeHandle input; TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), axis + 1, &input)); DimensionHandle depth; @@ -2901,7 +2913,10 @@ REGISTER_OP("QuantizeAndDequantizeV4Grad") ShapeHandle minmax; TF_RETURN_IF_ERROR(c->WithRank(c->input(2), minmax_rank, &minmax)); TF_RETURN_IF_ERROR(c->Merge(c->input(3), minmax, &minmax)); - if (axis != -1) { + if (axis < -1) { + return errors::InvalidArgument("axis should be at least -1, got ", + axis); + } else if (axis != -1) { ShapeHandle input; TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), axis + 1, &input)); DimensionHandle depth; @@ -2934,7 +2949,10 @@ REGISTER_OP("QuantizeAndDequantizeV3") ShapeHandle minmax; TF_RETURN_IF_ERROR(c->WithRank(c->input(1), minmax_rank, &minmax)); TF_RETURN_IF_ERROR(c->Merge(c->input(2), minmax, &minmax)); - if (axis != -1) { + if (axis < -1) { + return errors::InvalidArgument("axis should be at least -1, got ", + axis); + } else if (axis != -1) { ShapeHandle input; TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), axis + 1, &input)); DimensionHandle depth; diff --git a/tensorflow/core/ops/array_ops_test.cc b/tensorflow/core/ops/array_ops_test.cc index 412c926d3863a6..2fe45474bfaa44 100644 --- a/tensorflow/core/ops/array_ops_test.cc +++ b/tensorflow/core/ops/array_ops_test.cc @@ -1363,6 +1363,8 @@ TEST(ArrayOpsTest, QuantizeAndDequantizeV2_ShapeFn) { INFER_ERROR("Shapes must be equal rank, but are 1 and 0", op, "[1,2,?,4,5];[];[1]"); INFER_ERROR("Shape must be rank 0 but is rank 1", op, "[1,2,?,4,5];[1];[1]"); + (*op.node_def.mutable_attr())["axis"].set_i(-2); + INFER_ERROR("axis should be at least -1, got -2", op, "?;?;?"); } TEST(ArrayOpsTest, SpaceToBatch_ShapeFn) { diff --git a/tensorflow/core/ops/count_ops.cc b/tensorflow/core/ops/count_ops.cc index 8de0a2ef95459b..95e8026fd8b663 100644 --- a/tensorflow/core/ops/count_ops.cc +++ b/tensorflow/core/ops/count_ops.cc @@ -41,6 +41,8 @@ Status DenseCountSparseOutputShapeFn(InferenceContext *c) { } Status SparseCountSparseOutputShapeFn(InferenceContext *c) { + ShapeHandle unused; + TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 2, &unused)); auto rank = c->Dim(c->input(0), 1); auto nvals = c->UnknownDim(); c->set_output(0, c->Matrix(nvals, rank)); // out.indices diff --git a/tensorflow/core/ops/cudnn_rnn_ops.cc b/tensorflow/core/ops/cudnn_rnn_ops.cc index 1dd7659e137fe3..ff6d2852a66271 100644 --- a/tensorflow/core/ops/cudnn_rnn_ops.cc +++ b/tensorflow/core/ops/cudnn_rnn_ops.cc @@ -81,11 +81,17 @@ REGISTER_OP("CudnnRNN") .Attr("seed2: int = 0") .Attr("is_training: bool = true") .SetShapeFn([](InferenceContext* c) { + ShapeHandle unused; auto input_shape = c->input(0); auto input_h_shape = c->input(1); + TF_RETURN_IF_ERROR(c->WithRank(input_shape, 3, &unused)); + TF_RETURN_IF_ERROR(c->WithRank(input_h_shape, 3, &unused)); + TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 1, &unused)); + auto seq_length = c->Dim(input_shape, 0); auto batch_size = c->Dim(input_shape, 1); auto num_units = c->Dim(input_h_shape, 2); + string direction; TF_RETURN_IF_ERROR(c->GetAttr("direction", &direction)); string rnn_mode; @@ -124,8 +130,13 @@ REGISTER_OP("CudnnRNNV2") .Attr("seed2: int = 0") .Attr("is_training: bool = true") .SetShapeFn([](InferenceContext* c) { + ShapeHandle unused; auto input_shape = c->input(0); auto input_h_shape = c->input(1); + TF_RETURN_IF_ERROR(c->WithRank(input_shape, 3, &unused)); + TF_RETURN_IF_ERROR(c->WithRank(input_h_shape, 3, &unused)); + TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 1, &unused)); + auto seq_length = c->Dim(input_shape, 0); auto batch_size = c->Dim(input_shape, 1); auto num_units = c->Dim(input_h_shape, 2); @@ -171,16 +182,26 @@ REGISTER_OP("CudnnRNNV3") .Attr("is_training: bool = true") .Attr("time_major: bool = true") .SetShapeFn([](InferenceContext* c) { + ShapeHandle unused; auto input_shape = c->input(0); auto input_h_shape = c->input(1); auto input_c_shape = c->input(2); + TF_RETURN_IF_ERROR(c->WithRank(input_shape, 3, &unused)); + TF_RETURN_IF_ERROR(c->WithRank(input_h_shape, 3, &unused)); + TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 1, &unused)); + TF_RETURN_IF_ERROR(c->WithRank(c->input(4), 1, &unused)); + auto max_seq_length = c->Dim(input_shape, 0); auto batch_size = c->Dim(input_shape, 1); auto num_units = c->Dim(input_h_shape, 2); + string direction; TF_RETURN_IF_ERROR(c->GetAttr("direction", &direction)); string rnn_mode; TF_RETURN_IF_ERROR(c->GetAttr("rnn_mode", &rnn_mode)); + if (rnn_mode == "lstm") { + TF_RETURN_IF_ERROR(c->WithRank(input_c_shape, 3, &unused)); + } int dir_count = (direction == "bidirectional") ? 2 : 1; DimensionHandle output_size; TF_RETURN_IF_ERROR(c->Multiply(num_units, dir_count, &output_size)); diff --git a/tensorflow/core/ops/cudnn_rnn_ops_test.cc b/tensorflow/core/ops/cudnn_rnn_ops_test.cc index 8e8c8193a14a48..91043efa425a0b 100644 --- a/tensorflow/core/ops/cudnn_rnn_ops_test.cc +++ b/tensorflow/core/ops/cudnn_rnn_ops_test.cc @@ -68,6 +68,11 @@ TEST(CudnnRNNOpsTest, ForwardLstm_ShapeFn) { .Attr("direction", "unidirectional") .Finalize(&op.node_def)); INFER_OK(op, input_shapes_desc, output_shapes_desc); + INFER_ERROR("Shape must be rank 3 ", op, "[];[?,?,?];[?,?,?];[?]"); + INFER_ERROR("Shape must be rank 3 ", op, "[?,?,?];[];[?,?,?];[?]"); + // Disabled because the kernel does not check shape of input_c. + // INFER_ERROR("Shape must be rank 3 ", op, "[?,?,?];[?,?,?];[?];[?]"); + INFER_ERROR("Shape must be rank 1 ", op, "[?,?,?];[?,?,?];[?,?,?];[]"); } TEST(CudnnRNNOpsTest, ForwardV2Lstm_ShapeFn) { @@ -100,6 +105,11 @@ TEST(CudnnRNNOpsTest, ForwardV2Lstm_ShapeFn) { .Attr("direction", "unidirectional") .Finalize(&op.node_def)); INFER_OK(op, input_shapes_desc, output_shapes_desc); + INFER_ERROR("Shape must be rank 3 ", op, "[];[?,?,?];[?,?,?];[?]"); + INFER_ERROR("Shape must be rank 3 ", op, "[?,?,?];[];[?,?,?];[?]"); + // Disabled because the kernel does not check shape of input_c. + // INFER_ERROR("Shape must be rank 3 ", op, "[?,?,?];[?,?,?];[?];[?]"); + INFER_ERROR("Shape must be rank 1 ", op, "[?,?,?];[?,?,?];[?,?,?];[]"); } TEST(CudnnRNNOpsTest, ForwardV3Lstm_ShapeFn) { @@ -137,6 +147,52 @@ TEST(CudnnRNNOpsTest, ForwardV3Lstm_ShapeFn) { .Attr("direction", "unidirectional") .Finalize(&op.node_def)); INFER_OK(op, input_shapes_desc, output_shapes_desc); + INFER_ERROR("Shape must be rank 3 ", op, "[];[?,?,?];[?,?,?];[?];[?]"); + INFER_ERROR("Shape must be rank 3 ", op, "[?,?,?];[];[?,?,?];[?];[?]"); + INFER_ERROR("Shape must be rank 3 ", op, "[?,?,?];[?,?,?];[];[?];[?]"); + INFER_ERROR("Shape must be rank 1 ", op, "[?,?,?];[?,?,?];[?,?,?];[];[?]"); + INFER_ERROR("Shape must be rank 1 ", op, "[?,?,?];[?,?,?];[?,?,?];[?];[]"); +} + +TEST(CudnnRNNOpsTest, ForwardV3Gru) { + int max_seq_length = 2; + int batch_size = 3; + int num_units = 4; + int num_layers = 5; + int dir_count = 1; + std::vector input_shape = {max_seq_length, batch_size, num_units}; + std::vector input_h_shape = {num_layers * dir_count, batch_size, + num_units}; + std::vector input_c_shape = {num_layers * dir_count, batch_size, + num_units}; + std::vector output_shape = {max_seq_length, batch_size, + num_units * dir_count}; + std::vector seq_lengths_shape = {batch_size}; + auto shape_to_str = [](const std::vector& v) { + return strings::StrCat("[", absl::StrJoin(v, ","), "]"); + }; + string input_shapes_desc = strings::StrCat( + shape_to_str(input_shape), ";", shape_to_str(input_h_shape), ";", + shape_to_str(input_c_shape), ";", "[?]", ";", + shape_to_str(seq_lengths_shape)); + string output_shapes_desc = "[d0_0,d0_1,d1_2];in1;[];?;?"; + + ShapeInferenceTestOp op("CudnnRNNV3"); + TF_ASSERT_OK(NodeDefBuilder("test", "CudnnRNNV3") + .Input({"input", 0, DT_FLOAT}) + .Input({"input_h", 0, DT_FLOAT}) + .Input({"input_c", 0, DT_FLOAT}) + .Input({"params", 0, DT_FLOAT}) + .Input({"sequence_lengths", 0, DT_INT32}) + .Attr("rnn_mode", "gru") + .Attr("input_mode", "auto_select") + .Attr("direction", "unidirectional") + .Finalize(&op.node_def)); + INFER_OK(op, input_shapes_desc, output_shapes_desc); + INFER_ERROR("Shape must be rank 3 ", op, "[];[?,?,?];[];[?];[?]"); + INFER_ERROR("Shape must be rank 3 ", op, "[?,?,?];[];[];[?];[?]"); + INFER_ERROR("Shape must be rank 1 ", op, "[?,?,?];[?,?,?];[];[];[?]"); + INFER_ERROR("Shape must be rank 1 ", op, "[?,?,?];[?,?,?];[];[?];[]"); } } // end namespace tensorflow diff --git a/tensorflow/core/ops/ragged_array_ops.cc b/tensorflow/core/ops/ragged_array_ops.cc index 4eefa6181c2214..f113c779195002 100644 --- a/tensorflow/core/ops/ragged_array_ops.cc +++ b/tensorflow/core/ops/ragged_array_ops.cc @@ -64,6 +64,7 @@ REGISTER_OP("RaggedCross") .SetShapeFn([](shape_inference::InferenceContext* c) { std::vector ragged_values_types; std::vector ragged_splits_types; + std::vector sparse_values_types; std::vector dense_types; TF_RETURN_IF_ERROR( @@ -71,15 +72,21 @@ REGISTER_OP("RaggedCross") TF_RETURN_IF_ERROR( c->GetAttr("ragged_splits_types", &ragged_splits_types)); TF_RETURN_IF_ERROR(c->GetAttr("dense_types", &dense_types)); + TF_RETURN_IF_ERROR( + c->GetAttr("sparse_values_types", &sparse_values_types)); int num_ragged = ragged_values_types.size(); if (num_ragged != ragged_splits_types.size()) { return errors::InvalidArgument( - "Parameters `values` and `row_splits` must be the same length"); + "ragged values and splits must have the same length."); } int num_sparse; TF_RETURN_IF_ERROR(c->GetAttr("Nsparse", &num_sparse)); + if (num_sparse != sparse_values_types.size()) { + return errors::InvalidArgument( + "sparse indices and values must have the same length"); + } ShapeHandle out_values = c->UnknownShapeOfRank(1); ShapeHandle out_splits = c->UnknownShapeOfRank(1); @@ -99,7 +106,14 @@ REGISTER_OP("RaggedCross") int dense_start = num_ragged * 2 + num_sparse * 3; for (int i = 0; i < dense_types.size(); ++i) { ShapeHandle dense_input = c->input(i + dense_start); - int64 batch_size = c->Value(c->Dim(dense_input, 0)); + int32 rank = c->Rank(dense_input); + if (rank == InferenceContext::kUnknownRank) { + continue; + } else if (rank != 2) { + return errors::InvalidArgument( + "tf.ragged.cross only supports inputs with rank=2"); + } + int64_t batch_size = c->Value(c->Dim(dense_input, 0)); if (batch_size != InferenceContext::kUnknownDim) { ShapeHandle row_splits = c->Vector(batch_size + 1); if (!c->Merge(out_splits, row_splits, &out_splits).ok()) { diff --git a/tensorflow/core/ops/sparse_ops.cc b/tensorflow/core/ops/sparse_ops.cc index b1e40e66af8929..c79aeebd94bd33 100644 --- a/tensorflow/core/ops/sparse_ops.cc +++ b/tensorflow/core/ops/sparse_ops.cc @@ -16,6 +16,7 @@ limitations under the License. #include "tensorflow/core/framework/common_shape_fns.h" #include "tensorflow/core/framework/op.h" #include "tensorflow/core/framework/shape_inference.h" +#include "tensorflow/core/framework/types.pb.h" #include "tensorflow/core/platform/errors.h" namespace tensorflow { @@ -159,6 +160,8 @@ REGISTER_OP("DeserializeSparse") .Attr("Tserialized: {string, variant} = DT_STRING") .SetShapeFn([](InferenceContext* c) { // serialized sparse is [?, ..., ?, 3] vector. + ShapeHandle unused_shape; + TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 1, &unused_shape)); DimensionHandle unused; TF_RETURN_IF_ERROR(c->WithValue(c->Dim(c->input(0), -1), 3, &unused)); c->set_output(0, c->Matrix(InferenceContext::kUnknownDim, diff --git a/tensorflow/core/ops/tpu_cross_replica_ops.cc b/tensorflow/core/ops/tpu_cross_replica_ops.cc index 1f10fe3136dd1c..404c3e59eff941 100644 --- a/tensorflow/core/ops/tpu_cross_replica_ops.cc +++ b/tensorflow/core/ops/tpu_cross_replica_ops.cc @@ -32,17 +32,35 @@ REGISTER_OP("AllToAll") .Attr("split_count: int") .SetShapeFn([](InferenceContext* c) { ShapeHandle input = c->input(0); + ShapeHandle group_assignment = c->input(1); + int64 rank; if (c->RankKnown(input)) { rank = c->Rank(input); } else { return errors::InvalidArgument("input's rank is unknown."); } + int concat_dimension; int split_dimension; int split_count; TF_RETURN_IF_ERROR(c->GetAttr("split_count", &split_count)); + if (split_count < 1) { + return errors::InvalidArgument("split_count ", split_count, + " must at least be one."); + } + if (c->RankKnown(group_assignment) && c->Rank(group_assignment) != 2) { + return errors::InvalidArgument("group_assignment must have rank 2."); + } + DimensionHandle num_replicas_per_group = c->Dim(group_assignment, 1); + if (c->ValueKnown(num_replicas_per_group) && + (c->Value(num_replicas_per_group) != split_count)) { + return errors::InvalidArgument( + "split_count ", split_count, + " must equal the size of the second dimension of group_assignment ", + c->Value(num_replicas_per_group)); + } TF_RETURN_IF_ERROR(c->GetAttr("concat_dimension", &concat_dimension)); @@ -66,6 +84,12 @@ REGISTER_OP("AllToAll") dims[i] = c->MakeDim(c->Value(dims[i]) * split_count); } if (i == split_dimension) { + if (c->ValueKnown(dims[i]) && + (c->Value(dims[i]) % split_count != 0)) { + return errors::InvalidArgument( + "input dimension ", c->Value(dims[i]), + " not divisible by split_count ", split_count); + } dims[i] = c->MakeDim(c->Value(dims[i]) / split_count); } } diff --git a/tensorflow/core/platform/ctstring_internal.h b/tensorflow/core/platform/ctstring_internal.h index 9524267176c24e..9a01d48fb1e91f 100644 --- a/tensorflow/core/platform/ctstring_internal.h +++ b/tensorflow/core/platform/ctstring_internal.h @@ -63,9 +63,9 @@ static inline uint32_t TF_swap32(uint32_t host_int) { #endif #if TF_TSTRING_LITTLE_ENDIAN -#define TF_le32toh(x) TF_swap32(x) -#else // TF_TSTRING_LITTLE_ENDIAN #define TF_le32toh(x) x +#else // TF_TSTRING_LITTLE_ENDIAN +#define TF_le32toh(x) TF_swap32(x) #endif // TF_TSTRING_LITTLE_ENDIAN static inline size_t TF_align16(size_t i) { return (i + 0xF) & ~0xF; } diff --git a/tensorflow/core/platform/ctstring_test.cc b/tensorflow/core/platform/ctstring_test.cc index 4d82bcd87c327b..8624cc4ee73b49 100644 --- a/tensorflow/core/platform/ctstring_test.cc +++ b/tensorflow/core/platform/ctstring_test.cc @@ -18,6 +18,7 @@ limitations under the License. #include #include +#include "tensorflow/core/platform/ctstring_internal.h" #include "tensorflow/core/platform/test.h" static const char kLongString[] = @@ -329,3 +330,29 @@ TEST(TF_CTStringTest, ResizeReserve) { TF_TString_Dealloc(&s70); } } + +TEST(TF_CTStringTest, OffsetType) { + { + TF_TString s71; + + TF_TString_Init(&s71); + size_t header_length = 24; + size_t size = 8; + TF_TString_ResizeUninitialized(&s71, header_length + size); + uint32_t save_size = s71.u.offset.size; + uint32_t save_offset = s71.u.offset.offset; + uint32_t save_count = s71.u.offset.count; + + s71.u.offset.size = TF_TString_ToInternalSizeT(size, TF_TSTR_OFFSET); + s71.u.offset.offset = header_length; + s71.u.offset.count = 0; + EXPECT_EQ(size, TF_TString_GetSize(&s71)); + EXPECT_EQ(TF_TSTR_OFFSET, TF_TString_GetType(&s71)); + + // restore state so string can be deallocated + s71.u.offset.size = save_size; + s71.u.offset.offset = save_offset; + s71.u.offset.count = save_count; + TF_TString_Dealloc(&s71); + } +} diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index 58661de1b83495..799e359f46a82d 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -22,7 +22,7 @@ limitations under the License. // tensorflow/tools/pip_package/setup.py #define TF_MAJOR_VERSION 2 #define TF_MINOR_VERSION 5 -#define TF_PATCH_VERSION 1 +#define TF_PATCH_VERSION 2 // TF_VERSION_SUFFIX is non-empty for pre-releases (e.g. "-alpha", "-alpha.1", // "-beta", "-rc", "-rc.1") diff --git a/tensorflow/core/util/image_resizer_state.h b/tensorflow/core/util/image_resizer_state.h index 84459c9447edc2..e554b208e14fff 100644 --- a/tensorflow/core/util/image_resizer_state.h +++ b/tensorflow/core/util/image_resizer_state.h @@ -135,11 +135,16 @@ struct ImageResizerState { void ValidateAndCreateOutput(OpKernelContext* context, const Tensor& input) { ValidateAndCalculateOutputSize(context, input); if (!context->status().ok()) return; - OP_REQUIRES_OK(context, context->allocate_output( - 0, - TensorShape({input.dim_size(0), out_height, - out_width, input.dim_size(3)}), - &output)); + + TensorShape shape; + // Guard against shape overflow + OP_REQUIRES_OK(context, shape.AddDimWithStatus(batch_size)); + + OP_REQUIRES_OK(context, shape.AddDimWithStatus(out_height)); + OP_REQUIRES_OK(context, shape.AddDimWithStatus(out_width)); + OP_REQUIRES_OK(context, shape.AddDimWithStatus(channels)); + + OP_REQUIRES_OK(context, context->allocate_output(0, shape, &output)); } int64 batch_size; diff --git a/tensorflow/core/util/saved_tensor_slice_util.h b/tensorflow/core/util/saved_tensor_slice_util.h index 1f9768f5163d25..27916095bfb1fd 100644 --- a/tensorflow/core/util/saved_tensor_slice_util.h +++ b/tensorflow/core/util/saved_tensor_slice_util.h @@ -59,6 +59,9 @@ Status ParseShapeAndSlice(const string& shape_and_slice, TensorShape* shape, template struct SaveTypeTraits; +template +int TensorProtoDataSize(const TensorProto& t); + template const typename SaveTypeTraits::SavedType* TensorProtoData( const TensorProto& t); @@ -95,6 +98,10 @@ void Fill(T* data, size_t n, TensorProto* t); #define TENSOR_PROTO_EXTRACT_TYPE(TYPE, FIELD, FTYPE) \ TENSOR_PROTO_EXTRACT_TYPE_HELPER(TYPE, FIELD, FTYPE, FTYPE) \ template <> \ + inline int TensorProtoDataSize(const TensorProto& t) { \ + return t.FIELD##_val_size(); \ + } \ + template <> \ inline void Fill(const TYPE* data, size_t n, TensorProto* t) { \ typename protobuf::RepeatedField copy(data, data + n); \ t->mutable_##FIELD##_val()->Swap(©); \ @@ -104,6 +111,10 @@ void Fill(T* data, size_t n, TensorProto* t); #define TENSOR_PROTO_EXTRACT_TYPE_COMPLEX(TYPE, FIELD, FTYPE) \ TENSOR_PROTO_EXTRACT_TYPE_HELPER(TYPE, FIELD, FTYPE, TYPE) \ template <> \ + inline int TensorProtoDataSize(const TensorProto& t) { \ + return t.FIELD##_val_size() / 2; \ + } \ + template <> \ inline void Fill(const TYPE* data, size_t n, TensorProto* t) { \ const FTYPE* sub = reinterpret_cast(data); \ typename protobuf::RepeatedField copy(sub, sub + 2 * n); \ @@ -136,6 +147,11 @@ TENSOR_PROTO_EXTRACT_TYPE(quint16, int, int32); template <> struct SaveTypeTraits : SaveTypeTraits {}; +template <> +inline int TensorProtoDataSize(const TensorProto& t) { + return t.int_val_size(); +} + template <> inline const int32* TensorProtoData(const TensorProto& t) { static_assert(SaveTypeTraits::supported, @@ -158,6 +174,11 @@ struct SaveTypeTraits { typedef protobuf::RepeatedField RepeatedField; }; +template <> +inline int TensorProtoDataSize(const TensorProto& t) { + return t.half_val_size(); +} + template <> inline const int* TensorProtoData(const TensorProto& t) { return t.half_val().data(); @@ -187,6 +208,11 @@ struct SaveTypeTraits { typedef protobuf::RepeatedPtrField RepeatedField; }; +template <> +inline int TensorProtoDataSize(const TensorProto& t) { + return t.string_val_size(); +} + template <> inline const string* const* TensorProtoData(const TensorProto& t) { static_assert(SaveTypeTraits::supported, diff --git a/tensorflow/core/util/tensor_slice_reader.cc b/tensorflow/core/util/tensor_slice_reader.cc index 58c4c22ce7a9e2..fd98d1b4041a30 100644 --- a/tensorflow/core/util/tensor_slice_reader.cc +++ b/tensorflow/core/util/tensor_slice_reader.cc @@ -168,9 +168,13 @@ void TensorSliceReader::LoadShard(int shard) const { "checkpoint"); if (!status_.ok()) return; for (const SavedSliceMeta& ssm : sts.meta().tensor()) { - TensorShape ssm_shape(ssm.shape()); + TensorShape ssm_shape; + status_ = TensorShape::BuildTensorShapeBase(ssm.shape(), &ssm_shape); + if (!status_.ok()) return; for (const TensorSliceProto& tsp : ssm.slice()) { - TensorSlice ss_slice(tsp); + TensorSlice ss_slice; + status_ = TensorSlice::BuildTensorSlice(tsp, &ss_slice); + if (!status_.ok()) return; status_ = RegisterTensorSlice(ssm.name(), ssm_shape, ssm.type(), fname, ss_slice, &tensors_); if (!status_.ok()) return; @@ -248,7 +252,9 @@ Status TensorSliceReader::GetTensor( slice = tss->Slices().begin()->second.slice; } - std::unique_ptr t(new tensorflow::Tensor(type, shape)); + std::unique_ptr t(new tensorflow::Tensor); + Status s = tensorflow::Tensor::BuildTensor(type, shape, t.get()); + if (!s.ok()) return s; bool success = false; #define READER_COPY(dt) \ diff --git a/tensorflow/core/util/tensor_slice_reader.h b/tensorflow/core/util/tensor_slice_reader.h index 0fb2e11bf8dd08..bc0a91523fe36c 100644 --- a/tensorflow/core/util/tensor_slice_reader.h +++ b/tensorflow/core/util/tensor_slice_reader.h @@ -181,6 +181,22 @@ bool TensorSliceReader::CopySliceData(const string& name, << slice_s.DebugString() << ": computed key = " << key; return false; } + // Ensure the TensorSlice contains the expected amount of data. + TensorShape shp_s; + Status s = slice_s.SliceTensorShape(tss->shape(), &shp_s); + if (!s.ok()) { + VLOG(1) << "Failed to slice tensor " << name << ", slice " + << slice_s.DebugString() << ": " << s; + return false; + } + if (checkpoint::TensorProtoDataSize(sts.data().data()) != + shp_s.num_elements()) { + VLOG(1) << "Tensor " << name << ", slice " << slice_s.DebugString() + << " had an unexpected amount of data: expected = " + << shp_s.num_elements() << ", got = " + << checkpoint::TensorProtoDataSize(sts.data().data()); + return false; + } CopyDataFromTensorSliceToTensorSlice( tss->shape(), slice_s, slice, checkpoint::TensorProtoData(sts.data().data()), data); diff --git a/tensorflow/core/util/tensor_slice_reader_test.cc b/tensorflow/core/util/tensor_slice_reader_test.cc index fe617e8e30d67e..eb853886071541 100644 --- a/tensorflow/core/util/tensor_slice_reader_test.cc +++ b/tensorflow/core/util/tensor_slice_reader_test.cc @@ -13,15 +13,20 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include - #include "tensorflow/core/util/tensor_slice_reader.h" +#include +#include + +#include "tensorflow/core/framework/tensor_shape.pb.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/framework/versions.pb.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/lib/core/stringpiece.h" +#include "tensorflow/core/lib/io/iterator.h" #include "tensorflow/core/lib/io/path.h" +#include "tensorflow/core/lib/io/table.h" +#include "tensorflow/core/lib/io/table_builder.h" #include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/lib/strings/strcat.h" #include "tensorflow/core/platform/env.h" @@ -30,6 +35,7 @@ limitations under the License. #include "tensorflow/core/platform/test.h" #include "tensorflow/core/platform/types.h" #include "tensorflow/core/public/version.h" +#include "tensorflow/core/util/saved_tensor_slice.pb.h" #include "tensorflow/core/util/saved_tensor_slice_util.h" #include "tensorflow/core/util/tensor_slice_reader_cache.h" #include "tensorflow/core/util/tensor_slice_writer.h" @@ -309,6 +315,177 @@ TEST_SIMPLE_INT(int16, int32) TEST_SIMPLE_INT(int8, int32) TEST_SIMPLE_INT(uint8, int32) +// Modifies the SavedTensorSlices messages in a checkpoint to allow creating +// malformed or unsupported checkpoints. +void MutateSavedTensorSlices( + const std::string& fname, + const std::function& mutator) { + table::Options options; + options.compression = table::kNoCompression; + + // Read all entres from the table. + std::vector> entries; + { + std::unique_ptr file; + TF_CHECK_OK(Env::Default()->NewRandomAccessFile(fname, &file)); + uint64 file_size; + TF_CHECK_OK(Env::Default()->GetFileSize(fname, &file_size)); + table::Table* t; + TF_CHECK_OK(table::Table::Open(options, file.get(), file_size, &t)); + std::unique_ptr table(t); + std::unique_ptr it(table->NewIterator()); + for (it->Seek(""); it->Valid(); it->Next()) { + entries.emplace_back(it->key(), it->value()); + } + TF_CHECK_OK(it->status()); + } + + // Rewrite the table, mutating each value. + { + std::unique_ptr file; + TF_CHECK_OK(Env::Default()->NewWritableFile(fname, &file)); + table::TableBuilder builder(options, file.get()); + for (const auto& entry : entries) { + SavedTensorSlices sts; + CHECK(sts.ParseFromString(entry.second)); + builder.Add(entry.first, mutator(std::move(sts))); + } + TF_CHECK_OK(builder.Finish()); + TF_CHECK_OK(file->Close()); + } +} + +TEST(TensorSliceReaderTest, MissingTensorType) { + const string fname = io::JoinPath(testing::TmpDir(), "invalid_checkpoint"); + TensorSliceWriter writer(fname, CreateTableTensorSliceBuilder); + const int32 data[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + TensorShape shape({4, 5}); + TensorSlice slice = TensorSlice::ParseOrDie("0,2:-"); + TF_CHECK_OK(writer.Add("test", shape, slice, data)); + TF_CHECK_OK(writer.Finish()); + + MutateSavedTensorSlices(fname, [](SavedTensorSlices sts) { + if (sts.has_meta()) { + for (auto& tensor : *sts.mutable_meta()->mutable_tensor()) { + tensor.clear_type(); + } + } + return sts.SerializeAsString(); + }); + + TensorSliceReader reader(fname, OpenTableTensorSliceReader); + TF_CHECK_OK(reader.status()); + + // The tensor should be present, but loading it should fail due to the + // unset (invalid) type. + EXPECT_TRUE(reader.HasTensor("test", nullptr, nullptr)); + std::unique_ptr tensor; + EXPECT_FALSE(reader.GetTensor("test", &tensor).ok()); +} + +TEST(TensorSliceReaderTest, UnsupportedTensorType) { + const string fname = io::JoinPath(testing::TmpDir(), "int32_ref_checkpoint"); + TensorSliceWriter writer(fname, CreateTableTensorSliceBuilder); + const int32 data[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + TensorShape shape({4, 5}); + TensorSlice slice = TensorSlice::ParseOrDie("0,2:-"); + TF_CHECK_OK(writer.Add("test", shape, slice, data)); + TF_CHECK_OK(writer.Finish()); + + MutateSavedTensorSlices(fname, [](SavedTensorSlices sts) { + if (sts.has_meta()) { + for (auto& tensor : *sts.mutable_meta()->mutable_tensor()) { + tensor.set_type(DT_INT32_REF); + } + } + return sts.SerializeAsString(); + }); + + TensorSliceReader reader(fname, OpenTableTensorSliceReader); + TF_CHECK_OK(reader.status()); + + // The tensor should be present, but loading it should fail due to the + // unsupported type. + EXPECT_TRUE(reader.HasTensor("test", nullptr, nullptr)); + std::unique_ptr tensor; + EXPECT_FALSE(reader.GetTensor("test", &tensor).ok()); +} + +TEST(TensorSliceReaderTest, NegativeTensorShapeDimension) { + const string fname = + io::JoinPath(testing::TmpDir(), "negative_dim_checkpoint"); + TensorSliceWriter writer(fname, CreateTableTensorSliceBuilder); + const int32 data[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + TF_CHECK_OK(writer.Add("test", TensorShape({4, 5}), + TensorSlice::ParseOrDie("0,2:-"), data)); + TF_CHECK_OK(writer.Finish()); + + MutateSavedTensorSlices(fname, [](SavedTensorSlices sts) { + if (sts.has_meta()) { + for (auto& tensor : *sts.mutable_meta()->mutable_tensor()) { + for (auto& dim : *tensor.mutable_shape()->mutable_dim()) { + dim.set_size(-dim.size()); + } + } + } + return sts.SerializeAsString(); + }); + + TensorSliceReader reader(fname, OpenTableTensorSliceReader); + // The negative dimension should cause loading to fail. + EXPECT_FALSE(reader.status().ok()); +} + +TEST(TensorSliceReaderTest, InvalidTensorSlice) { + const string fname = + io::JoinPath(testing::TmpDir(), "invalid_slice_checkpoint"); + TensorSliceWriter writer(fname, CreateTableTensorSliceBuilder); + const int32 data[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + TF_CHECK_OK(writer.Add("test", TensorShape({4, 5}), + TensorSlice::ParseOrDie("0,2:-"), data)); + TF_CHECK_OK(writer.Finish()); + + MutateSavedTensorSlices(fname, [](SavedTensorSlices sts) { + if (sts.has_meta()) { + for (auto& tensor : *sts.mutable_meta()->mutable_tensor()) { + tensor.mutable_slice(0)->mutable_extent(0)->set_length(-10); + } + } + return sts.SerializeAsString(); + }); + + TensorSliceReader reader(fname, OpenTableTensorSliceReader); + // The negative exent length should cause loading to fail. + EXPECT_FALSE(reader.status().ok()); +} + +TEST(TensorSliceReaderTest, MissingTensorData) { + const string fname = + io::JoinPath(testing::TmpDir(), "missing_data_checkpoint"); + TensorSliceWriter writer(fname, CreateTableTensorSliceBuilder); + const int32 data[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + TF_ASSERT_OK(writer.Add("test", TensorShape({4, 5}), + TensorSlice::ParseOrDie("0,2:-"), data)); + TF_ASSERT_OK(writer.Finish()); + + MutateSavedTensorSlices(fname, [&](SavedTensorSlices sts) { + if (sts.has_data()) { + // Replace the data with only 4 elements. + Fill(data, 4, sts.mutable_data()->mutable_data()); + } + return sts.SerializeAsString(); + }); + + TensorSliceReader reader(fname, OpenTableTensorSliceReader); + TF_ASSERT_OK(reader.status()); + + // The tensor should be present, but loading it should fail due to the missing + // data. + EXPECT_TRUE(reader.HasTensor("test", nullptr, nullptr)); + std::unique_ptr tensor; + EXPECT_FALSE(reader.GetTensor("test", &tensor).ok()); +} + void CachedTensorSliceReaderTesterHelper( const TensorSliceWriter::CreateBuilderFunction& create_function, const TensorSliceReader::OpenTableFunction& open_function) { diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py index 1c90634095bc31..eb728abaa1d8e6 100644 --- a/tensorflow/python/eager/def_function.py +++ b/tensorflow/python/eager/def_function.py @@ -576,7 +576,7 @@ def __init__(self, ValueError: if `input_signature` is not None and the `python_function`'s argspec has keyword arguments. """ - self._lock = threading.Lock() + self._lock = threading.RLock() self._python_function = python_function self._function_spec = function_lib.FunctionSpec.from_function_and_signature( python_function, @@ -617,7 +617,7 @@ def __getstate__(self): def __setstate__(self, state): """Restore from pickled state.""" self.__dict__ = state - self._lock = threading.Lock() + self._lock = threading.RLock() self._descriptor_cache = weakref.WeakKeyDictionary() self._key_for_call_stats = self._get_key_for_call_stats() diff --git a/tensorflow/python/eager/def_function_test.py b/tensorflow/python/eager/def_function_test.py index bc41a070e0bc8d..b932dbda80bd76 100644 --- a/tensorflow/python/eager/def_function_test.py +++ b/tensorflow/python/eager/def_function_test.py @@ -28,6 +28,7 @@ from six.moves import range from tensorflow.python.autograph.core import converter +from tensorflow.python.eager import backprop from tensorflow.python.eager import def_function from tensorflow.python.eager import lift_to_graph from tensorflow.python.framework import constant_op @@ -38,6 +39,7 @@ from tensorflow.python.framework import test_util from tensorflow.python.module import module from tensorflow.python.ops import array_ops +from tensorflow.python.ops import cond_v2 from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import random_ops @@ -1000,6 +1002,117 @@ def testDouble(self, a): self.assertAllEqual(obj2.testDouble.experimental_get_tracing_count(), 3) self.assertAllEqual(obj1.testDouble.experimental_get_tracing_count(), 2) + def test_recursive_tf_function(self): + + @def_function.function + def recursive_fn(n): + if n > 0: + return recursive_fn(n - 1) + return 1 + + self.assertEqual(recursive_fn(5).numpy(), 1) + + def test_recursive_tf_function_with_gradients(self): + + @def_function.function + def recursive_fn(n, x): + if n > 0: + return n * recursive_fn(n - 1, x) + else: + return x + + x = variables.Variable(1.0) + with backprop.GradientTape() as tape: + g = recursive_fn(5, x) + + dg_dx = tape.gradient(g, x) + self.assertEqual(dg_dx.numpy(), 120) + + def test_recursive_python_function(self): + + def recursive_py_fn(n): + if n > 0: + return recursive_py_fn(n - 1) + return 1 + + @def_function.function + def recursive_fn(n): + return recursive_py_fn(n) + + self.assertEqual(recursive_fn(5).numpy(), 1) + + def test_recursive_python_function_with_gradients(self): + + def recursive_py_fn(n, x): + if n > 0: + return n * recursive_py_fn(n - 1, x) + return x + + @def_function.function + def recursive_fn(n, x): + return recursive_py_fn(n, x) + + x = variables.Variable(1.0) + with backprop.GradientTape() as tape: + g = recursive_fn(5, x) + + dg_dx = tape.gradient(g, x) + self.assertEqual(dg_dx.numpy(), 120) + + def test_recursive_tf_function_call_each_other(self): + + @def_function.function + def recursive_fn1(n): + if n <= 1: + return 1 + return recursive_fn2(n - 1) + + @def_function.function + def recursive_fn2(n): + if n <= 1: + return 2 + return recursive_fn1(n - 1) + + self.assertEqual(recursive_fn1(5).numpy(), 1) + self.assertEqual(recursive_fn1(6).numpy(), 2) + self.assertEqual(recursive_fn2(5).numpy(), 2) + self.assertEqual(recursive_fn2(6).numpy(), 1) + + def test_recursive_tf_function_call_each_other_with_gradients(self): + + @def_function.function + def recursive_fn1(n, x): + if n <= 1: + return x + return n * recursive_fn2(n - 1, x) + + @def_function.function + def recursive_fn2(n, x): + if n <= 1: + return 2 * x + return n * recursive_fn1(n - 1, x) + + x = variables.Variable(1.0) + with backprop.GradientTape() as tape: + g1 = recursive_fn1(5, x) + + dg1_dx = tape.gradient(g1, x) + self.assertEqual(dg1_dx.numpy(), 120) + + with backprop.GradientTape() as tape: + g2 = recursive_fn2(5, x) + + dg2_dx = tape.gradient(g2, x) + self.assertEqual(dg2_dx.numpy(), 240) + + def test_recursive_tf_function_with_cond(self): + @def_function.function(autograph=False) + def recursive_fn(n): + return cond_v2.cond_v2(n > 0, recursive_fn(n - 1), 1) + + with self.assertRaises(RecursionError): + recursive_fn(constant_op.constant(5)) + if __name__ == '__main__': ops.enable_eager_execution() diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py index 6a65aca1d2d13f..0af0e78ca43f45 100644 --- a/tensorflow/python/eager/function.py +++ b/tensorflow/python/eager/function.py @@ -3007,7 +3007,7 @@ def __init__(self, self._hashable_input_signature = _make_input_signature_hashable( self.flat_input_signature) - self._lock = threading.Lock() + self._lock = threading.RLock() # _descriptor_cache is a of instance of a class to an instance-specific # `Function`, used to make sure defun-decorated methods create different # functions for each instance. diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index 0f90494a95f68d..4e88cee70c1731 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -2359,7 +2359,7 @@ cuda_py_test( name = "pad_op_test", size = "small", srcs = ["pad_op_test.py"], - tags = ["no_mac"], # test is times out on mac b/186262388 + tags = ["no_oss"], # test is times out on mac b/186262388 xla_tags = [ "no_cuda_asan", # times out ], diff --git a/tensorflow/python/kernel_tests/array_ops/stack_op_test.py b/tensorflow/python/kernel_tests/array_ops/stack_op_test.py index f0e7db4a5ae127..a3b8f2524d8427 100644 --- a/tensorflow/python/kernel_tests/array_ops/stack_op_test.py +++ b/tensorflow/python/kernel_tests/array_ops/stack_op_test.py @@ -20,12 +20,16 @@ import numpy as np +from tensorflow.python import tf2 +from tensorflow.python.eager import def_function from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gen_array_ops from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import variables from tensorflow.python.platform import test @@ -76,6 +80,19 @@ def testSimpleParallelCPU(self): c = array_ops.parallel_stack(xs) self.assertAllEqual(c, data) + def testParallelConcatShapeZero(self): + if not tf2.enabled(): + self.skipTest("only fails in TF2") + + @def_function.function + def f(): + y = gen_array_ops.parallel_concat(values=[["tf"]], shape=0) + return y + + with self.assertRaisesRegex(errors.InvalidArgumentError, + r"0th dimension of value .* is less than"): + f() + def testSimpleParallelGPU(self): # tf.parallel_stack is only supported in graph mode. with ops.Graph().as_default(): diff --git a/tensorflow/python/kernel_tests/attention_ops_test.py b/tensorflow/python/kernel_tests/attention_ops_test.py index 804a0b20cc9dd4..fd83f4f6510c63 100644 --- a/tensorflow/python/kernel_tests/attention_ops_test.py +++ b/tensorflow/python/kernel_tests/attention_ops_test.py @@ -22,6 +22,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_image_ops from tensorflow.python.ops import image_ops @@ -301,6 +302,18 @@ def testGlimpseNonNormalizedNonCentered(self): np.asarray([[5, 6, 7], [10, 11, 12], [15, 16, 17]]), self.evaluate(result2)[0, :, :, 0]) + def testGlimpseNegativeInput(self): + img = np.arange(9).reshape([1, 3, 3, 1]) + with self.test_session(): + with self.assertRaises((errors.InternalError, ValueError)): + result = image_ops.extract_glimpse_v2( + img, + size=[1023, -63], + offsets=[1023, 63], + centered=False, + normalized=False) + self.evaluate(result) + if __name__ == '__main__': test.main() diff --git a/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py b/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py index 73098ed3084da6..2af570da73f815 100644 --- a/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py +++ b/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py @@ -21,9 +21,11 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import boosted_trees_ops +from tensorflow.python.ops import gen_boosted_trees_ops from tensorflow.python.ops import sparse_ops from tensorflow.python.platform import googletest @@ -1669,6 +1671,199 @@ def testMakeStatsSummaryNumericalPrecisionMegaBatch(self): """Tests numeric precision.""" self._verify_precision(length=50000000) + def testBoostedTreesCalculateBestGainsPerFeatureSecurity(self): + node_id_range = [1, 2] + stats_summary_list = [[[[]]]] + l1 = [1.0] + l2 = [1.0] + tree_complexity = [1.0] + min_node_weight = [1.17] + max_splits = 1 + with self.assertRaises((errors.InvalidArgumentError, ValueError)): + gen_boosted_trees_ops.boosted_trees_calculate_best_gains_per_feature( + node_id_range=node_id_range, + stats_summary_list=stats_summary_list, + l1=l1, + l2=l2, + tree_complexity=tree_complexity, + min_node_weight=min_node_weight, + max_splits=max_splits) + + def testBoostedTreesCalculateBestFeatureSplitSecurity(self): + node_id_range = [1, 2] + stats_summary = [[[[]]]] + split_type = 'equality' + l1 = [1.0] + l2 = [1.0] + tree_complexity = [1.0] + min_node_weight = [1.17] + logits_dimension = 5 + with self.assertRaises((errors.InvalidArgumentError, ValueError)): + gen_boosted_trees_ops.boosted_trees_calculate_best_feature_split( + node_id_range=node_id_range, + stats_summary=stats_summary, + l1=l1, + l2=l2, + tree_complexity=tree_complexity, + min_node_weight=min_node_weight, + logits_dimension=logits_dimension, + split_type=split_type) + + def testBoostedTreesCalculateBestFeatureSplitSecurity2(self): + with self.assertRaises((errors.InvalidArgumentError, ValueError)): + gen_boosted_trees_ops.boosted_trees_calculate_best_feature_split( + node_id_range=[0, 8], + stats_summary=[[[[1.0], [2.0], [3.0]]]], + l1=[0.5], + l2=[0.5], + tree_complexity=[0.1], + min_node_weight=[1.0], + logits_dimension=8) + + def testBoostedTreesCalculateBestFeatureSplitV2Security(self): + node_id_range = [1, 2] + stats_summaries_list = [[[[[]]]]] + split_types = ['inequality'] + candidate_feature_ids = [1, 2, 3, 4] + l1 = [1.0] + l2 = [1.0] + tree_complexity = [1.0] + min_node_weight = [1.17] + logits_dimension = 5 + with self.assertRaises((errors.InvalidArgumentError, ValueError)): + gen_boosted_trees_ops.boosted_trees_calculate_best_feature_split_v2( + node_id_range=node_id_range, + stats_summaries_list=stats_summaries_list, + split_types=split_types, + candidate_feature_ids=candidate_feature_ids, + l1=l1, + l2=l2, + tree_complexity=tree_complexity, + min_node_weight=min_node_weight, + logits_dimension=logits_dimension) + + def testBoostedTreesSparseCalculateBestFeatureSplitSecurity(self): + node_id_range = [] + stats_summary_indices = [[]] + stats_summary_values = [1.0] + stats_summary_shape = [1, 1, 1, 1] + l1 = [1.0] + l2 = [1.0] + tree_complexity = [0.5] + min_node_weight = [1.0] + logits_dimension = 3 + split_type = 'inequality' + with self.assertRaises((errors.InvalidArgumentError, ValueError)): + gen_boosted_trees_ops.boosted_trees_sparse_calculate_best_feature_split( + node_id_range=node_id_range, + stats_summary_indices=stats_summary_indices, + stats_summary_values=stats_summary_values, + stats_summary_shape=stats_summary_shape, + l1=l1, + l2=l2, + tree_complexity=tree_complexity, + min_node_weight=min_node_weight, + logits_dimension=logits_dimension, + split_type=split_type) + + def testBoostedTreesSparseCalculateBestFeatureSplitSecurity2(self): + with self.assertRaises((errors.InvalidArgumentError, ValueError)): + gen_boosted_trees_ops.boosted_trees_sparse_calculate_best_feature_split( + node_id_range=[0, 1], + stats_summary_indices=[[0, -1, -1, -1], [1, 0, -1, 0], [1, 0, 0, -1]], + stats_summary_values=[0.1, 0.2, 0.3], + stats_summary_shape=[1, 1, 1, 1], + l1=[0.5], + l2=[0.5], + tree_complexity=[0.1], + min_node_weight=[1.0], + logits_dimension=1) + + def testBoostedTreesMakeStatsSummarySecurity(self): + node_ids = [1, 2] + gradients = [[]] + hessians = [[0.2], [0.1]] + bucketized_features_list = [[1], [2]] + max_splits = 3 + num_buckets = 3 + with self.assertRaises((errors.InvalidArgumentError, ValueError)): + gen_boosted_trees_ops.boosted_trees_make_stats_summary( + node_ids=node_ids, + gradients=gradients, + hessians=hessians, + bucketized_features_list=bucketized_features_list, + max_splits=max_splits, + num_buckets=num_buckets) + + def testBoostedTreesMakeStatsSummarySecurity2(self): + node_ids = [1, 2, 3] + gradients = [[0.1], [0.2]] + hessians = [[0.2], [0.1]] + bucketized_features_list = [[1], [2]] + max_splits = 3 + num_buckets = 3 + with self.assertRaises((errors.InvalidArgumentError, ValueError)): + gen_boosted_trees_ops.boosted_trees_make_stats_summary( + node_ids=node_ids, + gradients=gradients, + hessians=hessians, + bucketized_features_list=bucketized_features_list, + max_splits=max_splits, + num_buckets=num_buckets) + + def testBoostedTreesAggregateStatsSecurity(self): + node_ids = [1, 2] + gradients = [[]] + hessians = [[100.0]] + feature = [[0, 0, 0]] + max_splits = 100 + num_buckets = 100 + with self.assertRaises((errors.InvalidArgumentError, ValueError)): + gen_boosted_trees_ops.boosted_trees_aggregate_stats( + node_ids=node_ids, + gradients=gradients, + hessians=hessians, + feature=feature, + max_splits=max_splits, + num_buckets=num_buckets) + + def testBoostedTreesAggregateStatsSecurity2(self): + node_ids = [-10] + gradients = [[0.0, 0.0]] + hessians = [[100.0]] + feature = [[0, 0, 0]] + max_splits = 100 + num_buckets = 100 + with self.assertRaises((errors.InvalidArgumentError, ValueError)): + self.evaluate( + gen_boosted_trees_ops.boosted_trees_aggregate_stats( + node_ids=node_ids, + gradients=gradients, + hessians=hessians, + feature=feature, + max_splits=max_splits, + num_buckets=num_buckets)) + + def testBoostedTreesSparseAggregateStatsSecurity(self): + node_ids = [] + gradients = [[1.0]] + hessians = [[100.0]] + feature_indices = [[0, 0, 0]] + feature_values = [0, 0, 0] + feature_shape = [0, 0, 0] + max_splits = 100 + num_buckets = 100 + with self.assertRaises((errors.InvalidArgumentError, ValueError)): + gen_boosted_trees_ops.boosted_trees_sparse_aggregate_stats( + node_ids=node_ids, + gradients=gradients, + hessians=hessians, + feature_indices=feature_indices, + feature_values=feature_values, + feature_shape=feature_shape, + max_splits=max_splits, + num_buckets=num_buckets) + class BestMultiDimFeatureSplitMultiClassV2Op(StatsOpsTest): """Tests multi-class/multi-regression for best splits using V2 op.""" diff --git a/tensorflow/python/kernel_tests/conv_ops_3d_test.py b/tensorflow/python/kernel_tests/conv_ops_3d_test.py index 5a7fa64d2bfbde..dabc61f8306bbe 100644 --- a/tensorflow/python/kernel_tests/conv_ops_3d_test.py +++ b/tensorflow/python/kernel_tests/conv_ops_3d_test.py @@ -24,6 +24,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors_impl from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import test_util from tensorflow.python.ops import gradient_checker @@ -460,6 +461,16 @@ def testKernelSizeMatchesInputSize(self): padding="VALID", expected=[1.5625, 1.875]) + def testZeroSizedFilterThrowsIllegalArgument(self): + tensor_in_sizes = [1, 1, 1, 1, 1] + x1 = self._CreateNumpyTensor(tensor_in_sizes) + filter_in = np.ones((1, 1, 0, 1, 1), dtype=np.float32) + with self.assertRaisesRegex( + errors_impl.InvalidArgumentError, "filter must not have zero elements" + "|has a non-positive dimension"): + self.evaluate( + nn_ops.conv3d(x1, filter_in, strides=[1, 1, 1, 1, 1], padding="SAME")) + def _ConstructAndTestGradientForConfig( self, batch, input_shape, filter_shape, in_depth, out_depth, stride, padding, test_input, data_format, use_gpu): diff --git a/tensorflow/python/kernel_tests/conv_ops_test.py b/tensorflow/python/kernel_tests/conv_ops_test.py index 424cdcb90fbc50..d753e74589db47 100644 --- a/tensorflow/python/kernel_tests/conv_ops_test.py +++ b/tensorflow/python/kernel_tests/conv_ops_test.py @@ -2531,139 +2531,83 @@ def testShapeFunctionEdgeCases(self): strides=[1, 1, 1, 1], padding=[0, 0, 0, 0]) - @test_util.deprecated_graph_mode_only def testOpEdgeCases(self): - with self.cached_session() as sess: - # Illegal strides. - with self.assertRaisesRegex(errors_impl.UnimplementedError, - "strides in the batch and depth"): - input_placeholder = array_ops.placeholder(dtypes.float32) - input_val = np.ones([10, 10]) - filter_placeholder = array_ops.placeholder(dtypes.float32) - filter_val = np.ones([10, 10]) - sess.run( - nn_ops.conv2d( - input_placeholder, - filter_placeholder, - strides=[2, 1, 1, 1], - padding="SAME"), - feed_dict={ - input_placeholder: input_val, - filter_placeholder: filter_val - }) - with self.assertRaisesRegex(errors_impl.UnimplementedError, - "strides in the batch and depth"): - input_placeholder = array_ops.placeholder(dtypes.float32) - filter_placeholder = array_ops.placeholder(dtypes.float32) - input_val = np.ones([10, 10]) - filter_val = np.ones([10, 10]) - sess.run( - nn_ops.conv2d( - input_placeholder, - filter_placeholder, - strides=[1, 1, 1, 2], - padding="SAME"), - feed_dict={ - input_placeholder: input_val, - filter_placeholder: filter_val - }) - - # Filter larger than input. - with self.assertRaisesRegex(ValueError, "Negative dimension size"): - input_placeholder = array_ops.placeholder( - dtypes.float32, shape=[32, 20, 20, 3]) - input_val = np.ones([32, 20, 20, 3]) - filter_placeholder = array_ops.placeholder( - dtypes.float32, shape=[20, 21, 3, 2]) - filter_val = np.ones([20, 21, 3, 2]) - - sess.run( - nn_ops.conv2d( - input_placeholder, - filter_placeholder, - strides=[1, 1, 1, 1], - padding="VALID"), - feed_dict={ - input_placeholder: input_val, - filter_placeholder: filter_val - }) - with self.assertRaisesRegex(ValueError, "Negative dimension size"): - input_placeholder = array_ops.placeholder( - dtypes.float32, shape=[32, 20, 20, 3]) - input_val = np.ones([32, 20, 20, 3]) - filter_placeholder = array_ops.placeholder( - dtypes.float32, shape=[21, 20, 3, 2]) - filter_val = np.ones([21, 20, 3, 2]) - sess.run( - nn_ops.conv2d( - input_placeholder, - filter_placeholder, - strides=[1, 1, 1, 1], - padding="VALID"), - feed_dict={ - input_placeholder: input_val, - filter_placeholder: filter_val - }) - - # Filter larger than input + padding. - with self.assertRaisesRegex(ValueError, "Negative dimension size"): - input_placeholder = array_ops.placeholder( - dtypes.float32, shape=[32, 20, 20, 3]) - input_val = np.ones([32, 20, 20, 3]) - filter_placeholder = array_ops.placeholder( - dtypes.float32, shape=[24, 25, 3, 2]) - filter_val = np.ones([24, 25, 3, 2]) - sess.run( - nn_ops.conv2d( - input_placeholder, - filter_placeholder, - strides=[1, 1, 1, 1], - padding=[[0, 0], [2, 2], [2, 2], [0, 0]]), - feed_dict={ - input_placeholder: input_val, - filter_placeholder: filter_val - }) - - # Negative padding during backprop. - with self.assertRaisesRegex( - errors_impl.InvalidArgumentError, - "All elements of explicit_paddings must be nonnegative"): - filter_placeholder = array_ops.placeholder( - dtypes.float32, shape=[18, 18, 3, 2]) - filter_val = np.ones([18, 18, 3, 2]) - out_backprop = array_ops.placeholder( - dtypes.float32, shape=[32, 3, 2, 2]) - out_backprop_val = np.ones([32, 3, 2, 2]) - sess.run( - nn_ops.conv2d_backprop_input([32, 20, 20, 3], - filter_placeholder, - out_backprop, - strides=[1, 1, 1, 1], - padding=[[0, 0], [-1, 0], [0, 0], - [0, 0]]), - feed_dict={ - filter_placeholder: filter_val, - out_backprop: out_backprop_val - }) - with self.assertRaisesRegex( - errors_impl.InvalidArgumentError, - "All elements of explicit_paddings must be nonnegative"): - input_placeholder = array_ops.placeholder( - dtypes.float32, shape=[32, 20, 20, 3]) - input_val = np.ones([32, 20, 20, 3]) - out_backprop = array_ops.placeholder( - dtypes.float32, shape=[32, 3, 2, 2]) - out_backprop_val = np.ones([32, 3, 2, 2]) - sess.run( - nn_ops.conv2d_backprop_filter( - input_placeholder, [18, 18, 3, 2], - out_backprop, - strides=[1, 1, 1, 1], - padding=[[0, 0], [-1, 0], [0, 0], [0, 0]]), - feed_dict={ - input_placeholder: input_val, - out_backprop: out_backprop_val - }) + # Illegal strides. + with self.assertRaisesRegex((ValueError, errors_impl.UnimplementedError), + "strides in the batch and depth"): + input_val = np.ones([2, 4, 10, 10]) + filter_val = np.ones([2, 4, 10, 10]) + self.evaluate( + nn_ops.conv2d( + input_val, filter_val, strides=[2, 1, 1, 1], padding="SAME")) + with self.assertRaisesRegex((ValueError, errors_impl.UnimplementedError), + "strides in the batch and depth"): + input_val = np.ones([2, 4, 10, 10]) + filter_val = np.ones([2, 4, 10, 10]) + self.evaluate( + nn_ops.conv2d( + input_val, filter_val, strides=[1, 1, 1, 2], padding="SAME")) + + # TODO(b/195689143): Will enable when fixed for V2 behavior + # # Filter larger than input. + # with self.assertRaisesRegex(ValueError, "Negative dimension size"): + # input_val = np.ones([32, 20, 20, 3]) + # filter_val = np.ones([20, 21, 3, 2]) + # self.evaluate( + # nn_ops.conv2d( + # input_val, filter_val, strides=[1, 1, 1, 1], padding="VALID")) + # with self.assertRaisesRegex(ValueError, "Negative dimension size"): + # input_val = np.ones([32, 20, 20, 3]) + # filter_val = np.ones([21, 20, 3, 2]) + # self.evaluate( + # nn_ops.conv2d( + # input_val, filter_val, strides=[1, 1, 1, 1], padding="VALID")) + # + # # Filter larger than input + padding. + # with self.assertRaisesRegex(ValueError, "Negative dimension size"): + # input_val = np.ones([32, 20, 20, 3]) + # filter_val = np.ones([24, 25, 3, 2]) + # self.evaluate( + # nn_ops.conv2d( + # input_val, + # filter_val, + # strides=[1, 1, 1, 1], + # padding=[[0, 0], [2, 2], [2, 2], [0, 0]])) + + # Filter dimensions must be greater than 0. + with self.assertRaisesRegex( + errors_impl.InvalidArgumentError, "filter must not have zero elements" + "|has a non-positive dimension"): + input_val = np.ones([1, 1, 1, 1]) + filter_val = np.ones([1, 0, 1, 1]) + self.evaluate( + nn_ops.conv2d( + input_val, filter_val, strides=[1, 1, 1, 1], padding="SAME")) + + # Negative padding during backprop. + with self.assertRaisesRegex( + errors_impl.InvalidArgumentError, + "All elements of explicit_paddings must be nonnegative"): + filter_val = np.ones([18, 18, 3, 2]) + out_backprop_val = np.ones([32, 3, 2, 2]) + self.evaluate( + nn_ops.conv2d_backprop_input([32, 20, 20, 3], + filter_val, + out_backprop_val, + strides=[1, 1, 1, 1], + padding=[[0, 0], [-1, 0], [0, 0], [0, + 0]])) + with self.assertRaisesRegex( + errors_impl.InvalidArgumentError, + "All elements of explicit_paddings must be nonnegative"): + input_val = np.ones([32, 20, 20, 3]) + out_backprop_val = np.ones([32, 3, 2, 2]) + self.evaluate( + nn_ops.conv2d_backprop_filter( + input_val, [18, 18, 3, 2], + out_backprop_val, + strides=[1, 1, 1, 1], + padding=[[0, 0], [-1, 0], [0, 0], [0, 0]])) class DepthwiseConv2DTest(test.TestCase): @@ -2673,10 +2617,10 @@ def _VerifyValues(self, tensor_in_sizes, filter_in_sizes, stride, padding, """Verifies the output values of the convolution function. Args: - tensor_in_sizes: Input tensor dimensions in - [batch, input_rows, input_cols, input_depth]. - filter_in_sizes: Filter tensor dimensions in - [filter_rows, filter_cols, input_depth, depth_multiplier]. + tensor_in_sizes: Input tensor dimensions in [batch, input_rows, + input_cols, input_depth]. + filter_in_sizes: Filter tensor dimensions in [filter_rows, filter_cols, + input_depth, depth_multiplier]. stride: Stride. padding: Padding type. expected: An array containing the expected operation outputs. diff --git a/tensorflow/python/kernel_tests/init_ops_test.py b/tensorflow/python/kernel_tests/init_ops_test.py index 898d6f3e9e3371..7d7a1907e42ac5 100644 --- a/tensorflow/python/kernel_tests/init_ops_test.py +++ b/tensorflow/python/kernel_tests/init_ops_test.py @@ -23,6 +23,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops from tensorflow.python.framework import random_seed from tensorflow.python.framework import test_util @@ -542,6 +543,19 @@ def testMixedDType(self): constant_op.constant(4, dtype=dtypes.int32), dtype=dtypes.int64) self.assertAllEqual(self.evaluate(tf_ans), np.array([0, 1, 2, 3])) + def testLargeStarts(self): + # Test case for GitHub issue 46899. + with self.session(): + with self.assertRaises(errors_impl.InternalError): + v = math_ops.range(start=-1e+38, limit=1) + self.evaluate(v) + + def testLargeLimits(self): + # Test case for GitHub issue 46913. + with self.session(): + with self.assertRaises(errors_impl.ResourceExhaustedError): + v = math_ops.range(0, 9223372036854775807) + self.evaluate(v) # TODO(vrv): move to sequence_ops_test? class LinSpaceTest(test.TestCase): diff --git a/tensorflow/python/kernel_tests/matrix_solve_op_test.py b/tensorflow/python/kernel_tests/matrix_solve_op_test.py index 0d149de2acb5e5..1739b2272be810 100644 --- a/tensorflow/python/kernel_tests/matrix_solve_op_test.py +++ b/tensorflow/python/kernel_tests/matrix_solve_op_test.py @@ -112,6 +112,12 @@ def testWrongDimensions(self): with self.assertRaises((ValueError, errors_impl.InvalidArgumentError)): self.evaluate(linalg_ops.matrix_solve(matrix, rhs)) + # The matrix and right-hand side should have the same batch dimensions + matrix = np.random.normal(size=(2, 6, 2, 2)) + rhs = np.random.normal(size=(2, 3, 2, 2)) + with self.assertRaises((ValueError, errors_impl.InvalidArgumentError)): + self.evaluate(linalg_ops.matrix_solve(matrix, rhs)) + def testNotInvertible(self): # The input should be invertible. with self.assertRaisesOpError("Input matrix is not invertible."): diff --git a/tensorflow/python/kernel_tests/pooling_ops_3d_test.py b/tensorflow/python/kernel_tests/pooling_ops_3d_test.py index 051f7e1168a72b..83d26e27d9849f 100644 --- a/tensorflow/python/kernel_tests/pooling_ops_3d_test.py +++ b/tensorflow/python/kernel_tests/pooling_ops_3d_test.py @@ -21,6 +21,7 @@ import numpy as np from tensorflow.python.framework import constant_op +from tensorflow.python.framework import errors from tensorflow.python.framework import test_util from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import gradients_impl @@ -505,6 +506,19 @@ def testAvgPoolGradSamePadding3_1_3d(self): strides=(1, 1, 1), padding="SAME") + def testMaxPool3DZeroPoolSize(self): + # Test case for GitHub issue 51936. + for f in [nn_ops.max_pool3d, nn_ops.avg_pool3d]: + with self.session(): + with self.assertRaises((errors.InvalidArgumentError, ValueError)): + input_sizes = [3, 4, 10, 11, 12] + + input_data = 1. + input_tensor = constant_op.constant( + input_data, shape=input_sizes, name="input") + pool_3d = f(input_tensor, ksize=[2, 2, 0], strides=1, padding="VALID") + self.evaluate(pool_3d) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/kernel_tests/segment_reduction_ops_test.py b/tensorflow/python/kernel_tests/segment_reduction_ops_test.py index d4ff43b8341917..4313fced7323ff 100644 --- a/tensorflow/python/kernel_tests/segment_reduction_ops_test.py +++ b/tensorflow/python/kernel_tests/segment_reduction_ops_test.py @@ -267,6 +267,20 @@ def testDataInvalid(self): data=np.uint16(10), segment_ids=np.array([]).astype("int64")) self.evaluate(s) + def testInvalidIds(self): + # Test case for GitHub issue 46888. + for op in [ + math_ops.segment_max, + math_ops.segment_min, + math_ops.segment_mean, + math_ops.segment_sum, + math_ops.segment_prod, + ]: + with self.cached_session(use_gpu=False): + with self.assertRaises((ValueError, errors_impl.InternalError)): + s = op(data=np.ones((1, 10, 1)), segment_ids=[1676240524292489355]) + self.evaluate(s) + class UnsortedSegmentTest(SegmentReductionHelper): diff --git a/tensorflow/python/kernel_tests/shape_ops_test.py b/tensorflow/python/kernel_tests/shape_ops_test.py index c5f6d02da64efc..cfd216f1d3fbe8 100644 --- a/tensorflow/python/kernel_tests/shape_ops_test.py +++ b/tensorflow/python/kernel_tests/shape_ops_test.py @@ -723,6 +723,17 @@ def testShapeFunctionEdgeCases(self): inp, array_ops.placeholder( dtypes.int32, shape=[3])) + def testLargeTensor(self): + # Test case for GItHub issue 46911. + if test_util.is_xla_enabled(): + # The following test fails with XLA enabled. + return + with self.assertRaises(errors_impl.InternalError): + with self.cached_session(): + tiled = array_ops.tile( + np.ones((1, 1, 1)), [100000000, 100000000, 100000000]) + self.evaluate(tiled) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/kernel_tests/sparse_serialization_ops_test.py b/tensorflow/python/kernel_tests/sparse_serialization_ops_test.py index 5a48eb825dbfa8..434287679181bb 100644 --- a/tensorflow/python/kernel_tests/sparse_serialization_ops_test.py +++ b/tensorflow/python/kernel_tests/sparse_serialization_ops_test.py @@ -20,10 +20,12 @@ import numpy as np +from tensorflow.python.eager import def_function from tensorflow.python.framework import dtypes from tensorflow.python.framework import sparse_tensor as sparse_tensor_lib from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gen_resource_variable_ops from tensorflow.python.ops import sparse_ops from tensorflow.python.platform import test @@ -464,6 +466,18 @@ def testDeserializeManyFailsInvalidProto(self): self._testDeserializeFailsInvalidProtoHelper( sparse_ops.serialize_sparse, sparse_ops.deserialize_many_sparse) + def testDeserializeInvalidVariant(self): + mu = gen_resource_variable_ops.mutex_v2() + mu_lock = gen_resource_variable_ops.mutex_lock(mutex=mu) + + @def_function.function + def f(): + return sparse_ops.deserialize_sparse( + serialized_sparse=mu_lock, dtype=dtypes.int32) + + with self.assertRaisesRegex(ValueError, r"Shape must be at least rank 1"): + f() + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/kernel_tests/split_op_test.py b/tensorflow/python/kernel_tests/split_op_test.py index 58674abd144df1..23f8f3878ca0cc 100644 --- a/tensorflow/python/kernel_tests/split_op_test.py +++ b/tensorflow/python/kernel_tests/split_op_test.py @@ -387,6 +387,24 @@ def testNonexistentDimTensor(self): "must have exactly one element"): sess.run(y, {x: np.array([], dtype=np.int32), splits: [4, 11, 15]}) + @test_util.run_in_graph_and_eager_modes + def testNegativeSizes(self): + x = constant_op.constant([1, 2, 3], dtypes.float32) + # A size of -1 signifies to determine size based on sum of other splits. + with self.assertRaisesRegex((ValueError, errors_impl.InvalidArgumentError), + "Split size at index 1 must be >= 0. Got: -2"): + splits = [-1, -2] + self.evaluate(array_ops.split(x, splits, axis=0)) + + @test_util.run_in_graph_and_eager_modes + def testBadSplitSizes(self): + x = constant_op.constant([1, 2], dtypes.float32) + with self.assertRaisesRegex((ValueError, errors_impl.InvalidArgumentError), + "Determined shape must either match input" + "|can't split axis"): + splits = [1, 2] + self.evaluate(array_ops.split(x, splits, axis=0)) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/kernel_tests/tridiagonal_matmul_op_test.py b/tensorflow/python/kernel_tests/tridiagonal_matmul_op_test.py index 38544000902359..3a4be581676fca 100644 --- a/tensorflow/python/kernel_tests/tridiagonal_matmul_op_test.py +++ b/tensorflow/python/kernel_tests/tridiagonal_matmul_op_test.py @@ -23,12 +23,15 @@ import numpy as np from tensorflow.python.client import session +from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import gradient_checker_v2 +from tensorflow.python.ops import linalg_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import variables from tensorflow.python.ops.linalg import linalg_impl @@ -179,6 +182,37 @@ def testGradientComplexWithBatches(self): rhs = self._randomComplexArray((b, m, n)) self._gradientTest(diags, rhs, dtype=dtypes.complex128) + def _testErrorWithShapesEager(self, exception_regex, superdiag_shape, + maindiag_shape, subdiag_shape, rhs_shape): + with context.eager_mode(): + superdiag = array_ops.ones(superdiag_shape) + maindiag = array_ops.ones(maindiag_shape) + subdiag = array_ops.ones(subdiag_shape) + rhs = array_ops.ones(rhs_shape) + with self.assertRaisesRegex(errors_impl.InvalidArgumentError, + exception_regex): + linalg_ops.tridiagonal_mat_mul(superdiag, maindiag, subdiag, rhs) + + def testInvalidShapesEagerGpu(self): + if not test.is_gpu_available(): + self.skipTest('Test requires GPU') + self._testErrorWithShapesEager('Input must have rank >= 2, but got ', + [2], [2], [2], [2]) + self._testErrorWithShapesEager( + 'superdiag must have same rank as rhs, but got 3 and 2', + [2, 1, 2], [2, 1], [2, 1], [2, 2]) + self._testErrorWithShapesEager( + 'maindiag must have same outer dimensions as rhs, but for index 0, got ' + '3 and 2', + [2, 1, 2], [3, 1, 2], [2, 1, 2], [2, 2, 2]) + self._testErrorWithShapesEager( + "subdiag's second-to-last dimension must be 1, but got 3", + [2, 1, 2], [2, 1, 2], [2, 3, 2], [2, 2, 2]) + self._testErrorWithShapesEager( + "subdiag's last dimension size must be rhs's second-to-last dimension " + "size, but got 3 and 2", + [2, 1, 2], [2, 1, 2], [2, 1, 3], [2, 2, 2]) + # Benchmark class TridiagonalMatMulBenchmark(test.Benchmark): sizes = [(100000, 1, 1), (1000000, 1, 1), (10000000, 1, 1), (100000, 10, 1), diff --git a/tensorflow/python/kernel_tests/xent_op_test.py b/tensorflow/python/kernel_tests/xent_op_test.py index b1adbd37e3e862..65be235cf77465 100644 --- a/tensorflow/python/kernel_tests/xent_op_test.py +++ b/tensorflow/python/kernel_tests/xent_op_test.py @@ -195,6 +195,13 @@ def testShapeMismatch(self): gen_nn_ops.softmax_cross_entropy_with_logits( [[0., 1.], [2., 3.]], [[0., 1., 0.], [1., 0., 0.]]) + tf_f = constant_op.constant(np.array([[1.]]).astype(np.float32)) + tf_l = constant_op.constant(np.array([[1.], [1.]]).astype(np.float32)) + tf_loss, tf_gradient = gen_nn_ops.softmax_cross_entropy_with_logits( + tf_f, tf_l) + self.assertAllClose([0, 0], tf_loss) + self.assertAllCloseAccordingToType([[0], [0]], tf_gradient) + @test_util.run_deprecated_v1 def testNotMatrix(self): with self.cached_session(): diff --git a/tensorflow/python/ops/bincount_ops_test.py b/tensorflow/python/ops/bincount_ops_test.py index 039f3452db85db..305791beca6c98 100644 --- a/tensorflow/python/ops/bincount_ops_test.py +++ b/tensorflow/python/ops/bincount_ops_test.py @@ -835,6 +835,25 @@ def test_ragged_input_different_shape_fails(self): self.evaluate(bincount_ops.sparse_bincount(x, weights=weights, axis=-1)) +class RawOpsHeapOobTest(test.TestCase, parameterized.TestCase): + + @test_util.run_v1_only("Test security error") + def testSparseCountSparseOutputBadIndicesShapeTooSmall(self): + indices = [1] + values = [[1]] + weights = [] + dense_shape = [10] + with self.assertRaisesRegex(ValueError, + "Shape must be rank 2 but is rank 1 for"): + self.evaluate( + gen_count_ops.SparseCountSparseOutput( + indices=indices, + values=values, + dense_shape=dense_shape, + weights=weights, + binary_output=True)) + + @test_util.run_all_in_graph_and_eager_modes @test_util.disable_tfrt class RawOpsTest(test.TestCase, parameterized.TestCase): diff --git a/tensorflow/python/ops/image_ops_test.py b/tensorflow/python/ops/image_ops_test.py index 82af4ce06f0ec3..67998d609eab04 100644 --- a/tensorflow/python/ops/image_ops_test.py +++ b/tensorflow/python/ops/image_ops_test.py @@ -2254,6 +2254,21 @@ def testNameScope(self): y = image_ops.pad_to_bounding_box(image, 0, 0, 55, 66) self.assertTrue(y.op.name.startswith("pad_to_bounding_box")) + def testInvalidInput(self): + # Test case for GitHub issue 46890. + if test_util.is_xla_enabled(): + # TODO(b/200850176): test fails with XLA. + return + with self.session(): + with self.assertRaises(errors_impl.InternalError): + v = image_ops.pad_to_bounding_box( + image=np.ones((1, 1, 1)), + target_height=5191549470, + target_width=5191549470, + offset_height=1, + offset_width=1) + self.evaluate(v) + class SelectDistortedCropBoxTest(test_util.TensorFlowTestCase): @@ -3161,6 +3176,14 @@ def testPreserveAspectRatioSquare(self): self._assertResizeCheckShape(x, x_shape, [320, 320], [320, 320, 3]) + def testLargeDim(self): + with self.session(): + with self.assertRaises(errors.InternalError): + x = np.ones((5, 1, 1, 2)) + v = image_ops.resize_images_v2(x, [1610637938, 1610637938], + image_ops.ResizeMethod.BILINEAR) + _ = self.evaluate(v) + class ResizeImagesTest(test_util.TensorFlowTestCase, parameterized.TestCase): @@ -6026,6 +6049,16 @@ def testImageCropAndResize(self): crop_size=[1, 1]) self.evaluate(op) + def DISABLED_testImageCropAndResizeWithInvalidInput(self): + with self.session(): + with self.assertRaises((errors.InternalError, ValueError)): + op = image_ops_impl.crop_and_resize_v2( + image=np.ones((1, 1, 1, 1)), + boxes=np.ones((11, 4)), + box_indices=np.ones((11)), + crop_size=[2065374891, 1145309325]) + self.evaluate(op) + @parameterized.named_parameters( ("_jpeg", "JPEG", "jpeg_merge_test1.jpg"), ("_png", "PNG", "lena_rgba.png"), diff --git a/tensorflow/python/ops/nn_fused_batchnorm_test.py b/tensorflow/python/ops/nn_fused_batchnorm_test.py index 0421829bff338b..1f4663d495a757 100644 --- a/tensorflow/python/ops/nn_fused_batchnorm_test.py +++ b/tensorflow/python/ops/nn_fused_batchnorm_test.py @@ -20,10 +20,13 @@ import numpy as np +from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors_impl from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gen_nn_ops from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import math_ops @@ -669,6 +672,126 @@ def testBatchNormGradGradConfig6(self): } self._testBatchNormGradGrad(config) + def testEagerShapeErrors(self): + with context.eager_mode(): + x = array_ops.ones((2, 2, 2, 2)) + scale = array_ops.ones((3,)) + offset = array_ops.ones((2,)) + with self.assertRaisesRegex( + errors_impl.InvalidArgumentError, + 'scale must have the same number of elements'): + nn_impl.fused_batch_norm(x, scale, offset) + + x = array_ops.ones((2, 2, 2, 2)) + scale = array_ops.ones((2,)) + offset = array_ops.ones((3,)) + with self.assertRaisesRegex( + errors_impl.InvalidArgumentError, + 'offset must have the same number of elements'): + nn_impl.fused_batch_norm(x, scale, offset) + + x = array_ops.ones((2, 2, 2, 2)) + scale = array_ops.ones((2,)) + offset = array_ops.ones((2,)) + mean = array_ops.ones((0,)) + variance = array_ops.ones((2,)) + with self.assertRaisesRegex( + errors_impl.InvalidArgumentError, + 'When is_training=false, mean must have the same number of elements'): + nn_impl.fused_batch_norm( + x, scale, offset, mean=mean, variance=variance, is_training=False) + + x = array_ops.ones((2, 2, 2, 2)) + scale = array_ops.ones((2,)) + offset = array_ops.ones((2,)) + mean = array_ops.ones((2,)) + variance = array_ops.ones((0,)) + with self.assertRaisesRegex( + errors_impl.InvalidArgumentError, + 'When is_training=false, variance must have the same number of ' + 'elements'): + nn_impl.fused_batch_norm( + x, scale, offset, mean=mean, variance=variance, is_training=False) + + x = array_ops.ones((2, 2, 2, 2)) + scale = array_ops.ones((2,)) + offset = array_ops.ones((2,)) + mean = array_ops.ones((0,)) + variance = array_ops.ones((2,)) + with self.assertRaisesRegex( + errors_impl.InvalidArgumentError, + 'When exponential_avg_factor != 1, mean must have the same number of ' + 'elements'): + nn_impl.fused_batch_norm( + x, + scale, + offset, + mean=mean, + variance=variance, + exponential_avg_factor=0.5) + + x = array_ops.ones((2, 2, 2, 2)) + scale = array_ops.ones((2,)) + offset = array_ops.ones((2,)) + mean = array_ops.ones((2,)) + variance = array_ops.ones((0,)) + with self.assertRaisesRegex( + errors_impl.InvalidArgumentError, + 'When exponential_avg_factor != 1, variance must have the same ' + 'number of elements'): + nn_impl.fused_batch_norm( + x, + scale, + offset, + mean=mean, + variance=variance, + exponential_avg_factor=0.5) + + def testEagerShapeGradErrors(self): + with context.eager_mode(): + y_backprop = array_ops.ones((2, 2, 2, 3)) + x = array_ops.ones((2, 2, 2, 2)) + scale = array_ops.ones((2,)) + reserve_space_1 = array_ops.ones((2,)) + reserve_space_2 = array_ops.ones((2,)) + with self.assertRaisesRegex(errors_impl.InvalidArgumentError, + 'x and y_backprop must have same shape,'): + gen_nn_ops.fused_batch_norm_grad_v2(y_backprop, x, scale, + reserve_space_1, reserve_space_2) + + y_backprop = array_ops.ones((2, 2, 2, 2)) + x = array_ops.ones((2, 2, 2, 2)) + scale = array_ops.ones((3,)) + reserve_space_1 = array_ops.ones((2,)) + reserve_space_2 = array_ops.ones((2,)) + with self.assertRaisesRegex( + errors_impl.InvalidArgumentError, + 'scale must have the same number of elements'): + gen_nn_ops.fused_batch_norm_grad_v2(y_backprop, x, scale, + reserve_space_1, reserve_space_2) + + y_backprop = array_ops.ones((2, 2, 2, 2)) + x = array_ops.ones((2, 2, 2, 2)) + scale = array_ops.ones((2,)) + reserve_space_1 = array_ops.ones((3,)) + reserve_space_2 = array_ops.ones((2,)) + with self.assertRaisesRegex( + errors_impl.InvalidArgumentError, + 'reserve_space_1 must have the same number of elements'): + gen_nn_ops.fused_batch_norm_grad_v2(y_backprop, x, scale, + reserve_space_1, reserve_space_2) + + y_backprop = array_ops.ones((2, 2, 2, 2)) + x = array_ops.ones((2, 2, 2, 2)) + scale = array_ops.ones((2,)) + reserve_space_1 = array_ops.ones((2,)) + reserve_space_2 = array_ops.ones((3,)) + with self.assertRaisesRegex( + errors_impl.InvalidArgumentError, + 'reserve_space_2 must have the same number of elements'): + gen_nn_ops.fused_batch_norm_grad_v2(y_backprop, x, scale, + reserve_space_1, reserve_space_2) + if __name__ == '__main__': test.main() diff --git a/tensorflow/python/ops/ragged/ragged_cross_op_test.py b/tensorflow/python/ops/ragged/ragged_cross_op_test.py index 07e5964ba83497..bfe7aa31a65325 100644 --- a/tensorflow/python/ops/ragged/ragged_cross_op_test.py +++ b/tensorflow/python/ops/ragged/ragged_cross_op_test.py @@ -22,11 +22,14 @@ import numpy as np +from tensorflow.python.eager import def_function from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import tensor_spec from tensorflow.python.framework import test_util +from tensorflow.python.ops import gen_ragged_array_ops from tensorflow.python.ops import sparse_ops from tensorflow.python.ops.ragged import ragged_array_ops from tensorflow.python.ops.ragged import ragged_factory_ops @@ -362,6 +365,16 @@ def testRaggedCrossLargeBatch(self): dense_const([[2], [3]])], exception=(ValueError, errors.InvalidArgumentError), message='inputs must all have the same batch dimension size'), + dict( + testcase_name='3DDenseTensor', + inputs=[dense_const([[[1]]])], + exception=(ValueError, errors.InvalidArgumentError), + message='tf.ragged.cross only supports inputs with rank=2'), + dict( + testcase_name='0DDenseTensor', + inputs=[dense_const(1)], + exception=(ValueError, errors.InvalidArgumentError), + message='tf.ragged.cross only supports inputs with rank=2'), ]) def testStaticError(self, inputs, exception=ValueError, message=None): with self.assertRaisesRegex(exception, message): @@ -372,17 +385,36 @@ def testStaticError(self, inputs, exception=ValueError, message=None): testcase_name='3DRaggedTensor', inputs=[ragged_const([[[1]]], ragged_rank=1)], message='tf.ragged.cross only supports inputs with rank=2'), + dict( + testcase_name='0DDenseTensor', + inputs=[dense_const(1)], + signature=[[tensor_spec.TensorSpec(None, dtypes.int32)]], + exception=(ValueError, errors.InvalidArgumentError), + message='tf.ragged.cross only supports inputs with rank=2'), + dict( + testcase_name='1DDenseTensor', + inputs=[dense_const([1])], + signature=[[tensor_spec.TensorSpec(None, dtypes.int32)]], + exception=(ValueError, errors.InvalidArgumentError), + message='tf.ragged.cross only supports inputs with rank=2'), dict( testcase_name='3DDenseTensor', inputs=[dense_const([[[1]]])], + signature=[[tensor_spec.TensorSpec(None, dtypes.int32)]], + exception=(ValueError, errors.InvalidArgumentError), message='tf.ragged.cross only supports inputs with rank=2'), ]) def testRuntimeError(self, inputs, exception=errors.InvalidArgumentError, - message=None): + message=None, + signature=None): + @def_function.function(input_signature=signature) + def fn(x): + return ragged_array_ops.cross(x) + with self.assertRaisesRegex(exception, message): - self.evaluate(ragged_array_ops.cross(inputs)) + self.evaluate(fn(inputs)) def _ragged_to_sparse(self, t): if ragged_tensor.is_ragged(t): @@ -392,6 +424,42 @@ def _ragged_to_sparse(self, t): else: return ops.convert_to_tensor(t) + def testSparseValuesAndIndicesMustMatch(self): + with self.assertRaisesRegex( + (ValueError, errors.InvalidArgumentError), + 'sparse indices and values must have the same length'): + self.evaluate(gen_ragged_array_ops.RaggedCross( + ragged_values=[], + ragged_row_splits=[], + sparse_indices=[[5]], + sparse_values=[], + sparse_shape=[5], + dense_inputs=[['a']], + input_order='RD', + hashed_output=False, + num_buckets=5, + hash_key=2, + out_values_type=dtypes.string, + out_row_splits_type=dtypes.int64)) + + def testRaggedValuesAndSplitsMustMatch(self): + with self.assertRaisesRegex( + (ValueError, errors.InvalidArgumentError), + 'ragged values and splits must have the same length'): + self.evaluate(gen_ragged_array_ops.RaggedCross( + ragged_values=[['a']], + ragged_row_splits=[], + sparse_indices=[], + sparse_values=[], + sparse_shape=[], + dense_inputs=[['a']], + input_order='RD', + hashed_output=False, + num_buckets=5, + hash_key=2, + out_values_type=dtypes.string, + out_row_splits_type=dtypes.int64)) + if __name__ == '__main__': googletest.main() diff --git a/tensorflow/python/summary/writer/writer_test.py b/tensorflow/python/summary/writer/writer_test.py index 19138b1372dea6..9fcac4952f6a25 100644 --- a/tensorflow/python/summary/writer/writer_test.py +++ b/tensorflow/python/summary/writer/writer_test.py @@ -34,6 +34,7 @@ from tensorflow.python.client import session from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors_impl from tensorflow.python.framework import meta_graph from tensorflow.python.framework import ops from tensorflow.python.framework import test_util @@ -685,6 +686,16 @@ def testSharing_withExplicitSummaryFileWriters(self): # No more files self.assertRaises(StopIteration, lambda: next(event_paths)) + def testSummaryFileWritersInvalidInput(self): + # Test case for GitHub issue 46909 + logdir = self.get_temp_dir() + with session.Session() as sess: + with self.assertRaises(errors_impl.InvalidArgumentError): + writer = summary_ops_v2.create_file_writer( + logdir=logdir, flush_millis=[1, 2]) + sess.run(writer.init()) + sess.run(writer.flush()) + class FileWriterCacheTest(test.TestCase): """FileWriterCache tests.""" diff --git a/tensorflow/python/tools/saved_model_cli.py b/tensorflow/python/tools/saved_model_cli.py index bbb03f7cc7fdd1..8565983a0f271b 100644 --- a/tensorflow/python/tools/saved_model_cli.py +++ b/tensorflow/python/tools/saved_model_cli.py @@ -24,6 +24,7 @@ from __future__ import print_function import argparse +import ast import os import re import sys @@ -518,7 +519,7 @@ def preprocess_inputs_arg_string(inputs_str): return input_dict -def preprocess_input_exprs_arg_string(input_exprs_str): +def preprocess_input_exprs_arg_string(input_exprs_str, safe=True): """Parses input arg into dictionary that maps input key to python expression. Parses input string in the format of 'input_key=' into a @@ -526,8 +527,10 @@ def preprocess_input_exprs_arg_string(input_exprs_str): Args: input_exprs_str: A string that specifies python expression for input keys. - Each input is separated by semicolon. For each input key: + Each input is separated by semicolon. For each input key: 'input_key=' + safe: Whether to evaluate the python expression as literals or allow + arbitrary calls (e.g. numpy usage). Returns: A dictionary that maps input keys to their values. @@ -542,8 +545,15 @@ def preprocess_input_exprs_arg_string(input_exprs_str): raise RuntimeError('--input_exprs "%s" format is incorrect. Please follow' '"="' % input_exprs_str) input_key, expr = input_raw.split('=', 1) - # ast.literal_eval does not work with numpy expressions - input_dict[input_key] = eval(expr) # pylint: disable=eval-used + if safe: + try: + input_dict[input_key] = ast.literal_eval(expr) + except: + raise RuntimeError( + f'Expression "{expr}" is not a valid python literal.') + else: + # ast.literal_eval does not work with numpy expressions + input_dict[input_key] = eval(expr) # pylint: disable=eval-used return input_dict @@ -656,7 +666,7 @@ def load_inputs_from_input_arg_string(inputs_str, input_exprs_str, tensor_key_feed_dict = {} inputs = preprocess_inputs_arg_string(inputs_str) - input_exprs = preprocess_input_exprs_arg_string(input_exprs_str) + input_exprs = preprocess_input_exprs_arg_string(input_exprs_str, safe=False) input_examples = preprocess_input_examples_arg_string(input_examples_str) for input_tensor_key, (filename, variable_name) in inputs.items(): @@ -920,8 +930,10 @@ def add_run_subparser(subparsers): parser_run.add_argument('--inputs', type=str, default='', help=msg) msg = ('Specifying inputs by python expressions, in the format of' ' "=\'\'", separated by \';\'. ' - 'numpy module is available as \'np\'. ' - 'Will override duplicate input keys from --inputs option.') + 'numpy module is available as \'np\'. Please note that the expression ' + 'will be evaluated as-is, and is susceptible to code injection. ' + 'When this is set, the value will override duplicate input keys from ' + '--inputs option.') parser_run.add_argument('--input_exprs', type=str, default='', help=msg) msg = ( 'Specifying tf.Example inputs as list of dictionaries. For example: ' diff --git a/tensorflow/python/tools/saved_model_cli_test.py b/tensorflow/python/tools/saved_model_cli_test.py index 2580cbd73ca0a9..df43ba0bbb65a0 100644 --- a/tensorflow/python/tools/saved_model_cli_test.py +++ b/tensorflow/python/tools/saved_model_cli_test.py @@ -380,7 +380,7 @@ def testInputPreProcessFormats(self): input_expr_str = 'input3=np.zeros([2,2]);input4=[4,5]' input_dict = saved_model_cli.preprocess_inputs_arg_string(input_str) input_expr_dict = saved_model_cli.preprocess_input_exprs_arg_string( - input_expr_str) + input_expr_str, safe=False) self.assertTrue(input_dict['input1'] == ('/path/file.txt', 'ab3')) self.assertTrue(input_dict['input2'] == ('file2', None)) print(input_expr_dict['input3']) @@ -416,6 +416,11 @@ def testInputPreProcessExamplesWithStrAndBytes(self): } """, feature) + def testInputPreprocessExampleWithCodeInjection(self): + input_examples_str = 'inputs=os.system("echo hacked")' + with self.assertRaisesRegex(RuntimeError, 'not a valid python literal.'): + saved_model_cli.preprocess_input_examples_arg_string(input_examples_str) + def testInputPreProcessFileNames(self): input_str = (r'inputx=C:\Program Files\data.npz[v:0];' r'input:0=c:\PROGRA~1\data.npy') @@ -432,8 +437,8 @@ def testInputPreProcessErrorBadFormat(self): with self.assertRaises(RuntimeError): saved_model_cli.preprocess_inputs_arg_string(input_str) input_str = 'inputx:np.zeros((5))' - with self.assertRaises(RuntimeError): - saved_model_cli.preprocess_input_exprs_arg_string(input_str) + with self.assertRaisesRegex(RuntimeError, 'format is incorrect'): + saved_model_cli.preprocess_input_exprs_arg_string(input_str, safe=False) def testInputParserNPY(self): x0 = np.array([[1], [2]]) diff --git a/tensorflow/python/tpu/tpu_test.py b/tensorflow/python/tpu/tpu_test.py index 8d10d9404d8bde..0858741a4ba491 100644 --- a/tensorflow/python/tpu/tpu_test.py +++ b/tensorflow/python/tpu/tpu_test.py @@ -36,6 +36,7 @@ from tensorflow.python.tpu import tpu from tensorflow.python.tpu import tpu_feed from tensorflow.python.tpu import training_loop +from tensorflow.python.tpu.ops import tpu_ops class TPUContextTest(test.TestCase): @@ -169,6 +170,51 @@ def test_prune_unconnected_ops(self): graph.get_operation_by_name("import/y").get_attr( tpu._TPU_REPLICATE_ATTR) + +class TPUOpsTest(test.TestCase): + + def test_all_to_all_zero_split_count(self): + with self.assertRaisesRegex( + ValueError, "split_count 0 must at least be one"): + tpu_ops.all_to_all( + x=[0.0, 0.1652, 0.6543], + group_assignment=[1, -1], + concat_dimension=0, + split_dimension=0, + split_count=0) + + def test_all_to_all_group_assignment_wrong_shape(self): + with self.assertRaisesRegex( + ValueError, "group_assignment must have rank 2"): + tpu_ops.all_to_all( + x=[0.0, 0.1652, 0.6543], + group_assignment=[1, -1], + concat_dimension=0, + split_dimension=0, + split_count=2) + + def test_all_to_all_split_count_not_equal_to_group_assignment_shape(self): + with self.assertRaisesRegex( + ValueError, "split_count 1 must equal the size of the second dimension " + "of group_assignment 2"): + tpu_ops.all_to_all( + x=[0.0, 0.1652, 0.6543], + group_assignment=[[0, 1], [2, 3]], + concat_dimension=0, + split_dimension=0, + split_count=1) + + def test_all_to_all_split_count_not_divide_input_shape(self): + with self.assertRaisesRegex( + ValueError, "input dimension 3 not divisible by split_count 2"): + tpu_ops.all_to_all( + x=[[0.0], [0.1652], [0.6543]], + group_assignment=[[0, 1], [2, 3]], + concat_dimension=1, + split_dimension=0, + split_count=2) + + def do_einsum(): a = array_ops.placeholder(dtype=dtypes.float32, name="a", shape=[2, 3, 4]) b = array_ops.placeholder(dtype=dtypes.float32, name="b", shape=[2, 4, 5]) diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl index 4962bebe56d4b8..975434f8fb2a2f 100644 --- a/tensorflow/tensorflow.bzl +++ b/tensorflow/tensorflow.bzl @@ -48,7 +48,7 @@ load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") # not contain rc or alpha, only numbers. # Also update tensorflow/core/public/version.h # and tensorflow/tools/pip_package/setup.py -VERSION = "2.5.1" +VERSION = "2.5.2" VERSION_MAJOR = VERSION.split(".")[0] two_gpu_tags = ["requires-gpu-nvidia:2", "notap", "manual", "no_pip"] diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index 15df1e70e8e900..bcf189447655b2 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -50,7 +50,7 @@ # result for pip. # Also update tensorflow/tensorflow.bzl and # tensorflow/core/public/version.h -_VERSION = '2.5.1' +_VERSION = '2.5.2' # We use the same setup.py for all tensorflow_* packages and for the nightly diff --git a/tensorflow/workspace2.bzl b/tensorflow/workspace2.bzl index df4b687d720525..5560251f00176f 100644 --- a/tensorflow/workspace2.bzl +++ b/tensorflow/workspace2.bzl @@ -622,12 +622,12 @@ def _tf_repositories(): tf_http_archive( name = "curl", build_file = "//third_party:curl.BUILD", - sha256 = "b0a3428acb60fa59044c4d0baae4e4fc09ae9af1d8a3aa84b2e3fbcd99841f77", - strip_prefix = "curl-7.77.0", + sha256 = "ed936c0b02c06d42cf84b39dd12bb14b62d77c7c4e875ade022280df5dcc81d7", + strip_prefix = "curl-7.78.0", system_build_file = "//third_party/systemlibs:curl.BUILD", urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/curl.haxx.se/download/curl-7.77.0.tar.gz", - "https://curl.haxx.se/download/curl-7.77.0.tar.gz", + "https://storage.googleapis.com/mirror.tensorflow.org/curl.haxx.se/download/curl-7.78.0.tar.gz", + "https://curl.haxx.se/download/curl-7.78.0.tar.gz", ], )